Make linter happy

Some of these things are now part of CI + there were some possible places for misunderstandings.

.env.template

@@ -0,0 +1,3 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+PATE_TOKEN=

.eslintrc.json

@@ -8,17 +8,10 @@
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
   "rules": {
-    "no-console": "off",
+    "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error"

.github/workflows/release-oodikone.yaml

@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'
 
 on:
   release:
@@ -6,23 +6,23 @@ on:
 
 jobs:
   build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -38,26 +38,26 @@ jobs:
           target: production
 
   build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
         id: get_current_time
         run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -72,11 +72,11 @@ jobs:
           push: true
           tags: toska/oodikone2-frontend:latest
           target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
         run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
         run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -87,52 +87,19 @@ jobs:
           environment: production
           set_commits: 'skip'
           sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
+          url_prefix: '~/static/js'
           version: ${{ steps.extract_version.outputs.VERSION }}
 
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
-        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
-        # This steps extracts release version from GITHUB_REF env var
-        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
-        id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
-            SENTRY_ENVIRONMENT=production
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
-
   report_success:
     runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: success()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -141,15 +108,14 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: failure()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/test-oodikone.yaml

@@ -1,4 +1,4 @@
-name: "Test oodikone and release to staging"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
@@ -18,55 +18,41 @@ jobs:
   # === Static tests / linting ===
 
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
-      - name: "Run backend linter"
+      - name: 'Run backend linter'
         run: npm run eslint services/backend
 
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run concurrently \
-              'eslint --ext .jsx,.js services/frontend' \
-              'stylelint services/frontend/src/**/*.css'
-
-  lint_userservice:
-    name: "Lint userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: "Install dependencies"
-        run: npm ci && cd services/userservice && npm ci
-      - name: "Run frontend linter"
-        run: npm run eslint services/userservice
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
 
   # === Build images ===
-
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -81,21 +67,21 @@ jobs:
           tags: toska/oodikone2-backend:staging
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
 
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Get current time"
+      - name: 'Get current time'
         id: get_current_time
         run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -111,334 +97,395 @@ jobs:
           tags: toska/oodikone2-frontend:staging
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
 
-  build_userservice:
-    name: "Build userservice"
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
         with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:staging
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: "Upload image for other jobs"
-        uses: actions/upload-artifact@v2
-        with:
-          path: /tmp/userservice.tar
-
-  # === Integration tests ===
-  #
-  run_userservice_integration_tests:
-    name: "Run userservice integration tests"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: "Run tests included in docker build"
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
-
-  # === End-to-end tests ===
-
-  test_course_statistics:
-    name: "Test course statistics"
-    needs:
-      - build_backend
-      - build_frontend
-      - build_userservice
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: "Download images"
-        uses: actions/download-artifact@v2
-        with:
-          path: /tmp/
-      - name: "Load built images"
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
+
+  # === End-to-end tests ===
+  test_course_statistics:
+    name: 'Test course statistics'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built images'
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
-    name: "Test population statistics"
+    name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Population_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
-    name: "Test filters"
+    name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
-    name: "Test student statistics"
+    name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Student_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
-    name: "Test studyprogramme overview"
+    name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Studyprogramme_overview.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
-    name: "Test teachers and users"
+    name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -447,94 +494,92 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Check that exit statuses are not errors"
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   # === Release ===
 
   release_images:
-    name: "Release built images"
+    name: 'Release built images'
     runs-on: ubuntu-latest
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+      - lint_backend
+      - lint_frontend
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Create a container from frontend"
+      - name: 'Create a container from frontend'
         run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
         run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -545,19 +590,21 @@ jobs:
           environment: staging
           set_commits: 'skip'
           sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
+          url_prefix: '~/static/js'
           version: ${{ github.sha }}
-      - name: "Push to docker hub"
-        run: docker-compose --file docker-compose.ci.yml push
-      - name: "Send deployment success notification to slack"
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+      - name: 'Send deployment success notification to slack'
         if: success()
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
           softa-url: https://oodikone-staging.cs.helsinki.fi/
           deployment-target: staging
-      - name: "Send deployment failure notification to slack"
+      - name: 'Send deployment failure notification to slack'
         if: failure()
         uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
@@ -567,25 +614,23 @@ jobs:
 
   report_test_failure:
     runs-on: ubuntu-latest
-    name: "Report test failure to oodikone notifications"
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+      - lint_backend
+      - lint_frontend
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
       - uses: actions/checkout@v2
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

README.md

@@ -10,12 +10,19 @@ To run Oodikone locally, you will need the following:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
    - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
-3. (Optional) For real data setup, access to Toskas production servers.
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
 ## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
@@ -36,8 +43,8 @@ What different CLI options do:
    - Cleans up any previous real data databases
    - Downloads needed database dumps from production servers.
    - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.
 
 ## ⌨️ Development
 
@@ -72,7 +79,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
 ### User types for development
@@ -117,7 +124,15 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
+
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
 
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
@@ -128,3 +143,7 @@ Just do clean install by launching cli with `npm cli` and running option 1: _Set
 Toska of course.
 
 University of Helsinki
+
+## Tietosuoja / data protection
+
+See [tietosuoja.md](tietosuoja.md).

anonyymioodi/.gitignore

@@ -0,0 +1,2 @@
+*.sqz
+

anonyymioodi/README.md

@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone

anonyymioodi/enrich_sampledata.sql

@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573

cli.sh

@@ -44,20 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
   msg ""
 }
 
-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "
 
-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}
 
-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt
 
 while true; do
   select opt in "${options[@]}"; do
@@ -66,8 +68,10 @@ while true; do
         set_up_oodikone_from_scratch;;
       "Reset all real data.")
         reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress/integration/Course_statistics.js

@@ -186,6 +186,18 @@ describe('Course Statistics tests', () => {
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
+    it.skip('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
     describe('When searching unified course stats', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')

cypress/integration/Filters.js

@@ -9,23 +9,24 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }
 
 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }
 
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    cy.get(`[data-cy="${filterName}-header"]`).click()
     checkFilteringResult(amountWithoutFiltering)
     testStepFunctionToRun()
     checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
   }
 }
 
@@ -41,90 +42,120 @@ describe('Population Statistics', () => {
 
   it('Transfer filter is on not transferred by default', () => {
     cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+    const card = cy.cs('TransferredToProgramme-filter-card')
+    card.cs('TransferredToProgramme-header').click()
+    card.get('[data-cy="option-havenot"] input').should('be.checked')
+    card.cs('TransferredToProgramme-header').click()
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
   })
 
   it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const card = cy.cs('GraduatedFromProgramme-filter-card')
+
       const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      card.cs('option-graduated-true').click()
       checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      card.cs('option-graduated-false').click()
       checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
     })
   })
 
   it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
       const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('option-have').click()
       checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('option-all').click()
       checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('option-havenot').click()
     })
   })
 
   it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
       checkFilteringResult(137)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
 
   it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
       cy.cs('credit-filter-min').type('50')
       cy.cs('credit-filter-max').type('150')
       checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
     })
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(41)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
       checkFilteringResult(121)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
-  it('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(10)
       clearSingleDropdownSelection('admissionTypeFilter-dropdown')
     })
@@ -134,31 +165,32 @@ describe('Population Statistics', () => {
   it.skip('Courses filter works', () => {
     // courses takes some time to load, wait for it to complete
     cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
       cy.cs('courseFilter-course-dropdown').click()
       const courses = ['TKT20001', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
       checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
       checkFilteringResult(131)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(2)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const card = cy.cs('GraduatedFromProgramme-filter-card')
+        card.cs('option-graduated-true').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
         checkFilteringResult(11)
         cy.cs('ageFilter-min').find('input').clear()
         cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        card.cs('option-all').click()
       })
     })
   })
@@ -175,7 +207,7 @@ describe('Course Statistics', () => {
   })
 
   it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
       cy.cs('gradeFilter-5').click()
       checkFilteringResult(20)
       cy.cs('gradeFilter-3').click()
@@ -185,44 +217,68 @@ describe('Course Statistics', () => {
     })
   })
 
+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(32)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
       checkFilteringResult(29)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
         cy.cs('gradeFilter-3').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
@@ -244,13 +300,14 @@ describe('Custom Population Statistics', () => {
     cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
     })
     cy.cs('search-button').click()
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('30')
       checkFilteringResult(1)
@@ -259,36 +316,59 @@ describe('Custom Population Statistics', () => {
     })
   })
 
-  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
     })
   })
 
-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(4)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(4)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
+    })
+  })
+
+  it.skip('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
+      checkFilteringResult(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
       checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })

cypress/integration/Population_statistics.js

@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />
 
+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
   includeSettings.forEach(setting => {
@@ -13,6 +16,8 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
   describe('when using basic user', () => {
     beforeEach(() => {
       cy.init('/populations')
@@ -59,6 +64,7 @@ describe('Population Statistics tests', () => {
 
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
       setPopStatsUntil('toukokuu 2020')
 
       cy.get('.card').within(() => {
@@ -69,7 +75,8 @@ describe('Population Statistics tests', () => {
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
-      cy.contains('Courses of population').click({ force: true })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -123,18 +130,62 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit accumulation (for 170 students)')
     })
 
-    it('Credit Statistics, Statistics pane works', () => {
+    it('Credit Statistics, Credits Gained tab works', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
       cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
       cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
       cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
       cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })

cypress/integration/Student_statistics.js

@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
     studentnumber: '010654019',
   }
 
-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
     cy.contains('Student number').should('not.exist')
   })
 
-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
   })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -79,7 +79,7 @@ describe('Studyprogramme overview', () => {
     let originalProductivityCalculatedText
     const doRecalculation = () => {
       // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(2000)
+      cy.wait(5000)
       cy.get('.attached').contains('Admin').click()
       cy.contains('recalculate productivity').click()
       cy.contains('recalculate throughput').click()

cypress/integration/Users.js

@@ -4,16 +4,16 @@ describe('Users tests', () => {
   describe('Using user with just grp-oodikone-user, no other rights', () => {
     it('shows only frontpage, trends feedback', () => {
       cy.init('', 'norights') // login with norights user
-      cy.get('.navBar').contains('Trends').should('exist')
-      cy.get('.navBar').contains('Give feedback').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
 
-      cy.get('.navBar').contains('Study programme').should('not.exist')
-      cy.get('.navBar').contains('Student statistics').should('not.exist')
-      cy.get('.navBar').contains('Course statistics').should('not.exist')
-      cy.get('.navBar').contains('Teachers').should('not.exist')
-      cy.get('.navBar').contains('Users').should('not.exist')
-      cy.get('.navBar').contains('Faculty').should('not.exist')
-      cy.get('.navBar').contains('Updater').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
     })
   })
 
@@ -29,32 +29,27 @@ describe('Users tests', () => {
     })
 
     it('should see more stuff than others', () => {
-      cy.get('.navBar').contains('Study programme').should('exist')
-      cy.get('.navBar').contains('Student statistics').should('exist')
-      cy.get('.navBar').contains('Course statistics').should('exist')
-      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
-      cy.get('.navBar').contains('Users').should('exist')
-      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
     })
 
     it("mocking normal user shows only the mocked user's programmes", () => {
       cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
+      cy.contains('tr', 'basic').within(() => {
         cy.contains('.button', 'Edit').click()
       })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
       cy.get('i.spy').click()
-      cy.wait('@superlogin')
       cy.contains('mocking as basic')
       cy.wait(1000)
       cy.contains('Study programme').click().siblings().contains('Search by class').click()
       cy.contains('label', 'Study programme')
       cy.contains('label', 'Study programme')
         .siblings()
-        .within($row => {
+        .within(() => {
           cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
         })
     })

cypress/support/commands.js

@@ -10,6 +10,7 @@ const adminUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
 }
 
 const basicUserHeaders = {
@@ -19,6 +20,7 @@ const basicUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
@@ -28,6 +30,7 @@ const onlycoursestatisticsUserHeaders = {
   hyGroupCn: 'grp-oodikone-basic-users',
   eduPersonAffiliation: 'member',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
 }
 
 const norightsUserHeaders = {
@@ -37,6 +40,7 @@ const norightsUserHeaders = {
   hygroupcn: 'grp-oodikone-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
 }
 
 const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
@@ -50,8 +54,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
     if (!headersToUse) throw Error(`${userId} is not valid user id!`)
     req.headers = headersToUse
   })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
   cy.visit(baseUrl.concat(path))
 })
@@ -59,24 +61,33 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const p = subject === undefined ? cy : cy.wrap(subject)
+
+  return p.get(`[data-cy='${name}']`)
 })
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
   const indexes = Array.isArray(index) ? index : [index]
 
   indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
   })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
   }
 })
 

docker-compose.ci.yml

@@ -2,7 +2,6 @@ x-defaults: &defaults
   restart: unless-stopped
 
 services:
-
   backend:
     <<: *defaults
     container_name: backend
@@ -10,17 +9,16 @@ services:
       - kone-db
       - redis
       - sis-db
+      - user-db
     environment:
-      CI: "true"
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     image: toska/oodikone2-backend:staging
 
   frontend:
@@ -31,11 +29,15 @@ services:
   kone-db:
     <<: *defaults
     container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
     image: registry.toska.cs.helsinki.fi/kone-db
 
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
@@ -45,23 +47,20 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
 
   user-db:
     <<: *defaults
     container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
     environment:
-      CI: "true"
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:staging
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
 
   nginx:
     image: nginx:latest
@@ -69,8 +68,45 @@ services:
     depends_on:
       - frontend
       - backend
-      - userservice
     ports:
-      -  1337:80
+      - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,7 +1,7 @@
 services:
-
   backend:
     environment:
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 
@@ -18,9 +18,5 @@ services:
       DB_URL: postgres://postgres@sis-db:5432/sis-db-real
       SIS_IMPORTER_DATABASE: sis-importer-db-real
 
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
   redis-data-real:

docker-compose.test.yml

@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -5,7 +5,6 @@ x-build_defaults: &build_defaults
   target: development
 
 services:
-
   adminer:
     <<: *defaults
     container_name: adminer
@@ -27,16 +26,17 @@ services:
       - kone-db
       - redis
       - sis-db
+      - user-db
+    env_file:
+      - .env
     environment:
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     ports:
       - 8080:8080
     profiles:
@@ -70,11 +70,11 @@ services:
 
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -97,7 +97,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -164,26 +164,6 @@ services:
     volumes:
       - user-db-data:/data
 
-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
   backend-nodemod:
   frontend-nodemod:
@@ -194,4 +174,3 @@ volumes:
   sis-updater-scheduler-nodemod:
   sis-updater-worker-nodemod:
   user-db-data:
-  userservice-nodemod:

package.json

@@ -2,10 +2,11 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "concurrently": "concurrently",
     "cypress": "cypress",
+    "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
@@ -13,29 +14,8 @@
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install",
-    "lint:docker-compose": "docker-compose config -q && :",
-    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
-    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
-    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
-  },
-  "lint-staged": {
-    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
-    "*.{js,json,md}": "prettier --write",
-    "*.css": "stylelint --fix",
-    ".github/workflows/*": "actionlint",
-    "Dockerfile": "hadolint",
-    "*.sh": "shellcheck -x",
-    "docker-compose.yml": "npm run lint:docker-compose",
-    "docker-compose.real.yml": "npm run lint:docker-compose:real",
-    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
-    "docker-compose.test.yml": "npm run lint:docker-compose:test"
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "prepare": "husky install"
   },
   "repository": {
     "type": "git",
@@ -50,24 +30,28 @@
     "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
     "concurrently": "^6.2.1",
-    "cypress": "^8.3.0",
+    "cypress": "^8.3.1",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.1",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^3.4.1",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.1",
+    "husky": "^7.0.2",
     "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",
     "stylelint-config-prettier": "^8.0.2",
     "stylelint-config-standard": "^22.0.0",
     "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
   }
 }

run.sh

@@ -64,8 +64,7 @@ parse_profiles() {
   return 0
 }
 
-# Set which docker-compose files to use based on version. In ci, profiles are also
-# emptied, since they're passed from github actions.
+# Set which docker-compose files to use based on version.
 parse_env() {
   env=""
   if [[ "$version" == "real" ]]; then
@@ -85,7 +84,7 @@ parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
 elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${compose_command}"
+  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

scripts/functions_for_setup.sh

@@ -20,10 +20,10 @@ USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
 
 # Source utility functions
@@ -71,7 +71,7 @@ reset_databases() {
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
-  docker-compose down
+  npm run docker:down
   docker-compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
@@ -110,12 +110,54 @@ reset_all_real_data() {
   reset_databases "${DATABASES[@]}"
 }
 
-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
+
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
+  )
+
+  local database=""
+  local url=""
+
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
 }
 
 set_up_oodikone_from_scratch() {
@@ -123,7 +165,6 @@ set_up_oodikone_from_scratch() {
   folders_to_set_up=(
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
     "$PROJECT_ROOT/services/backend"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"
@@ -152,10 +193,10 @@ linting and formatting"
   npm run prepare
 
   infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon pull
+  "$PROJECT_ROOT"/run.sh both anon pull
 
   infomsg "Building images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon build
 
   successmsg "Setup ready, oodikone can be started! See README for more info."
 }

services/backend/.dockerignore

@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log

services/backend/.gitignore

@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json

services/backend/index.js

@@ -1,5 +1,6 @@
 require('./src/app')
+const logger = require('./src/util/logger')
 
 process.on('unhandledRejection', reason => {
-  console.log(reason)
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
 })

services/backend/package.json

@@ -13,65 +13,28 @@
   "dependencies": {
     "@sentry/node": "^5.30.0",
     "@sentry/tracing": "^6.11.0",
-    "async": "^3.2.0",
-    "async-lock": "^1.2.8",
-    "axios": "^0.21.1",
+    "async": "^3.2.1",
+    "async-lock": "^1.3.0",
+    "axios": "^0.21.2",
     "bluebird": "^3.7.2",
-    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.17.1",
-    "jsonwebtoken": "^8.5.1",
-    "kafka-node": "^4.1.3",
+    "express-async-errors": "^3.1.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
-    "mkdirp": "^0.5.5",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
-    "nodemailer": "^4.7.0",
-    "pg": "^8.6.0",
-    "redis": "^3.1.1",
-    "sequelize": "^6.6.2",
+    "pg": "^8.7.1",
+    "redis": "^3.1.2",
+    "sequelize": "^6.6.5",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0",
-    "winston-transport": "^4.4.0"
+    "winston": "^3.3.3"
   },
   "devDependencies": {
-    "ava": "^2.4.0",
-    "faker": "^4.1.0",
-    "jest": "^24.9.0",
-    "nock": "^10.0.6",
-    "node-status": "^1.0.0",
-    "nodemon": "^1.19.4",
-    "nyc": "^13.3.0",
-    "supertest": "^3.4.2"
-  },
-  "jest": {
-    "setupFilesAfterEnv": [
-      "./test/jest/globals.js"
-    ],
-    "testEnvironment": "node"
-  },
-  "ava": {
-    "files": [
-      "test/**/*_test.js"
-    ],
-    "source": [
-      "**/*.{js,jsx}",
-      "!test/**/*"
-    ],
-    "concurrency": 5,
-    "failFast": true,
-    "failWithoutAssertions": true,
-    "tap": true,
-    "powerAssert": false,
-    "require": [
-      "babel-register"
-    ],
-    "babel": "inherit"
+    "nodemon": "^1.19.4"
   }
 }

services/backend/requests/courses_yearly.rest

@@ -1,7 +0,0 @@
-GET /api/courseyearlystats/?start=2013&end=2018&code=TKT10002 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token generated from login.rest to use

services/backend/requests/enableUser.rest

@@ -1,6 +0,0 @@
-POST http://localhost:8080/api/users/enable
-Content-Type: application/json
-
-{
-  "id": 2
-}

services/backend/requests/get_all_duplicate_codes.rest

@@ -1,6 +0,0 @@
-GET /api/courses/duplicatecodes/all HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_duplicate_codes.rest

@@ -1,7 +0,0 @@
-GET /api/courses/duplicatecodes/?code=TKT10005 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_mandatory_courses.rest

@@ -1,7 +0,0 @@
-GET /api/v2/studyprogrammes/{insertstudyprogrammecodehere}/mandatory_courses
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/get_student.rest

@@ -1,7 +0,0 @@
-GET /api/students/{insert student id}
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/login.rest

@@ -1,7 +0,0 @@
-POST /api/login HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-Cache-Control: no-cache
-Postman-Token: d8a05d0f-fd91-e4e0-54a0-172d18ff3fad
-Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW

services/backend/requests/newUser.rest

@@ -1,4 +0,0 @@
-GET http://localhost:8080/api/login
-Content-Type: application/json
-eduPersonPrincipalName: totutotu@
-shib-session-id: IDEEE

services/backend/requests/post_duplicate_codes.rest

@@ -1,7 +0,0 @@
-POST /api/courses/duplicatecodes/TKT10005/205044 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/remove_duplicate_code.rest

@@ -1,7 +0,0 @@
-DELETE /api/courses/duplicatecodes/TKT10005/58123 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/update.rest

@@ -1,11 +0,0 @@
-POST /api/updatedatabase HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-content-type: application/json
-
-[
-  "01234562",
-  "12302459"
-]

services/backend/src/app.js

@@ -1,58 +1,35 @@
+require('express-async-errors')
 const express = require('express')
-const cors = require('cors')
-const bodyParser = require('body-parser')
-const Sentry = require('@sentry/node')
-const conf = require('./conf-backend')
+const { baseUrl, backendPort } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
-const { PORT } = conf
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-const initializeSentry = require('./util/sentry')
+const logger = require('./util/logger')
 
 initializeDatabaseConnection()
   .then(() => {
     dbConnections.connect()
 
     dbConnections.on('connect', () => {
-      console.log('Connected to sis db successfully')
+      logger.info('Connected to sis db successfully')
     })
 
     dbConnections.on('error', () => {
-      console.log('Failed connecting to sis db')
+      logger.error('Failed to connect to sis db!')
     })
-
-    const app = express()
-    initializeSentry(app)
-    app.use(Sentry.Handlers.requestHandler())
-    app.use(Sentry.Handlers.tracingHandler())
-
     startCron()
 
-    app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-    app.use(bodyParser.json())
+    const app = express()
+    routes(app, baseUrl)
 
-    let BASE_URL = ''
-    if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test') {
-      BASE_URL = '/api'
-    }
-
-    routes(app, BASE_URL)
-
-    app.get('*', async (req, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+    const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
     process.on('SIGTERM', () => {
       server.close(() => {
-        console.log('Process terminated')
+        logger.info('Process terminated')
       })
     })
   })
   .catch(e => {
     process.exitCode = 1
-    console.log(e)
+    logger.error(e)
   })

services/backend/src/conf-backend.js

@@ -1,6 +1,7 @@
 // Node env to use
-const isTest = process.env.NODE_ENV === 'test'
 const isStaging = process.env.NODE_ENV === 'staging'
+const isDev = process.env.NODE_ENV === 'development'
+const isProduction = process.env.NODE_ENV === 'production'
 
 // Sentry
 const sentryRelease = process.env.SENTRY_RELEASE || ''
@@ -12,8 +13,30 @@ const requiredGroup = isStaging
   ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
   : ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
+// Pate
+const pateToken = process.env.PATE_API_TOKEN || ''
+
+// Importer client
+const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
+
+// Networking: Urls & ports
+const baseUrl = isDev ? '/api' : ''
+const frontUrl = process.env.FRONT_URL
+const backendPort = 8080
+
+// Analytics
+const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
+const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
+const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
+const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
+
+// Matomo
+const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
+const MATOMO_URL = process.env.MATOMO_URL
+
 // Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
+const DB_URL_USER = process.env.DB_URL_USER
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -24,20 +47,10 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
   DB_MAX_CRON_CONNECTIONS = 1
 }
 
-const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
-const PORT = isTest ? 8079 : 8080
-const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
-const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
-const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
-const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
-const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
-const MATOMO_URL = process.env.MATOMO_URL
 
 const formatURL = url => {
   return !!url && !url.startsWith('http') ? `http://${url}` : url
@@ -45,22 +58,15 @@ const formatURL = url => {
 
 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
 
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-
 module.exports = {
-  frontend_addr,
+  DB_URL_USER,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
   redis,
-  TOKEN_SECRET,
   SECRET_TOKEN,
   DB_SCHEMA_KONE,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
-  PORT,
   requiredGroup,
-  isTest,
   SIS_UPDATER_URL,
   ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
   ANALYTICS_INFLUXDB_USER,
@@ -71,4 +77,10 @@ module.exports = {
   runningInCI,
   sentryRelease,
   sentryEnvironment,
+  isProduction,
+  pateToken,
+  baseUrl,
+  frontUrl,
+  backendPort,
+  importerToken,
 }

services/backend/src/database/connection.js

@@ -3,6 +3,8 @@ const EventEmitter = require('events')
 const Umzug = require('umzug')
 const conf = require('../conf-backend')
 const { SIS_DB_URL } = process.env
+const logger = require('../util/logger')
+
 class DbConnection extends EventEmitter {
   constructor() {
     super()
@@ -31,76 +33,79 @@ class DbConnection extends EventEmitter {
         this.emit('error', e)
         return
       }
-      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
-      console.log(e)
+      logger.error({ message: `Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`, meta: e })
       setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
     }
   }
 }
 
+// Old-style kone + user db connections
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
   logging: false,
 })
+
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
+const sequelizeUser = new Sequelize(conf.DB_URL_USER, {
+  logging: false,
+})
+
 const initializeDatabaseConnection = async () => {
   const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
+  const rounds = 60
+
+  for (const [seq, dbName] of [
+    [sequelizeKone, 'kone-db'],
+    [sequelizeUser, 'user-db'],
+  ]) {
+    logger.info(`Connecting to ${dbName}...`)
+    for (let round = 1; round <= rounds; round++) {
+      try {
+        await seq.authenticate()
+        break
+      } catch (error) {
+        if (round === rounds) {
+          logger.error(`${dbName} database connection failed!`)
+          throw error
+        }
+        await sleep(1000)
       }
-      console.log('.')
-      await sleep(1000)
     }
-  }
-  if (!conf.isTest) {
+    logger.info(`${dbName} database connection established`)
+
+    const schema = dbName === 'kone-db' ? conf.DB_SCHEMA_KONE : undefined
+    const migrationsFolder = dbName === 'kone-db' ? 'migrations_kone' : 'migrations_user'
+    const migrator = new Umzug({
+      storage: 'sequelize',
+      storageOptions: {
+        sequelize: seq,
+        tableName: 'migrations',
+        schema,
+      },
+      migrations: {
+        params: [seq.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/${migrationsFolder}`,
+        pattern: /\.js$/,
+        schema,
+      },
+    })
     try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelizeKone,
-          tableName: 'migrations',
-          schema: conf.DB_SCHEMA_KONE,
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelizeKone.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/src/database/migrations_kone`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA_KONE,
-        },
-      })
       const migrations = await migrator.up()
-
-      console.log('Kone Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
+      logger.info({ message: `${dbName} migrations up to date: `, meta: migrations.map(m => m.file) })
+    } catch (error) {
+      logger.error({ message: `${dbName} migrations failed`, meta: error })
+      throw error
     }
   }
 }
 
-const forceSyncDatabase = async () => {
-  try {
-    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
-  } catch (e) {
-    //console.log(e)
-  }
-  await sequelizeKone.sync({ force: true })
-}
-
 const dbConnections = new DbConnection()
+
 module.exports = {
   dbConnections,
   sequelizeKone,
+  sequelizeUser,
   initializeDatabaseConnection,
-  forceSyncDatabase,
 }

services/backend/src/database/migrations_kone/20211029_00_add_study_guidance_group_tags.js

@@ -0,0 +1,34 @@
+const { BIGINT, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable(
+      'study_guidance_group_tags',
+      {
+        id: {
+          primaryKey: true,
+          type: BIGINT,
+          autoIncrement: true,
+        },
+        study_guidance_group_id: {
+          type: STRING,
+          unique: true,
+        },
+        study_programme: {
+          type: STRING,
+        },
+        year: {
+          type: STRING,
+        },
+        created_at: {
+          type: DATE,
+        },
+        updated_at: {
+          type: DATE,
+        },
+      },
+      { underscored: true, timestamps: true }
+    )
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20210829_00_add_sisu_person_id.js

@@ -0,0 +1,10 @@
+const { STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('users', 'sisu_person_id', STRING)
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('users', 'sisu_person_id')
+  },
+}

services/backend/src/database/migrations_user/20210829_01_add_studyguidance_groups.js

@@ -0,0 +1,17 @@
+module.exports = {
+  up: queryInterface => {
+    return queryInterface.bulkInsert('access_groups', [
+      {
+        group_code: 'studyGuidanceGroups',
+        group_info: 'grants access to person groups feature',
+      },
+    ])
+  },
+  down: queryInterface => {
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: 'studyGuidanceGroups',
+      },
+    ])
+  },
+}

services/backend/src/database/migrations_user/20210831_00_remove_usage_and_coursegroups.js

@@ -0,0 +1,10 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const Op = Sequelize.Op
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: { [Op.in]: ['usage', 'coursegroups'] },
+      },
+    ])
+  },
+}

services/backend/src/database/migrations_user/20210910_00_remove_moar_access_groups.js

@@ -0,0 +1,10 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const Op = Sequelize.Op
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: { [Op.in]: ['dev', 'users', 'cooldata', 'studyprogramme'] },
+      },
+    ])
+  },
+}

services/backend/src/database/migrations_user/20210922_00_add_last_login.js

@@ -0,0 +1,10 @@
+const { DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('users', 'last_login', DATE)
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('users', 'last_login')
+  },
+}

services/backend/src/database/migrations_user/20211013_00_add_lisensiaatti_to_elainlaakis.js

@@ -0,0 +1,8 @@
+module.exports = {
+  up: async queryInterface =>
+    queryInterface.sequelize.query(`
+      --- add lisensiaatti to lääkis faculty
+      INSERT INTO faculty_programmes VALUES ('H90', 'MH90_001', 'NOW()', 'NOW()');
+    `),
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20211020_00_drop_faculty_programmes.js

@@ -0,0 +1,6 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('faculty_programmes')
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20211227_00_drop_affiliations.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('user_affiliation')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS user_affilation_id_seq')
+    await queryInterface.dropTable('affiliations')
+    await queryInterface.sequelize.query('DROP SEQUENCE IF EXISTS affiliations_id_seq')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -9,204 +9,168 @@ const {
   getStartYears,
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
-const { getAllProgrammes, nonGraduatedStudentsOfElementDetail } = require('./services/studyrights')
+const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const {
-  setProductivity,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-  patchNonGraduatedStudents,
-} = require('./services/analyticsService')
-const { isNewHYStudyProgramme } = require('./util')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
+const { isProduction } = require('./conf-backend')
+const { getCurrentSemester } = require('./services/semesters')
+const logger = require('./util/logger')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
 const refreshStudyrightAssociations = async () => {
-  try {
-    console.log('Refreshing studyright associations...')
-    await refreshAssociationsInRedis()
-  } catch (e) {
-    console.error(e)
-  }
+  await refreshAssociationsInRedis()
+  logger.info('Studyright associations refreshed!')
 }
 
 const refreshOverview = async () => {
-  try {
-    console.log('Refreshing throughput and productivity for programmes...')
-    const codes = (await getAllProgrammes()).map(p => p.code)
-    let ready = 0
-    for (const code of codes) {
-      let programmeStatsSince = new Date('2017-07-31')
-      if (code.includes('MH') || code.includes('KH')) {
-        programmeStatsSince = new Date('2017-07-31')
-      } else {
-        programmeStatsSince = new Date('2000-07-31')
-      }
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-        const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-        await setThroughput(data)
-      } catch (e) {
-        try {
-          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-      }
-      try {
-        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-        const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-        await setProductivity(data)
-      } catch (e) {
-        try {
-          await patchProductivity({
-            [code]: { status: 'RECALCULATION ERRORED' },
-          })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-      }
-      ready += 1
-      console.log(`${ready}/${codes.length} programmes done`)
+  logger.info('Refreshing throughput and productivity for programmes...')
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+  let ready = 0
+  for (const code of codes) {
+    let programmeStatsSince = new Date('2017-07-31')
+    if (code.includes('MH') || code.includes('KH')) {
+      programmeStatsSince = new Date('2017-07-31')
+    } else {
+      programmeStatsSince = new Date('2000-07-31')
     }
-  } catch (e) {
-    console.error(e)
+    try {
+      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
+      await setThroughput(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
+      try {
+        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+      } catch (e) {
+        logger.error({ message: 'failed to update throughtput status to error', meta: e })
+      }
+    }
+    try {
+      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
+      await setProductivity(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
+      try {
+        await patchProductivity({
+          [code]: { status: 'RECALCULATION ERRORED' },
+        })
+      } catch (e) {
+        logger.error({ message: 'failed to update productivity status to error', meta: e })
+      }
+    }
+    ready += 1
+    logger.info(`${ready}/${codes.length} programmes done`)
   }
+  logger.info('Throughput and productivity for programmes refreshed')
 }
 
 const refreshTeacherLeaderboard = async () => {
-  try {
-    const startyearcode = new Date().getFullYear() - 1950
-    const endyearcode = startyearcode + 1
-    console.log('Refreshing teacher leaderboard...')
-    await findAndSaveTeachers(startyearcode, endyearcode)
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const refreshNonGraduatedStudentsOfOldProgrammes = async () => {
-  try {
-    const oldProgrammeCodes = (await getAllProgrammes()).map(p => p.code).filter(c => !isNewHYStudyProgramme(c))
-    let i = 0
-    console.log('Refreshing non-graduated students of old programmes...')
-    await Promise.all(
-      oldProgrammeCodes.map(
-        c =>
-          new Promise(async res => {
-            try {
-              const [nonGraduatedStudents, studentnumbers] = await nonGraduatedStudentsOfElementDetail(c)
-              await patchNonGraduatedStudents({ [c]: { formattedData: nonGraduatedStudents, studentnumbers } })
-              console.log(`${++i}/${oldProgrammeCodes.length}`)
-            } catch (e) {
-              console.log(`Failed refreshing non-graduated students of programme ${c}!`)
-            }
-            res()
-          })
-      )
-    )
-  } catch (e) {
-    console.log(e)
-  }
+  // refresh this and previous year
+  const currentSemestersYearCode = (await getCurrentSemester()).getDataValue('yearcode')
+  await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
 const refreshProtoCtoRedis = async () => {
-  try {
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-    console.log('Refreshing CDS ProtoC')
-    await refreshProtoC(defaultQuery)
-    await refreshProtoC(onlyOld)
-    await refreshProtoC(onlyEnr)
-    await refreshProtoC(bothToggles)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS ProtoC')
+  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
+  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
+  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
+  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
+  await refreshProtoC(defaultQuery)
+  await refreshProtoC(onlyOld)
+  await refreshProtoC(onlyEnr)
+  await refreshProtoC(bothToggles)
+  logger.info('Refreshing CDS ProtoC doned')
 }
 
 const refreshStatusToRedis = async () => {
-  try {
-    const unixMillis = moment().valueOf()
-    const date = new Date(Number(unixMillis))
+  const unixMillis = moment().valueOf()
+  const date = new Date(Number(unixMillis))
 
-    date.setHours(23, 59, 59, 999)
-    const showByYearOff = 'false'
-    const showByYear = 'true'
-    console.log('Refreshing CDS Status')
-    await refreshStatus(date.getTime(), showByYearOff)
-    await refreshStatus(date.getTime(), showByYear)
+  date.setHours(23, 59, 59, 999)
+  const showByYearOff = 'false'
+  const showByYear = 'true'
+  logger.info('Refreshing CDS Status')
+  await refreshStatus(date.getTime(), showByYearOff)
+  await refreshStatus(date.getTime(), showByYear)
+  logger.info('Refreshing CDS Status doned')
 
-    console.log('Refreshing CDS Graduated')
-    await refreshStatusGraduated(date.getTime(), showByYearOff)
-    await refreshStatusGraduated(date.getTime(), showByYear)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS Graduated')
+  await refreshStatusGraduated(date.getTime(), showByYearOff)
+  await refreshStatusGraduated(date.getTime(), showByYear)
+  logger.info('Refreshing CDS graduated doned')
 }
 
 const refreshUberToRedis = async () => {
-  try {
-    const years = await getStartYears()
-    const mappedYears = years.map(({ studystartdate }) => studystartdate)
-    mappedYears.forEach(async year => {
-      console.log('Refreshing CDS Uber data for date', year)
-      const defaultQuery = { include_old_attainments: 'false', start_date: year }
-      const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+  const years = (await getStartYears()).map(({ studystartdate }) => studystartdate)
+  for (const year of years) {
+    const formattedYear = new Date(year).getFullYear()
+    logger.info(`Refreshing CDS uber year ${formattedYear}`)
+    const defaultQuery = { include_old_attainments: 'false', start_date: year }
+    const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+    try {
       await refreshUber(defaultQuery)
       await refreshUber(oldAttainmentsQuery)
-    })
-  } catch (e) {
-    console.log(e)
+      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
+    } catch (e) {
+      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: e })
+    }
   }
+  logger.info(`Refreshing CDS Uber data doned`)
 }
 
 const refreshProtoCProgrammeToRedis = async () => {
-  try {
-    const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-    codes.forEach(async code => {
-      console.log('refreshing code', code)
-      const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-      const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-      const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-      const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-      console.log('Refreshing CDS ProtoCProgramme for code ', code)
-      await refreshProtoCProgramme(defaultQuery)
-      await refreshProtoCProgramme(onlyOld)
-      await refreshProtoCProgramme(onlyEnr)
-      await refreshProtoCProgramme(bothToggles)
-    })
-  } catch (e) {
-    console.log(e)
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
+  for (const code of codes) {
+    logger.info(`Refreshing ProtoCProgramme code ${code}`)
+    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
+    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
+    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
+    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
+    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
+    for (const query of queries) {
+      try {
+        await refreshProtoCProgramme(query)
+      } catch (e) {
+        logger.error({
+          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
+          meta: e,
+        }) // object.keys is hax to print query variable name
+      }
+    }
+    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
   }
+  logger.info(`Refreshing protocprogramme doned`)
 }
 
 const refreshStatistics = async () => {
-  await refreshStudyrightAssociations()
-  await refreshOverview()
-  await refreshNonGraduatedStudentsOfOldProgrammes()
-  await refreshTeacherLeaderboard()
+  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  logger.info('Refreshing statistics')
+  for (const func of statfuncs) {
+    await func()
+  }
+  logger.info('Statistics refreshed!')
 }
 
 const refreshTrends = async () => {
-  await refreshProtoCtoRedis()
-  await refreshStatusToRedis()
-  await refreshUberToRedis()
-  await refreshProtoCProgrammeToRedis()
+  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  logger.info('Refreshing trends')
+  for (const func of trendfuncs) {
+    await func()
+  }
+  logger.info('Trends refreshed!')
 }
 
 const startCron = () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 6 * * *', async () => {
-      await refreshStatistics()
-      await refreshTrends()
+  if (isProduction) {
+    // refresh 3am every day
+    schedule('0 3 * * *', async () => {
+      for (const func of [refreshStatistics, refreshTrends]) {
+        await func()
+      }
     })
   }
 }
@@ -214,4 +178,5 @@ const startCron = () => {
 module.exports = {
   startCron,
   refreshStatistics,
+  refreshTrends,
 }

services/backend/src/middleware/accessLogger.js

@@ -7,7 +7,8 @@ const logger = require('../util/logger')
 // so don't e.g. make it into an async function or it'll print [object Promise] on every request
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const meta = { ...req.decodedToken }
+  const { user } = req
+  const meta = { ...user }
   fields.forEach(field => (meta[field] = tokens[field](req, res)))
   meta['time'] = tokens['date'](req, res, 'iso')
   if (req.route) {
@@ -15,7 +16,7 @@ const accessLogger = morgan((tokens, req, res) => {
   }
 
   const message = [
-    req.decodedToken.name,
+    user.name,
     ':',
     tokens['method'](req, res),
     tokens['url'](req, res),

services/backend/src/middleware/auth.js

@@ -1,42 +1,5 @@
-const jwt = require('jsonwebtoken')
-const conf = require('../conf-backend')
-const blacklist = require('../services/blacklist')
-const { getUserDataFor } = require('../services/userService')
-const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
-const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
-
-const TOKEN_VERSION = 1 // When token structure changes, increment in userservice, backend and frontend
-
-const checkAuth = async (req, res, next) => {
-  const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
-  const uid = req.headers['uid']
-
-  if (!token) {
-    return res.status(403).json({ error: 'No token in headers' })
-  }
-
-  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) {
-      return res.status(403).json(err)
-    }
-
-    if (decoded.version !== TOKEN_VERSION) {
-      return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    }
-
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
-      return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-    }
-
-    const userData = await getUserDataFor(decoded.userId)
-    req.decodedToken = decoded
-    Object.assign(req, userData)
-    next()
-  })
-}
-
 const roles = requiredRoles => async (req, res, next) => {
-  if (req.decodedToken) {
+  if (req.user) {
     const { roles } = req
 
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
@@ -47,37 +10,6 @@ const roles = requiredRoles => async (req, res, next) => {
   res.status(403).json({ error: 'missing required roles' })
 }
 
-const checkRequiredGroup = async (req, res, next) => {
-  const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-  const enabled = hasRequiredGroup(hyGroups)
-  const tokenOutdated = req.decodedToken.enabled !== enabled
-
-  if (tokenOutdated) {
-    return res.status(401).json({
-      error: 'Token needs to be refreshed - enabled does not match hy-group requirement',
-    })
-  }
-
-  if (!enabled) {
-    return res.status(403).json({ error: 'User is not enabled' })
-  }
-  next()
-}
-
-const checkUserBlacklisting = async (req, res, next) => {
-  const { userId, createdAt } = req.decodedToken
-  const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
-
-  if (isBlacklisted) {
-    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  }
-
-  next()
-}
-
 module.exports = {
-  checkAuth,
   roles,
-  checkRequiredGroup,
-  checkUserBlacklisting,
 }

services/backend/src/middleware/currentUserMiddleware.js

@@ -0,0 +1,57 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const { getUser, getUserDataFor, getMockedUser } = require('../services/userService')
+const { requiredGroup } = require('../conf-backend')
+const _ = require('lodash')
+
+const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
+
+const hasRequiredIamGroup = iamGroups => _.intersection(iamGroups, requiredGroup).length > 0
+
+const currentUserMiddleware = async (req, _, next) => {
+  const { uid: username, 'shib-session-id': sessionId, shib_logout_url: logoutUrl } = req.headers
+
+  if (!sessionId) {
+    throw new ApplicationError('No session id found in request headers', 403, { logoutUrl })
+  }
+
+  if (!username) {
+    throw new ApplicationError('No username found in request headers', 403, { logoutUrl })
+  }
+  const { displayname: name, mail: email, hygroupcn: iamGroups, hypersonsisuid: sisId } = req.headers
+  const parsedIamGroups = parseIamGroups(iamGroups)
+
+  req.user = await getUser({
+    username,
+    name,
+    email,
+    iamGroups: parsedIamGroups,
+    sisId,
+  })
+
+  if (!hasRequiredIamGroup(parsedIamGroups)) {
+    throw new ApplicationError('User is not enabled', 403, { logoutUrl })
+  }
+
+  const showAsUser = req.headers['x-show-as-user']
+
+  if (showAsUser) {
+    const mockedUser = await getMockedUser(username, showAsUser)
+    if (mockedUser) {
+      req.user = mockedUser
+    }
+  }
+  req.logoutUrl = logoutUrl
+
+  const { userId, mockedBy } = req.user
+
+  Sentry.setUser({ username: mockedBy ?? userId })
+
+  // TODO: remove this garbo
+  const userData = await getUserDataFor(userId)
+  Object.assign(req, userData)
+
+  next()
+}
+
+module.exports = currentUserMiddleware

services/backend/src/middleware/errorMiddleware.js

@@ -0,0 +1,19 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const logger = require('../util/logger')
+
+const errorHandler = (error, req, res, next) => {
+  logger.error(`${error.message} ${error.name} ${error.stack}`)
+
+  Sentry.captureException(error)
+
+  if (res.headersSent) {
+    return next(error)
+  }
+
+  const normalizedError = error instanceof ApplicationError ? error : new ApplicationError(error.message)
+
+  return res.status(normalizedError.status).json(normalizedError)
+}
+
+module.exports = errorHandler

services/backend/src/middleware/sentryUserId.js

@@ -1,13 +0,0 @@
-const Sentry = require('@sentry/node')
-
-const sentryUserIdMiddleware = (req, res, next) => {
-  Sentry.getCurrentHub().configureScope(scope => {
-    scope.setUser({
-      username: req.decodedToken && req.decodedToken.userId,
-    })
-  })
-
-  next()
-}
-
-module.exports = sentryUserIdMiddleware

services/backend/src/middleware/shibbolethCharsetMiddleware.js

@@ -0,0 +1,5 @@
+const headersMiddleware = require('unfuck-utf8-headers-middleware')
+
+const headers = ['uid', 'displayName', 'shib-session-id', 'hyGroupCn', 'mail', 'hyPersonSisuId']
+
+module.exports = headersMiddleware(headers)

services/backend/src/models/course.js

@@ -47,6 +47,9 @@ Course.init(
     substitutions: {
       type: JSONB,
     },
+    course_unit_type: {
+      type: STRING,
+    },
   },
   {
     underscored: false,

services/backend/src/models/credit.js

@@ -75,6 +75,12 @@ Credit.init(
     updatedAt: {
       type: DATE,
     },
+    language: {
+      type: STRING,
+    },
+    is_open: {
+      type: BOOLEAN,
+    },
   },
   {
     underscored: false,

services/backend/src/models/index.js

@@ -30,6 +30,9 @@ Student.hasMany(SemesterEnrollment, { foreignKey: 'studentnumber', sourceKey: 's
 SemesterEnrollment.belongsTo(Semester, { foreignKey: 'semestercomposite', targetKey: 'composite' })
 Semester.hasMany(SemesterEnrollment, { foreignKey: 'semestercomposite', sourceKey: 'composite' })
 
+ProgrammeModule.belongsTo(Organization, { foreignKey: 'organization_id' })
+Organization.hasMany(ProgrammeModule, { foreignKey: 'organization_id' })
+
 Credit.notUnnecessary = credit => {
   return credit.credits > 0 && credit.credits <= 12
 }
@@ -96,6 +99,8 @@ ProgrammeModule.belongsToMany(ProgrammeModule, {
   foreignKey: 'parent_id',
 })
 
+Organization.hasMany(Organization, { foreignKey: 'parent_id', as: 'children' })
+
 module.exports = {
   Organization,
   Course,

services/backend/src/models/models_kone.js

@@ -135,6 +135,28 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
   },
 })
 
+const StudyGuidanceGroupTag = sequelizeKone.define(
+  'study_guidance_group_tags',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    studyGuidanceGroupId: {
+      type: Sequelize.STRING,
+      unique: true,
+    },
+    studyProgramme: {
+      type: Sequelize.STRING,
+    },
+    year: {
+      type: Sequelize.STRING,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 MandatoryCourse.belongsTo(MandatoryCourseLabels, {
   foreignKey: 'label',
   sourceKey: 'id',
@@ -157,4 +179,5 @@ module.exports = {
   CustomPopulationSearch,
   sequelizeKone,
   ExcludedCourse,
+  StudyGuidanceGroupTag,
 }

services/backend/src/models/models_user.js

@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
+const { sequelizeUser } = require('../database/connection')
 
-const UserElementDetails = sequelize.define(
+const UserElementDetails = sequelizeUser.define(
   'user_elementdetails',
   {
     userId: {
@@ -18,7 +18,7 @@ const UserElementDetails = sequelize.define(
   }
 )
 
-const AccessGroup = sequelize.define('access_group', {
+const AccessGroup = sequelizeUser.define('access_group', {
   id: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -39,7 +39,7 @@ const AccessGroup = sequelize.define('access_group', {
   },
 })
 
-const UserFaculties = sequelize.define('user_faculties', {
+const UserFaculties = sequelizeUser.define('user_faculties', {
   userId: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -59,24 +59,8 @@ const UserFaculties = sequelize.define('user_faculties', {
     type: Sequelize.DATE,
   },
 })
-const FacultyProgrammes = sequelize.define('faculty_programmes', {
-  faculty_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  programme_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
 
-const User = sequelize.define(
+const User = sequelizeUser.define(
   'users',
   {
     id: {
@@ -91,31 +75,15 @@ const User = sequelize.define(
     },
     email: { type: Sequelize.STRING },
     language: { type: Sequelize.STRING },
+    sisu_person_id: { type: Sequelize.STRING },
+    last_login: { type: Sequelize.DATE },
   },
   {
     tableName: 'users',
     timestamps: false,
   }
 )
-const HyGroup = sequelize.define('hy_group', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true,
-  },
-
-  code: { type: Sequelize.STRING },
-
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
-const Affiliation = sequelize.define('affiliations', {
+const HyGroup = sequelizeUser.define('hy_group', {
   id: {
     primaryKey: true,
     type: Sequelize.BIGINT,
@@ -145,26 +113,13 @@ AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 User.belongsToMany(HyGroup, { through: 'user_hy_group', as: 'hy_group' })
 HyGroup.belongsToMany(User, { through: 'user_hy_group' })
 
-User.belongsToMany(Affiliation, {
-  through: 'user_affiliation',
-  as: 'affiliation',
-})
-Affiliation.belongsToMany(User, { through: 'user_affiliation' })
-
 User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
-UserFaculties.hasMany(FacultyProgrammes, {
-  as: 'programme',
-  foreignKey: 'faculty_code',
-  sourceKey: 'faculty_code',
-})
 
 module.exports = {
   User,
   UserElementDetails,
   AccessGroup,
   HyGroup,
-  Affiliation,
   UserFaculties,
-  FacultyProgrammes,
-  sequelize,
+  sequelizeUser,
 }

services/backend/src/models/organization.js

@@ -19,6 +19,9 @@ Organization.init(
     name: {
       type: JSONB,
     },
+    parent_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/programmeModule.js

@@ -18,6 +18,16 @@ ProgrammeModule.init(
     type: {
       type: STRING,
     },
+    studyLevel: {
+      type: STRING,
+    },
+    organization_id: {
+      type: STRING,
+      references: {
+        model: 'organization',
+        key: 'id',
+      },
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/student.js

@@ -17,7 +17,6 @@ Student.init(
     dateofuniversityenrollment: { type: DATE },
     email: { type: STRING },
     national_student_number: { type: STRING },
-    home_county_id: { type: INTEGER },
     country_fi: { type: STRING },
     country_sv: { type: STRING },
     country_en: { type: STRING },
@@ -25,6 +24,7 @@ Student.init(
     home_country_sv: { type: STRING },
     home_country_en: { type: STRING },
     gender_code: { type: INTEGER },
+    sis_person_id: { type: STRING },
     createdAt: {
       type: DATE,
     },

services/backend/src/routes.js

@@ -1,38 +1,52 @@
-const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
-const accessLogger = require('./middleware/accesslogger')
-const sentryUserId = require('./middleware/sentryUserId')
+const Sentry = require('@sentry/node')
+const express = require('express')
+const cors = require('cors')
+const { frontUrl } = require('./conf-backend')
+const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
+const matomoInit = require('./routes/matomo-init')
+const currentUserMiddleware = require('./middleware/currentUserMiddleware')
+const accessLogger = require('./middleware/accessLogger')
+const auth = require('./middleware/auth')
+
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
-const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
-const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const mandatoryCourses = require('./routes/mandatorycourses')
 const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
-const ping = require('./routes/ping')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
 const updater = require('./routes/updater')
 const tsaAnalytics = require('./routes/tsaAnalytics')
-const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
+const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
+const studyProgramme = require('./routes/studyProgramme')
+const initializeSentry = require('./util/sentry')
+
+const errorMiddleware = require('./middleware/errorMiddleware')
 
 module.exports = (app, url) => {
-  app.use(url, ping)
-  app.use(shibbolethHeadersFix(['hyGroupCn', 'SHIB_LOGOUT_URL', 'eduPersonAffiliation', 'uid', 'displayName', 'mail']))
+  initializeSentry(app)
+  app.use(Sentry.Handlers.requestHandler())
+  app.use(Sentry.Handlers.tracingHandler())
+
+  app.use(cors({ credentials: true, origin: frontUrl }))
+  app.use(express.json())
+
+  app.use(shibbolethCharsetMiddleware)
   app.use(url, matomoInit)
+  app.use(currentUserMiddleware)
+  app.use(accessLogger)
   app.use(url, login)
-  app.use(`${url}/superlogin`, superlogin)
-  app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
   app.use(url, elementdetails)
   app.use(url, courses)
   app.use(url, students)
@@ -42,14 +56,22 @@ module.exports = (app, url) => {
   app.use(url, semesters)
   app.use(url, tags)
   app.use(url, programmeModules)
+  app.use(url, studyProgramme)
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
-  app.use(`${url}/users`, auth.roles(['users']), users)
+  app.use(`${url}/users`, auth.roles(['admin']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/mandatory_courses`, mandatoryCourses)
   app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
   app.use(`${url}/tsa`, tsaAnalytics)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/cool-data-science`, trends)
+  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.get('*', async (_, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+  app.use(Sentry.Handlers.errorHandler())
+  app.use(errorMiddleware)
 }

services/backend/src/routes/courses.js

@@ -2,7 +2,6 @@ const router = require('express').Router()
 const crypto = require('crypto')
 const Course = require('../services/courses')
 const { validateParamLength } = require('../util')
-const logger = require('../util/logger')
 
 router.get('/v2/coursesmulti', async (req, res) => {
   let results = { courses: [] }
@@ -17,38 +16,30 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  try {
-    const { rights, roles } = req
+  const { rights, roles } = req
 
-    const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
+  const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 
-    // If user has rights to see at least one programme, then they are allowed
-    // to see all of them
-    if (!allowedRoles && rights.length < 1) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
+  // If user has rights to see at least one programme, then they are allowed
+  // to see all of them
+  if (!allowedRoles && rights.length < 1) {
+    return res.status(403).json({ error: 'No programmes so no access to course stats' })
+  }
 
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+  const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
 
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      // Studentnumbers should be obfuscated to all other users except admins
-      // and users with rights to any specific study programmes
-      const anonymize = allowedRoles !== 'admin' && rights.length < 1
-      const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
-      res.json(results)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
+  const separate = !sep ? false : JSON.parse(sep)
+  const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
+  if (!codes) {
+    res.status(422).send('Missing required query parameters')
+  } else {
+    // Studentnumbers should be obfuscated to all other users except admins
+    // and users with rights to any specific study programmes
+    const anonymize = allowedRoles !== 'admin' && rights.length < 1
+    const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
+    const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
+    res.json(results)
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/customPopulationSearch.js

@@ -7,76 +7,56 @@ const {
 } = require('../services/customPopulationSearch')
 
 router.get('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-    } = req
-    const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
-    res.json(customPopulationSearches)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const {
+    user: { id },
+  } = req
+  const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
+  res.json(customPopulationSearches)
 })
 
 router.post('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-      body: { name, students },
-    } = req
+  const {
+    user: { id },
+    body: { name, students },
+  } = req
 
-    if (!name) return res.status(400).json({ error: 'name missing' })
-    if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!name) return res.status(400).json({ error: 'name missing' })
+  if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
 
-    const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
-    res.json(customPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
+  res.json(customPopulationSearch)
 })
 
 router.put('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-      body: { students },
-    } = req
-    const { id } = req.params
+  const {
+    user: { id: userId },
+    body: { students },
+  } = req
+  const { id } = req.params
 
-    if (!id) return res.status(400).json({ error: 'id missing' })
-    if (!students) return res.status(400).json({ error: 'students missing' })
-    if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!students) return res.status(400).json({ error: 'students missing' })
+  if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
 
-    const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
-    if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
+  const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
+  if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
 
-    res.json(updatedPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(updatedPopulationSearch)
 })
 
 router.delete('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-    } = req
-    const { id } = req.params
+  const {
+    user: { id: userId },
+  } = req
+  const { id } = req.params
 
-    if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
 
-    const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
+  const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
 
-    if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
+  if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
 
-    res.json(id)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(id)
 })
 
 module.exports = router

services/backend/src/routes/elementdetails.js

@@ -1,13 +1,11 @@
 const router = require('express').Router()
-const { getStudentsUserCanAccess } = require('../services/userService')
 const MandatoryCourses = require('../services/mandatoryCourses')
 const {
   productivityStatsForStudytrack,
   throughputStatsForStudytrack,
   optionData,
 } = require('../services/studyprogramme')
-const { findProgrammeTheses } = require('../services/thesis')
-
+const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
 const {
   getProductivity,
   setProductivity,
@@ -15,11 +13,22 @@ const {
   setThroughput,
   patchProductivity,
   patchThroughput,
-  getNonGraduatedStudents,
 } = require('../services/analyticsService')
+const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
+const logger = require('../util/logger')
 
 const programmeStatsSince = new Date('2017-07-31')
 
+router.get('/elementdetails/all', async (req, res) => {
+  const elementdetails = await getAllElementDetails()
+  res.json(elementdetails)
+})
+
+router.get('/studyprogrammes', async (req, res) => {
+  const studyrights = await getAllProgrammes()
+  res.json(studyrights)
+})
+
 router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   const code = req.params.id
 
@@ -36,34 +45,6 @@ router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   res.json(data)
 })
 
-router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
-  try {
-    const {
-      roles,
-      decodedToken,
-      params: { id },
-    } = req
-    if (!id) return res.status(400).json({ error: 'programme id missing' })
-
-    const nonGraduatedStudents = await getNonGraduatedStudents(id)
-    if (!nonGraduatedStudents) return res.status(200).json({})
-    const {
-      data: { formattedData, studentnumbers },
-    } = nonGraduatedStudents
-
-    const filteredData = { ...formattedData }
-    const studentsUserCanAccess = await getStudentsUserCanAccess(studentnumbers, roles, decodedToken.userId)
-    Object.keys(filteredData).forEach(year => {
-      filteredData[year] = filteredData[year].filter(({ studentNumber }) => studentsUserCanAccess.has(studentNumber))
-    })
-
-    res.json(filteredData)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -80,14 +61,14 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
     try {
       data = await getProductivity(code)
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} productivity`)
     }
     if (!data) {
       try {
         const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
         data = await setProductivity(stats)
       } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       }
     }
     return res.json(data)
@@ -99,14 +80,8 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
 router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
   const code = req.query.code
 
-  console.log('Productivity stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()
 
   try {
     if (code.includes('MH') || code.includes('KH')) {
@@ -122,13 +97,13 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
         [code]: { status: 'RECALCULATION ERRORED' },
       })
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       return
     }
-    console.error(e)
-    console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+
+    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
   }
-  console.log(`Productivity stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
 })
 
 router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
@@ -140,14 +115,14 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
     try {
       data = await getThroughput(code)
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} throughput`)
     }
     if (!data) {
       try {
         const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
         data = await setThroughput(result)
       } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       }
     }
     return res.json({ ...data, thesis: await thesisPromise })
@@ -159,14 +134,8 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
 router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   const code = req.query.code
 
-  console.log('Throughput stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()
 
   try {
     if (code.includes('MH') || code.includes('KH')) {
@@ -180,15 +149,43 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
     try {
       await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       return
     }
-    console.error(e)
-    console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
+    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
   }
-  console.log(`Throughput stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
 })
 
-router.use('*', (req, res, next) => next())
+router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
+  const { id } = req.params
+  if (id) {
+    const thesis = await findProgrammeTheses(id)
+    res.json(thesis)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
+  const { id } = req.params
+  const { course, thesisType } = req.body
+  if (id && course && thesisType) {
+    const thesis = await createThesisCourse(id, course, thesisType)
+    res.status(201).json(thesis)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
+  const { id, course } = req.params
+  if (id && course) {
+    const deleted = await deleteThesisCourse(id, course)
+    res.status(204).json(deleted)
+  } else {
+    res.status(422).end()
+  }
+})
 
 module.exports = router

services/backend/src/routes/faculties.js

@@ -3,7 +3,6 @@ const { faculties } = require('../services/organisations')
 
 router.get('/', async (req, res) => {
   const facultyList = await faculties()
-  console.log('faculties')
   res.json(facultyList)
 })
 

services/backend/src/routes/feedback.js

@@ -1,29 +1,28 @@
 const router = require('express').Router()
-const mailservice = require('../services/mailservice')
-const userService = require('../services/userService')
+const { getUserDataFor } = require('../services/userService')
+const { sendFeedbackToToska } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 router.post('/email', async (req, res) => {
   const { content } = req.body
-  const { uid } = req.headers
-  const { email, full_name } = await userService.byUsername(uid)
-  const formattedEmail = content
-    .split('\n\n')
-    .map(line => `<p> ${line} </p>`)
-    .join('')
-  const feedback = mailservice.feedback(formattedEmail, uid, email, full_name)
+  const {
+    user: { userId },
+  } = req
 
-  if (process.env.SMTP !== undefined && content) {
-    await mailservice.transporter.sendMail(feedback, error => {
-      if (error) {
-        console.log('Error occured')
-        res.status(400).json(error).end()
-      } else {
-        console.log('Message sent succesfully!')
-        res.status(200).json('success').end()
-      }
-      mailservice.transporter.close()
-    })
+  const { email, full_name } = await getUserDataFor(userId)
+
+  const result = await sendFeedbackToToska({
+    feedbackContent: content,
+    userId,
+    userEmail: email,
+    userFullName: full_name,
+  })
+  if (result.error) {
+    return res.status(400).json(result).end()
   }
+  logger.info(`${userId} succesfully sent some feedback mail to toska`)
+
+  res.status(200).json('success').end()
 })
 
 module.exports = router

services/backend/src/routes/language.js

@@ -2,8 +2,10 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/language', async (req, res) => {
-  const { language } = req.body
-  const { userId } = req.decodedToken
+  const {
+    body: { language },
+    user: { userId },
+  } = req
   if (!['fi', 'sv', 'en'].includes(language)) {
     return res.status(400).json('invalid language')
   }

services/backend/src/routes/login.js

@@ -1,66 +1,22 @@
 const router = require('express').Router()
-const mailservice = require('../services/mailservice')
-const userService = require('../services/userService')
-const { parseHyGroups } = require('../util/utils')
+const { ApplicationError } = require('../util/customErrors')
 
-const sendEmail = async uid => {
-  if (process.env.SMTP !== undefined) {
-    const message = mailservice.message1(uid)
-    await mailservice.transporter.sendMail(message, error => {
-      if (error) {
-        console.log('Error occurred')
-        console.log(error.message)
-      } else {
-        console.log('Message sent successfully!')
-      }
-      // only needed when using pooled connections
-      mailservice.transporter.close()
-    })
+router.get('/login', async (req, res) => {
+  const { user: userFromReq, logoutUrl } = req
+
+  if (!userFromReq) {
+    throw new ApplicationError('User not found', 404)
   }
-}
-
-router.post('/login', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const full_name = req.headers.displayname || ''
-      const mail = req.headers.mail || ''
-      const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-      const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-
-      console.log(uid, 'trying to login, referring to userservice.')
-      let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail)
-      isNew && sendEmail(uid)
-      res.status(200).json({ token })
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(401).json({ message: 'problem with login', err })
+  const roles = userFromReq.roles.map(role => role.group_code)
+  const user = {
+    ...userFromReq,
+    roles,
+    isAdmin: roles.includes('admin'),
   }
+
+  res.send({
+    user,
+    logoutUrl,
+  })
 })
-
-router.delete('/logout', async (req, res) => {
-  try {
-    const logoutUrl = req.headers.shib_logout_url
-    const { returnUrl } = req.body
-    if (logoutUrl) {
-      return res
-        .status(200)
-        .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-        .end()
-    }
-    res.status(200).send({ logoutUrl: returnUrl }).end()
-  } catch (err) {
-    res.status(500).json({ message: 'Error with logout', err })
-  }
-})
-
 module.exports = router

services/backend/src/routes/mandatorycourselabels.js

@@ -1,5 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
+const logger = require('../util/logger')
 
 router.get('/:programme/labels', async (req, res) => {
   const { programme } = req.params
@@ -8,7 +9,7 @@ router.get('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -25,7 +26,7 @@ router.delete('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -42,7 +43,7 @@ router.post('/:programme/move', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -59,7 +60,7 @@ router.post('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {

services/backend/src/routes/mandatorycourses.js

@@ -5,47 +5,32 @@ router.delete('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.remove(programme, course)
-    res.status(200).json(course)
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.remove(programme, course)
+  res.status(200).json(course)
 })
 
 router.post('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.create(programme, course)
-    const newCourse = await MandatoryCourses.find(programme, course)
-    if (!newCourse) return res.status(400).end()
-    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.create(programme, course)
+  const newCourse = await MandatoryCourses.find(programme, course)
+  if (!newCourse) return res.status(400).end()
+  res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
 })
 
 router.post('/:programme/label/:course', async (req, res) => {
   const { programme, course } = req.params
   const { label } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.updateLabel(programme, course, label)
-    const updatedCourse = await MandatoryCourses.find(programme, course)
-    if (!updatedCourse) return res.status(400).end()
-    res.status(200).json({
-      name: updatedCourse.course.name,
-      code: updatedCourse.course_code,
-      label: updatedCourse.mandatory_course_label,
-    })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.updateLabel(programme, course, label)
+  const updatedCourse = await MandatoryCourses.find(programme, course)
+  if (!updatedCourse) return res.status(400).end()
+  res.status(200).json({
+    name: updatedCourse.course.name,
+    code: updatedCourse.course_code,
+    label: updatedCourse.mandatory_course_label,
+  })
 })
 
 module.exports = router

services/backend/src/routes/matomo-init.js

@@ -28,7 +28,7 @@ _paq.push(["enableLinkTracking"]);
 
 const script = config.MATOMO_URL ? acual : ''
 
-router.get('/matomo-init', (req, res) => {
+router.get('/matomo-init', (_, res) => {
   res.contentType('text/javascript; charset=UTF-8')
   res.send(script)
 })

services/backend/src/routes/ping.js

@@ -1,85 +0,0 @@
-/* eslint-disable */
-
-const { router, wrapper } = require('./router').routerWithWrapper()
-const axios = require('axios')
-
-wrapper.get('/ping', async (req, res) => {
-  res.json({
-    message: 'pong'
-  })
-})
-
-wrapper.get('/v3/mandatory_courses/:code', async (req, res) => {
-  const courses = []
-  const included = new Set()
-
-  const dumb_flatten = module => {
-    if (module.children) {
-      module.children.forEach(child => {
-        dumb_flatten(child)
-      })
-    } else if (module.code && !module.code.startsWith('KK') && !courses.map(c => c.code).includes(module.code)) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label: module.code.replace('-', '').slice(0, 4), orderNumber: courses.length }
-      })
-    }
-  }
-
-  const better_flatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        better_flatten(child, module.module ? module.module.name : label)
-      })
-    } else if (module.code && !module.code.startsWith('KK')) {
-      courses.push({
-        name: {
-          fi: module.name,
-          en: module.name
-        },
-        code: module.code,
-        label: { id: `${courses.length}`, label, orderNumber: courses.length }
-      })
-    }
-  }
-
-  const superFlattenFlatten = (module, label) => {
-    if (module.children) {
-      module.children.forEach(child => {
-        superFlattenFlatten(child, module.module ? module.module.name : label)
-      })
-    } else if (Array.isArray(module)) {
-      module.forEach(elem => superFlattenFlatten(elem, label))
-    } else if (module.code && !module.code.startsWith('KK')) {
-      if (!included.has(`${label}${module.code}`)) {
-        included.add(`${label}${module.code}`)
-        courses.push({
-          name: {
-            fi: module.name,
-            en: module.name
-          },
-          code: module.code,
-          label: { id: `${courses.length}`, label, orderNumber: courses.length }
-        })
-      }
-    }
-  }
-
-  const code = req.params.code
-  const response = await axios.get(`http://sis-updater-scheduler:8082/v1/courses/${code}?token=dev&superFlatten=true`)
-
-  // dumb_flatten(response.data)
-  // better_flatten(response.data)
-  superFlattenFlatten(response.data)
-
-  const byCode = (c1, c2) => (c1.code < c2.code ? -1 : 1)
-
-  courses.sort(byCode)
-  res.json(courses)
-})
-
-module.exports = router

services/backend/src/routes/population.js

@@ -8,6 +8,8 @@ const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
+const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
+const { logger } = require('../util/logger')
 
 const filterPersonalTags = (population, userId) => {
   return {
@@ -21,282 +23,107 @@ const filterPersonalTags = (population, userId) => {
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
-  try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
-      Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
-      res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+  if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
+    Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
+    res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+    return
+  }
+  if (!Array.isArray(req.body.studyRights)) {
+    // studyRights should always be an array
+    req.body.studyRights = [req.body.studyRights]
+  }
+
+  if (req.body.months == null) {
+    req.body.months = 12
+  }
+
+  if (req.body.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multicoursestatPromises = Promise.all(
+      req.body.years.map(year => {
+        if (req.body.selectedStudentsByYear) {
+          req.body.selectedStudents = req.body.selectedStudentsByYear[year]
+        }
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const query = { ...req.body, year, months: newMonths }
+        const coursestatistics = Population.bottlenecksOf(query)
+        return coursestatistics
+      })
+    )
+    const multicoursestats = await multicoursestatPromises
+    const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
+    if (result.error) {
+      res.status(400).json(result)
       return
     }
-    if (!Array.isArray(req.body.studyRights)) {
-      // studyRights should always be an array
-      req.body.studyRights = [req.body.studyRights]
+
+    res.json(result)
+  } else {
+    const result = await Population.bottlenecksOf(req.body)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).json(result)
+      return
     }
 
-    if (req.body.months == null) {
-      req.body.months = 12
-    }
-
-    if (req.body.years) {
-      console.log('years väylä')
-      const upperYearBound = new Date().getFullYear() + 1
-      const multicoursestatPromises = Promise.all(
-        req.body.years.map(year => {
-          if (req.body.selectedStudentsByYear) {
-            req.body.selectedStudents = req.body.selectedStudentsByYear[year]
-          }
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const query = { ...req.body, year, months: newMonths }
-          const coursestatistics = Population.bottlenecksOf(query)
-          return coursestatistics
-        })
-      )
-      const multicoursestats = await multicoursestatPromises
-      const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
-      console.log('result: ', result)
-      if (result.error) {
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    } else {
-      console.log('bottleneck väylä')
-      const result = await Population.bottlenecksOf(req.body)
-
-      if (result.error) {
-        Sentry.captureException(new Error(result.error))
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    }
-  } catch (e) {
-    console.log(e)
-    Sentry.captureException(e)
-    res.status(500).json({ error: e })
+    res.json(result)
   }
 })
 
 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  try {
-    const { from, to, coursecodes } = req.body
-    if (!from || !to || !coursecodes) {
-      return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-    }
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-    if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-      return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { from, to, coursecodes } = req.body
+  if (!from || !to || !coursecodes) {
+    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
   }
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
+  if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
+    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
+  }
+  let studentnumberlist
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const {
+    user: { userId },
+    roles,
+  } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })
 
 router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
-  try {
-    const { tag } = req.body
-    if (!tag) {
-      res.status(400).json({ error: 'The body should have a tag defined' })
-      return
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByTag(tag)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-        tag,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { tag } = req.body
+  if (!tag) {
+    res.status(400).json({ error: 'The body should have a tag defined' })
+    return
   }
-})
-
-router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  try {
-    if (!req.body.studentnumberlist) {
-      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-      return
-    }
-    let studentnumberlist
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = req.body.studentnumberlist
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: err })
-  }
-})
-
-router.get('/v3/populationstatistics', async (req, res) => {
-  const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { decodedToken } = req
-
-  try {
-    if (!year || !semesters || !studyRightsJSON) {
-      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
-      return
-    }
-    let studyRights = null
-    try {
-      studyRights = JSON.parse(studyRightsJSON)
-      const { rights, roles } = req
-
-      if (!roles || !roles.includes('admin')) {
-        if (!rights.includes(studyRights.programme)) {
-          res.status(403).json([])
-          return
-        }
-      }
-    } catch (e) {
-      console.error(e)
-      res.status(400).json({ error: 'The query had invalid studyRights' })
-      return
-    }
-
-    if (req.query.months == null) {
-      req.query.months = 12
-    }
-
-    if (req.query.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multipopulationstudentPromises = Promise.all(
-        req.query.years.map(year => {
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const populationStudents = Population.optimizedStatisticsOf({
-            ...req.query,
-            studyRights,
-            year,
-            months: newMonths,
-          })
-          return populationStudents
-        })
-      )
-      const multipopulationstudents = await multipopulationstudentPromises
-
-      const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
-
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    } else {
-      const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    }
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { decodedToken } = req
-
-  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
-  const foundTag = await TagService.findTagById(tag)
-  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
-
-  const semesters = ['FALL', 'SPRING']
   let studentnumberlist
   const studentnumbers = await Student.findByTag(tag)
   const {
-    decodedToken: { userId },
+    user: { userId },
     roles,
   } = req
   if (roles && roles.includes('admin')) {
@@ -306,39 +133,180 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     const codes = unitsUserCanAccess.map(unit => unit.id)
     studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
   }
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+      tag,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(result)
+})
+
+router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
+  if (!req.body.studentnumberlist) {
+    res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
+    return
+  }
+  let studentnumberlist
+  const {
+    user: { userId, sisPersonId },
+    roles,
+  } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = req.body.studentnumberlist
+  } else if (req.body.usingStudyGuidanceGroups) {
+    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
+    studentnumberlist = req.body.studentnumberlist.filter(s => studentsUserCanAccess.has(s))
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+  }
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
+})
+
+router.get('/v3/populationstatistics', async (req, res) => {
+  const { year, semesters, studyRights: studyRightsJSON } = req.query
+  const { user } = req
+
+  if (!year || !semesters || !studyRightsJSON) {
+    res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
+    return
+  }
+  let studyRights = null
   try {
-    const studyRights = JSON.parse(studyRightsJSON)
-    const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-    const yearDifference = Number(year) - Number(newStartYear)
-    const newMonths = Number(months) + 12 * yearDifference
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: newStartYear,
-        studyRights,
-        semesters,
-        months: newMonths,
-        tag: foundTag,
-      },
-      studentnumberlist
+    studyRights = JSON.parse(studyRightsJSON)
+    const { rights, roles } = req
+
+    if (!roles || !roles.includes('admin')) {
+      if (!rights.includes(studyRights.programme)) {
+        res.status(403).json([])
+        return
+      }
+    }
+  } catch (e) {
+    res.status(400).json({ error: 'The query had invalid studyRights' })
+    return
+  }
+
+  if (req.query.months == null) {
+    req.query.months = 12
+  }
+
+  if (req.query.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multipopulationstudentPromises = Promise.all(
+      req.query.years.map(year => {
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const populationStudents = Population.optimizedStatisticsOf({
+          ...req.query,
+          studyRights,
+          year,
+          months: newMonths,
+        })
+        return populationStudents
+      })
     )
+    const multipopulationstudents = await multipopulationstudentPromises
+
+    const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
 
     if (result.error) {
-      console.log(result.error)
+      Sentry.captureException(new Error(result.error))
       res.status(400).end()
       return
     }
 
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+    res.json(filterPersonalTags(result, user.id))
+  } else {
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, user.id))
   }
 })
 
+router.get('/v3/populationstatisticsbytag', async (req, res) => {
+  const { tag, studyRights: studyRightsJSON, months, year } = req.query
+  const { user } = req
+
+  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
+  const foundTag = await TagService.findTagById(tag)
+  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
+
+  const semesters = ['FALL', 'SPRING']
+  let studentnumberlist
+  const studentnumbers = await Student.findByTag(tag)
+  const {
+    user: { userId },
+    roles,
+  } = req
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+  const studyRights = JSON.parse(studyRightsJSON)
+  const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
+  const yearDifference = Number(year) - Number(newStartYear)
+  const newMonths = Number(months) + 12 * yearDifference
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: newStartYear,
+      studyRights,
+      semesters,
+      months: newMonths,
+      tag: foundTag,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(filterPersonalTags(result, user.id))
+})
+
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   const { coursecodes, from, to, separate: sep } = req.query
-  const { decodedToken, roles } = req
+  const { user, roles } = req
   const separate = sep ? JSON.parse(sep) : false
 
   if (!coursecodes || !from || !to) {
@@ -355,128 +323,106 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 
   const semesters = ['FALL', 'SPRING']
   const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        // Useless, because studentnumbers are already filtered above by from & to.
-        // We should probably refactor this to avoid more confusement.
-        year: 1900,
-        studyRights: [],
-        semesters,
-        months: 10000,
-      },
-      studentnumbers
-    )
+  const result = await Population.optimizedStatisticsOf(
+    {
+      // Useless, because studentnumbers are already filtered above by from & to.
+      // We should probably refactor this to avoid more confusement.
+      year: 1900,
+      studyRights: [],
+      semesters,
+      months: 10000,
+    },
+    studentnumbers
+  )
 
-    let studentsUserCanAccess
-    if (roles && roles.includes('admin')) {
-      studentsUserCanAccess = new Set(studentnumbers)
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-    }
-
-    const randomHash = crypto.randomBytes(12).toString('hex')
-    const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
-      courses,
-      studyrights,
-      gender_code,
-      studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
-      firstnames: '',
-      lastname: '',
-      name: '',
-      email: '',
-      started: null,
-      credits: '',
-      tags: [],
-      obfuscated: true,
-    })
-
-    result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  let studentsUserCanAccess
+  if (roles && roles.includes('admin')) {
+    studentsUserCanAccess = new Set(studentnumbers)
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(user.userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
   }
+
+  const randomHash = crypto.randomBytes(12).toString('hex')
+  const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
+    courses,
+    studyrights,
+    gender_code,
+    studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
+    firstnames: '',
+    lastname: '',
+    name: '',
+    email: '',
+    started: null,
+    credits: '',
+    tags: [],
+    obfuscated: true,
+  })
+
+  result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(filterPersonalTags(result, user.id))
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist } = req.body
-  const { roles, decodedToken } = req
+  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
+  const { roles, user } = req
 
-  const studentsUserCanAccess = await UserService.getStudentsUserCanAccess(
-    studentnumberlist,
-    roles,
-    decodedToken.userId
-  )
+  if (usingStudyGuidanceGroups) {
+    if (!user.sisPersonId) {
+      logger.error(`User ${user.userId} tried to get person groups but personId was ${user.sisPersonId} in header`)
+      return res.status(400).json({ error: 'Not possible to get groups without personId' })
+    }
+  }
+
+  const studentsUserCanAccess = usingStudyGuidanceGroups
+    ? await getAllStudentsUserHasInGroups(user.sisPersonId)
+    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, user.userId)
   const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
 
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      filteredStudentNumbers
-    )
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-    res.status(200).json(filterPersonalTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    filteredStudentNumbers
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    return
   }
+  res.status(200).json(filterPersonalTags(result, user.id))
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  try {
-    const { rights, roles } = req
-    if (roles && roles.includes('admin')) {
-      const studyrights = await StudyrightService.getAssociations()
-      res.json(studyrights)
-    } else {
-      const studyrights = await StudyrightService.getFilteredAssociations(rights)
-      res.json(studyrights)
-    }
-  } catch (err) {
-    res.status(500).json(err)
+  const { rights, roles } = req
+  if (roles && roles.includes('admin')) {
+    const studyrights = await StudyrightService.getAssociations()
+    res.json(studyrights)
+  } else {
+    const studyrights = await StudyrightService.getFilteredAssociations(rights)
+    res.json(studyrights)
   }
 })
 
 router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, res) => {
-  try {
-    const studyrights = await StudyrightService.getAssociations()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
+  const studyrights = await StudyrightService.getAssociations()
+  res.json(studyrights)
 })
 
 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  try {
-    const { courseCodes } = req.query
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-    return res.json(maxYearsToCreatePopulationFrom)
-  } catch (err) {
-    console.log('err', err)
-    res.status(500).json(err)
-  }
+  const { courseCodes } = req.query
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
+  return res.json(maxYearsToCreatePopulationFrom)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/programmeModules.js

@@ -8,17 +8,20 @@ const {
 
 router.get('/v3/programme_modules/:code', async (req, res) => {
   const { code } = req.params
-  const module = await byProgrammeCode(code)
-  if (!module) {
-    res.status(400).end()
-    return
-  }
-  res.json(module)
+  const result = await byProgrammeCode(code)
+  res.json(result)
 })
 
 router.get('/v3/programme_modules/:code/modules', async (req, res) => {
   const { code } = req.params
   const result = await modulesByProgrammeCode(code)
+  res.json(result)
+})
+
+router.delete('/v3/programme_modules', async (req, res) => {
+  const { programmecode, ids } = req.body
+  await removeExcludedCourses(ids)
+  const result = await byProgrammeCode(programmecode)
   if (!result) {
     res.status(400).end()
     return
@@ -26,36 +29,15 @@ router.get('/v3/programme_modules/:code/modules', async (req, res) => {
   res.json(result)
 })
 
-router.delete('/v3/programme_modules', async (req, res) => {
-  const { programmecode, ids } = req.body
-  try {
-    await removeExcludedCourses(ids)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
-  }
-})
-
 router.post('/v3/programme_modules/:programmecode/', async (req, res) => {
   const { programmecode, coursecodes } = req.body
-  try {
-    await addExcludedCourses(programmecode, coursecodes)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
+  await addExcludedCourses(programmecode, coursecodes)
+  const result = await byProgrammeCode(programmecode)
+  if (!result) {
+    res.status(400).end()
+    return
   }
+  res.json(result)
 })
 
 module.exports = router

services/backend/src/routes/semesters.js

@@ -6,6 +6,4 @@ router.get('/semesters/codes', async (req, res) => {
   res.json(providers)
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/students.js

@@ -2,7 +2,7 @@ const router = require('express').Router()
 const Student = require('../services/students')
 const userService = require('../services/userService')
 const Unit = require('../services/units')
-const { mismatchedStudents } = require('../services/student_credit_total_mismatches')
+const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
 
 const filterStudentTags = (student, userId) => {
   return {
@@ -11,17 +11,10 @@ const filterStudentTags = (student, userId) => {
   }
 }
 
-const creditTotalMismatch = student => {
-  return {
-    ...student,
-    mismatch: mismatchedStudents.has(student.studentNumber),
-  }
-}
-
 router.get('/students', async (req, res) => {
   const {
     roles,
-    decodedToken: { userId },
+    user: { userId, sisPersonId },
     query: { searchTerm },
   } = req
 
@@ -45,26 +38,29 @@ router.get('/students', async (req, res) => {
   } else {
     const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
     const codes = unitsUserCanAccess.map(unit => unit.id)
+    let extraStudents = []
+    if (roles.includes('studyGuidanceGroups')) {
+      const matchingOnlyByTerm = await Student.bySearchTerm(trimmedSearchTerm)
+      const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
+      extraStudents.push(...matchingOnlyByTerm.filter(s => studentsUserCanAccess.has(s.studentNumber)))
+    }
     const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json(matchingStudents)
+    res.json([...matchingStudents, ...extraStudents])
   }
 })
 
 router.get('/students/:id', async (req, res) => {
   const { id: studentId } = req.params
-  const { roles, decodedToken } = req
+  const { roles, user } = req
 
-  if (roles && roles.includes('admin')) {
+  if (roles?.includes('admin')) {
     const results = await Student.withId(studentId)
     return results.error
       ? res.status(400).json({ error: 'error finding student' }).end()
-      : res
-          .status(200)
-          .json(creditTotalMismatch(filterStudentTags(results, decodedToken.id)))
-          .end()
+      : res.status(200).json(filterStudentTags(results, user.id)).end()
   }
 
-  const uid = req.decodedToken.userId
+  const uid = req.user.userId
   const student = await Student.withId(studentId)
   if (student.error) {
     return res.status(400).json({ error: 'error finding student' }).end()
@@ -78,13 +74,18 @@ router.get('/students/:id', async (req, res) => {
     })
   )
 
+  if (roles?.includes('studyGuidanceGroups')) {
+    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(user.sisPersonId)
+    if (studentsUserCanAccess.has(studentId)) {
+      return res.status(200).json(filterStudentTags(student, user.id)).end()
+    }
+  }
+
   if (rights.some(right => right !== null)) {
-    res.status(200).json(filterStudentTags(student, decodedToken.id)).end()
+    res.status(200).json(filterStudentTags(student, user.id)).end()
   } else {
     res.status(400).json({ error: 'error finding student' }).end()
   }
 })
 
-router.use('*', (req, res, next) => next())
-
 module.exports = router

services/backend/src/routes/studyGuidanceGroups.js

@@ -0,0 +1,39 @@
+const router = require('express').Router()
+const { logger } = require('../util/logger')
+const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyGuidanceGroups')
+
+router.get('/', async (req, res) => {
+  const {
+    user: { uid, sisPersonId },
+  } = req
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to get groups without personId' })
+  }
+  return res.json(await getAllGroupsAndStudents(sisPersonId))
+})
+
+router.put('/:id/tags', async (req, res) => {
+  const {
+    user: { uid, sisPersonId },
+    body: tags,
+    params: { id: groupId },
+  } = req
+
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to change study guidance group tags but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to change groups without personId' })
+  }
+
+  const groupsUserHasAccessTo = (await getAllGroupsAndStudents(sisPersonId)).map(group => group.id)
+
+  if (!groupsUserHasAccessTo.includes(groupId)) {
+    logger.error(
+      `User ${uid} tried to change study guidance group tags, but didn't have permission for the group ${groupId}`
+    )
+    return res.status(400).json({ error: 'Access denied' })
+  }
+  return res.status(200).json(await changeGroupTags({ groupId, tags }))
+})
+
+module.exports = router