temp console log shibbo headers

body-parser now included in express, see:
https://expressjs.com/en/changelog/4x.html#4.16.0

.env.template

@@ -0,0 +1,4 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+IMPORTER_DB_API_URL=
+PATE_TOKEN=

.github/workflows/release-oodikone-staging.yaml

@@ -1,143 +0,0 @@
-name: "Release oodikone to staging"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-oodikone-staging.yaml'
-      - 'services/**'
-
-jobs:
-
-  build_and_release_backend:
-    name: "Build and release backend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:staging
-          target: production
-
-  build_and_release_frontend:
-    name: "Build and release frontend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
-        id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:staging
-          target: production
-      - name: "Create a container from built image "
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: "Copy assets from the container"
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-          version: ${{ github.sha }}
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:staging
-          target: production
-
-  report_success:
-    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: success()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -29,7 +29,6 @@ jobs:
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
@@ -66,7 +65,6 @@ jobs:
             SENTRY_ENVIRONMENT=production
             BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
@@ -103,11 +101,19 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
       - name: "Build and push"
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
             type=registry,ref=toska/oodikone2-userservice:staging
             type=registry,ref=toska/oodikone2-userservice:latest
           cache-to: type=inline

.github/workflows/release-sis-updater-scheduler.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-scheduler"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-scheduler.yaml'
       - 'updater/sis-updater-scheduler/**'

.github/workflows/release-sis-updater-worker.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-worker"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-worker.yaml'
       - 'updater/sis-updater-worker/**'

.github/workflows/test-oodikone.yaml

@@ -1,149 +1,161 @@
-name: "Test oodikone"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
-    branches-ignore:
+    branches:
       - master
     paths:
-      - .github/workflows/test-oodikone.yaml
-      - cypress/**
-      - cypress.json
-      - services/**
-      - docker-compose.ci.yml
-      - nginx.conf
-      - package*.json
+      - '.github/workflows/test-oodikone.yaml'
+      - 'cypress/**'
+      - 'cypress.json'
+      - 'services/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'nginx.conf'
+      - 'package*.json'
 
 jobs:
   # === Static tests / linting ===
 
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
-      - name: "Run backend linter"
+      - name: 'Run backend linter'
         run: npm run eslint services/backend
 
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run concurrently \
-              'eslint --ext .jsx,.js services/frontend' \
-              'stylelint services/frontend/src/**/*.css'
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
 
   lint_userservice:
-    name: "Lint userservice"
+    name: 'Lint userservice'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/userservice && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run eslint services/userservice
 
   # === Build images ===
 
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
           context: services/backend
           push: false
-          tags: toska/oodikone2-backend:trunk
+          tags: toska/oodikone2-backend:staging
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
 
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
           context: services/frontend
           push: false
-          tags: toska/oodikone2-frontend:trunk
+          tags: toska/oodikone2-frontend:staging
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
 
   build_userservice:
-    name: "Build userservice"
+    name: 'Build userservice'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
             type=registry,ref=toska/oodikone2-userservice:staging
             type=registry,ref=toska/oodikone2-userservice:latest
           cache-to: type=inline
           context: services/userservice
           push: false
-          tags: toska/oodikone2-userservice:trunk
+          tags: toska/oodikone2-userservice:staging
           target: production
           outputs: type=docker,dest=/tmp/userservice.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/userservice.tar
 
   # === Integration tests ===
+  #
   run_userservice_integration_tests:
-    name: "Run userservice integration tests"
+    name: 'Run userservice integration tests'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Run tests included in docker build"
+      - name: 'Run tests included in docker build'
         run: docker-compose --file docker-compose.test.yml up --build userservice-test
 
   # === End-to-end tests ===
 
   test_course_statistics:
-    name: "Test course statistics"
+    name: 'Test course statistics'
     needs:
       - build_backend
       - build_frontend
@@ -151,42 +163,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
@@ -194,42 +206,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
-    name: "Test population statistics"
+    name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
@@ -237,42 +249,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Population_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
-    name: "Test filters"
+    name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
@@ -280,42 +292,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
-    name: "Test student statistics"
+    name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
@@ -323,42 +335,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Student_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
-    name: "Test studyprogramme overview"
+    name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
@@ -366,42 +378,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Studyprogramme_overview.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
-    name: "Test teachers and users"
+    name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
@@ -409,24 +421,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -435,20 +447,20 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
       - build_backend
       - build_frontend
@@ -456,91 +468,127 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Check that exit statuses are not errors"
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
-  push_to_dockerhub:
-    name: "Push images to docker hub for caching"
-    needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+  # === Release ===
+
+  release_images:
+    name: 'Release built images'
     runs-on: ubuntu-latest
+    needs:
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Push to docker hub"
-        run: docker-compose --file docker-compose.ci.yml push
+      - name: 'Create a container from frontend'
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: 'Copy assets from the container'
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+          docker push toska/oodikone2-userservice:staging
+      - name: 'Send deployment success notification to slack'
+        if: success()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+      - name: 'Send deployment failure notification to slack'
+        if: failure()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
 
-  report_failure:
+  report_test_failure:
     runs-on: ubuntu-latest
-    name: "Report test failure to oodikone notifications"
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
-    - push_to_dockerhub
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
       - uses: actions/checkout@v2
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

README.md

@@ -10,12 +10,19 @@ To run Oodikone locally, you will need the following:
    - [Docker](https://docs.docker.com/install/) (version 20.10+),
    - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
-3. (Optional) For real data setup, access to Toskas production servers.
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
 ## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
@@ -108,10 +115,10 @@ There are three types of tests in this project: static tests with eslint/prettie
 Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
 - oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
-- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
-- after successful merge to `master`, oodikone is deployed to staging
+- tests are run on every push.
+- after successful test run, oodikone is deployed to staging
 - after creating a release, oodikone is deployed to production
-- updater is deployed to production on every push to `trunk`.
+- updater is deployed to production on every push, no tests included
 
 ## ❓FAQ
 

cypress/integration/Filters.js

@@ -283,9 +283,9 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('courseFilter-header', () => {
       const courses = ['MAT11001', 'TKT20004']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(3)
       cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)

docker-compose.ci.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -13,6 +11,7 @@ services:
       - redis
       - sis-db
     environment:
+      CI: "true"
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
@@ -22,12 +21,12 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:trunk
+    image: toska/oodikone2-backend:staging
 
   frontend:
     <<: *defaults
     container_name: frontend
-    image: toska/oodikone2-frontend:trunk
+    image: toska/oodikone2-frontend:staging
 
   kone-db:
     <<: *defaults
@@ -58,10 +57,11 @@ services:
     depends_on:
       - user-db
     environment:
+      CI: "true"
       DB_URL: postgres://postgres@user-db:5432/user-db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:trunk
+    image: toska/oodikone2-userservice:staging
 
   nginx:
     image: nginx:latest

docker-compose.yml

@@ -5,7 +5,6 @@ x-build_defaults: &build_defaults
   target: development
 
 services:
-
   adminer:
     <<: *defaults
     container_name: adminer
@@ -27,6 +26,8 @@ services:
       - kone-db
       - redis
       - sis-db
+    env_file:
+      - .env
     environment:
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:3000
@@ -70,11 +71,11 @@ services:
 
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -84,6 +85,7 @@ services:
     image: registry.toska.cs.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
+    shm_size: 2gb
 
   sis-importer-db:
     <<: *defaults
@@ -96,7 +98,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:

package-lock.json

@@ -5,24 +5,25 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "oodikone",
       "license": "ISC",
       "devDependencies": {
         "@babel/eslint-parser": "^7.15.0",
         "@babel/plugin-syntax-jsx": "^7.14.5",
         "concurrently": "^6.2.1",
-        "cypress": "^8.2.0",
+        "cypress": "^8.3.1",
         "eslint": "^7.32.0",
         "eslint-config-airbnb": "^18.2.1",
         "eslint-config-prettier": "^8.3.0",
         "eslint-plugin-babel": "^5.3.1",
         "eslint-plugin-cypress": "^2.11.3",
-        "eslint-plugin-import": "^2.24.0",
+        "eslint-plugin-import": "^2.24.2",
         "eslint-plugin-jsx-a11y": "^6.4.1",
         "eslint-plugin-prefer-object-spread": "^1.2.1",
-        "eslint-plugin-prettier": "^3.4.0",
+        "eslint-plugin-prettier": "^3.4.1",
         "eslint-plugin-react": "^7.24.0",
         "eslint-plugin-react-hooks": "^4.2.0",
-        "husky": "^7.0.1",
+        "husky": "^7.0.2",
         "lint-staged": "^11.1.2",
         "prettier": "^2.3.2",
         "stylelint": "^13.13.1",
@@ -535,9 +536,9 @@
       }
     },
     "node_modules/@cypress/request": {
-      "version": "2.88.5",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz",
-      "integrity": "sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA==",
+      "version": "2.88.6",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.6.tgz",
+      "integrity": "sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ==",
       "dev": true,
       "dependencies": {
         "aws-sign2": "~0.7.0",
@@ -553,13 +554,12 @@
         "isstream": "~0.1.2",
         "json-stringify-safe": "~5.0.1",
         "mime-types": "~2.1.19",
-        "oauth-sign": "~0.9.0",
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
         "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
-        "uuid": "^3.3.2"
+        "uuid": "^8.3.2"
       },
       "engines": {
         "node": ">= 6"
@@ -686,6 +686,12 @@
         "postcss-syntax": ">=0.36.2"
       }
     },
+    "node_modules/@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
+      "dev": true
+    },
     "node_modules/@types/mdast": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
@@ -1468,19 +1474,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/concurrently/node_modules/date-fns": {
-      "version": "2.22.1",
-      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz",
-      "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.11"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/date-fns"
-      }
-    },
     "node_modules/concurrently/node_modules/supports-color": {
       "version": "8.1.1",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
@@ -1580,13 +1573,13 @@
       }
     },
     "node_modules/cypress": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
-      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.3.1.tgz",
+      "integrity": "sha512-1v6pfx+/5cXhaT5T6QKOvnkawmEHWHLiVzm3MYMoQN1fkX2Ma1C32STd3jBStE9qT5qPSTILjGzypVRxCBi40g==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@cypress/request": "^2.88.5",
+        "@cypress/request": "^2.88.6",
         "@cypress/xvfb": "^1.2.4",
         "@types/node": "^14.14.31",
         "@types/sinonjs__fake-timers": "^6.0.2",
@@ -1665,6 +1658,19 @@
         "node": ">=0.10"
       }
     },
+    "node_modules/date-fns": {
+      "version": "2.23.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.23.0.tgz",
+      "integrity": "sha512-5ycpauovVyAk0kXNZz6ZoB9AYMZB4DObse7P3BPWmyEjXNORTI8EJ6X0uaSAq4sCHzM1uajzrkr6HnsLQpxGXA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.11"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/date-fns"
+      }
+    },
     "node_modules/dayjs": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.5.tgz",
@@ -2067,9 +2073,9 @@
       }
     },
     "node_modules/eslint-import-resolver-node": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
-      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz",
+      "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==",
       "dev": true,
       "dependencies": {
         "debug": "^3.2.7",
@@ -2144,26 +2150,26 @@
       }
     },
     "node_modules/eslint-plugin-import": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
-      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
+      "version": "2.24.2",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz",
+      "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==",
       "dev": true,
       "dependencies": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-import-resolver-node": "^0.3.6",
         "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
-        "is-core-module": "^2.4.0",
+        "is-core-module": "^2.6.0",
         "minimatch": "^3.0.4",
-        "object.values": "^1.1.3",
+        "object.values": "^1.1.4",
         "pkg-up": "^2.0.0",
         "read-pkg-up": "^3.0.0",
         "resolve": "^1.20.0",
-        "tsconfig-paths": "^3.9.0"
+        "tsconfig-paths": "^3.11.0"
       },
       "engines": {
         "node": ">=4"
@@ -2237,9 +2243,9 @@
       }
     },
     "node_modules/eslint-plugin-prettier": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz",
-      "integrity": "sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.1.tgz",
+      "integrity": "sha512-htg25EUYUeIhKHXjOinK4BgCcDwtLHjqaxCDsMy5nbnUMkKFvIhMVCp+5GFUXQ4Nr8lBsPqtGAqBenbpFqAA2g==",
       "dev": true,
       "dependencies": {
         "prettier-linter-helpers": "^1.0.0"
@@ -3034,6 +3040,7 @@
       "version": "5.1.5",
       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
       "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "deprecated": "this library is no longer supported",
       "dev": true,
       "dependencies": {
         "ajv": "^6.12.3",
@@ -3148,9 +3155,9 @@
       }
     },
     "node_modules/husky": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.1.tgz",
-      "integrity": "sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.2.tgz",
+      "integrity": "sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg==",
       "dev": true,
       "bin": {
         "husky": "lib/bin.js"
@@ -3355,9 +3362,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz",
+      "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==",
       "dev": true,
       "dependencies": {
         "has": "^1.0.3"
@@ -4345,21 +4352,21 @@
       }
     },
     "node_modules/mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
       "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
     },
     "node_modules/mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
       "dev": true,
       "dependencies": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.49.0"
       },
       "engines": {
         "node": ">= 0.6"
@@ -4487,15 +4494,6 @@
       "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
       "dev": true
     },
-    "node_modules/oauth-sign": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
-      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
-      "dev": true,
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -4761,9 +4759,9 @@
       }
     },
     "node_modules/path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "node_modules/path-type": {
@@ -5625,6 +5623,11 @@
         "safer-buffer": "^2.0.2",
         "tweetnacl": "~0.14.0"
       },
+      "bin": {
+        "sshpk-conv": "bin/sshpk-conv",
+        "sshpk-sign": "bin/sshpk-sign",
+        "sshpk-verify": "bin/sshpk-verify"
+      },
       "engines": {
         "node": ">=0.10.0"
       }
@@ -6135,16 +6138,29 @@
       }
     },
     "node_modules/tsconfig-paths": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz",
-      "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==",
+      "version": "3.11.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz",
+      "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==",
       "dev": true,
       "dependencies": {
-        "json5": "^2.2.0",
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.1",
         "minimist": "^1.2.0",
         "strip-bom": "^3.0.0"
       }
     },
+    "node_modules/tsconfig-paths/node_modules/json5": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+      "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.0"
+      },
+      "bin": {
+        "json5": "lib/cli.js"
+      }
+    },
     "node_modules/tslib": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
@@ -6327,12 +6343,12 @@
       "dev": true
     },
     "node_modules/uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true,
       "bin": {
-        "uuid": "bin/uuid"
+        "uuid": "dist/bin/uuid"
       }
     },
     "node_modules/v8-compile-cache": {
@@ -6928,9 +6944,9 @@
       }
     },
     "@cypress/request": {
-      "version": "2.88.5",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz",
-      "integrity": "sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA==",
+      "version": "2.88.6",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.6.tgz",
+      "integrity": "sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ==",
       "dev": true,
       "requires": {
         "aws-sign2": "~0.7.0",
@@ -6946,13 +6962,12 @@
         "isstream": "~0.1.2",
         "json-stringify-safe": "~5.0.1",
         "mime-types": "~2.1.19",
-        "oauth-sign": "~0.9.0",
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
         "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
-        "uuid": "^3.3.2"
+        "uuid": "^8.3.2"
       }
     },
     "@cypress/xvfb": {
@@ -7055,6 +7070,12 @@
         "unist-util-find-all-after": "^3.0.2"
       }
     },
+    "@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
+      "dev": true
+    },
     "@types/mdast": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
@@ -7653,12 +7674,6 @@
         "yargs": "^16.2.0"
       },
       "dependencies": {
-        "date-fns": {
-          "version": "2.22.1",
-          "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz",
-          "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==",
-          "dev": true
-        },
         "supports-color": {
           "version": "8.1.1",
           "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
@@ -7736,12 +7751,12 @@
       "dev": true
     },
     "cypress": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
-      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.3.1.tgz",
+      "integrity": "sha512-1v6pfx+/5cXhaT5T6QKOvnkawmEHWHLiVzm3MYMoQN1fkX2Ma1C32STd3jBStE9qT5qPSTILjGzypVRxCBi40g==",
       "dev": true,
       "requires": {
-        "@cypress/request": "^2.88.5",
+        "@cypress/request": "^2.88.6",
         "@cypress/xvfb": "^1.2.4",
         "@types/node": "^14.14.31",
         "@types/sinonjs__fake-timers": "^6.0.2",
@@ -7810,6 +7825,12 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "date-fns": {
+      "version": "2.23.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.23.0.tgz",
+      "integrity": "sha512-5ycpauovVyAk0kXNZz6ZoB9AYMZB4DObse7P3BPWmyEjXNORTI8EJ6X0uaSAq4sCHzM1uajzrkr6HnsLQpxGXA==",
+      "dev": true
+    },
     "dayjs": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.5.tgz",
@@ -8142,9 +8163,9 @@
       "requires": {}
     },
     "eslint-import-resolver-node": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
-      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz",
+      "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==",
       "dev": true,
       "requires": {
         "debug": "^3.2.7",
@@ -8210,26 +8231,26 @@
       }
     },
     "eslint-plugin-import": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
-      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
+      "version": "2.24.2",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz",
+      "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==",
       "dev": true,
       "requires": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-import-resolver-node": "^0.3.6",
         "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
-        "is-core-module": "^2.4.0",
+        "is-core-module": "^2.6.0",
         "minimatch": "^3.0.4",
-        "object.values": "^1.1.3",
+        "object.values": "^1.1.4",
         "pkg-up": "^2.0.0",
         "read-pkg-up": "^3.0.0",
         "resolve": "^1.20.0",
-        "tsconfig-paths": "^3.9.0"
+        "tsconfig-paths": "^3.11.0"
       },
       "dependencies": {
         "debug": {
@@ -8285,9 +8306,9 @@
       "requires": {}
     },
     "eslint-plugin-prettier": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz",
-      "integrity": "sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.1.tgz",
+      "integrity": "sha512-htg25EUYUeIhKHXjOinK4BgCcDwtLHjqaxCDsMy5nbnUMkKFvIhMVCp+5GFUXQ4Nr8lBsPqtGAqBenbpFqAA2g==",
       "dev": true,
       "requires": {
         "prettier-linter-helpers": "^1.0.0"
@@ -8960,9 +8981,9 @@
       "dev": true
     },
     "husky": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.1.tgz",
-      "integrity": "sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.2.tgz",
+      "integrity": "sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg==",
       "dev": true
     },
     "ignore": {
@@ -9091,9 +9112,9 @@
       }
     },
     "is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz",
+      "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==",
       "dev": true,
       "requires": {
         "has": "^1.0.3"
@@ -9817,18 +9838,18 @@
       }
     },
     "mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
       "dev": true,
       "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.49.0"
       }
     },
     "mimic-fn": {
@@ -9932,12 +9953,6 @@
       "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
       "dev": true
     },
-    "oauth-sign": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
-      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
-      "dev": true
-    },
     "object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -10127,9 +10142,9 @@
       "dev": true
     },
     "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "path-type": {
@@ -11174,14 +11189,26 @@
       "dev": true
     },
     "tsconfig-paths": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz",
-      "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==",
+      "version": "3.11.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz",
+      "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==",
       "dev": true,
       "requires": {
-        "json5": "^2.2.0",
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.1",
         "minimist": "^1.2.0",
         "strip-bom": "^3.0.0"
+      },
+      "dependencies": {
+        "json5": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+          "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.0"
+          }
+        }
       }
     },
     "tslib": {
@@ -11333,9 +11360,9 @@
       "dev": true
     },
     "uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true
     },
     "v8-compile-cache": {

package.json

@@ -2,8 +2,8 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "concurrently": "concurrently",
     "cypress": "cypress",
+    "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
     "cli": "./cli.sh",
@@ -13,29 +13,8 @@
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install",
-    "lint:docker-compose": "docker-compose config -q && :",
-    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
-    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
-    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
-  },
-  "lint-staged": {
-    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
-    "*.{js,json,md}": "prettier --write",
-    "*.css": "stylelint --fix",
-    ".github/workflows/*": "actionlint",
-    "Dockerfile": "hadolint",
-    "*.sh": "shellcheck -x",
-    "docker-compose.yml": "npm run lint:docker-compose",
-    "docker-compose.real.yml": "npm run lint:docker-compose:real",
-    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
-    "docker-compose.test.yml": "npm run lint:docker-compose:test"
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "prepare": "husky install"
   },
   "repository": {
     "type": "git",
@@ -50,19 +29,19 @@
     "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
     "concurrently": "^6.2.1",
-    "cypress": "^8.2.0",
+    "cypress": "^8.3.1",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.0",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-prettier": "^3.4.1",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.1",
+    "husky": "^7.0.2",
     "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",

run.sh

@@ -85,7 +85,7 @@ parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
 elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${compose_command}"
+  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

scripts/functions_for_setup.sh

@@ -71,7 +71,7 @@ reset_databases() {
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
-  docker-compose down
+  npm run docker:down
   docker-compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do

services/backend/package.json

@@ -12,26 +12,23 @@
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
-    "async": "^3.2.0",
-    "async-lock": "^1.2.8",
+    "@sentry/tracing": "^6.11.0",
+    "async": "^3.2.1",
+    "async-lock": "^1.3.0",
     "axios": "^0.21.1",
     "bluebird": "^3.7.2",
-    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.17.1",
     "jsonwebtoken": "^8.5.1",
-    "kafka-node": "^4.1.3",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
-    "mkdirp": "^0.5.5",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
-    "nodemailer": "^4.7.0",
-    "pg": "^8.6.0",
-    "redis": "^3.1.1",
-    "sequelize": "^6.6.2",
+    "pg": "^8.7.1",
+    "redis": "^3.1.2",
+    "sequelize": "^6.6.5",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
@@ -40,37 +37,6 @@
     "winston-transport": "^4.4.0"
   },
   "devDependencies": {
-    "ava": "^2.4.0",
-    "faker": "^4.1.0",
-    "jest": "^24.9.0",
-    "nock": "^10.0.6",
-    "node-status": "^1.0.0",
-    "nodemon": "^1.19.4",
-    "nyc": "^13.3.0",
-    "supertest": "^3.4.2"
-  },
-  "jest": {
-    "setupFilesAfterEnv": [
-      "./test/jest/globals.js"
-    ],
-    "testEnvironment": "node"
-  },
-  "ava": {
-    "files": [
-      "test/**/*_test.js"
-    ],
-    "source": [
-      "**/*.{js,jsx}",
-      "!test/**/*"
-    ],
-    "concurrency": 5,
-    "failFast": true,
-    "failWithoutAssertions": true,
-    "tap": true,
-    "powerAssert": false,
-    "require": [
-      "babel-register"
-    ],
-    "babel": "inherit"
+    "nodemon": "^1.19.4"
   }
 }

services/backend/requests/courses_yearly.rest

@@ -1,7 +0,0 @@
-GET /api/courseyearlystats/?start=2013&end=2018&code=TKT10002 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token generated from login.rest to use

services/backend/requests/enableUser.rest

@@ -1,6 +0,0 @@
-POST http://localhost:8080/api/users/enable
-Content-Type: application/json
-
-{
-  "id": 2
-}

services/backend/requests/get_all_duplicate_codes.rest

@@ -1,6 +0,0 @@
-GET /api/courses/duplicatecodes/all HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_duplicate_codes.rest

@@ -1,7 +0,0 @@
-GET /api/courses/duplicatecodes/?code=TKT10005 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_mandatory_courses.rest

@@ -1,7 +0,0 @@
-GET /api/v2/studyprogrammes/{insertstudyprogrammecodehere}/mandatory_courses
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/get_student.rest

@@ -1,7 +0,0 @@
-GET /api/students/{insert student id}
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/login.rest

@@ -1,7 +0,0 @@
-POST /api/login HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-Cache-Control: no-cache
-Postman-Token: d8a05d0f-fd91-e4e0-54a0-172d18ff3fad
-Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW

services/backend/requests/newUser.rest

@@ -1,4 +0,0 @@
-GET http://localhost:8080/api/login
-Content-Type: application/json
-eduPersonPrincipalName: totutotu@
-shib-session-id: IDEEE

services/backend/requests/post_duplicate_codes.rest

@@ -1,7 +0,0 @@
-POST /api/courses/duplicatecodes/TKT10005/205044 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/remove_duplicate_code.rest

@@ -1,7 +0,0 @@
-DELETE /api/courses/duplicatecodes/TKT10005/58123 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/update.rest

@@ -1,11 +0,0 @@
-POST /api/updatedatabase HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-content-type: application/json
-
-[
-  "01234562",
-  "12302459"
-]

services/backend/src/app.js

@@ -1,15 +1,11 @@
 const express = require('express')
 const cors = require('cors')
-const bodyParser = require('body-parser')
 const Sentry = require('@sentry/node')
-const conf = require('./conf-backend')
+const { baseUrl, backendPort, frontUrl } = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
-const { PORT } = conf
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
-
-const SENTRY_ENVIRONMENT = process.env.SENTRY_ENVIRONMENT || ''
-const SENTRY_RELEASE = process.env.SENTRY_ENVIRONMENT || ''
+const initializeSentry = require('./util/sentry')
 
 initializeDatabaseConnection()
   .then(() => {
@@ -24,27 +20,16 @@ initializeDatabaseConnection()
     })
 
     const app = express()
-
-    if (SENTRY_ENVIRONMENT && SENTRY_RELEASE) {
-      Sentry.init({
-        dsn: 'https://020b79f0cbb14aad94cc9d69a1ea9d52@sentry.cs.helsinki.fi/2',
-        environment: SENTRY_ENVIRONMENT,
-        release: SENTRY_RELEASE,
-      })
-      app.use(Sentry.Handlers.requestHandler())
-    }
+    initializeSentry(app)
+    app.use(Sentry.Handlers.requestHandler())
+    app.use(Sentry.Handlers.tracingHandler())
 
     startCron()
 
-    app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-    app.use(bodyParser.json())
+    app.use(cors({ credentials: true, origin: frontUrl }))
+    app.use(express.json())
 
-    let BASE_URL = ''
-    if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test') {
-      BASE_URL = '/api'
-    }
-
-    routes(app, BASE_URL)
+    routes(app, baseUrl)
 
     app.get('*', async (req, res) => {
       const results = { error: 'unknown endpoint' }
@@ -53,7 +38,7 @@ initializeDatabaseConnection()
 
     app.use(Sentry.Handlers.errorHandler())
 
-    const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+    const server = app.listen(backendPort, () => console.log(`Backend listening on port ${backendPort}!`))
     process.on('SIGTERM', () => {
       server.close(() => {
         console.log('Process terminated')

services/backend/src/conf-backend.js

@@ -1,4 +1,41 @@
-const isTest = process.env.NODE_ENV === 'test'
+// Node env to use
+const isStaging = process.env.NODE_ENV === 'staging'
+const isDev = process.env.NODE_ENV === 'development'
+const isProduction = process.env.NODE_ENV === 'production'
+
+// Sentry
+const sentryRelease = process.env.SENTRY_RELEASE || ''
+const sentryEnvironment = process.env.SENTRY_ENVIRONMENT || ''
+const runningInCI = process.env.CI === 'true'
+
+// IAM group
+const requiredGroup = isStaging
+  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
+  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
+
+// Pate
+const pateToken = process.env.PATE_API_TOKEN || ''
+
+// Importer client
+const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
+const importerUrl = process.env.IMPORTER_DB_API_URL || ''
+
+// Networking: Urls & ports
+const baseUrl = isDev ? '/api' : ''
+const frontUrl = process.env.FRONT_URL
+const backendPort = 8080
+
+// Analytics
+const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
+const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
+const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
+const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
+
+// Matomo
+const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
+const MATOMO_URL = process.env.MATOMO_URL
+
+// Other stuff
 const DB_URL_KONE = process.env.DB_URL_KONE
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
@@ -10,20 +47,12 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
   DB_MAX_CRON_CONNECTIONS = 1
 }
 
-const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const USERSERVICE_URL = process.env.USERSERVICE_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
-const PORT = isTest ? 8079 : 8080
-const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
-const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
-const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
-const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
-const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
-const MATOMO_URL = process.env.MATOMO_URL
 
 const formatURL = url => {
   return !!url && !url.startsWith('http') ? `http://${url}` : url
@@ -33,14 +62,7 @@ const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
 
 const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
 
-const isStaging = process.env.NODE_ENV === 'staging'
-
-const requiredGroup = isStaging
-  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
-
 module.exports = {
-  frontend_addr,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
@@ -50,9 +72,7 @@ module.exports = {
   DB_SCHEMA_KONE,
   USERSERVICE_URL: formatURL(USERSERVICE_URL),
   ACCESS_TOKEN_HEADER_KEY,
-  PORT,
   requiredGroup,
-  isTest,
   SIS_UPDATER_URL,
   ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
   ANALYTICS_INFLUXDB_USER,
@@ -60,4 +80,14 @@ module.exports = {
   ANALYTICS_INFLUXDB_DB,
   MATOMO_SITE_ID,
   MATOMO_URL: MATOMO_URL && addSlashToEnd(MATOMO_URL),
+  runningInCI,
+  sentryRelease,
+  sentryEnvironment,
+  isProduction,
+  pateToken,
+  baseUrl,
+  frontUrl,
+  backendPort,
+  importerToken,
+  importerUrl,
 }

services/backend/src/database/connection.js

@@ -61,30 +61,28 @@ const initializeDatabaseConnection = async () => {
       await sleep(1000)
     }
   }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelizeKone,
-          tableName: 'migrations',
-          schema: conf.DB_SCHEMA_KONE,
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelizeKone.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/src/database/migrations_kone`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA_KONE,
-        },
-      })
-      const migrations = await migrator.up()
+  try {
+    const migrator = new Umzug({
+      storage: 'sequelize',
+      storageOptions: {
+        sequelize: sequelizeKone,
+        tableName: 'migrations',
+        schema: conf.DB_SCHEMA_KONE,
+      },
+      logging: console.log,
+      migrations: {
+        params: [sequelizeKone.getQueryInterface(), Sequelize],
+        path: `${process.cwd()}/src/database/migrations_kone`,
+        pattern: /\.js$/,
+        schema: conf.DB_SCHEMA_KONE,
+      },
+    })
+    const migrations = await migrator.up()
 
-      console.log('Kone Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
+    console.log('Kone Migrations up to date', migrations)
+  } catch (e) {
+    console.log('Kone Migration error')
+    throw e
   }
 }
 

services/backend/src/database/migrations_kone/20210823_00_remove_programme_modules_and_programme_module_children.js

@@ -0,0 +1,7 @@
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.dropTable('programme_module_children')
+    await queryInterface.dropTable('programme_modules')
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -20,6 +20,7 @@ const {
   patchNonGraduatedStudents,
 } = require('./services/analyticsService')
 const { isNewHYStudyProgramme } = require('./util')
+const { isProduction } = require('./conf-backend')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -189,21 +190,25 @@ const refreshProtoCProgrammeToRedis = async () => {
 }
 
 const refreshStatistics = async () => {
+  console.log('Starting refreshing statistics')
   await refreshStudyrightAssociations()
   await refreshOverview()
   await refreshNonGraduatedStudentsOfOldProgrammes()
   await refreshTeacherLeaderboard()
+  console.log('Statistics refreshed!')
 }
 
 const refreshTrends = async () => {
+  console.log('Starting refreshing trends')
   await refreshProtoCtoRedis()
   await refreshStatusToRedis()
   await refreshUberToRedis()
   await refreshProtoCProgrammeToRedis()
+  console.log('Trends refreshed!')
 }
 
 const startCron = () => {
-  if (process.env.NODE_ENV === 'production') {
+  if (isProduction) {
     schedule('0 6 * * *', async () => {
       await refreshStatistics()
       await refreshTrends()

services/backend/src/routes.js

@@ -25,10 +25,22 @@ const matomoInit = require('./routes/matomo-init')
 const customPopulationSearch = require('./routes/customPopulationSearch')
 const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
+const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 
 module.exports = (app, url) => {
   app.use(url, ping)
-  app.use(shibbolethHeadersFix(['hyGroupCn', 'SHIB_LOGOUT_URL', 'eduPersonAffiliation', 'uid', 'displayName', 'mail']))
+  app.use(
+    shibbolethHeadersFix([
+      'hyGroupCn',
+      'SHIB_LOGOUT_URL',
+      'eduPersonAffiliation',
+      'uid',
+      'displayName',
+      'mail',
+      'hyPersonSisuId',
+      'preferredLanguage',
+    ])
+  )
   app.use(url, matomoInit)
   app.use(url, login)
   app.use(`${url}/superlogin`, superlogin)
@@ -52,4 +64,5 @@ module.exports = (app, url) => {
   app.use(`${url}/tsa`, tsaAnalytics)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/cool-data-science`, trends)
+  app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
 }

services/backend/src/routes/elementdetails.js

@@ -6,7 +6,7 @@ const {
   throughputStatsForStudytrack,
   optionData,
 } = require('../services/studyprogramme')
-const { findProgrammeTheses } = require('../services/thesis')
+const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
 
 const {
   getProductivity,
@@ -17,9 +17,28 @@ const {
   patchThroughput,
   getNonGraduatedStudents,
 } = require('../services/analyticsService')
+const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
 
 const programmeStatsSince = new Date('2017-07-31')
 
+router.get('/elementdetails/all', async (req, res) => {
+  try {
+    const elementdetails = await getAllElementDetails()
+    res.json(elementdetails)
+  } catch (e) {
+    res.status(500).json(e)
+  }
+})
+
+router.get('/studyprogrammes', async (req, res) => {
+  try {
+    const studyrights = await getAllProgrammes()
+    res.json(studyrights)
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
 router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   const code = req.params.id
 
@@ -189,6 +208,37 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   console.log(`Throughput stats recalculation for studyprogramme ${code} done`)
 })
 
+router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
+  const { id } = req.params
+  if (id) {
+    const thesis = await findProgrammeTheses(id)
+    res.json(thesis)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
+  const { id } = req.params
+  const { course, thesisType } = req.body
+  if (id && course && thesisType) {
+    const thesis = await createThesisCourse(id, course, thesisType)
+    res.status(201).json(thesis)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
+  const { id, course } = req.params
+  if (id && course) {
+    const deleted = await deleteThesisCourse(id, course)
+    res.status(204).json(deleted)
+  } else {
+    res.status(422).end()
+  }
+})
+
 router.use('*', (req, res, next) => next())
 
 module.exports = router

services/backend/src/routes/faculties.js

@@ -3,7 +3,6 @@ const { faculties } = require('../services/organisations')
 
 router.get('/', async (req, res) => {
   const facultyList = await faculties()
-  console.log('faculties')
   res.json(facultyList)
 })
 

services/backend/src/routes/feedback.js

@@ -1,29 +1,23 @@
 const router = require('express').Router()
-const mailservice = require('../services/mailservice')
 const userService = require('../services/userService')
+const { sendFeedbackToToska } = require('../services/mailservice')
 
 router.post('/email', async (req, res) => {
   const { content } = req.body
   const { uid } = req.headers
   const { email, full_name } = await userService.byUsername(uid)
-  const formattedEmail = content
-    .split('\n\n')
-    .map(line => `<p> ${line} </p>`)
-    .join('')
-  const feedback = mailservice.feedback(formattedEmail, uid, email, full_name)
 
-  if (process.env.SMTP !== undefined && content) {
-    await mailservice.transporter.sendMail(feedback, error => {
-      if (error) {
-        console.log('Error occured')
-        res.status(400).json(error).end()
-      } else {
-        console.log('Message sent succesfully!')
-        res.status(200).json('success').end()
-      }
-      mailservice.transporter.close()
-    })
+  const result = await sendFeedbackToToska({
+    feedbackContent: content,
+    userId: uid,
+    userEmail: email,
+    userFullName: full_name,
+  })
+  if (result.error) {
+    return res.status(400).json(result).end()
   }
+  console.log('Message sent succesfully!')
+  res.status(200).json('success').end()
 })
 
 module.exports = router

services/backend/src/routes/login.js

@@ -1,23 +1,7 @@
 const router = require('express').Router()
-const mailservice = require('../services/mailservice')
 const userService = require('../services/userService')
 const { parseHyGroups } = require('../util/utils')
-
-const sendEmail = async uid => {
-  if (process.env.SMTP !== undefined) {
-    const message = mailservice.message1(uid)
-    await mailservice.transporter.sendMail(message, error => {
-      if (error) {
-        console.log('Error occurred')
-        console.log(error.message)
-      } else {
-        console.log('Message sent successfully!')
-      }
-      // only needed when using pooled connections
-      mailservice.transporter.close()
-    })
-  }
-}
+const { sendNotificationAboutNewUser } = require('../services/mailservice')
 
 router.post('/login', async (req, res) => {
   try {
@@ -27,10 +11,16 @@ router.post('/login', async (req, res) => {
       const mail = req.headers.mail || ''
       const hyGroups = parseHyGroups(req.headers['hygroupcn'])
       const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-
+      const hyPersonSisuId = req.headers.hypersonsisuid || ''
+      console.log('headers', JSON.stringify(req.headers, null, 2))
       console.log(uid, 'trying to login, referring to userservice.')
-      let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail)
-      isNew && sendEmail(uid)
+      let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
+      if (isNew) {
+        const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
+        if (result.error) {
+          return res.status(400).json(result).end()
+        }
+      }
       res.status(200).json({ token })
     } else {
       res

services/backend/src/routes/population.js

@@ -37,7 +37,6 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
     }
 
     if (req.body.years) {
-      console.log('years väylä')
       const upperYearBound = new Date().getFullYear() + 1
       const multicoursestatPromises = Promise.all(
         req.body.years.map(year => {

services/backend/src/routes/studyGuidanceGroups.js

@@ -0,0 +1,21 @@
+const router = require('express').Router()
+const { getImporterClient } = require('../util/importerClient')
+
+router.get('/', async (req, res) => {
+  const personId = req.headers.hypersonsisuid
+  if (!personId) {
+    const errorMessage = 'Not possible to get groups without personId header'
+    console.log(errorMessage)
+    return res.status(400).json({ error: errorMessage })
+  }
+  const importerClient = getImporterClient()
+  try {
+    const { data } = await importerClient.get(`/person-groups/person/${personId}`)
+    return res.json(Object.values(data))
+  } catch (e) {
+    console.log(e)
+    return res.status(500).json(e)
+  }
+})
+
+module.exports = router

services/backend/src/routes/users.js

@@ -1,8 +1,8 @@
 const router = require('express').Router()
 const userService = require('../services/userService')
-const mailservice = require('../services/mailservice')
 const blacklist = require('../services/blacklist')
 const { userDataCache } = require('../services/cache')
+const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
 
 const addUserToBlacklist = async (user, decodedToken) => {
   if (user) {
@@ -35,29 +35,22 @@ router.post('/modifyaccess', async (req, res) => {
 })
 
 router.get('/email/preview', (req, res) => {
-  const { subject, html } = mailservice.message2(null)
-  res.json({ subject, html })
+  const { accessMessageSubject, accessMessageText } = previewNotificationAboutAccessToUser()
+  res.json({ subject: accessMessageSubject, html: accessMessageText })
 })
 
 router.post('/email', async (req, res) => {
-  if (!process.env.SMTP) {
-    return res.status(500).json({ error: 'Email system has not been configured' })
-  }
-
-  const email = req.body.email
-  if (!email) {
+  const userEmail = req.body.email
+  if (!userEmail) {
     return res.status(400).json({ error: 'email is missing' })
   }
 
-  const message = mailservice.message2(email)
-  try {
-    const info = await mailservice.transporter.sendMail(message)
-    console.log('Message sent successfully', info)
-    res.status(200).end()
-  } catch (e) {
-    console.error('Error occurred while sending user email', e)
-    res.status(500).json({ error: e.message })
+  const result = await sendNotificationAboutAccessToUser(userEmail)
+  if (result.error) {
+    return res.status(400).json(result).end()
   }
+  console.log('Message sent successfully')
+  res.status(200).end()
 })
 
 router.post('/:uid/elements', async (req, res) => {

services/backend/src/services/analyticsService.js

@@ -91,7 +91,8 @@ const patchThroughput = async data => {
 const patchNonGraduatedStudents = async data => {
   const [id, dataToPatch] = Object.entries(data)[0]
   const redisKey = createRedisKeyForNonGraduatedStudents(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
+  const dataFromRedisRaw = await redisClient.getAsync(redisKey)
+  const dataFromRedis = dataFromRedisRaw ? JSON.parse(dataFromRedisRaw) : { data: null }
   const patchedData = {
     data: {
       ...dataFromRedis.data,

services/backend/src/services/courses.js

@@ -424,7 +424,6 @@ const byNameAndOrCodeLike = async (name, code) => {
       organizeSubgroups(course)
     }
   })
-  console.log(courses)
   return { courses }
 }
 

services/backend/src/services/mailservice.js

@@ -1,81 +1,109 @@
-const nodemailer = require('nodemailer')
+const axios = require('axios')
+const { isProduction, pateToken } = require('../conf-backend')
 
-// NB! Store the account object values somewhere if you want
-// to re-use the same account for future mail deliveries
-
-// Create a SMTP transporter object
-const transporter = nodemailer.createTransport(
-  {
-    from: 'Oodikone Robot <noreply@helsinki.fi>',
-    host: `${process.env.SMTP}`,
-    port: 587,
-    secure: false, // false -> TLS, true -> SSL
-    logger: false,
-    debug: false, // include SMTP traffic in the logs
+const pateClient = axios.create({
+  baseURL: 'https://pate.toska.cs.helsinki.fi',
+  params: {
+    token: pateToken,
   },
-  {
-    from: 'Oodikone Robot <noreply@helsinki.fi>',
-  }
-)
+})
 
-const message1 = user => {
-  return {
-    to: 'Toska <grp-toska@helsinki.fi>',
-    subject: 'New user in Oodikone ✔',
-    text: `${user} just logged into oodikone for the first time!`,
-    html: `<p>${user} just logged into oodikone for the first time! </p><br />
-          <img src="cid:toskalogoustcid"/>`,
-    attachments: [
-      {
-        filename: 'toska.png',
-        path: `${process.cwd()}/assets/toska.png`,
-        cid: 'toskalogoustcid',
-      },
-    ],
-  }
-}
-const message2 = email => {
-  return {
-    to: [email, 'grp-toska@helsinki.fi'],
-    replyTo: 'Toska <grp-toska@helsinki.fi>',
-    subject: 'Authorized in oodikone',
-    text: "You've been authorized to use oodikone.",
-    html: `<p>You've been authorized to use <a href="https://oodikone.cs.helsinki.fi">oodikone</a> or 
-    your privileges in <a href="https://oodikone.cs.helsinki.fi">oodikone</a> have changed.</p>
-    <p>You may need to log out and back in before your changes take place.</p>
-    <p>NOTE! If the automatic logging out from all services was not successful when
-    logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
-    <p>If you have defined your browser to restore your previous session in connection
-    with starting the browser, logging out may also require deleting the cookies
-    of the listed services in addition to closing the browser windows.</p>
-    <p>See the instructions for clearing the browser cache: <a href="https://helpdesk.it.helsinki.fi/en/help/1002">
-    https://helpdesk.it.helsinki.fi/en/help/1002</a></p>
-    <br />
-    <hr />
-    <br />
-    <p>Helsingin Yliopisto, TOSKA.</p>
-    <p>grp-toska@helsinki.fi</p>
-    <p>Pietari Kalmin katu 5, Exactum BK113</p>
-    <img style="max-width: 13.5%;height: auto;" src="https://i.imgur.com/tnNDAJk.png" /> `,
-  }
-}
-const feedback = (content, user, email, full_name) => {
-  return {
-    to: 'Toska <grp-toska@helsinki.fi>',
-    replyTo: email,
-    subject: 'New message from Oodikone feedback',
-    text: `New message from user ${user}`,
-    html: `${content}
-    <p>sent by ${full_name}, userid: ${user}, email: ${email}</p>
-    <img src="cid:toskalogoustcid"/>`,
-    attachments: [
-      {
-        filename: 'toska.png',
-        path: `${process.cwd()}/assets/toska.png`,
-        cid: 'toskalogoustcid',
-      },
-    ],
-  }
+const template = {
+  from: 'Oodikone Robot',
 }
 
-module.exports = { transporter, message1, message2, feedback }
+const baseSettings = {
+  color: 'orange',
+  header: 'Sent by Oodikone',
+  dryrun: false,
+}
+
+const sendEmail = async (options = {}) => {
+  if (!isProduction) {
+    const errorMessage = 'Skipped sending email in non-production environment'
+    console.log(errorMessage, options)
+    return { error: errorMessage }
+  }
+
+  if (!pateToken) {
+    const errorMessage = 'Skipped sending email since pate token was not available'
+    console.log(errorMessage, options)
+    return { error: errorMessage }
+  }
+
+  const { data } = await pateClient.post('/', options)
+
+  return data
+}
+
+const sendFeedbackToToska = ({ feedbackContent, userId, userEmail, userFullName }) => {
+  const userDetails = `Sent by ${userFullName}, userid: ${userId}, email: ${userEmail}`
+  const text = [feedbackContent, userDetails].join('<br />')
+  return sendEmail({
+    template,
+    emails: [
+      {
+        to: 'Toska <grp-toska@helsinki.fi>',
+        replyTo: userEmail,
+        subject: `Oodikone feedback from ${userFullName}`,
+        text,
+      },
+    ],
+    settings: {
+      ...baseSettings,
+      hideToska: true,
+      disableToska: true,
+    },
+  })
+}
+
+const sendNotificationAboutNewUser = ({ userId, userFullName }) =>
+  sendEmail({
+    template,
+    emails: [
+      {
+        to: 'Toska <grp-toska@helsinki.fi>',
+        subject: 'New user in Oodikone ✔',
+        text: `${userFullName} (${userId}) just logged into Oodikone for the first time!`,
+      },
+    ],
+    settings: {
+      ...baseSettings,
+      hideToska: true,
+      disableToska: true,
+    },
+  })
+
+const accessMessageSubject = 'Authorized in Oodikone'
+const accessMessageText = `<p>You've been authorized to use <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> or your privileges in <a href="https://oodikone.cs.helsinki.fi">Oodikone</a> have changed.</p>
+<p>You may need to log out and back in before your changes take place.</p>
+<p>NOTE! If the automatic logging out from all services was not successful when logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
+<p>If you have defined your browser to restore your previous session in connection with starting the browser, logging out may also require deleting the cookies of the listed services in addition to closing the browser windows.</p>
+<p>See the instructions for clearing the browser cache: <a href="https://helpdesk.it.helsinki.fi/en/help/1002">https://helpdesk.it.helsinki.fi/en/help/1002</a></p>`
+
+const previewNotificationAboutAccessToUser = () => ({ accessMessageSubject, accessMessageText })
+
+const sendNotificationAboutAccessToUser = userEmail =>
+  sendEmail({
+    template,
+    emails: [
+      {
+        to: userEmail,
+        replyTo: 'Toska <grp-toska@helsinki.fi>',
+        subject: accessMessageSubject,
+        text: accessMessageText,
+      },
+    ],
+    settings: {
+      ...baseSettings,
+      hideToska: false,
+      disableToska: false,
+    },
+  })
+
+module.exports = {
+  sendFeedbackToToska,
+  sendNotificationAboutNewUser,
+  sendNotificationAboutAccessToUser,
+  previewNotificationAboutAccessToUser,
+}

services/backend/src/services/populations.js

@@ -19,7 +19,7 @@ const {
   dbConnections: { sequelize },
 } = require('../database/connection')
 const { Tag, TagStudent } = require('../models/models_kone')
-const { allCodeAltenatives, findOneByCode } = require('./courses')
+const { /*allCodeAltenatives, */ findOneByCode } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
@@ -842,6 +842,14 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date,
 })
 
+const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+const unifyOpenUniversity = code => {
+  const regexresult = isOpenUniCourseCode(code)
+  if (!regexresult) return code
+  return regexresult[1]
+}
+
 const bottlenecksOf = async (query, studentnumberlist) => {
   const isValidRequest = async (query, params) => {
     const {
@@ -853,6 +861,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
       nondegreeStudents,
       transferredStudents,
     } = params
+
     if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
     }
@@ -883,6 +892,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     }
     return null
   }
+
   const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
     if (!studentnumberlist) {
       const {
@@ -943,13 +953,15 @@ const bottlenecksOf = async (query, studentnumberlist) => {
 
   for (const course of courses) {
     let { course_type } = course
-    const substitutions = allCodeAltenatives(course.code)
     let maincourse = course
+    if (isOpenUniCourseCode) {
+      const nonOpen = unifyOpenUniversity(course.code)
+      const response = await findOneByCode(nonOpen)
 
-    if (course.code !== substitutions[0]) {
-      maincourse = await findOneByCode(course.code)
+      if (response) {
+        maincourse = response
+      }
     }
-
     if (!stats[maincourse.code]) {
       stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudentslength)
     }

services/backend/src/services/students.js

@@ -11,6 +11,7 @@ const {
   StudyrightElement,
   ElementDetail,
   SemesterEnrollment,
+  Semester,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
@@ -75,6 +76,21 @@ const byId = async id => {
       },
     },
   })
+
+  const semesters = await Semester.findAll()
+
+  const mappedEnrollments = student.semester_enrollments.map(enrollment => {
+    const semester = semesters.find(sem => sem.semestercode === enrollment.semestercode)
+    return {
+      ...enrollment.dataValues,
+      name: semester.name,
+      yearname: semester.yearname,
+      startYear: semester.startYear,
+    }
+  })
+
+  student.semester_enrollments = mappedEnrollments
+
   student.tags = tags.map(t => ({
     ...t.get(),
     programme: tagprogrammes.find(p => p.code === t.tag.studytrack),
@@ -170,10 +186,15 @@ const formatStudent = ({
 
   studyrights = studyrights || []
   semester_enrollments = semester_enrollments || []
-  const semesterenrollments = semester_enrollments.map(({ semestercode, enrollmenttype }) => ({
-    semestercode,
-    enrollmenttype,
-  }))
+  const semesterenrollments = semester_enrollments.map(
+    ({ semestercode, enrollmenttype, name, yearname, startYear }) => ({
+      yearname,
+      name,
+      startYear,
+      semestercode,
+      enrollmenttype,
+    })
+  )
 
   const courseByDate = (a, b) => {
     return moment(a.attainment_date).isSameOrBefore(b.attainment_date) ? -1 : 1

services/backend/src/services/studyrights.js

@@ -402,7 +402,6 @@ const getAssociations = async (doRefresh = false) => {
 }
 
 const getFilteredAssociations = async codes => {
-  console.log(codes)
   const associations = await getAssociations()
   associations.programmes = _.pick(associations.programmes, codes)
 

services/backend/src/services/trends/statusAttainments.js

@@ -95,7 +95,10 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     yearRange,
     diff => ({
       from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to: new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
+      to:
+        showByYear === 'true'
+          ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
+          : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
     }),
     'acc',
     'students'

services/backend/src/services/trends/statusGraduated.js

@@ -56,7 +56,7 @@ const calculateStatusGraduated = async (unixMillis, showByYear) => {
 
   const yearlyRange = diff => ({
     from: new Date(startTime - diff * Y_TO_MS),
-    to: new Date(unixMillis - diff * Y_TO_MS),
+    to: showByYear === 'true' ? new Date(startTime - (diff - 1) * Y_TO_MS) : new Date(unixMillis - diff * Y_TO_MS),
   })
 
   const totalRange = diff => ({

services/backend/src/services/userService.js

@@ -17,13 +17,14 @@ const findAll = async () => {
   return response.data
 }
 
-const login = async (uid, full_name, hyGroups, affiliations, email) => {
+const login = async (uid, full_name, hyGroups, affiliations, email, hyPersonSisuId) => {
   const response = await client.post('/login', {
     uid,
     full_name,
     hyGroups,
     affiliations,
     email,
+    hyPersonSisuId,
   })
   return response.data
 }

services/backend/src/util/add_json_duplicates_to_redis.js

@@ -1,18 +0,0 @@
-const { setDuplicateCode } = require('../services/courses')
-const newToOld = require('./newToOld.json')
-const oldToNew = require('./oldToNew.json')
-
-const mapJson = async () => {
-  for (const i in newToOld) {
-    await setDuplicateCode(i, newToOld[i])
-    await setDuplicateCode(newToOld[i], i)
-  }
-  for (const i in oldToNew) {
-    await setDuplicateCode(i, oldToNew[i])
-    await setDuplicateCode(oldToNew[i], i)
-  }
-  console.log('Mapped codes from newToOld.json and oldToNew.json')
-  process.exit()
-}
-
-mapJson()

services/backend/src/util/importerClient.js

@@ -0,0 +1,22 @@
+const axios = require('axios')
+
+const { importerToken, importerUrl } = require('../conf-backend')
+
+const importerClient = axios.create({
+  headers: {
+    token: importerToken,
+  },
+  baseURL: importerUrl,
+})
+
+const getImporterClient = () => {
+  if (!importerToken || !importerUrl) {
+    console.log("Importer token or importer url not set, can't return client!")
+    return null
+  }
+  return importerClient
+}
+
+module.exports = {
+  getImporterClient,
+}

services/backend/src/util/index.js

@@ -1,82 +1,8 @@
-const LRU = require('lru-cache')
-const oldToNewMap = require('./oldToNew.json')
-
-const arrayUnique = (value, index, self) => {
-  return self.indexOf(value) === index
-}
-
-const getStudentNumberChecksum = studentNumber => {
-  let checksumNumbers = [7, 3, 1]
-  let checksum = 0
-
-  for (let i = 0; i < studentNumber.length; i++) {
-    // go from end t start
-    let currentNumber = studentNumber[studentNumber.length - (i + 1)]
-    checksum += currentNumber * checksumNumbers[i % checksumNumbers.length]
-  }
-
-  return (10 - (checksum % 10)) % 10
-}
-
-const isValidStudentId = id => {
-  if (/^0\d{8}$/.test(id)) {
-    // is a 9 digit number
-    const multipliers = [7, 1, 3, 7, 1, 3, 7]
-    const checksum = id
-      .substring(1, 8)
-      .split('')
-      .reduce((sum, curr, index) => {
-        return (sum + curr * multipliers[index]) % 10
-      }, 0)
-    return (10 - checksum) % 10 == id[8]
-  }
-  return false
-}
-
-// helpers to print complex sequelizedata
-const plainPrint = sequelizeData => console.log(JSON.parse(JSON.stringify(sequelizeData)))
-
-const plainify = sequelizeData => JSON.parse(JSON.stringify(sequelizeData))
-
-const oldToNew = code => {
-  const mappedCode = oldToNewMap[code]
-  return mappedCode ? mappedCode : code
-}
-
 const validateParamLength = (param, minLength) => param && param.trim().length >= minLength
 
 const isNewHYStudyProgramme = code => !!(code && code.match(/^[A-Z]*[0-9]*_[0-9]*$/))
 
-/**
- * @param {(...args: any[]) => Promise<any>} fn
- * @param {(...args: any[]) => string | number} keyPicker
- * @param {LRU.Options} cacheOpts
- * @returns {(...args: any[]) => Promise<any>}
- */
-const lruMemoize = (fn, keyPicker, cacheOpts) => {
-  const cache = new LRU(cacheOpts)
-
-  return async (...args) => {
-    const key = keyPicker(args)
-    const cached = cache.get(key)
-    if (cached) {
-      return cached
-    }
-
-    const ret = await fn(...args)
-    cache.set(key, ret)
-    return ret
-  }
-}
-
 module.exports = {
-  arrayUnique,
-  getStudentNumberChecksum,
-  plainPrint,
-  plainify,
-  isValidStudentId,
-  oldToNew,
   validateParamLength,
   isNewHYStudyProgramme,
-  lruMemoize,
 }

services/backend/src/util/init_redis.js

@@ -1,9 +0,0 @@
-const { redisClient } = require('../services/redis')
-
-const initializeDuplicates = async () => {
-  await redisClient.setAsync('duplicates', '{}')
-  console.log('Initiated redis')
-  process.exit()
-}
-
-initializeDuplicates()

services/backend/src/util/logger.js

@@ -10,7 +10,7 @@ if (process.env.LOG_PORT && process.env.LOG_HOST) {
       host: process.env.LOG_HOST,
       port: process.env.LOG_PORT,
       protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
+      environment: 'production',
       protocolOptions: {
         path: process.env.LOG_PATH || '/gelf',
       },
@@ -18,9 +18,7 @@ if (process.env.LOG_PORT && process.env.LOG_HOST) {
   )
 }
 
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
+transports.push(new winston.transports.File({ filename: 'debug.log' }))
 
 transports.push(new winston.transports.Console({ level: 'debug' }))
 

services/backend/src/util/map_course_codes.js

@@ -1,36 +0,0 @@
-const Course = require('../services/courses')
-const { setDuplicateCode } = require('../services/courses')
-const { redisClient } = require('../services/redis')
-
-// two tables of prefixes, for example new prefixes of  Matlu courses:
-// ['CSM', 'DATA', 'FYS', 'MAT', 'TKT', 'KEK']
-// and old prefixes
-// ['5', '7']
-
-const mapCourseCodes = async (newPrefixes, oldPrefixes) => {
-  const all = await redisClient.getAsync('duplicates')
-  if (!all) {
-    await redisClient.setAsync('duplicates', '{}')
-  }
-  const res = await Course.findDuplicates(newPrefixes, oldPrefixes)
-  const mappedCodes = res ? res[0] : []
-  const formattedCodes = mappedCodes.map(r => {
-    return { [r.code1]: r.code2 }
-  })
-  await updateRedis(formattedCodes)
-  console.log('Mapped course codes with similar names with codes starting with ', newPrefixes, ' to  ', oldPrefixes)
-  process.exit()
-}
-
-const updateRedis = async codes => {
-  for (const i in codes) {
-    let code = codes[i]
-    const newCode = Object.keys(code)[0]
-    const oldCode = code[newCode]
-
-    await setDuplicateCode(newCode, oldCode)
-    await setDuplicateCode(oldCode, newCode)
-  }
-}
-
-mapCourseCodes(['CSM', 'TKT', 'MAT', 'FYS', 'KEK'], ['5', '7'])

services/backend/src/util/newToOld.json

@@ -1,114 +0,0 @@
-{
-  "TKT20002": "582101",
-  "TKT20002": "582104",
-  "TKT20001": "58131",
-  "TKT20010": "58161",
-  "TKT10005": "581305",
-  "TKT10004": "581328",
-  "TKT20006": "581259",
-  "TKT20007": "581260",
-  "TKT20005": "582206",
-  "MAT11001": "57033",
-  "TKT50003": "581324",
-  "TKT10002": "581325",
-  "80088": "582514",
-  "TKT10001": "582102",
-  "TKT10003": "582103",
-  "TKT20004": "582202",
-  "TKT20011": "582203",
-  "MAT11002": "57043",
-  "MAT21014": "57160",
-  "TKT21015": "582335",
-  "TKT20003": "582219",
-  "TKT21002": "582733",
-  "MAT21001": "57047",
-  "MAT11003": "57116",
-  "DATA15001": "582216",
-  "MAT21015": "57161",
-  "TKT50005": "582509",
-  "MAT11004": "57117",
-  "FYS2041": "53861",
-  "TKT50002": "582506",
-  "TKT20013": "582204",
-  "TKT50001": "582505",
-  "TKT1": "584111",
-  "TKT21001": "582482",
-  "TKT21013": "582326",
-  "CSM14101": "581358",
-  "CSM14203": "581362",
-  "CSM14103": "582663",
-  "FYS4012": "53905",
-  "CSM14105": "581360",
-  "KEK101": "55052",
-  "65122": "64162",
-  "KEK223": "55059",
-  "MAT21003": "57121",
-  "FYS1001": "530281",
-  "MAT21020": "57122",
-  "CSM13202": "582749",
-  "ON-160": "200017",
-  "770270": "770044",
-  "MAT21012": "57044",
-  "MAT21007": "57101",
-  "MAT21013": "57048",
-  "99501Mat-lu": "99501Kaikki",
-  "57595": "530146",
-  "FYS1002": "530282",
-  "78134": "57714",
-  "582221": "58160",
-  "KEK103": "55053",
-  "KEK110": "55395",
-  "68153Mat": "68153Fys",
-  "PED111": "68152",
-  "MFK-403": "57169",
-  "55521": "55056",
-  "FYS1013": "53398",
-  "FYS2010": "53723",
-  "MFK-M303": "57124",
-  "FYS2011": "53724",
-  "MFK-M305": "57136",
-  "FYS2083": "530177",
-  "530148": "300123",
-  "55328": "55055",
-  "FYS2013": "53726",
-  "FYS2012": "53725",
-  "200230_WT": "200230T",
-  "90093": "3750501",
-  "90096": "3750502",
-  "20012_WasaT": "20012",
-  "FYS2077": "530289",
-  "200250_WasaT": "200250",
-  "55306": "55077",
-  "50031": "30300",
-  "53112": "530217",
-  "ME-004": "80144",
-  "992993": "99299",
-  "200251_WasaT": "200251",
-  "200203C_WT": "200203C",
-  "MAT11005": "57118",
-  "MAT21002": "57119",
-  "MAT21005": "57152",
-  "590190": "50099",
-  "FYS1003": "530283",
-  "FYS1010": "53704",
-  "KK-ENVALT": "KK-ENMALU",
-  "KK-RUMAME": "KK-RUMALU",
-  "FYS1011": "53705",
-  "57084": "50000",
-  "FYS2001": "530286",
-  "KK-ENMALU1": "KK-ENLAAK1",
-  "20016T": "20016",
-  "20014T": "20014",
-  "FYS2004": "530000",
-  "BIO-104": "52073",
-  "FYS2031": "535026",
-  "ON-330": "MAT20001",
-  "99291Oik": "99291Mat-lu",
-  "KK-ENMALU2": "KK-ENLAAK2",
-  "FYS2085": "53399",
-  "PROV-907": "590367",
-  "790Y330": "732290",
-  "KEK225": "55075",
-  "68207": "68166",
-  "TKT20008": "DATA15001"
-}

services/backend/src/util/oldToNew.json

@@ -1,116 +0,0 @@
-{
-  "582101": "TKT20002",
-  "582104": "TKT20002",
-  "58131": "TKT20001",
-  "58161": "TKT20010",
-  "581305": "TKT10005",
-  "581328": "TKT10004",
-  "581259": "TKT20006",
-  "581260": "TKT20007",
-  "582206": "TKT20005",
-  "57033": "MAT11001",
-  "581324": "TKT50003",
-  "581325": "TKT10002",
-  "582514": "80088",
-  "582102": "TKT10001",
-  "582103": "TKT10003",
-  "582202": "TKT20004",
-  "582203": "TKT20011",
-  "57043": "MAT11002",
-  "57160": "MAT21014",
-  "582335": "TKT21015",
-  "582733": "TKT21002",
-  "57047": "MAT21001",
-  "57116": "MAT11003",
-  "582216": "DATA15001",
-  "57161": "MAT21015",
-  "582509": "TKT50005",
-  "57117": "MAT11004",
-  "53861": "FYS2041",
-  "582506": "TKT50002",
-  "582204": "TKT20013",
-  "582505": "TKT50001",
-  "584111": "TKT1",
-  "582482": "TKT21001",
-  "582326": "TKT21013",
-  "581358": "CSM14101",
-  "581362": "CSM14203",
-  "582663": "CSM14103",
-  "53905": "FYS4012",
-  "581360": "CSM14105",
-  "582219": "TKT20003",
-  "55052": "KEK101",
-  "64162": "65122",
-  "55059": "KEK223",
-  "57121": "MAT21003",
-  "530281": "FYS1001",
-  "57122": "MAT21020",
-  "582749": "CSM13202",
-  "200017": "ON-160",
-  "770044": "770270",
-  "57044": "MAT21012",
-  "57101": "MAT21007",
-  "57048": "MAT21013",
-  "99501Kaikki": "99501Mat-lu",
-  "530146": "57595",
-  "530282": "FYS1002",
-  "57714": "78134",
-  "58160": "582221",
-  "55053": "KEK103",
-  "55395": "KEK110",
-  "68153Fys": "68153Mat",
-  "68152": "PED111",
-  "57169": "MFK-403",
-  "55056": "55521",
-  "53398": "FYS1013",
-  "53723": "FYS2010",
-  "57124": "MFK-M303",
-  "53724": "FYS2011",
-  "57136": "MFK-M305",
-  "530177": "FYS2083",
-  "300123": "530148",
-  "55055": "55328",
-  "53726": "FYS2013",
-  "53725": "FYS2012",
-  "200230T": "200230_WT",
-  "3750501": "90093",
-  "3750502": "90096",
-  "20012": "20012_WasaT",
-  "530289": "FYS2077",
-  "200250": "200250_WasaT",
-  "55077": "55306",
-  "30300": "50031",
-  "530217": "53112",
-  "80144": "ME-004",
-  "99299": "992993",
-  "200251": "200251_WasaT",
-  "200203C": "200203C_WT",
-  "57118": "MAT11005",
-  "57119": "MAT21002",
-  "57152": "MAT21005",
-  "99501Hum": "99501Mat-lu",
-  "50099": "590190",
-  "530283": "FYS1003",
-  "53704": "FYS1010",
-  "KK-ENMALU": "KK-ENVALT",
-  "KK-RUMALU": "KK-RUMAME",
-  "53705": "FYS1011",
-  "530148": "80088",
-  "50000": "57084",
-  "68153Kem": "68153Mat",
-  "530286": "FYS2001",
-  "KK-ENLAAK1": "KK-ENMALU1",
-  "20016": "20016T",
-  "20014": "20014T",
-  "530000": "FYS2004",
-  "52073": "BIO-104",
-  "535026": "FYS2031",
-  "MAT20001": "ON-330",
-  "99291Mat-lu": "99291Oik",
-  "KK-ENLAAK2": "KK-ENMALU2",
-  "53399": "FYS2085",
-  "590367": "PROV-907",
-  "732290": "790Y330",
-  "55075": "KEK225",
-  "68166": "68207"
-}

services/backend/src/util/sentry.js

@@ -0,0 +1,18 @@
+const Sentry = require('@sentry/node')
+const Tracing = require('@sentry/tracing')
+
+const { sentryRelease, sentryEnvironment, runningInCI } = require('../conf-backend')
+
+const initializeSentry = app => {
+  if (!sentryRelease || !sentryEnvironment || runningInCI) return
+
+  Sentry.init({
+    dsn: 'https://020b79f0cbb14aad94cc9d69a1ea9d52@sentry.cs.helsinki.fi/2',
+    environment: sentryEnvironment,
+    release: sentryRelease,
+    integrations: [new Sentry.Integrations.Http({ tracing: true }), new Tracing.Integrations.Express({ app })],
+    tracesSampleRate: 1.0,
+  })
+}
+
+module.exports = initializeSentry

services/backend/src/util/taskpool.js

@@ -1,63 +0,0 @@
-const taskpool = (poolsize = 5) => {
-  const queue = []
-  let hasPendingTask = false
-  let getPendingTask = undefined
-  let sleep = true
-
-  const nextTask = () => {
-    if (queue.length <= poolsize && hasPendingTask) {
-      const task = getPendingTask()
-      queue.push(task)
-    }
-    return queue.pop()
-  }
-
-  const execute = async () => {
-    sleep = false
-    while (queue.length > 0 || hasPendingTask) {
-      const task = nextTask()
-      await task()
-    }
-    sleep = true
-  }
-
-  const enqueue = task => {
-    return new Promise(resolve => {
-      if (queue.length < poolsize) {
-        queue.push(task)
-        resolve()
-      } else {
-        hasPendingTask = true
-        getPendingTask = () => {
-          resolve()
-          hasPendingTask = false
-          return task
-        }
-      }
-      if (sleep === true) {
-        execute()
-      }
-    })
-  }
-
-  const complete = () => {
-    return new Promise(resolve => {
-      queue.unshift(() => {
-        resolve()
-      })
-      if (sleep === true) {
-        execute()
-      }
-    })
-  }
-
-  return {
-    execute,
-    enqueue,
-    complete,
-  }
-}
-
-module.exports = {
-  taskpool,
-}

services/frontend/package.json

@@ -7,6 +7,7 @@
   },
   "dependencies": {
     "@sentry/browser": "^6.9.0",
+    "@sentry/tracing": "^6.11.0",
     "axios": "^0.21.1",
     "highcharts": "^9.1.2",
     "highcharts-custom-events": "^3.0.10",

services/frontend/src/apiConnection/index.js

@@ -12,6 +12,7 @@ const getHeaders = () => {
     hyGroupCn: 'grp-oodikone-users;grp-oodikone-basic-users',
     eduPersonAffiliation: 'member;employee;faculty',
     mail: 'grp-toska+mockmluukkai@helsinki.fi',
+    hyPersonSisuId: 'hy-hlo-1441871',
   }
   return isDev ? { ...devUserHeaders } : {}
 }

services/frontend/src/components/CoolDataScience/Status.jsx

@@ -154,15 +154,16 @@ const Status = ({ getStatusDispatch, data, loading }) => {
   const [showSettings, setShowSettings] = useState(true)
   const [selectedDate, setSelectedDate] = useState(moment())
   const [codes, setCodes] = useState([])
+  const [showByStudents, setShowByStudents] = useState(false)
   const { CoolDataScience } = InfoToolTips
 
   const isValidDate = d => moment.isMoment(d) && moment().diff(d) > 0
 
   useEffect(() => {
     if (selectedDate && isValidDate(selectedDate)) {
-      getStatusDispatch({ date: selectedDate.valueOf(), showByYear })
+      getStatusDispatch({ date: selectedDate.valueOf(), showByYear, showByStudents })
     }
-  }, [selectedDate, showByYear])
+  }, [selectedDate, showByYear, showByStudents])
 
   useEffect(() => {
     if (codes.length > 0) {
@@ -191,6 +192,12 @@ const Status = ({ getStatusDispatch, data, loading }) => {
     sendAnalytics(`S Show by year toggle ${!byYear ? 'on' : 'off'}`, 'Status')
   }
 
+  const handleShowByStudentsToggled = () => {
+    const byStudents = showByStudents
+    setShowByStudents(!showByStudents)
+    sendAnalytics(`S Show by students toggle ${!byStudents ? 'on' : 'off'}`, 'Status')
+  }
+
   const pushToDrillStack = (values, code) => {
     const updatedCodes = [...codes].concat(code)
     const updatedDrillStack = [...drillStack].concat(values)
@@ -247,6 +254,13 @@ const Status = ({ getStatusDispatch, data, loading }) => {
               checked={showByYear}
             />
             <VerticalLine />
+            <Checkbox
+              style={{ fontSize: '0.9em', fontWeight: 'normal' }}
+              label="Näytä suhteessa opiskelijamäärään"
+              onChange={handleShowByStudentsToggled}
+              checked={showByStudents}
+            />
+            <VerticalLine />
             <Form>
               <Form.Field error={!isValidDate(selectedDate)} style={{ display: 'flex', alignItems: 'center' }}>
                 <span style={{ fontSize: '0.9em' }}>Näytä päivänä:</span>

services/frontend/src/components/CustomPopulation/CustomPopulationCourses/index.jsx

@@ -5,11 +5,11 @@ import PopulationCourseStatsFlat from '../../PopulationCourseStats/PopulationCou
 import useCourseFilter from '../../FilterTray/filters/Courses/useCourseFilter'
 
 const CustomPopulationCourses = ({ courses, pending, selectedStudents, query, error, showFilter }) => {
-  const { setCoursesOnce, resetCourses } = useCourseFilter()
+  const { setCourses, resetCourses } = useCourseFilter()
 
   useEffect(() => {
     if (!pending && !error && courses.coursestatistics) {
-      setCoursesOnce(courses.coursestatistics)
+      setCourses(courses.coursestatistics)
     }
   }, [courses])
 

services/frontend/src/components/FilterTray/filters/AdmissionType.jsx

@@ -10,7 +10,8 @@ import useAnalytics from '../useAnalytics'
 const AdmissionType = ({ code }) => {
   const { addFilter, removeFilter, activeFilters, withoutFilter } = useFilters()
   const analytics = useAnalytics()
-  const [value, setValue] = useState(null)
+  // Using undefined as default, since students can be filtered with null value
+  const [value, setValue] = useState(undefined)
   const name = 'admissionTypeFilter'
 
   // Naming follows convention from SIS API (e.g urn:code:admissiont-type:m for "Muu")
@@ -21,10 +22,14 @@ const AdmissionType = ({ code }) => {
     AV: 'Avoin väylä',
     KP: 'Koepisteet',
     YP: 'Yhteispisteet',
+    N: null,
   }
 
-  const filterFunction = value => student =>
-    student.studyrights.some(sr => sr.studyright_elements.some(e => e.code === code) && value === sr.admission_type)
+  const filterFunction = value => student => {
+    return value === 'Ei valintatapaa'
+      ? student.studyrights.some(sr => sr.studyright_elements.some(e => e.code === code) && !sr.admission_type)
+      : student.studyrights.some(sr => sr.studyright_elements.some(e => e.code === code) && value === sr.admission_type)
+  }
 
   useEffect(() => {
     if (!value) {
@@ -38,11 +43,14 @@ const AdmissionType = ({ code }) => {
 
   const count = admissionType => withoutFilter(name).filter(filterFunction(admissionType)).length
 
-  const options = Object.entries(admissionTypes).map(([key, admissionType]) => ({
-    key,
-    text: `${admissionType} (${count(admissionType)})`,
-    value: admissionType,
-  }))
+  const options = Object.entries(admissionTypes).map(([key, admissionType]) => {
+    const value = admissionType || 'Ei valintatapaa'
+    return {
+      key,
+      text: `${value} (${count(admissionType)})`,
+      value,
+    }
+  })
 
   return (
     <FilterCard

services/frontend/src/components/PopulationStudents/StudentTable/GeneralTab.jsx

@@ -3,6 +3,7 @@ import { shape, bool, string, arrayOf } from 'prop-types'
 import { Item, Icon, Popup } from 'semantic-ui-react'
 import { Link } from 'react-router-dom'
 import { flatten } from 'lodash'
+import moment from 'moment'
 import { connect } from 'react-redux'
 import SortableTable from '../../SortableTable'
 import {
@@ -255,7 +256,8 @@ const GeneralTab = ({
       getRowVal: s => {
         const grade = s.courses.filter(c => coursecode.includes(c.course_code))
         if (grade) {
-          return grade[grade.length - 1].grade
+          grade.sort((a, b) => (moment(a.date).isBefore(b.date) ? 1 : -1))
+          return grade[0].grade
         }
         return ''
       },
@@ -332,6 +334,18 @@ const GeneralTab = ({
     })
   }
 
+  if (!(coursePopulation || customPopulation)) {
+    const code = queryStudyrights[0]
+    columns.push({
+      key: 'admission type',
+      title: 'admission type',
+      getRowVal: s => {
+        const studyright = s.studyrights.find(sr => sr.studyright_elements.some(e => e.code === code))
+        return studyright && studyright.admission_type ? studyright.admission_type : 'Ei valintatapaa'
+      },
+    })
+  }
+
   if (coursePopulation || customPopulation) {
     columns.push(
       {

services/frontend/src/components/Routes/index.jsx

@@ -37,6 +37,7 @@ const CoursePopulation = React.lazy(() => retry(() => import('../CoursePopulatio
 const CustomPopulation = React.lazy(() => retry(() => import('../CustomPopulation')))
 const Updater = React.lazy(() => retry(() => import('../Updater')))
 const Trends = React.lazy(() => retry(() => import('../Trends')))
+const StudyGuidanceGroups = React.lazy(() => retry(() => import('../StudyGuidanceGroups')))
 
 const routes = {
   students: '/students/:studentNumber?',
@@ -48,6 +49,7 @@ const routes = {
   custompopulation: '/custompopulation',
   updater: '/updater',
   trends: '/trends',
+  studyGuidanceGroups: '/studyguidancegroups/:groupid?',
 }
 
 const Routes = () => (
@@ -65,6 +67,7 @@ const Routes = () => (
       <Route exact path={routes.custompopulation} component={CustomPopulation} />
       <Route exact path={routes.updater} component={Updater} />
       <Route path={routes.trends} component={Trends} />
+      <Route path={routes.studyGuidanceGroups} component={StudyGuidanceGroups} />
       <Redirect from="/cool-data-science" to="/trends" />
       <Redirect to="/" />
     </Switch>

services/frontend/src/components/StudentStatistics/StudentInfoCard/EnrollmentAccordion.jsx

@@ -0,0 +1,77 @@
+import React, { useState } from 'react'
+import { Icon, Accordion, Table } from 'semantic-ui-react'
+import { array } from 'prop-types'
+import _ from 'lodash'
+
+import './studentInfoCard.css'
+
+const getEnrollmentStatus = (semester, season) => {
+  if (semester.name.en.includes('Spring') && season === 'SPRING') {
+    return semester.enrollmenttype === 1 ? 'Present' : 'Absent'
+  }
+  if (semester.name.en.includes('Autumn') && season === 'FALL') {
+    return semester.enrollmenttype === 1 ? 'Present' : 'Absent'
+  }
+  return 'Absent'
+}
+
+const EnrollmentAccordion = ({ semesterEnrollments }) => {
+  const [active, setActive] = useState(false)
+
+  const handleAccordionClick = () => {
+    setActive(!active)
+  }
+
+  const groupedEnrollments = _.groupBy(semesterEnrollments, 'yearname')
+
+  const sortedKeys = Object.keys(groupedEnrollments).sort((a, b) => {
+    return groupedEnrollments[b][0].semestercode - groupedEnrollments[a][0].semestercode
+  })
+
+  return (
+    <div className={active ? 'enrollmentAccordion' : ''}>
+      <Accordion>
+        <Accordion.Title active={active} onClick={handleAccordionClick}>
+          <Icon name="dropdown" />
+          Enrollments
+        </Accordion.Title>
+        <Accordion.Content active={active}>
+          <Table celled>
+            <Table.Header>
+              <Table.Row>
+                <Table.HeaderCell>Semester</Table.HeaderCell>
+                <Table.HeaderCell>Autumn</Table.HeaderCell>
+                <Table.HeaderCell>Spring</Table.HeaderCell>
+              </Table.Row>
+            </Table.Header>
+            <Table.Body>
+              {sortedKeys.map(key => {
+                const semester = groupedEnrollments[key]
+                const date = new Date()
+                return semester[0].startYear <= date.getFullYear() ? (
+                  <Table.Row key={key}>
+                    <Table.Cell>{key}</Table.Cell>
+
+                    <Table.Cell>{getEnrollmentStatus(semester[0], 'FALL')}</Table.Cell>
+
+                    <Table.Cell>
+                      {semester.length > 1
+                        ? getEnrollmentStatus(semester[1], 'SPRING')
+                        : getEnrollmentStatus(semester[0], 'SPRING')}
+                    </Table.Cell>
+                  </Table.Row>
+                ) : null
+              })}
+            </Table.Body>
+          </Table>
+        </Accordion.Content>
+      </Accordion>
+    </div>
+  )
+}
+
+EnrollmentAccordion.propTypes = {
+  semesterEnrollments: array.isRequired,
+}
+
+export default EnrollmentAccordion

services/frontend/src/components/StudentStatistics/StudentInfoCard/index.jsx

@@ -9,6 +9,7 @@ import { DISPLAY_DATE_FORMAT, DISPLAY_DATE_FORMAT_DEV } from '../../../constants
 import './studentInfoCard.css'
 import { removeStudentSelection, resetStudent } from '../../../redux/students'
 import { callApi } from '../../../apiConnection'
+import EnrollmentAccordion from './EnrollmentAccordion'
 
 const StudentInfoCard = props => {
   const { student, showName, updating } = props
@@ -51,9 +52,10 @@ const StudentInfoCard = props => {
               {student.mismatch && '*There is difference between the total credits of this student in Oodi and Sisu'}
             </i>
           </p>
-          <p style={{ fontSize: 14 }}>{`Updated at ${formattedTimestamp}`}</p>
         </Card.Description>
         <div style={{ paddingTop: '4px' }}>
+          <EnrollmentAccordion semesterEnrollments={student.semesterenrollments} />
+          <p style={{ fontSize: 14 }}>{`Updated at ${formattedTimestamp}`}</p>
           <Button disabled={updating} compact size="medium" labelPosition="left" onClick={updateStudent}>
             <Icon loading={updating} name="refresh" />
             update student

services/frontend/src/components/StudentStatistics/StudentInfoCard/studentInfoCard.css

@@ -10,3 +10,7 @@
 .controlIcon {
   cursor: pointer;
 }
+
+.enrollmentAccordion {
+  margin-bottom: 10px;
+}

services/frontend/src/components/StudyGuidanceGroups/index.js

@@ -0,0 +1,239 @@
+/* eslint-disable react/prop-types */
+// temp disable prop types
+import React, { useEffect, useState, useRef } from 'react'
+import { useDispatch, useSelector } from 'react-redux'
+import { Link, useRouteMatch } from 'react-router-dom'
+import { Loader, Segment, Message, Header, Accordion } from 'semantic-ui-react'
+import scrollToComponent from 'react-scroll-to-component'
+import { useProgress, useTitle } from '../../common/hooks'
+import { getStudyGuidanceGroups } from '../../redux/studyGuidanceGroups'
+import SortableTable from '../SortableTable'
+import { getTextIn } from '../../common'
+import { getCustomPopulation } from '../../redux/populations'
+import { getCustomPopulationCoursesByStudentnumbers } from '../../redux/populationCourses'
+import FilterTray from '../FilterTray'
+import useFilters from '../FilterTray/useFilters'
+import { CustomPopulationFilters } from '../FilterTray/FilterSets'
+import useFilterTray from '../FilterTray/useFilterTray'
+import CreditAccumulationGraphHighCharts from '../CreditAccumulationGraphHighCharts'
+import PopulationStudents from '../PopulationStudents'
+import CustomPopulationCourses from '../CustomPopulation/CustomPopulationCourses'
+import CustomPopulationProgrammeDist from '../CustomPopulation/CustomPopulationProgrammeDist'
+import InfoBox from '../InfoBox'
+import infotooltips from '../../common/InfoToolTips'
+
+const SingleStudyGroupView = ({ group }) => {
+  const dispatch = useDispatch()
+  const { data, pending } = useSelector(state => state.populations)
+  const { language } = useSelector(state => state.settings)
+  const custompop = data.students || []
+  const { onProgress } = useProgress(pending)
+  const { setAllStudents, filteredStudents } = useFilters()
+  const [trayOpen] = useFilterTray()
+  const selectedStudents = filteredStudents.map(stu => stu.studentNumber)
+  const creditGainRef = useRef()
+  const programmeRef = useRef()
+  const coursesRef = useRef()
+  const studentRef = useRef()
+  const refs = [creditGainRef, programmeRef, coursesRef, studentRef]
+  const [activeIndex, setIndex] = useState([])
+  const [newestIndex, setNewest] = useState(null)
+
+  const handleClick = index => {
+    const indexes = [...activeIndex].sort()
+    if (indexes.includes(index)) {
+      indexes.splice(
+        indexes.findIndex(ind => ind === index),
+        1
+      )
+    } else {
+      indexes.push(index)
+    }
+    if (activeIndex.length < indexes.length) setNewest(index)
+    else setNewest(null)
+    setIndex(indexes)
+  }
+
+  useEffect(() => {
+    if (!custompop || custompop.length === 0) return
+    setAllStudents(custompop)
+    handleClick(0)
+  }, [custompop])
+
+  useEffect(() => {
+    if (!group) return
+    const studentnumberlist = group.members.map(({ personStudentNumber }) => personStudentNumber)
+    dispatch(getCustomPopulation({ studentnumberlist, onProgress }))
+    dispatch(getCustomPopulationCoursesByStudentnumbers({ studentnumberlist }))
+  }, [group])
+
+  useEffect(() => {
+    if (newestIndex) {
+      scrollToComponent(refs[newestIndex].current, { align: 'bottom' })
+    }
+  }, [activeIndex])
+
+  const panels = [
+    {
+      key: 0,
+      title: {
+        content: (
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Credit accumulation (for {selectedStudents.length} students)
+          </span>
+        ),
+      },
+      onTitleClick: () => handleClick(0),
+      content: {
+        content: (
+          <div ref={creditGainRef}>
+            <CreditAccumulationGraphHighCharts
+              students={custompop}
+              selectedStudents={selectedStudents}
+              render={false}
+              trayOpen={trayOpen}
+              language={language}
+            />
+          </div>
+        ),
+      },
+    },
+    {
+      key: 1,
+      title: {
+        content: (
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Programme distribution
+          </span>
+        ),
+      },
+      onTitleClick: () => handleClick(1),
+      content: {
+        content: (
+          <div ref={programmeRef}>
+            <InfoBox content={infotooltips.PopulationStatistics.ProgrammeDistributionCoursePopulation} />
+            <CustomPopulationProgrammeDist samples={custompop} selectedStudents={selectedStudents} />
+          </div>
+        ),
+      },
+    },
+    {
+      key: 2,
+      title: {
+        content: (
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Courses of population
+          </span>
+        ),
+      },
+      onTitleClick: () => handleClick(2),
+      content: {
+        content: (
+          <div ref={coursesRef}>
+            <CustomPopulationCourses selectedStudents={selectedStudents} />
+          </div>
+        ),
+      },
+    },
+    {
+      key: 3,
+      title: {
+        content: (
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Students ({selectedStudents.length})
+          </span>
+        ),
+      },
+      onTitleClick: () => handleClick(3),
+      content: {
+        content: (
+          <div ref={studentRef}>
+            <PopulationStudents language={language} filteredStudents={filteredStudents} customPopulation />
+          </div>
+        ),
+      },
+    },
+  ]
+
+  if (!group) return null
+
+  return (
+    <FilterTray filterSet={<CustomPopulationFilters />}>
+      <div className="segmentContainer">
+        <Segment className="contentSegment">
+          {pending ? <Loader active>Loading</Loader> : null}
+          {!pending && custompop ? (
+            <>
+              <Header className="segmentTitle" size="medium" textAlign="center">
+                Population of group {getTextIn(group.name, language)}
+              </Header>
+              <Accordion activeIndex={activeIndex} exclusive={false} styled fluid panels={panels} />
+            </>
+          ) : null}
+        </Segment>
+      </div>
+    </FilterTray>
+  )
+}
+
+const StudyGuidanceGroupOverview = ({ groups }) => {
+  const { language } = useSelector(state => state.settings)
+  const headers = [
+    {
+      key: 'name',
+      title: 'name',
+      getRowVal: group => getTextIn(group.name, language),
+      getRowContent: group => (
+        <Link
+          style={{
+            color: 'black',
+            display: 'inline-block',
+            width: '100%',
+            height: '100%',
+            padding: '.78571429em .78571429em',
+          }}
+          to={`studyguidancegroups/${group.id}`}
+        >
+          {getTextIn(group.name, language)}
+        </Link>
+      ),
+      cellProps: {
+        style: {
+          padding: '0',
+        },
+      },
+    },
+  ]
+  return !groups || groups.length === 0 ? (
+    <Message>You do not have access to any study guidance groups</Message>
+  ) : (
+    <SortableTable columns={headers} getRowKey={group => group.id} data={groups} />
+  )
+}
+
+const StudyGuidanceGroups = () => {
+  useTitle('Stugy guidance groups')
+  const dispatch = useDispatch()
+  const { pending, data } = useSelector(state => state.studyGuidanceGroups)
+  const match = useRouteMatch('/studyguidancegroups/:groupid')
+  const groupid = match?.params?.groupid
+
+  useEffect(() => {
+    dispatch(getStudyGuidanceGroups())
+  }, [])
+
+  return (
+    <div className="segmentContainer">
+      <Header className="segmentTitle" size="large">
+        Study guidance groups
+      </Header>
+      <Segment className="contentSegment">
+        {pending ? <Loader active>Loading</Loader> : null}
+        {!pending && !groupid ? <StudyGuidanceGroupOverview groups={data} /> : null}
+        {!pending && groupid ? <SingleStudyGroupView group={data.find(g => g.id === groupid)} /> : null}
+      </Segment>
+    </div>
+  )
+}
+
+export default StudyGuidanceGroups

services/frontend/src/conf.js

@@ -3,8 +3,9 @@ const isProduction = process.env.NODE_ENV === 'production'
 const isDev = process.env.NODE_ENV === 'development'
 
 // Sentry
-const SENTRY_RELEASE = process.env.REACT_APP_SENTRY_RELEASE || ''
-const SENTRY_ENVIRONMENT = process.env.REACT_APP_SENTRY_ENVIRONMENT || ''
+const sentryRelease = process.env.REACT_APP_SENTRY_RELEASE || ''
+const sentryEnvironment = process.env.REACT_APP_SENTRY_ENVIRONMENT || ''
+const runningInCypress = !!window.Cypress
 
 // Adminer is only used in dev mode, imo hardcoding this url here is ok.
 const adminerBaseUrl = 'http://localhost:5050'
@@ -12,7 +13,7 @@ const databaseNames = ['kone-db', 'sis-db', 'sis-importer-db', 'user-db']
 const adminerUrls = databaseNames.map(db => ({ url: `${adminerBaseUrl}/?pgsql=${db}&username=postgres`, text: db }))
 
 // Base paths
-const basePath = process.env.REACT_APP_PUBLIC_URL || ''
+const basePath = process.env.PUBLIC_URL || ''
 const apiBasePath = `${basePath}/api`
 
 // Update time for frontpage
@@ -25,6 +26,7 @@ module.exports = {
   basePath,
   apiBasePath,
   builtAt,
-  SENTRY_RELEASE,
-  SENTRY_ENVIRONMENT,
+  runningInCypress,
+  sentryRelease,
+  sentryEnvironment,
 }

services/frontend/src/redux/coolDataScience.js

@@ -14,10 +14,10 @@ export const getProtoCProgramme = ({ includeOldAttainments, excludeNonEnrolled,
   return callController(route, prefix, [], 'get', null, params)
 }
 
-export const getStatus = ({ date, showByYear }) => {
+export const getStatus = ({ date, showByYear, showByStudents }) => {
   const route = '/cool-data-science/status'
   const prefix = 'GET_STATUS_'
-  const params = { date, showByYear }
+  const params = { date, showByYear, showByStudents }
   return callController(route, prefix, [], 'get', null, params)
 }
 

services/frontend/src/redux/index.js

@@ -40,6 +40,7 @@ import userAccessEmail from './userAccessEmail'
 import customPopulationSearch from './customPopulationSearch'
 import presentStudents from './presentStudents'
 import coolDataScience from './coolDataScience'
+import studyGuidanceGroups from './studyGuidanceGroups'
 
 export default combineReducers({
   actionHistory,
@@ -83,4 +84,5 @@ export default combineReducers({
   customPopulationSearch,
   presentStudents,
   coolDataScience,
+  studyGuidanceGroups,
 })

services/frontend/src/redux/studyGuidanceGroups.js

@@ -0,0 +1,35 @@
+import { callController } from '../apiConnection'
+
+const prefix = 'GET_STUDY_GUIDANCE_GROUPS_'
+
+export const getStudyGuidanceGroups = () => {
+  const route = '/studyguidancegroups'
+  return callController(route, prefix)
+}
+
+const reducer = (state = { data: [] }, action) => {
+  switch (action.type) {
+    case `${prefix}ATTEMPT`:
+      return {
+        ...state,
+        pending: true,
+      }
+    case `${prefix}FAILURE`:
+      return {
+        ...state,
+        pending: false,
+        error: true,
+      }
+    case `${prefix}SUCCESS`:
+      return {
+        ...state,
+        pending: false,
+        error: false,
+        data: action.response,
+      }
+    default:
+      return state
+  }
+}
+
+export default reducer

services/frontend/src/util/sentry.js

@@ -1,13 +1,16 @@
 import * as Sentry from '@sentry/browser'
-import { SENTRY_RELEASE, SENTRY_ENVIRONMENT } from '../conf'
+import { Integrations } from '@sentry/tracing'
+import { sentryRelease, sentryEnvironment, runningInCypress } from '../conf'
 
 const initializeSentry = () => {
-  if (!SENTRY_RELEASE || !SENTRY_ENVIRONMENT) return
+  if (!sentryRelease || !sentryEnvironment || runningInCypress) return
 
   Sentry.init({
     dsn: 'https://020b79f0cbb14aad94cc9d69a1ea9d52@sentry.cs.helsinki.fi/2',
-    environment: SENTRY_ENVIRONMENT,
-    release: SENTRY_RELEASE,
+    environment: sentryEnvironment,
+    release: sentryRelease,
+    integrations: [new Integrations.BrowserTracing()],
+    tracesSampleRate: 1.0,
   })
 }
 

services/userservice/Dockerfile

@@ -21,6 +21,11 @@ CMD ["npm", "run", "test"]
 
 FROM base as production
 ENV NODE_ENV=production
+ENV NODE_ENV=production
+ARG SENTRY_ENVIRONMENT
+ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+ARG SENTRY_RELEASE
+ENV SENTRY_RELEASE=$SENTRY_RELEASE
 EXPOSE 4567
 COPY --chown=node:node ./package* /usr/src/app/
 RUN npm ci --production

services/userservice/index.js

@@ -1,18 +1,22 @@
 const express = require('express')
+const Sentry = require('@sentry/node')
 
 const User = require('./src/services/users')
 const AccessGroup = require('./src/services/accessgroups')
 const FacultyProgrammes = require('./src/services/facultyprogrammes')
 const { initializeDatabaseConnection } = require('./src/database/connection')
+const initializeSentry = require('./src/util/sentry')
 
 initializeDatabaseConnection()
   .then(() => {
     const app = express()
+    initializeSentry(app)
+    app.use(Sentry.Handlers.requestHandler())
+    app.use(Sentry.Handlers.tracingHandler())
     const port = 4567
-    const bodyParser = require('body-parser')
     const checkSecret = require('./src/middlewares/secret')
 
-    app.use(bodyParser.json())
+    app.use(express.json())
     app.use(checkSecret)
 
     app.get('/ping', (req, res) => res.json({ message: 'pong ' }))
@@ -59,9 +63,9 @@ initializeDatabaseConnection()
     })
 
     app.post('/login', async (req, res) => {
-      const { uid, full_name, hyGroups, affiliations, email } = req.body
+      const { uid, full_name, hyGroups, affiliations, email, hyPersonSisuId } = req.body
       console.log(uid, full_name, 'logging in!')
-      const { token, isNew } = await User.login(uid, full_name, hyGroups, affiliations, email)
+      const { token, isNew } = await User.login(uid, full_name, hyGroups, affiliations, email, hyPersonSisuId)
       res.status(200).json({ token, isNew })
     })
 

services/userservice/package.json

@@ -9,10 +9,12 @@
     "test": "jest --verbose --runInBand --silent=false"
   },
   "dependencies": {
+    "@sentry/node": "^6.11.0",
+    "@sentry/tracing": "^6.11.0",
     "express": "^4.17.1",
     "jsonwebtoken": "^8.5.1",
     "moment": "^2.24.0",
-    "pg": "^8.6.0",
+    "pg": "^8.7.1",
     "sequelize": "^6.6.2",
     "umzug": "^2.2.0"
   },
@@ -21,4 +23,4 @@
     "nodemon": "^2.0.7",
     "sqlite3": "^5.0.2"
   }
-}
+}

services/userservice/src/conf.js

@@ -1,4 +1,9 @@
-const { NODE_ENV, DB_URL, SECRET, TOKEN_SECRET } = process.env
+const { NODE_ENV, DB_URL, SECRET, TOKEN_SECRET, SENTRY_RELEASE, SENTRY_ENVIRONMENT, CI } = process.env
+
+// Sentry
+const sentryRelease = SENTRY_RELEASE || ''
+const sentryEnvironment = SENTRY_ENVIRONMENT || ''
+const runningInCI = CI === 'true'
 
 const isStaging = NODE_ENV === 'staging'
 
@@ -14,4 +19,7 @@ module.exports = {
   TOKEN_SECRET,
   requiredGroup,
   courseStatisticsGroup,
+  sentryRelease,
+  sentryEnvironment,
+  runningInCI,
 }

services/userservice/src/database/migrations/20210818_00_add_new_special_programmes_to_faculties.js

@@ -0,0 +1,13 @@
+module.exports = {
+  up: async queryInterface =>
+    queryInterface.sequelize.query(`
+      --- Some lääkis special programmes
+      INSERT INTO faculty_programmes VALUES ('H30', '447000', 'NOW()', 'NOW()');
+      INSERT INTO faculty_programmes VALUES ('H30', '447001', 'NOW()', 'NOW()');
+      INSERT INTO faculty_programmes VALUES ('H30', '447002', 'NOW()', 'NOW()');
+      --- Some kasvis special programmes
+      INSERT INTO faculty_programmes VALUES ('H60', 'EDUK730', 'NOW()', 'NOW()');
+      INSERT INTO faculty_programmes VALUES ('H60', 'K-MUUT-ERIL', 'NOW()', 'NOW()');
+    `),
+  down: async () => {},
+}

services/userservice/src/database/migrations/20210823_00_add_moar_special_programmes.js

@@ -0,0 +1,34 @@
+module.exports = {
+  up: async queryInterface =>
+    queryInterface.sequelize.query(`
+    --- Some kasvis special programmes
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED400', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED200', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED700', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED100', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED300', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED600', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '8009', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '8118', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '8600', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'DUK-PED', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '65', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '1102', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '20018', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '8601', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '9290', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '8172', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '83600', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'DUM-PED', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', 'ED500', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '83601', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '0130', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '20005-e', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '1040', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '8703', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '0005', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '7900', 'NOW()', 'NOW()');
+    INSERT INTO faculty_programmes VALUES ('H60', '0152 ', 'NOW()', 'NOW()');
+    `),
+  down: async () => {},
+}

services/userservice/src/database/migrations/20210829_00_add_sisu_person_id.js

@@ -0,0 +1,10 @@
+const { STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('users', 'sisu_person_id', STRING)
+  },
+  down: async queryInterface => {
+    await queryInterface.removeColumn('users', 'sisu_person_id')
+  },
+}

services/userservice/src/database/migrations/20210829_01_add_studyguidance_groups.js

@@ -0,0 +1,17 @@
+module.exports = {
+  up: queryInterface => {
+    return queryInterface.bulkInsert('access_groups', [
+      {
+        group_code: 'studyGuidanceGroups',
+        group_info: 'grants access to person groups feature',
+      },
+    ])
+  },
+  down: queryInterface => {
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: 'studyGuidanceGroups',
+      },
+    ])
+  },
+}

services/userservice/src/models/index.js

@@ -91,6 +91,7 @@ const User = sequelize.define(
     },
     email: { type: Sequelize.STRING },
     language: { type: Sequelize.STRING },
+    sisu_person_id: { type: Sequelize.STRING },
   },
   {
     tableName: 'users',

services/userservice/src/services/users.js

@@ -88,7 +88,7 @@ const updateGroups = async (user, affiliations, hyGroups) => {
   await user.removeHy_group(await HyGroupService.byCodes(hyGroupsToDelete))
 }
 
-const login = async (uid, full_name, hyGroups, affiliations, mail) => {
+const login = async (uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId) => {
   let user = await byUsername(uid)
   let isNew = false
   await createMissingGroups(hyGroups, HyGroupService)
@@ -96,7 +96,7 @@ const login = async (uid, full_name, hyGroups, affiliations, mail) => {
 
   if (!user) {
     console.log('New user')
-    user = await createUser(uid, full_name, mail)
+    user = await createUser(uid, full_name, mail, hyPersonSisuId)
 
     const userHyGroups = await HyGroupService.byCodes(hyGroups)
     await user.addHy_group(userHyGroups)
@@ -106,7 +106,7 @@ const login = async (uid, full_name, hyGroups, affiliations, mail) => {
 
     isNew = true
   } else {
-    console.log('mail', mail)
+    user = await updateUser(user, { full_name, email: mail, sisu_person_id: hyPersonSisuId })
     user = await updateUser(user, { full_name, email: mail })
     await updateGroups(user, affiliations, hyGroups)
   }
@@ -214,11 +214,12 @@ const byUsername = async username => {
   return user
 }
 
-const createUser = async (username, fullname, email) => {
+const createUser = async (username, fullname, email, hyPersonSisuId) => {
   return User.create({
     username: username,
     full_name: fullname,
     email,
+    sisu_person_id: hyPersonSisuId,
   })
 }
 

services/userservice/src/util/sentry.js

@@ -0,0 +1,18 @@
+const Sentry = require('@sentry/node')
+const Tracing = require('@sentry/tracing')
+
+const { sentryRelease, sentryEnvironment, runningInCI } = require('../conf')
+
+const initializeSentry = app => {
+  if (!sentryRelease || !sentryEnvironment || runningInCI) return
+
+  Sentry.init({
+    dsn: 'https://020b79f0cbb14aad94cc9d69a1ea9d52@sentry.cs.helsinki.fi/2',
+    environment: sentryEnvironment,
+    release: sentryRelease,
+    integrations: [new Sentry.Integrations.Http({ tracing: true }), new Tracing.Integrations.Express({ app })],
+    tracesSampleRate: 1.0,
+  })
+}
+
+module.exports = initializeSentry

updater/sis-updater-scheduler/Dockerfile

@@ -5,7 +5,7 @@ EXPOSE 8082
 
 FROM base as development
 ENV NODE_ENV=development
-COPY package* /usr/src/app
+COPY ./package* /usr/src/app/
 RUN npm ci
 COPY . /usr/src/app
 CMD ["npm", "run", "dev"]

updater/sis-updater-worker/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /usr/src/app
 
 FROM base as development
 ENV NODE_ENV=development
-COPY package* /usr/src/app
+COPY ./package* /usr/src/app/
 RUN npm ci
 COPY . /usr/src/app
 CMD ["npm", "run", "dev"]

updater/sis-updater-worker/src/db/migrations/20210819_00_add_study_levels_to_programme_modules.js

@@ -0,0 +1,8 @@
+const { STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('programme_modules', 'study_level', STRING)
+  },
+  down: async () => {},
+}

updater/sis-updater-worker/src/db/models/programmeModule.js

@@ -21,6 +21,9 @@ ProgrammeModule.init(
     order: {
       type: INTEGER,
     },
+    studyLevel: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

updater/sis-updater-worker/src/updater/updateProgrammeModules.js

@@ -58,14 +58,28 @@ async function moduleResolver(rule, n) {
   if (mod.type == 'StudyModule') {
     const result = await resolver(mod.rule, n)
     if (mod.code.slice(0, 3) === 'KK-') return null
-    const moduleCourses = { id: mod.group_id, code: mod.code, name: mod.name, type: 'module', children: result }
+    const moduleCourses = {
+      id: mod.group_id,
+      code: mod.code,
+      name: mod.name,
+      type: 'module',
+      study_level: mod.study_level,
+      children: result,
+    }
     return moduleCourses
   }
 
   if (mod.type == 'GroupingModule') {
     const module = await moduleRuleResolver(mod, n)
     if (superFlatten) return customFlatten(module)
-    return { id: mod.group_id, code: mod.code, name: mod.name, type: 'module', children: module }
+    return {
+      id: mod.group_id,
+      code: mod.code,
+      name: mod.name,
+      type: 'module',
+      study_level: mod.study_level,
+      children: module,
+    }
   }
 
   return {
@@ -93,6 +107,7 @@ async function courseResolver(rule) {
     id: course.group_id,
     code: course.code,
     name: course.name,
+    study_level: course.study_level,
     type: 'course',
   }
 }
@@ -138,6 +153,11 @@ const recursiveWrite = async (module, parentId) => {
     code: module.code,
     name: module.name,
     type: module.type,
+    studyLevel: module.study_level,
+  }
+  if (module.type !== 'course') {
+    console.log('new module', newModule)
+    console.log('new modules orig details', module)
   }
 
   let join = {
@@ -168,6 +188,7 @@ const updateProgrammeModules = async (entityIds = []) => {
       code: module.code,
       name: module.name,
       type: 'module',
+      studyLevel: module.study_level,
     }
     programmes[module.group_id] = topModule
     const submodule = await resolver(module.rule)