Make linter happy

Some of these things are now part of CI + there were some possible places for misunderstandings.

.env.template

@@ -0,0 +1,3 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+PATE_TOKEN=

.eslintrc.json

@@ -8,17 +8,10 @@
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
   "rules": {
-    "no-console": "off",
+    "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error"

.github/workflows/release-oodikone-staging.yaml

@@ -1,142 +0,0 @@
-name: "Release oodikone to staging"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-oodikone-staging.yaml'
-      - 'services/**'
-
-jobs:
-
-  build_and_release_backend:
-    name: "Build and release backend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:staging
-          target: production
-
-  build_and_release_frontend:
-    name: "Build and release frontend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
-        id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:staging
-          target: production
-      - name: "Create a container from built image "
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: "Copy assets from the container"
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:staging
-          target: production
-
-  report_success:
-    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: success()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'
 
 on:
   release:
@@ -6,30 +6,29 @@ on:
 
 jobs:
   build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
@@ -39,26 +38,26 @@ jobs:
           target: production
 
   build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
+      - name: 'Get current time'
         id: get_current_time
         run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
@@ -66,7 +65,6 @@ jobs:
             SENTRY_ENVIRONMENT=production
             BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
@@ -74,11 +72,11 @@ jobs:
           push: true
           tags: toska/oodikone2-frontend:latest
           target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
         run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
         run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -89,43 +87,19 @@ jobs:
           environment: production
           set_commits: 'skip'
           sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
+          url_prefix: '~/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
 
   report_success:
     runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: success()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -134,15 +108,14 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: failure()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/release-sis-updater-scheduler.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-scheduler"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-scheduler.yaml'
       - 'updater/sis-updater-scheduler/**'

.github/workflows/release-sis-updater-worker.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-worker"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-worker.yaml'
       - 'updater/sis-updater-worker/**'

.github/workflows/test-oodikone.yaml

@@ -1,432 +1,491 @@
-name: "Test oodikone"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
-    branches-ignore:
+    branches:
       - master
     paths:
-      - .github/workflows/test-oodikone.yaml
-      - cypress/**
-      - cypress.json
-      - services/**
-      - docker-compose.ci.yml
-      - nginx.conf
-      - package*.json
+      - '.github/workflows/test-oodikone.yaml'
+      - 'cypress/**'
+      - 'cypress.json'
+      - 'services/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'nginx.conf'
+      - 'package*.json'
 
 jobs:
   # === Static tests / linting ===
 
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
-      - name: "Run backend linter"
+      - name: 'Run backend linter'
         run: npm run eslint services/backend
 
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run concurrently \
-              'eslint --ext .jsx,.js services/frontend' \
-              'stylelint services/frontend/src/**/*.css'
-
-  lint_userservice:
-    name: "Lint userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: "Install dependencies"
-        run: npm ci && cd services/userservice && npm ci
-      - name: "Run frontend linter"
-        run: npm run eslint services/userservice
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
 
   # === Build images ===
-
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
           context: services/backend
           push: false
-          tags: toska/oodikone2-backend:trunk
+          tags: toska/oodikone2-backend:staging
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
 
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
           context: services/frontend
           push: false
-          tags: toska/oodikone2-frontend:trunk
+          tags: toska/oodikone2-frontend:staging
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
 
-  build_userservice:
-    name: "Build userservice"
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
         with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:trunk
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: "Upload image for other jobs"
-        uses: actions/upload-artifact@v2
-        with:
-          path: /tmp/userservice.tar
-
-  # === Integration tests ===
-  run_userservice_integration_tests:
-    name: "Run userservice integration tests"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: "Run tests included in docker build"
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
-
-  # === End-to-end tests ===
-
-  test_course_statistics:
-    name: "Test course statistics"
-    needs:
-      - build_backend
-      - build_frontend
-      - build_userservice
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: "Download images"
-        uses: actions/download-artifact@v2
-        with:
-          path: /tmp/
-      - name: "Load built images"
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
+
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
+
+  # === End-to-end tests ===
+  test_course_statistics:
+    name: 'Test course statistics'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built images'
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
-    name: "Test population statistics"
+    name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Population_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
-    name: "Test filters"
+    name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
-    name: "Test student statistics"
+    name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Student_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
-    name: "Test studyprogramme overview"
+    name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Studyprogramme_overview.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
-    name: "Test teachers and users"
+    name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -435,112 +494,143 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Check that exit statuses are not errors"
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
-  push_to_dockerhub:
-    name: "Push images to docker hub for caching"
-    needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+  # === Release ===
+
+  release_images:
+    name: 'Release built images'
     runs-on: ubuntu-latest
+    needs:
+      - lint_backend
+      - lint_frontend
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Push to docker hub"
-        run: docker-compose --file docker-compose.ci.yml push
+      - name: 'Create a container from frontend'
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: 'Copy assets from the container'
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ github.sha }}
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+      - name: 'Send deployment success notification to slack'
+        if: success()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+      - name: 'Send deployment failure notification to slack'
+        if: failure()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
 
-  report_failure:
+  report_test_failure:
     runs-on: ubuntu-latest
-    name: "Report test failure to oodikone notifications"
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
-    - push_to_dockerhub
+      - lint_backend
+      - lint_frontend
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
       - uses: actions/checkout@v2
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

README.md

@@ -7,15 +7,22 @@ An application for analyzing university data, running at [https://oodikone.cs.he
 To run Oodikone locally, you will need the following:
 
 1. Applications:
-    - [Docker](https://docs.docker.com/install/) (version 20.10+),
-    - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
-    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
-3. (Optional) For real data setup, access to Toskas production servers.
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
 ## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
@@ -26,20 +33,18 @@ _Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary
 What different CLI options do:
 
 1. Set up oodikone from scratch:
-    - Cleans up any previous installations
-    - Installs all needed npm packages locally
-    - Sets up pre-commit linting hooks.
-    - Pulls and builds all needed Docker images and sets up dockerized development environment. 
-    - Setup happens with anonymized data, which developers should use by default.
-    - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
 2. Reset all real data:
-    - Cleans up any previous real data databases
-    - Downloads needed database dumps from production servers.
-    - Creates real data databases and populates them with downloaded dumps.
-3. Reset sis-importer data
-    - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
-4. Reset old oodi data:
-    - Like option 2, but for old oodi data, which is not refreshed anymore. Is only needed when setting up real databases after running option 1.
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.
 
 ## ⌨️ Development
 
@@ -74,7 +79,7 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
 ### User types for development
@@ -85,40 +90,49 @@ Anon data contains some preset user types, corresponding to most usual user type
 
 ### Linting
 
-Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`. 
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question. 
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually. 
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
 However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-## 🔨 Testing & CI 
+## 🔨 Testing & CI
 
 There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
 - Cypress
-    - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
-    - are defined in cypress -folder and cypress.json
-    - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
-    - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
 - Jest
-    - are ran with `docker-compose.test.yml`
-    - test command and environment themselves are in Dockerfile for service in question
-    - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
 Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
+
 - oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
-- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
-- after successful merge to `master`, oodikone is deployed to staging
+- tests are run on every push.
+- after successful test run, oodikone is deployed to staging
 - after creating a release, oodikone is deployed to production
-- updater is deployed to production on every push to `trunk`.
+- updater is deployed to production on every push, no tests included
 
 ## ❓FAQ
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
+
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
 
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
@@ -126,6 +140,10 @@ Just do clean install by launching cli with `npm cli` and running option 1: _Set
 
 ## ✌🏼 Maintainers and contribution
 
-Toska of course. 
+Toska of course.
 
 University of Helsinki
+
+## Tietosuoja / data protection
+
+See [tietosuoja.md](tietosuoja.md).

adminer-plugins/login-servers.php

@@ -8,7 +8,6 @@ $driver = "pgsql";
 return new AdminerLoginServers(
     $servers = array(
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
-        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),
         "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
         "user-db" => array("server" => "user-db", "driver" => $driver)

anonyymioodi/.gitignore

@@ -0,0 +1,2 @@
+*.sqz
+

anonyymioodi/README.md

@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This folder contains a guide and scripts to create anonymous database images for oodikone, mainly used for testing.
+
+## How to create / modify database images
+
+Creating / modifying images varies a bit depending on the db. Generally, to modify images, you can either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose. 
+
+By default, previous db images from Toska Hub are used as a base. If needed, comment out Toska hub image and uncomment corresponding clean postgres image in docker-compose before running other steps.
+
+See more details on how to create / modify databases below.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich db with command in file `enrich_data.sql` either by piping command to container or by running command inside container.
+
+### Sis-db:
+
+The following steps are for building a clean sis-db image from scratch using oodikone's updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+(or run these from frontend. You might also want to follow sis-updater-workers logs to ensure things are running smoothly.)
+- Clean up any extra stuff you don't want to keep in sis-db.
+- Bonus: check that cypress tests run okay
+
+### User-db / kone-db
+
+- Temporarily replace the db service in oodikone's docker-compose.yml with the same db service from this folder's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker-compose pull`
+- Modify db to wanted state. Tips:
+  - Run oodikone if you want to upgrade something from frontend (e.g. tags in kone-db, courses in sis-db etc.).
+  - Easy way to create new users to user-db is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+## Publishing the image
+
+After completing the steps above and making sure your local version has the correct data, run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the db image to Toska Hub.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone

anonyymioodi/enrich_sampledata.sql

@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573

cli.sh

@@ -44,21 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
   msg ""
 }
 
-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "
 
-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Reset old oodi data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}
 
-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt
 
 while true; do
   select opt in "${options[@]}"; do
@@ -67,10 +68,10 @@ while true; do
         set_up_oodikone_from_scratch;;
       "Reset all real data.")
         reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
-      "Reset old oodi data.")
-        reset_old_oodi_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress/integration/Course_statistics.js

@@ -119,6 +119,42 @@ describe('Course Statistics tests', () => {
       cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
     })
 
+    it('Searching course by name displays right courses, 10 credit courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
+      cy.contains('AYTKT20001')
+      cy.contains('TKT20001, 58131')
+      cy.contains('td', /^TKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get('.right').click()
+      cy.contains('No results')
+
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', /^AYTKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
+
+      cy.get('.right').click()
+      cy.contains('No results')
+      cy.get("input[name='unifyOpenUniCourses']").parent().click()
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+    })
+
     it('Can find course population', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
@@ -150,6 +186,18 @@ describe('Course Statistics tests', () => {
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
+    it.skip('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
     describe('When searching unified course stats', () => {
       beforeEach(() => {
         cy.url().should('include', '/coursestatistics')

cypress/integration/Filters.js

@@ -1,6 +1,6 @@
 /// <reference types="Cypress" />
 
-const baseUrl = Cypress.config().baseUrl
+// const baseUrl = Cypress.config().baseUrl
 
 const checkFilteringResult = studentCount => {
   cy.contains(`Students (${studentCount})`)
@@ -9,23 +9,24 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }
 
 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }
 
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    cy.get(`[data-cy="${filterName}-header"]`).click()
     checkFilteringResult(amountWithoutFiltering)
     testStepFunctionToRun()
     checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
   }
 }
 
@@ -41,127 +42,155 @@ describe('Population Statistics', () => {
 
   it('Transfer filter is on not transferred by default', () => {
     cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+    const card = cy.cs('TransferredToProgramme-filter-card')
+    card.cs('TransferredToProgramme-header').click()
+    card.get('[data-cy="option-havenot"] input').should('be.checked')
+    card.cs('TransferredToProgramme-header').click()
+  })
+
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
   })
 
   it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const card = cy.cs('GraduatedFromProgramme-filter-card')
+
       const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      card.cs('option-graduated-true').click()
       checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      card.cs('option-graduated-false').click()
       checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
     })
   })
 
   it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
       const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('option-have').click()
       checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('option-all').click()
       checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('option-havenot').click()
     })
   })
 
   it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
       checkFilteringResult(137)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
 
   it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
       cy.cs('credit-filter-min').type('50')
       cy.cs('credit-filter-max').type('150')
       checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
     })
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(41)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
       checkFilteringResult(121)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
-  it('Admission type filter works', () => {
-    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(5)
-      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(10)
       clearSingleDropdownSelection('admissionTypeFilter-dropdown')
     })
   })
 
-  // TODO: this fails in CI, but not locally. Investigate and fix
+  // plz fix
   it.skip('Courses filter works', () => {
     // courses takes some time to load, wait for it to complete
-    cy.intercept({ path: '**/courses' }).as('coursesOfPopulation')
-    cy.wait('@coursesOfPopulation')
-    // moar waiting hack
     cy.wait(10000)
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
       cy.cs('courseFilter-course-dropdown').click()
       const courses = ['TKT20001', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
       checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
       checkFilteringResult(131)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(2)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const card = cy.cs('GraduatedFromProgramme-filter-card')
+        card.cs('option-graduated-true').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
         checkFilteringResult(11)
         cy.cs('ageFilter-min').find('input').clear()
         cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        card.cs('option-all').click()
       })
     })
   })
@@ -178,7 +207,7 @@ describe('Course Statistics', () => {
   })
 
   it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
       cy.cs('gradeFilter-5').click()
       checkFilteringResult(20)
       cy.cs('gradeFilter-3').click()
@@ -188,44 +217,68 @@ describe('Course Statistics', () => {
     })
   })
 
+  it('Programme defaults to "Attainment" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Attainment')
+    })
+  })
+
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(116)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Study track filter works', () => {
+    runTestStepWithPreAndPostParts('StudyTrack', () => {
+      const card = cy.cs('StudyTrack-filter-card')
+      const programmeDropdown = card.cs('StudyTrack-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(1)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(32)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
       checkFilteringResult(29)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
         cy.cs('gradeFilter-3').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
@@ -247,13 +300,14 @@ describe('Custom Population Statistics', () => {
     cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
     })
     cy.cs('search-button').click()
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('30')
       checkFilteringResult(1)
@@ -262,36 +316,59 @@ describe('Custom Population Statistics', () => {
     })
   })
 
-  it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
-  })
-
-  it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
-      checkFilteringResult(1)
-      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+  it('Programme defaults to "Active Study Right" mode', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
     })
   })
 
-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
-      const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(4)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(4)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+  it('Programme filter works', () => {
+    runTestStepWithPreAndPostParts('Programme', () => {
+      const card = cy.cs('Programme-filter-card')
+      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      programmeDropdown.get('i.delete').click()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
+    })
+  })
+
+  it.skip('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
+      checkFilteringResult(3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
       checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })

cypress/integration/Population_statistics.js

@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />
 
+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
   includeSettings.forEach(setting => {
@@ -13,6 +16,8 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
   describe('when using basic user', () => {
     beforeEach(() => {
       cy.init('/populations')
@@ -59,6 +64,7 @@ describe('Population Statistics tests', () => {
 
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
       setPopStatsUntil('toukokuu 2020')
 
       cy.get('.card').within(() => {
@@ -69,11 +75,12 @@ describe('Population Statistics tests', () => {
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
-      cy.contains('Courses of population').click({ force: true })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(3000) // a bit hacky way, wait until ui is ready
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
       cy.get('[data-cy=expand-TKT1]').click()
       cy.get('[data-cy=coursestats-link-TKT10002]').click()
       cy.wait('@coursePage')
@@ -123,18 +130,62 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit accumulation (for 170 students)')
     })
 
-    it('Credit Statistics, Statistics pane works', () => {
+    it('Credit Statistics, Credits Gained tab works', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
       cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
       cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
       cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
       cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })

cypress/integration/Student_statistics.js

@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
     studentnumber: '010654019',
   }
 
-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
     cy.contains('Student number').should('not.exist')
   })
 
-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
   })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -13,33 +13,33 @@ describe('Studyprogramme overview', () => {
     // Column amount depends on user. Note that with dev user, you'll see even more
     // stuff than with admin or basic.
     const CSBachPopulationProgress2018ForAdmin = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
-      '11',
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
       '1',
-      '0',
-      '150',
-      '123',
-      '76',
-      '41',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
       '16',
     ]
     const CSBachPopulationProgress2018ForBasic = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
       '1',
-      '150',
-      '123',
-      '76',
-      '41',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
       '16',
     ] // cancelled and transferred from are missing
 
@@ -52,7 +52,7 @@ describe('Studyprogramme overview', () => {
       .each((elem, index) => {
         cy.wrap(elem).contains(population[index])
       })
-    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
     cy.get('table')
       .eq(1)
       .contains('2019')
@@ -79,7 +79,7 @@ describe('Studyprogramme overview', () => {
     let originalProductivityCalculatedText
     const doRecalculation = () => {
       // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(2000)
+      cy.wait(5000)
       cy.get('.attached').contains('Admin').click()
       cy.contains('recalculate productivity').click()
       cy.contains('recalculate throughput').click()

cypress/integration/Users.js

@@ -4,17 +4,16 @@ describe('Users tests', () => {
   describe('Using user with just grp-oodikone-user, no other rights', () => {
     it('shows only frontpage, trends feedback', () => {
       cy.init('', 'norights') // login with norights user
-      cy.get('.navBar').contains('Trends').should('exist')
-      cy.get('.navBar').contains('Give feedback').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
 
-      cy.get('.navBar').contains('Study programme').should('not.exist')
-      cy.get('.navBar').contains('Student statistics').should('not.exist')
-      cy.get('.navBar').contains('Course statistics').should('not.exist')
-      cy.get('.navBar').contains('Teachers').should('not.exist')
-      cy.get('.navBar').contains('Users').should('not.exist')
-      cy.get('.navBar').contains('Faculty').should('not.exist')
-      cy.get('.navBar').contains('Updater').should('not.exist')
-      cy.get('.navBar').contains('Sandbox').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
     })
   })
 
@@ -30,32 +29,27 @@ describe('Users tests', () => {
     })
 
     it('should see more stuff than others', () => {
-      cy.get('.navBar').contains('Study programme').should('exist')
-      cy.get('.navBar').contains('Student statistics').should('exist')
-      cy.get('.navBar').contains('Course statistics').should('exist')
-      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
-      cy.get('.navBar').contains('Users').should('exist')
-      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
     })
 
     it("mocking normal user shows only the mocked user's programmes", () => {
       cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'basic').within($row => {
+      cy.contains('tr', 'basic').within(() => {
         cy.contains('.button', 'Edit').click()
       })
-      cy.intercept({
-        method: 'POST',
-        path: '/api/superlogin/basic',
-      }).as('superlogin')
       cy.get('i.spy').click()
-      cy.wait('@superlogin')
       cy.contains('mocking as basic')
       cy.wait(1000)
       cy.contains('Study programme').click().siblings().contains('Search by class').click()
       cy.contains('label', 'Study programme')
       cy.contains('label', 'Study programme')
         .siblings()
-        .within($row => {
+        .within(() => {
           cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
         })
     })

cypress/support/commands.js

@@ -10,6 +10,7 @@ const adminUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
 }
 
 const basicUserHeaders = {
@@ -19,6 +20,7 @@ const basicUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
@@ -28,6 +30,7 @@ const onlycoursestatisticsUserHeaders = {
   hyGroupCn: 'grp-oodikone-basic-users',
   eduPersonAffiliation: 'member',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
 }
 
 const norightsUserHeaders = {
@@ -37,6 +40,7 @@ const norightsUserHeaders = {
   hygroupcn: 'grp-oodikone-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
 }
 
 const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
@@ -50,8 +54,6 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
     if (!headersToUse) throw Error(`${userId} is not valid user id!`)
     req.headers = headersToUse
   })
-  // Hide SIS warning flag to avoid screen size problems when running headlessly
-  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
   cy.visit(baseUrl.concat(path))
 })
@@ -59,24 +61,33 @@ Cypress.Commands.add('init', (path = '', userId = 'basic') => {
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const p = subject === undefined ? cy : cy.wrap(subject)
+
+  return p.get(`[data-cy='${name}']`)
 })
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
   const indexes = Array.isArray(index) ? index : [index]
 
   indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+
+    if (typeof i === 'number') {
+      subject.children('.menu').children().eq(i).click({ force: true })
+    } else {
+      subject.children('.menu').containing(i).eq(0).click({ force: true })
+    }
   })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
   }
 })
 

docker-compose.ci.yml

@@ -1,49 +1,43 @@
-version: "3.9"
-
 x-defaults: &defaults
   restart: unless-stopped
 
 services:
-
   backend:
     <<: *defaults
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
+      - user-db
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:trunk
+    image: toska/oodikone2-backend:staging
 
   frontend:
     <<: *defaults
     container_name: frontend
-    image: toska/oodikone2-frontend:trunk
+    image: toska/oodikone2-frontend:staging
 
   kone-db:
     <<: *defaults
     container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
     image: registry.toska.cs.helsinki.fi/kone-db
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
@@ -53,22 +47,20 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
 
   user-db:
     <<: *defaults
     container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
     environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:trunk
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
 
   nginx:
     image: nginx:latest
@@ -76,8 +68,45 @@ services:
     depends_on:
       - frontend
       - backend
-      - userservice
     ports:
-      -  1337:80
+      - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,8 +1,7 @@
 services:
-
   backend:
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db-real
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 
@@ -19,9 +18,5 @@ services:
       DB_URL: postgres://postgres@sis-db:5432/sis-db-real
       SIS_IMPORTER_DATABASE: sis-importer-db-real
 
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
   redis-data-real:

docker-compose.test.yml

@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -5,7 +5,6 @@ x-build_defaults: &build_defaults
   target: development
 
 services:
-
   adminer:
     <<: *defaults
     container_name: adminer
@@ -25,20 +24,19 @@ services:
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
+      - user-db
+    env_file:
+      - .env
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     ports:
       - 8080:8080
     profiles:
@@ -70,22 +68,13 @@ services:
     volumes:
       - kone-db-data:/data
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    profiles:
-      - oodikone
-    volumes:
-      - oodi-db-data:/data
-
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -95,6 +84,7 @@ services:
     image: registry.toska.cs.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
+    shm_size: 2gb
 
   sis-importer-db:
     <<: *defaults
@@ -107,7 +97,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -174,35 +164,13 @@ services:
     volumes:
       - user-db-data:/data
 
-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
   backend-nodemod:
   frontend-nodemod:
   kone-db-data:
-  oodi-db-data: # TODO: Remove when oodi stuff has been removed from oodikone
   redis-data:
   sis-db-data:
   sis-importer-db-data:
   sis-updater-scheduler-nodemod:
   sis-updater-worker-nodemod:
   user-db-data:
-  userservice-nodemod:

old_oodikone_stuff/old-backend-tests/course_stats_counter.test.js

@@ -1,197 +0,0 @@
-const { CourseStatsCounter } = require('./course_stats_counter')
-
-describe('CourseStatsCounter tests', () => {
-  const studentnumbers = ['1', '2', '3', '4', '5']
-  const coursecode = 'COURSE_01'
-  const coursename = 'COURSE NAME'
-
-  const createCounter = () => new CourseStatsCounter(coursecode, coursename, studentnumbers.length)
-
-  describe('counter constructor should set course values correctly', () => {
-    const counter = createCounter()
-
-    test('course name should be set after creation', () => {
-      expect(counter.course.name).toBe(coursename)
-    })
-
-    test('course code should be set after creation', () => {
-      expect(counter.course.code).toBe(coursecode)
-    })
-  })
-
-  describe('mark passed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '5'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, true, false, false)
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '0'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, false, true, false)
-
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('failed students should have studentnumber as key', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber as key', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then passed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '5', true, false, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber as key', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have student', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { failed, passed, failedMany } = counter.students
-
-    test('failed students should have studentnumber', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-
-    test('failed many students should have students', () => {
-      expect(failedMany).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then failed then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, failedMany, retryPassed } = counter.students
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('failedMany should not have studentnumber', () => {
-      expect(failedMany).not.toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then improved grade for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', true, false, false)
-    counter.markCredit(student, '5', false, false, true)
-
-    const { passed, improvedPassedGrade } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should have student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-  })
-
-  describe('mark improved grade then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', false, false, true)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, improvedPassedGrade, failed } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should contain student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-
-    test('failed should not contain student', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/globals.js

@@ -1,17 +0,0 @@
-const { sequelize } = require('../../src/models/index')
-const { sequelizeKone } = require('../../src/models/models_kone')
-const { redisClient } = require('../../src/services/redis')
-const { forceSyncDatabase } = require('../../src/database/connection')
-const { seedAllMigrations } = require('../../src/database/seed_migrations')
-
-beforeAll(async () => {
-  await forceSyncDatabase()
-  await seedAllMigrations()
-})
-
-afterAll(async () => {
-  await sequelize.close()
-  await sequelizeKone.close()
-  // https://stackoverflow.com/a/54560610
-  await new Promise(res => redisClient.quit(() => setImmediate(res)))
-})

old_oodikone_stuff/old-backend-tests/services/population.test.js

@@ -1,227 +0,0 @@
-const {
-  Student,
-  Course,
-  ElementDetails,
-  StudyrightElement,
-  Studyright,
-  Credit,
-  StudyrightExtent,
-  Semester,
-  SemesterEnrollment
-} = require('../../../src/models/index')
-const { optimizedStatisticsOf } = require('../../../src/services/populations')
-jest.mock('../../../src/services/semesters')
-const semesterService = require('../../../src/services/semesters')
-jest.setTimeout(10000)
-
-const langify = name => ({
-  en: `${name}_en`,
-  fi: `${name}_fi`,
-  sv: `${name}_sv`
-})
-
-const SEMESTER = { SPRING: 'SPRING', FALL: 'FALL' }
-
-const elementdetails = {
-  bsc: {
-    code: 'Element_BSC',
-    type: 10,
-    name: langify('Bachelor of Science')
-  },
-  maths: {
-    code: 'Element_MATH',
-    type: 20,
-    name: langify('Mathematics')
-  },
-  cs: {
-    code: 'ELEMENT_CS',
-    type: 20,
-    name: langify('Computer Science')
-  }
-}
-
-const courses = {
-  elements_of_ai: {
-    code: 'COURSE-1',
-    name: langify('ELEMENTS_OF_AI')
-  }
-}
-
-const semesters = {
-  fall: {
-    semestercode: 123,
-    name: { en: 'Autumn 2011', fi: 'Syksy 2011', sv: 'Hösten 2011' },
-    startdate: new Date('2011-07-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  spring: {
-    semestercode: 124,
-    name: { en: 'Spring 2012', fi: 'Kevät 2012', sv: 'Våren 2012' },
-    startdate: new Date('2011-12-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  // used for mocking current
-  current: {
-    semestercode: 125,
-    name: { en: 'Autumn 2012', fi: 'Syksy 2012', sv: 'Hösten 2012' },
-    startdate: new Date('2012-08-01'),
-    enddate: new Date('2012-12-31'),
-    yearcode: 63,
-    yearname: '2012-13'
-  }
-}
-
-const createQueryObject = (year, semester, codes, months) => ({
-  studyRights: codes,
-  year,
-  semesters: [semester],
-  months
-})
-
-afterAll(async () => {})
-
-describe('optimizedStatisticsOf tests', () => {
-  const courseinstanceFall = {
-    id: 1,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2011-09-31 21:00:00+00')
-  }
-
-  const courseinstanceSpring = {
-    id: 2,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2012-02-31 21:00:00+00')
-  }
-
-  const student = {
-    studentnumber: '1234'
-  }
-
-  const studyrightextents = {
-    bachelors: {
-      extentcode: 1,
-      name: langify("bachelor's degree extent")
-    }
-  }
-
-  const studyright = {
-    studyrightid: 1,
-    studystartdate: new Date('2011-07-31 21:00:00+00'),
-    student_studentnumber: student.studentnumber,
-    extentcode: studyrightextents.bachelors.extentcode
-  }
-
-  const creditFall = {
-    id: 'CREDIT-1',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceFall.course_code,
-    attainment_date: courseinstanceFall.coursedate,
-    semestercode: 123
-  }
-
-  const creditSpring = {
-    id: 'CREDIT-2',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceSpring.course_code,
-    attainment_date: courseinstanceSpring.coursedate,
-    semestercode: 124
-  }
-
-  const semesterEnrollments = {
-    fall: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.fall.semestercode,
-      enrollment_date: semesters.fall.startdate
-    },
-    spring: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.spring.semestercode,
-      enrollment_date: semesters.spring.startdate
-    },
-    current: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.current.semestercode,
-      enrollment_date: semesters.current.startdate
-    }
-  }
-
-  const studyrightelements = {
-    bsc: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.bsc.code,
-      studentnumber: student.studentnumber
-    },
-    maths: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.maths.code,
-      studentnumber: student.studentnumber
-    }
-  }
-
-  describe(`
-    Student with:
-    - BSc studyright (2011-07-31 – 2016-12-20),
-    - Mathematics studyright (2011-07-31 – 2016-12-20),
-    - Two credits in 2011-09-31 and 2012-02-31.
-    `, () => {
-    beforeAll(async () => {
-      await Semester.bulkCreate([semesters.fall, semesters.spring, semesters.current])
-      await Student.create(student)
-      await Course.create(courses.elements_of_ai)
-      await Credit.bulkCreate([creditFall, creditSpring])
-      await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
-      await StudyrightExtent.create(studyrightextents.bachelors)
-      await Studyright.create(studyright)
-      await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
-      await SemesterEnrollment.bulkCreate([
-        semesterEnrollments.fall,
-        semesterEnrollments.spring,
-        semesterEnrollments.current
-      ])
-      // Mock current semester implementation
-      semesterService.getCurrentSemester.mockImplementation(async () => {
-        const currentSemester = await Semester.findOne({
-          where: {
-            semestercode: semesters.current.semestercode
-          }
-        })
-        return currentSemester
-      })
-    })
-
-    test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
-      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/services/test_data.js

@@ -1,42 +0,0 @@
-const faculties = [
-  {
-    code: 'FAC1',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 1 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 1 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 1 SV'
-      }
-    ]
-  },
-  {
-    code: 'FAC2',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 2 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 2 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 3 SV'
-      }
-    ]
-  }
-]
-
-module.exports = {
-  faculties
-}

old_oodikone_stuff/old-backend-tests/studyprogramme.test.js

@@ -1,264 +0,0 @@
-const {
-  getCreditsForProvider,
-  productivityStatsFromCredits,
-  productivityStatsForProvider,
-  findGraduated,
-  graduatedStatsFromStudyrights,
-  combineStatistics,
-  productivityStatsForCode,
-  findProgrammeThesisCredits,
-  thesisProductivityFromCredits,
-  thesisProductivityForStudytrack
-} = require('./studyprogramme')
-const { sequelize } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { readFileSync } = require('fs')
-const { mapToProviders } = require('../util/utils')
-
-const { MASTER, BACHELOR } = ThesisTypeEnums
-
-const provider = '500-M010'
-const studytrack = 'MH50_010'
-
-const thesisCourse = {
-  // from studyprogramme.test.sql
-  programmeCode: 'MH50_010',
-  courseCode: 'THESIS_01',
-  thesisType: MASTER
-}
-
-beforeAll(async () => {
-  const query = readFileSync('./src/services/studyprogramme.test.sql', 'utf8')
-  await sequelize.query(query)
-  await ThesisCourse.create(thesisCourse)
-})
-
-test('Credits for provider should contain passed course credit', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({ id: 'CREDIT_01' })
-})
-
-test('Credits for provider should return object in correct format', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({
-    credits: 5,
-    year: 2016,
-    id: 'CREDIT_01'
-  })
-})
-
-test('Credits for provider should not contain passed credits from other providers', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_04')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit with failed credit typecode 10', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_02')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit for passed study module', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_03')
-  expect(credit).toBeFalsy()
-})
-
-test('KH50_005 track code maps to 500-K005 provider code', () => {
-  const track = 'KH50_005'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-K005')
-})
-
-test('MH50_010 track code maps to 500-M010 provider code', () => {
-  const track = 'MH50_010'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-M010')
-})
-
-test('productivityStatsFromCredits calculates stats correctly', () => {
-  const credits = [
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 10 }
-  ]
-  const stats = productivityStatsFromCredits(credits)
-  expect(stats).toMatchObject({
-    2016: { year: 2016, credits: 15 },
-    2015: { year: 2015, credits: 20 }
-  })
-})
-
-test('productivityStats integrates correctly', async () => {
-  const stats = await productivityStatsForProvider(provider, '2001-01-01')
-  expect(stats).toMatchObject({
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 }
-  })
-})
-
-test('findGraduated finds graduated studyright', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.studyrightid === '10')
-  expect(match).toBeTruthy()
-})
-
-test('findGraduated formats graduated studyrights correctly', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  expect(studyrights).toContainEqual({ studyrightid: '10', year: 2016, timeToGraduation: NaN })
-})
-
-test('findGraduated does not return studyrights that are not graduated', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '11')
-  expect(match).toBeFalsy()
-})
-
-test('findGraduated does return graduated studyrights from other studytracks', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '12')
-  expect(match).toBeFalsy()
-})
-
-test('graduatedStatsFromStudyrights calculates stats correctly', () => {
-  const studyrights = [{ year: 2015 }, { year: 2015 }, { year: 2014 }]
-  const stats = graduatedStatsFromStudyrights(studyrights)
-  expect(stats).toMatchObject({
-    2015: { graduated: 2 },
-    2014: { graduated: 1 }
-  })
-})
-
-test('combineStatistics returns correctly formatted array', () => {
-  const creditStats = {
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 },
-    2014: { year: 2014, credits: 20 }
-  }
-  const studyrightStats = {
-    2015: { graduated: 2 },
-    2016: { graduated: 1 }
-  }
-  const thesisStats = {
-    2014: { mThesis: 1 },
-    2015: { mThesis: 2, bThesis: 1 }
-  }
-  const creditsForMajors = {
-    2014: 10,
-    2015: 22
-  }
-  const transferredCredits = {
-    2014: 2,
-    2015: 4
-  }
-
-  const stats = combineStatistics(creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits)
-  expect(stats).toContainEqual({
-    year: 2015,
-    mThesis: 2,
-    bThesis: 1,
-    credits: 40,
-    graduated: 2,
-    creditsForMajors: 22,
-    transferredCredits: 4
-  })
-  expect(stats).toContainEqual({
-    year: 2014,
-    mThesis: 1,
-    bThesis: 0,
-    credits: 20,
-    graduated: 0,
-    creditsForMajors: 10,
-    transferredCredits: 2
-  })
-  expect(stats).toContainEqual({
-    year: 2016,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    graduated: 1,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('productivityStatsForCode integrates', async () => {
-  const stats = await productivityStatsForCode(studytrack, '2001-01-01')
-  expect(stats.data).toContainEqual({
-    year: 2015,
-    graduated: 0,
-    bThesis: 0,
-    mThesis: 1,
-    credits: 40,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-  expect(stats.data).toContainEqual({
-    year: 2016,
-    graduated: 1,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('findProgrammeThesisCredits returns correct credit', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toBeTruthy()
-  expect(credits.length).toBe(1)
-  expect(credits.find(c => c.id === 'CREDIT_07')).toBeTruthy()
-})
-
-test('findProgrammeThesisCredits format credit correctly', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toContainEqual({
-    id: 'CREDIT_07',
-    code: 'THESIS_01',
-    type: MASTER,
-    year: 2015
-  })
-})
-
-test('thesisProductivityFromCredits', async () => {
-  const credits = [
-    { type: MASTER, year: 2015 },
-    { type: MASTER, year: 2015 },
-    { type: BACHELOR, year: 2015 },
-    { type: BACHELOR, year: 2014 },
-    { type: MASTER, year: 2013 }
-  ]
-  const stats = thesisProductivityFromCredits(credits)
-  expect(stats).toHaveProperty('2015', '2014', '2013')
-  expect(stats).toMatchObject({
-    '2015': {
-      mThesis: 2,
-      bThesis: 1
-    },
-    '2014': {
-      mThesis: 0,
-      bThesis: 1
-    },
-    '2013': {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})
-
-test('thesisProductivityForStudytrack integrates', async () => {
-  const stats = await thesisProductivityForStudytrack(studytrack)
-  expect(stats).toMatchObject({
-    2015: {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})

old_oodikone_stuff/old-backend-tests/studyprogramme.test.sql

@@ -1,93 +0,0 @@
-DO $$
-DECLARE
-  ts TIMESTAMP = CURRENT_TIMESTAMP;
-  studytrk_01 VARCHAR(50) = 'MH50_010';
-  studytrk_02 VARCHAR(50) = 'KH50_005';
-  provider_01 VARCHAR(50) = '500-M010';
-  provider_02 VARCHAR(50) = '500-K005';
-  course_01 VARCHAR(50) = 'M010_COURSE';
-  module_01 VARCHAR(50) = 'M010_MODULE';
-  course_02 VARCHAR(50) = 'K005_COURSE';
-  thesis VARCHAR(50) = 'THESIS_01';
-  date_2016 TIMESTAMPTZ = '2016-09-28 00:00:00+00';
-  date_2015 TIMESTAMPTZ = '2015-09-28 00:00:00+00';
-BEGIN
-  INSERT INTO semesters ("semestercode", "createdAt", "updatedAt")
-  VALUES
-  (5, ts, ts)
-  ;
-
-  INSERT INTO course
-  ("code", "name", "is_study_module")
-  VALUES
-  (course_01, '{ "en": "COURSE_EN" }', 'false'),
-  (module_01, '{ "en": "MODULE_EN" }', 'true'),
-  (course_02, '{ "en": "COURSE_EN" }', 'false'),
-  (thesis, '{ "en": "Bachelor thesis" }', 'false')
-  ;
-
-  INSERT INTO student
-  ("studentnumber", "createdAt", "updatedAt")
-  VALUES
-  ('012111517', ts, ts)
-  ;
-
-  INSERT INTO credit_types
-  ("credittypecode", "createdAt", "updatedAt")
-  VALUES
-  (10, ts, ts),
-  (4, ts, ts)
-  ;
-
-  INSERT INTO credit
-  ("id", "createddate", "lastmodifieddate", "semestercode", "course_code", "credittypecode", "attainment_date", "credits", "student_studentnumber")
-  VALUES
-  ('CREDIT_01', ts, ts, 5, course_01, 4, date_2016, 5, '012111517'),
-  ('CREDIT_02', ts, ts, 5, course_01, 10, date_2016, 5, '012111517'),
-  ('CREDIT_03', ts, ts, 5, module_01, 4, date_2016, 20, '012111517'),
-  ('CREDIT_04', ts, ts, 5, course_02, 4, date_2016, 5, '012111517'),
-  ('CREDIT_05', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_06', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_07', ts, ts, 5, thesis, 4, date_2015, 30, '012111517')
-  ;
-
-  INSERT INTO providers
-  ("providercode", "createdAt", "updatedAt")
-  VALUES
-  (provider_01, ts, ts),
-  (provider_02, ts, ts)
-  ;
-
-  INSERT INTO course_providers
-  ("coursecode", "providercode", "createdAt", "updatedAt")
-  VALUES
-  (course_01, provider_01, ts, ts),
-  (module_01, provider_01, ts, ts),
-  (course_02, provider_02, ts, ts),
-  (thesis, provider_01, ts, ts)
-  ;
-
-  INSERT INTO element_details
-  ("code", "createdAt", "updatedAt")
-  VALUES
-  (studytrk_01, ts, ts),
-  (studytrk_02, ts, ts)
-  ;
-
-  INSERT INTO studyright
-  ("studyrightid", "graduated", "enddate")
-  VALUES
-  (10, 1, date_2016),
-  (11, 0, date_2016),
-  (12, 1, date_2016)
-  ;
-
-  INSERT INTO studyright_elements
-  ("studyrightid", "code", "createdAt", "updatedAt", "studentnumber")
-  VALUES
-  (10, studytrk_01, ts, ts, '012111517'),
-  (11, studytrk_01, ts, ts, '012111517'),
-  (12, studytrk_02, ts, ts, '012111517')
-  ;
-
-END $$;

old_oodikone_stuff/old-backend-tests/teachers.test.js

@@ -1,53 +0,0 @@
-const { Teacher } = require('../models/index')
-const service = require('./teachers')
-
-const teacher = {
-  id: 'TID_001',
-  code: 'usercode',
-  name: 'Lastname Firstname Middlename'
-}
-
-beforeAll(async () => {
-  await Teacher.create(teacher)
-})
-
-describe('teacher service tests', () => {
-  describe('teacher by search term tests', () => {
-    const teacherIsInResults = (id, matches) => matches.some(teacher => teacher.id === 'TID_001')
-
-    test('Search by `usercode` finds teacher', async () => {
-      const matches = await service.bySearchTerm('usercode')
-      expect(teacherIsInResults('TID_001', matches)).toBe(true)
-    })
-
-    test('Search by `usercodeEXTRA` does not find teacher', async () => {
-      const matches = await service.bySearchTerm('usercodeEXTRA')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(false)
-    })
-
-    test('Search by just `Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Lastname Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `id` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('TID_001')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/thesis.test.js

@@ -1,85 +0,0 @@
-const { forceSyncDatabase } = require('../database/connection')
-const { Course, ElementDetails } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { createThesisCourse, deleteThesisCourse, findProgrammeTheses } = require('./thesis')
-
-const courseOne = {
-  code: 'COURSE',
-  name: { en: 'Course_EN' }
-}
-
-const courseTwo = {
-  code: 'COURSE_2',
-  name: { en: 'Course_EN_2' }
-}
-
-const progOne = {
-  code: 'CS_BSC',
-  name: { en: 'CS_BSC' }
-}
-
-const progTwo = {
-  code: 'CS_MSC',
-  name: { en: 'CS_MSC' }
-}
-
-beforeEach(async () => {
-  await forceSyncDatabase()
-  await Course.create(courseOne)
-  await Course.create(courseTwo)
-  await ElementDetails.create(progOne)
-  await ElementDetails.create(progTwo)
-})
-
-test('createThesisCourse should save thesis correctly', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-})
-
-test('createThesisCourse should create separate bachelor and thesis for same programme and course', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  const master = courses.find(c => c.thesisType === 'MASTER')
-  const bachelor = courses.find(c => c.thesisType === 'BACHELOR')
-  expect(master).toBeTruthy()
-  expect(master).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-  expect(bachelor).toBeTruthy()
-  expect(bachelor).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE_2', thesisType: 'BACHELOR' })
-})
-
-test('createThesisCourse should error for creating a thesis for existing (course, prog) pair', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const promise = createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.BACHELOR)
-  await expect(promise).rejects.toThrow('Validation error')
-})
-
-test('deleteThesisCourse removes thesis from table', async () => {
-  let courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(0)
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  await deleteThesisCourse(progOne.code, courseOne.code)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ thesisType: 'BACHELOR' })
-})
-
-test('findProgrammeTheses returns correct courses', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  await createThesisCourse(progTwo.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const theses = await findProgrammeTheses(progOne.code)
-  expect(theses.length).toBe(2)
-})
-
-test('createThesisCourse should return created thesis', async () => {
-  const thesis = await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  expect(thesis).toBeTruthy()
-  expect(thesis).toMatchObject({ programmeCode: progOne.code, courseCode: courseOne.code })
-})

old_oodikone_stuff/old-backend-tests/utils.js

@@ -1,248 +0,0 @@
-const faker = require('faker')
-
-const numberFromTo = (from, to) => Math.round(Math.random() * (to - from)) + from
-const daysAgo = daysAgo => new Date(new Date() - 1000 * 60 * 60 * 24 * daysAgo)
-const idFromTwoIds = (a, b) => (1 / 2) * (a + b) * (a + b + 1) + b
-
-const gradeArray = [
-  '2',
-  'CL',
-  'Eisa',
-  'L',
-  'Hyv.',
-  'TT',
-  'LUB',
-  'HT',
-  'MCLA',
-  '4',
-  '0',
-  'A',
-  'NSLA',
-  'Luop',
-  '5',
-  '3',
-  'Hyl.',
-  'ECLA',
-  '1',
-  'KH'
-]
-const statusArray = [
-  { status: 'Suoritettu', statuscode: 4 },
-  { status: 'Korotettu', statuscode: 7 },
-  { status: 'Hyväksiluettu', statuscode: 9 },
-  { status: 'Hylätty', statuscode: 10 }
-]
-const teacherRoleArray = ['Responsible', 'Approver', 'Teacher']
-
-const generateCourses = async amount => {
-  const number = amount || numberFromTo(10, 100)
-  const courses = []
-  for (let i = 0; i < number; i++) {
-    courses.push({
-      code: i.toString(10),
-      name: faker.lorem.words(3)
-    })
-  }
-  return courses
-}
-
-const generateCourseInstances = async (courses, amount) => {
-  const instances = []
-  let id = 0
-  courses.forEach(course => {
-    const number = amount || numberFromTo(5, 40)
-    for (let i = 0; i < number; i++) {
-      instances.push({
-        id,
-        coursedate: daysAgo(numberFromTo(10, 365)).toUTCString(),
-        course_code: course.code
-      })
-      id++
-    }
-  })
-  return instances
-}
-
-const generateStudents = async amount => {
-  const students = []
-  const number = amount || numberFromTo(5, 40)
-
-  const now = new Date()
-
-  for (let i = 0; i < number; i++) {
-    const birthdate = daysAgo(numberFromTo(365 * 18, 365 * 70))
-    const dateoffirstcredit = faker.date.between(birthdate, now)
-    const dateoflastcredit = faker.date.between(dateoffirstcredit, now)
-    const dateofuniversityenrollment = faker.date.between(birthdate, dateoflastcredit)
-    const lastname = faker.name.lastName()
-    const firstnames = faker.name.firstName(2)
-    students.push({
-      studentnumber: `0${numberFromTo(10000000, 15000000)}`,
-      lastname,
-      firstnames,
-      abbreviatedname: `${lastname} ${firstnames}`,
-      birthdate,
-      communicationlanguage: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      country: faker.address.country(),
-      creditcount: numberFromTo(0, 600),
-      dateoffirstcredit,
-      dateoflastcredit,
-      dateofuniversityenrollment,
-      gradestudent: `${numberFromTo(0, 1)}`,
-      nationalities: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      semesterenrollmenttypecode: [null, '1.0', '2.0'][numberFromTo(0, 2)],
-      sex: ['female', 'male'][numberFromTo(0, 1)],
-      studentstatuscode: numberFromTo(2, 8)
-    })
-  }
-
-  return students
-}
-
-const generateCredits = async (courseInstances, students, amount) => {
-  const credits = []
-  students.forEach((student, studentIdx) => {
-    courseInstances.forEach(courseInstance => {
-      const number = amount || numberFromTo(5, 40)
-      for (let i = 0; i < number; i++) {
-        const statusObject = statusArray[numberFromTo(0, statusArray.length - 1)]
-        credits.push({
-          id: idFromTwoIds(idFromTwoIds(studentIdx, courseInstance.id), i),
-          grade: gradeArray[numberFromTo(0, gradeArray.length - 1)],
-          student_studentnumber: student.studentnumber,
-          credits: [5, 10][numberFromTo(0, 1)],
-          ordering: faker.date.between(student.dateoffirstcredit, student.dateoflastcredit),
-          status: statusObject.status,
-          statuscode: statusObject.statuscode,
-          courseinstance_id: courseInstance.id
-        })
-      }
-    })
-  })
-  return credits
-}
-
-const generateTeachers = async amount => {
-  const teachers = []
-  const number = amount || numberFromTo(5, 40)
-
-  for (let i = 0; i < number; i++) {
-    teachers.push({
-      id: i,
-      code: faker.lorem.word(),
-      name: `${faker.name.lastName()} ${faker.name.firstName()}`
-    })
-  }
-
-  return teachers
-}
-
-const generateCourseTeachers = async (courseInstances, teachers, maximumAmount) => {
-  const courseTeachers = []
-  courseInstances.forEach((courseInstance, index) => {
-    const number = numberFromTo(0, maximumAmount)
-    for (let i = 0; i < number; i++) {
-      const teacher = teachers[numberFromTo(0, teachers.length - 1)]
-      courseTeachers.push({
-        id: idFromTwoIds(index, i),
-        teacherrole: teacherRoleArray[numberFromTo(0, teacherRoleArray.length - 1)],
-        courseinstance_id: courseInstance.id,
-        teacher_id: teacher.id
-      })
-    }
-  })
-
-  return courseTeachers
-}
-
-const generateOrganizations = async amount => {
-  const organizations = []
-
-  const number = amount || numberFromTo(5, 40)
-  for (let i = 0; i < number; i++) {
-    organizations.push({
-      code: faker.lorem.words(3),
-      name: faker.company.companyName()
-    })
-  }
-  return organizations
-}
-
-const generateStudyrights = async (students, organization, amount, date, prioritycode) => {
-  const studyrights = []
-  const possibleStudyrightNames = []
-  if (amount) {
-    for (let i = 0; i < amount; i++) {
-      possibleStudyrightNames.push(faker.company.catchPhrase())
-    }
-  }
-  students.forEach(student => {
-    studyrights.push({
-      studyrightid: student.studentnumber,
-      canceldate: null,
-      cancelorganisation: null,
-      enddate: new Date().toUTCString(),
-      extentcode: numberFromTo(2, 99),
-      givendate: daysAgo(365 * numberFromTo(1, 100)),
-      graduated: null,
-      prioritycode: prioritycode || numberFromTo(1, 30),
-      startdate: date,
-      studystartdate: date,
-      organization_code: organization.code,
-      student_studentnumber: student.studentnumber
-    })
-  })
-  return studyrights
-}
-
-const generateUsers = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const users = []
-  for (let i = 0; i < number; i++) {
-    users.push({
-      full_name: `${faker.name.lastName()}, ${faker.name.firstName(2)}`,
-      username: faker.lorem.word(),
-      is_enabled: numberFromTo(0, 1) === 1
-    })
-  }
-  return users
-}
-
-const generateTags = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const tags = []
-  for (let i = 0; i < number; i++) {
-    tags.push({
-      tagname: `${faker.company.catchPhraseAdjective()} ${i}` //tagname should be unique, add i just in case
-    })
-  }
-  return tags
-}
-
-// adds all tags to all students
-const generateTagStudents = (students, tags) => {
-  let tag_students = []
-  students.forEach(student => {
-    tags.forEach(tag => {
-      tag_students.push({
-        taggedstudents_studentnumber: student.studentnumber,
-        tags_tagname: tag.tagname
-      })
-    })
-  })
-  return tag_students
-}
-
-module.exports = {
-  generateCourses,
-  generateCourseInstances,
-  generateStudents,
-  generateStudyrights,
-  generateCredits,
-  generateTeachers,
-  generateCourseTeachers,
-  generateOrganizations,
-  generateUsers,
-  generateTags,
-  generateTagStudents
-}

old_oodikone_stuff/old-backend-tests/utils/semester.test.js

@@ -1,14 +0,0 @@
-const moment = require('moment')
-const semesterUtils = require('../../../src/util/semester')
-
-describe('Semester utils tests', () => {
-  test('Get passing semester', () => {
-    expect(semesterUtils.getPassingSemester(2018, moment('2017-09-09'))).toBe('BEFORE')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-09-09'))).toBe('0-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-05-01'))).toBe('0-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-09-01'))).toBe('1-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2020-05-01'))).toBe('1-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2026-05-01'))).toBe('LATER')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-05-01'))).toBe('BEFORE')
-  })
-})

old_oodikone_stuff/updater_api/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_api/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_api/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_api/.gitignore

@@ -1 +0,0 @@
-node_modules/

old_oodikone_stuff/updater_api/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_api/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_api/doo_api_database_updater/db_studentnumbers_to_file.js

@@ -1,15 +0,0 @@
-const fs = require('fs')
-const { Student } = require('../../models/index')
-
-const getStudentNumbers = async (writefilepath = './studentnumbers.txt') => {
-  const file = fs.openSync(writefilepath, 'w')
-  const students = await Student.findAll()
-  for (let student of students) {
-    const data = `${student.studentnumber} \n`
-    fs.appendFileSync(file, data)
-  }
-  fs.closeSync(file)
-  process.exit(0)
-}
-
-getStudentNumbers()

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.js

@@ -1,350 +0,0 @@
-const moment = require('moment')
-const _ = require('lodash')
-
-const getStudyRightIdStrings = data => data['data'].map(elements => elements[0])
-
-const getTextsByLanguage = texts => {
-  const names = {}
-  texts.forEach(text => (names[text.langcode] = text.text))
-  return { fi: null, sv: null, en: null, ...names }
-}
-
-const defaultNameFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names.fi || names.en || names.sv
-}
-
-const jsonNamesFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names
-}
-
-const universityEnrollmentDateFromStudyRights = studyRightArray => {
-  return _.sortBy(studyRightArray.map(s => s.start_date), n => moment(n).valueOf())[0]
-}
-
-const getStudentFromData = (student, studyrights) => {
-  const city = getTextsByLanguage(student.city)
-  const language = getTextsByLanguage(student.language)
-  const country = getTextsByLanguage(student.country)
-  const gender = getTextsByLanguage(student.gender)
-  const home_country = getTextsByLanguage(student.home_country)
-  return {
-    studentnumber: student.student_number,
-    email: student.email,
-    phone: student.phone,
-    city_fi: city.fi,
-    city_sv: city.sv,
-    national_student_number: student.national_student_number,
-    zipcode: student.zipcode,
-    address: student.address1,
-    creditcount: student.studyattainments,
-    address2: student.address2,
-    birthdate: getDate(student.birth_date),
-    language_fi: language.fi,
-    language_sv: language.sv,
-    language_en: language.en,
-    age: student.age_years,
-    lastname: student.last_name,
-    mobile: student.mobile_phone,
-    home_county_id: student.home_county_id,
-    country_fi: country.fi,
-    country_sv: country.sv,
-    country_en: country.en,
-    home_country_fi: home_country.fi,
-    home_country_sv: home_country.sv,
-    home_country_en: home_country.en,
-    firstnames: student.first_names,
-    communicationlanguage: language.en || language.fi || language.sv,
-    dateofuniversityenrollment: universityEnrollmentDateFromStudyRights(studyrights),
-    abbreviatedname: [student.last_name, student.first_names].join(' '),
-    gender_code: student.gender_code,
-    gender_fi: gender.fi,
-    gender_sv: gender.sv,
-    gender_en: gender.en
-  }
-}
-
-const getSemesterCode = attainment_date => moment(attainment_date).diff(moment('1950', 'YYYY'), 'months') / 6
-
-const getDate = (date, format = 'DD.MM.YYYY') => {
-  if (!date) return null
-  return moment(date, format).format('YYYY-MM-DD')
-}
-
-const getOrganisationFromData = ({ name, code }) => {
-  return {
-    code,
-    name: jsonNamesFromTexts(name)
-  }
-}
-
-const attainmentDataToCredit = (attainment, studentnumber) => {
-  return {
-    id: String(attainment.studyattainment_id),
-    grade: defaultNameFromTexts(attainment.grade),
-    credits: attainment.credits,
-    ordering: getDate(attainment.attainment_date, null),
-    credittypecode: attainment.attainment_status_code,
-    student_studentnumber: studentnumber,
-    attainment_date: parseDate(attainment.attainment_date),
-    course_code: `${attainment.learningopportunity_id}`,
-    semestercode: attainment.semester_code,
-    isStudyModule: attainment.is_leaf === 'false'
-  }
-}
-
-const attainmentDataToCourse = attainment => {
-  const { learningopportunity_name, attainment_date } = attainment
-  return {
-    code: attainment.learningopportunity_id,
-    name: jsonNamesFromTexts(learningopportunity_name),
-    latest_instance_date: parseDate(attainment_date),
-    is_study_module: attainment.is_leaf === 'false'
-  }
-}
-
-const getTeacherFromData = teacher => {
-  if (teacher) {
-    return {
-      id: teacher.teacher_id,
-      code: teacher.userid,
-      name: teacher.full_name
-    }
-  }
-}
-
-const parseDate = (date, format = 'YYYY-MM-DD HH:mm') => date && moment.utc(date, format).toDate()
-
-const getStudyRightFromData = (data, studentNumber) => {
-  return {
-    studyrightid: `${data.studyright_id}`,
-    canceldate: data.cancel_date,
-    cancelorganisation: data.organisation_code,
-    enddate: parseDate(data.degree_date) || parseDate(data.end_date),
-    extentcode: data.extent_code,
-    givendate: parseDate(data.admission_date),
-    graduated: Number(!!data.degree_date),
-    graduation_date: data.degree_date,
-    prioritycode: data.priority,
-    startdate: parseDate(data.start_date),
-    studystartdate: parseDate(data.study_start_date),
-    organization_code: data.organisation_code,
-    student_studentnumber: studentNumber,
-    faculty_code: data.faculty_code
-  }
-}
-
-const elementDetailFromData = element => {
-  const { code, name, element_id } = element
-  const names = getTextsByLanguage(name)
-  return {
-    code,
-    name: names,
-    type: element_id
-  }
-}
-
-const studyrightElementFromData = (element, studyrightid, studentnumber, degree_date) => {
-  const { start_date, end_date, code } = element
-  return {
-    code,
-    studyrightid,
-    studentnumber,
-    startdate: start_date,
-    enddate: degree_date || end_date
-  }
-}
-
-const attainmentDataToTeachers = data =>
-  data.teachers.map(teacherdata => ({
-    id: teacherdata.teacher_id
-  }))
-
-const studyrightDataToExtent = data => ({
-  extentcode: data.extent_code,
-  name: getTextsByLanguage(data.extent)
-})
-
-const courseTypeFromData = data => ({
-  coursetypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const disciplineFromData = data => ({
-  discipline_id: data.discipline_id,
-  name: getTextsByLanguage(data.name)
-})
-
-const learningOpportunityDataToCourseDisciplines = data => {
-  if (data) {
-    return data.disciplines.map(discipline => ({
-      discipline_id: discipline.discipline_id,
-      course_id: data.learningopportunity_id
-    }))
-  }
-}
-
-const learningOpportunityDataToCourse = data => {
-  if (data) {
-    return {
-      code: data.learningopportunity_id,
-      coursetypecode: data.learningopportunity_type_code,
-      disciplines: data.disciplines,
-      name: getTextsByLanguage(data.names),
-      startdate: parseDate(data.start_date),
-      enddate: parseDate(data.end_date)
-    }
-  }
-}
-
-const learningOpportunityDataToCourseProviders = data => {
-  if (!data) {
-    return { providers: [], courseproviders: [] }
-  }
-  const providers = data.organisations.map(organisation => ({
-    providercode: organisation.code,
-    name: getTextsByLanguage(organisation.name)
-  }))
-  const courseproviders = providers.map(provider => ({
-    providercode: provider.providercode,
-    coursecode: data.learningopportunity_id
-  }))
-  return {
-    providers,
-    courseproviders
-  }
-}
-
-const studyattainmentStatusCodeToCreditType = data => ({
-  credittypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const semesterFromData = data => ({
-  semestercode: data.semester_code,
-  name: getTextsByLanguage(data.name),
-  startdate: parseDate(data.start_date),
-  enddate: parseDate(data.end_date),
-  yearcode: data.year_code,
-  yearname: data.year_name
-})
-
-const semesterEnrollmentFromData = (data, studentnumber) => ({
-  enrollmenttype: data.semester_enrollment_type_code,
-  semestercode: data.semester_code,
-  enrollment_date: parseDate(data.semester_enrollment_date),
-  studentnumber
-})
-
-const getTransfersFromData = (data, studentnumber) => {
-  const studytracks = data.elements.filter(element => element.element_id === 20)
-  const sorted = _.sortBy(studytracks, 'end_date')
-  let transfers = []
-  let i = 0
-  while (i < sorted.length) {
-    if (i === 0) {
-      i++
-      continue
-    }
-    let target = sorted[i].code
-    let source = sorted[i - 1].code
-    let transferdate = sorted[i - 1].end_date
-
-    transfers = transfers.concat({
-      sourcecode: source,
-      targetcode: target,
-      transferdate,
-      studentnumber,
-      studyrightid: `${data.studyright_id}`
-    })
-    i++
-  }
-  return transfers
-}
-
-const courseRealisationTypeFromData = data => ({
-  realisationtypecode: `${data.code}`,
-  name: getTextsByLanguage(data.name)
-})
-
-const studentEnrollmentToModels = (data, studentnumber) => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.name),
-    realisationtypecode: `${data.type_code}`,
-    parent: `${data.parent_id}`,
-    root: `${data.root_id}`
-  }
-  const courseenrollment = {
-    studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }
-  const course = {
-    code: data.learningopportunity_id
-  }
-  return {
-    courserealisation,
-    courseenrollment,
-    course
-  }
-}
-
-const courseUnitRealisationDataToModels = data => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.realisation_name),
-    realisationtypecode: `${data.realisation_type_code}`,
-    parent: `${data.parent_id}`
-  }
-  const course = {
-    code: courserealisation.coursecode
-  }
-  const students = data.students.map(({ student_number }) => ({
-    studentnumber: student_number
-  }))
-  const courseenrollments = students.map(student => ({
-    studentnumber: student.studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }))
-  return {
-    courserealisation,
-    course,
-    students,
-    courseenrollments
-  }
-}
-
-module.exports = {
-  getStudentFromData,
-  getStudyRightIdStrings,
-  getStudyRightFromData,
-  getDate,
-  getTeacherFromData,
-  elementDetailFromData,
-  studyrightElementFromData,
-  attainmentDataToCourse,
-  attainmentDataToCredit,
-  getOrganisationFromData,
-  attainmentDataToTeachers,
-  studyrightDataToExtent,
-  courseTypeFromData,
-  learningOpportunityDataToCourse,
-  studyattainmentStatusCodeToCreditType,
-  disciplineFromData,
-  learningOpportunityDataToCourseDisciplines,
-  semesterFromData,
-  semesterEnrollmentFromData,
-  learningOpportunityDataToCourseProviders,
-  getTransfersFromData,
-  courseRealisationTypeFromData,
-  studentEnrollmentToModels,
-  courseUnitRealisationDataToModels,
-  getSemesterCode
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.test.js

@@ -1,199 +0,0 @@
-const mapper = require('./oodi_data_mapper')
-
-describe('studyattainment data mapping tests', () => {
-  const data = {
-    attainment_status_code: 4,
-    attainment_date: '2018-05-30T21:00:00.000Z',
-    studyattainment_id: 123456789,
-    attainment_lang: null,
-    person_id: 1000002,
-    textual_assessment: null,
-    teachers: [
-      {
-        teacher_id: 'TID_01',
-        person_id: 1000001,
-        short_name: 'Matti Meikäläinen',
-        last_names: 'Meikäläinen',
-        first_names: 'Matti Teppo'
-      }
-    ],
-    studyright_id: null,
-    semester_code: 136,
-    grade: [
-      {
-        langcode: 'fi',
-        text: 'Hyv.'
-      },
-      {
-        langcode: 'sv',
-        text: 'Godk'
-      },
-      {
-        langcode: 'en',
-        text: 'Pass'
-      }
-    ],
-    credits: 10,
-    expiry_date: null,
-    learningopportunity_name: [
-      {
-        langcode: 'fi',
-        text: 'learningopportunity_name FI'
-      },
-      {
-        langcode: 'sv',
-        text: 'learningopportunity_name SV'
-      },
-      {
-        langcode: 'en',
-        text: 'learningopportunity_name EN'
-      }
-    ],
-    learningopportunity_id: 'LID_001'
-  }
-
-  describe('credit mapping tests', () => {
-    const studentnumber = '0876654321'
-    const credit = mapper.attainmentDataToCredit(data, studentnumber)
-
-    describe('studyattainment date tests', () => {
-      const { attainment_date } = credit
-      const MAY = 4
-
-      test('attainment date is truthy', () => {
-        expect(attainment_date).toBeTruthy()
-      })
-
-      test('attainment date is instance of Date', () => {
-        expect(attainment_date instanceof Date).toBe(true)
-      })
-
-      test('attainment date has correct month, year and date', () => {
-        expect(attainment_date.getMonth()).toBe(MAY)
-        expect(attainment_date.getFullYear()).toBe(2018)
-        expect(attainment_date.getDate()).toBe(30)
-      })
-    })
-
-    test('course code matches learningopportunity_id', () => {
-      expect(credit.course_code).toBe('LID_001')
-    })
-
-    test('credit id should attainment id as string', () => {
-      expect(credit.id).toBe('123456789')
-    })
-
-    test('studentnumber is correct', () => {
-      expect(credit.student_studentnumber).toBe(studentnumber)
-    })
-
-    test('semestercode is correct', () => {
-      expect(credit.semestercode).toBe(136)
-    })
-  })
-
-  describe('teacher mapping tests', () => {
-    const teachers = mapper.attainmentDataToTeachers(data)
-
-    test('amount of mapped teachers is correct (1)', () => {
-      expect(teachers.length).toBe(1)
-    })
-
-    test('teacher should have correct id', () => {
-      expect(teachers[0].id).toBe('TID_01')
-    })
-  })
-})
-
-describe('semester enrollment data mapping tests', () => {
-  const studentnumber = '12345678'
-
-  const data = {
-    full_time_student: 'true',
-    yths_code: null,
-    semester_enrollment_type_code: 1,
-    absence_reason_code: null,
-    payment_amount: null,
-    yths_description: null,
-    student_union_member: 'true',
-    semester_enrollment_date: '2011-08-30T21:00:00.000Z',
-    payment_date: null,
-    semester_code: 123,
-    doctoral_current_year_estimate: null,
-    doctoral_previous_year: null
-  }
-
-  describe('mapping of enrolled data', () => {
-    const mapping = mapper.semesterEnrollmentFromData(data, studentnumber)
-
-    test('mapped semestercode should be integer 123', () => {
-      expect(mapping.semestercode).toBe(123)
-    })
-
-    test('mapped enrollmenttype code should be integer 1', () => {
-      expect(mapping.enrollmenttype).toBe(1)
-    })
-
-    test('mapped studentnumber should be correct', () => {
-      expect(mapping.studentnumber).toBe(studentnumber)
-    })
-
-    test('mapped enrollment_date should be instance of Date', () => {
-      expect(mapping.enrollment_date instanceof Date).toBe(true)
-    })
-
-    test('mapped enrollment_date should be equal to Date object with same day, month and year', () => {
-      expect(mapping.enrollment_date).toEqual(new Date('2011-08-30'))
-    })
-  })
-})
-
-describe('semester code data mapping tests', () => {
-  const data = {
-    teaching_start_date: '1950-09-14T22:00:00.000Z',
-    name: [
-      {
-        langcode: 'fi',
-        text: 'Syksy 1950'
-      },
-      {
-        langcode: 'sv',
-        text: 'Hösten 1950'
-      },
-      {
-        langcode: 'en',
-        text: 'Autumn 1950'
-      }
-    ],
-    year_name: '1950-51',
-    enroll_end_date: '1950-09-29T22:00:00.000Z',
-    enroll_start_date: '1950-05-31T22:00:00.000Z',
-    end_date: '1950-12-30T22:00:00.000Z',
-    semester_code: 1,
-    start_date: '1900-07-31T22:20:11.000Z',
-    teaching_end_date: '1950-12-20T22:00:00.000Z',
-    year_code: 1
-  }
-
-  const semester = mapper.semesterFromData(data)
-
-  test('semester code should be correct', () => {
-    expect(semester.semestercode).toBe(1)
-  })
-
-  test('semester name should have all language keys set as properties', () => {
-    expect(semester.name).toHaveProperty('fi')
-    expect(semester.name).toHaveProperty('en')
-    expect(semester.name).toHaveProperty('sv')
-  })
-
-  test('semester start and end date should be instances of Date', () => {
-    expect(semester.startdate instanceof Date).toBe(true)
-    expect(semester.enddate instanceof Date).toBe(true)
-  })
-
-  test('semester yearcode and yearname should be correct', () => {
-    expect(semester.yearcode).toBe(1)
-    expect(semester.yearname).toBe('1950-51')
-  })
-})

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_interface.js

@@ -1,253 +0,0 @@
-require('dotenv').config()
-const axios = require('axios')
-const LRU = require('lru-cache')
-const https = require('https')
-const fs = require('fs')
-
-const { OODI_ADDR, KEY_PATH, CERT_PATH } = process.env
-const base_url = OODI_ADDR
-
-const requestCache = new LRU({
-  max: 25000,
-  length: () => 1,
-  maxAge: 60 * 60 * 1000
-})
-
-const agent =
-  KEY_PATH && CERT_PATH
-    ? new https.Agent({
-        cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-        key: fs.readFileSync(process.env.KEY_PATH, 'utf8')
-      })
-    : new https.Agent({
-        rejectUnauthorized: false
-      })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = async url => {
-  const cacheHit = requestCache.get(url)
-  if (cacheHit) return cacheHit
-
-  const res = await instance.get(url)
-
-  // Sometimes oodi seems to randomly return a response with status 200
-  // that in reality contains an error. Check this out:
-  // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550167612
-  if (res.data.exception) {
-    throw new Error(res.data.exception.message)
-  }
-
-  if (!url.includes('/students/')) requestCache.set(url, res.data.data)
-  return res.data.data
-}
-
-const getOodiApi = async relative => {
-  const route = base_url + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-const attemptGetFor = async (url, attempts = 6) => {
-  for (let attempt = 1; attempt <= attempts; ++attempt) {
-    try {
-      return await getUrl(url)
-    } catch (error) {
-      if (attempt === attempts) {
-        console.log('ATTEMPT GET FOR FAILURE')
-        throw error
-      }
-      await sleep(attempt * 5000)
-    }
-  }
-}
-
-if (process.env.NODE_ENV === 'development') {
-  // axios.defaults.auth = {
-  //   username: 'tktl',
-  //   password: process.env.OODI_PW
-  // }
-  instance.interceptors.request.use(config => {
-    config.params = { token: process.env.TOKEN }
-    return config
-  })
-}
-console.log(process.env.NODE_ENV)
-
-const getStudent = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDENT FAILED')
-    throw e
-  }
-}
-
-const getStudentStudyRights = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyrights`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDYRIGHT FAIELD')
-    throw e
-  }
-}
-
-const getFaculties = async () => {
-  const url = `${base_url}/codes/faculties`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET FACULTIE S FAIELD')
-    throw e
-  }
-}
-
-const getStudyAttainments = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyattainments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STSDU ATTAINEMNT FIALE')
-    throw e
-  }
-}
-
-const getTeacherInfo = async id => {
-  const url = `${base_url}/teachers/${id}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET TCHR FIAIELd', id)
-    throw e
-  }
-}
-
-const getCourseTypeCodes = async () => {
-  const url = `${base_url}/codes/learningopportunities/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('CORUSE TTYPE CODE FAIEL')
-    throw e
-  }
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${base_url}/learningopportunities/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET LEARNIGN OEPPRITUNCITY FAIELD', id)
-    throw e
-  }
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${base_url}/codes/studyattainments/statuses`
-  return await attemptGetFor(url)
-}
-
-const getCourseDisciplines = async () => {
-  const url = `${base_url}/codes/learningopportunities/disciplines`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET COURS EDISCIIPLINES FIALED')
-    throw e
-  }
-}
-const getSemesters = async () => {
-  const url = `${base_url}/codes/semesters`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET SEMESTERS FAIELD')
-    throw e
-  }
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/semesterenrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GEST EMESTER ENROLLMENTS FAIELD')
-    throw e
-  }
-}
-
-const getCourseEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/enrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EENREOLMETNS FAIELD')
-    throw e
-  }
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${base_url}/codes/courseunitrealisations/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURSE REALSITASITON TYPES')
-    throw e
-  }
-}
-
-const courseUnitRealisationsSince = async sinceDate => {
-  const url = `${base_url}/courseunitrealisations/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('COURSEU UNTI REALISTIATION')
-    throw e
-  }
-}
-
-const learningOpportunitiesSince = async (sinceDate = '0000-01-01') => {
-  const url = `${base_url}/learningopportunities/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('LRNG OPRRTN SINCE')
-    throw e
-  }
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${base_url}/courseunitrealisations/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EUNTI GREALSISTAITOn', id)
-    throw e
-  }
-}
-
-module.exports = {
-  getStudentStudyRights,
-  getStudent,
-  getFaculties,
-  getStudyAttainments,
-  getTeacherInfo,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getCourseDisciplines,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisationsSince,
-  getCourseUnitRealisation,
-  learningOpportunitiesSince,
-  getOodiApi
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/updater_formatter.js

@@ -1,158 +0,0 @@
-const Oodi = require('./oodi_interface')
-const mapper = require('./oodi_data_mapper')
-
-const getAllStudentInformationFromApi = async studentnumber => {
-  const [student, studyrights, studyattainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-    Oodi.getStudent(studentnumber),
-    Oodi.getStudentStudyRights(studentnumber),
-    Oodi.getStudyAttainments(studentnumber),
-    Oodi.getSemesterEnrollments(studentnumber),
-    Oodi.getCourseEnrollments(studentnumber)
-  ])
-  return {
-    student,
-    studyrights,
-    studyattainments,
-    studentnumber,
-    semesterEnrollments,
-    courseEnrollments
-  }
-}
-
-const formatStudyrights = async (api, studentnumber) => {
-  if (api.studyrights.length === 0) {
-    console.log(`No studyrights for ${studentnumber}`)
-    return []
-  }
-  return api.studyrights.map(studyrightData => {
-    const studyRightExtent = mapper.studyrightDataToExtent(studyrightData)
-    const studyright = mapper.getStudyRightFromData(studyrightData, studentnumber)
-
-    const elementDetails = studyrightData.elements.map(element => mapper.elementDetailFromData(element))
-    const studyRightElements = studyrightData.elements.map(element =>
-      mapper.studyrightElementFromData(element, studyright.studyrightid, studentnumber, studyrightData.degree_date)
-    )
-
-    const transfers = mapper.getTransfersFromData(studyrightData, studentnumber)
-    return { studyRightExtent, studyright, elementDetails, studyRightElements, transfers }
-  })
-}
-
-const formatCourseEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.courseEnrollments.map(enrollment => mapper.studentEnrollmentToModels(enrollment, studentnumber))
-  )
-
-const parseAttainmentData = (data, studentnumber) => {
-  return {
-    credit: mapper.attainmentDataToCredit(data, studentnumber),
-    teachers: mapper.attainmentDataToTeachers(data),
-    course: mapper.attainmentDataToCourse(data)
-  }
-}
-
-const createCreditTeachers = async (credit, teachers) => {
-  const creditTeachers = teachers.map(teacher => ({
-    credit_id: credit.id,
-    teacher_id: teacher.id
-  }))
-  return creditTeachers
-}
-
-const formatStudyattainments = async (api, studentnumber) => {
-  let studyAttainments = []
-  for (let data of api.studyattainments) {
-    const { credit, teachers, course } = parseAttainmentData(data, studentnumber)
-    let learningOpportunity
-    // For some learning opportunities oodi returns an SQL error.
-    // F.ex. /learningopportunities/590202B. Therefore, until the
-    // bug is fixed on Oodi's side, this is what we need to do.
-    try {
-      learningOpportunity = await Oodi.getLearningOpportunity(course.code)
-    } catch (e) {
-      if (e.message && e.message.includes('PreparedStatementCallback')) continue
-      throw e
-    }
-    const { providers, courseproviders } = mapper.learningOpportunityDataToCourseProviders(learningOpportunity)
-
-    studyAttainments = [
-      ...studyAttainments,
-      {
-        credit: credit.semestercode
-          ? credit
-          : { ...credit, semestercode: mapper.getSemesterCode(credit.attainment_date) },
-        creditTeachers: await createCreditTeachers(credit, teachers),
-        teachers: (await Promise.all(
-          teachers.map(async t => {
-            const teacher = await Oodi.getTeacherInfo(t.id)
-            // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550184131
-            // Sometimes Oodi returns a null teacher... We need to figure out why.
-            if (!teacher) {
-              console.log('TEACHER MISSING!', t.id)
-              return null
-            }
-            return mapper.getTeacherFromData(teacher)
-          })
-        )).filter(t => t),
-        course: {
-          ...course,
-          ...mapper.learningOpportunityDataToCourse(learningOpportunity),
-          disciplines: mapper.learningOpportunityDataToCourseDisciplines(learningOpportunity),
-          providers,
-          courseproviders
-        }
-      }
-    ]
-  }
-  return studyAttainments
-}
-const formatSemesterEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.semesterEnrollments.map(apiEnrollment => mapper.semesterEnrollmentFromData(apiEnrollment, studentnumber))
-  )
-
-const getStudent = async studentnumber => {
-  const api = await getAllStudentInformationFromApi(studentnumber)
-  if (api.student == null) {
-    const error = new Error(`API returned ${api.student} for studentnumber ${studentnumber}.`)
-    error.name = 'NO_STUDENT'
-    throw error
-  }
-  try {
-    const studentInfo = await mapper.getStudentFromData(api.student, api.studyrights)
-    const [studyRights, studyAttainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-      formatStudyrights(api, studentnumber),
-      formatStudyattainments(api, studentnumber),
-      formatSemesterEnrollments(api, studentnumber),
-      formatCourseEnrollments(api, studentnumber)
-    ])
-    return { studentInfo, studyRights, studyAttainments, semesterEnrollments, courseEnrollments }
-  } catch (e) {
-    console.error('getStudent failed', api)
-    throw e
-  }
-}
-
-const getMeta = async () => {
-  const [
-    faculties,
-    courseRealisationsTypes,
-    semesters,
-    creditTypeCodes,
-    courseTypeCodes,
-    disciplines
-  ] = await Promise.all([
-    (await Oodi.getFaculties()).map(mapper.getOrganisationFromData),
-    (await Oodi.getCourseRealisationTypes()).map(mapper.courseRealisationTypeFromData),
-    (await Oodi.getSemesters()).map(mapper.semesterFromData),
-    (await Oodi.getStudyattainmentStatusCodes()).map(mapper.studyattainmentStatusCodeToCreditType),
-    (await Oodi.getCourseTypeCodes()).map(mapper.courseTypeFromData),
-    (await Oodi.getCourseDisciplines()).map(mapper.disciplineFromData)
-  ])
-  return { faculties, courseRealisationsTypes, semesters, creditTypeCodes, courseTypeCodes, disciplines }
-}
-
-module.exports = {
-  getStudent,
-  getMeta
-}

old_oodikone_stuff/updater_api/index.js

@@ -1,72 +0,0 @@
-const { stan, opts } = require('./stan')
-const { getStudent, getMeta } = require('./doo_api_database_updater/updater_formatter')
-const logger = require('./logger')
-const moment = require('moment')
-
-const hasActiveEnrollment = semesterEnrollments => {
-  if (semesterEnrollments) {
-    const currentSemester = Math.floor(moment().diff(moment('1950', 'YYYY'), 'months') / 6)
-    const active = semesterEnrollments.find(e => e.semestercode === currentSemester && e.enrollmenttype === 1)
-    return active ? true : false
-  }
-  return false
-}
-
-const fetchData = async (priority, msg) => {
-  try {
-    const task = JSON.parse(msg.getData())
-    if (task.task === 'meta') {
-      const data = await getMeta()
-      stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', priority }), err => {
-        if (err) console.log('STATUS PUBLISH FAILED', err)
-      })
-      stan.publish(priority ? 'PriorityWrite' : 'UpdateWrite', JSON.stringify({ task: task.task, data }))
-    } else {
-      try {
-        const data = await getStudent(task.task)
-        // TODO: check that data is properly structured(?)
-        const active = hasActiveEnrollment(data.semesterEnrollments)
-        stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', active, priority }), err => {
-          if (err) console.log('STATUS PUBLISH FAIELD', err)
-        })
-        stan.publish(
-          priority ? 'PriorityWrite' : 'UpdateWrite',
-          JSON.stringify({ task: task.task, data, active }),
-          err => {
-            if (err) console.log('STATUS PUBLISH FAILED', err)
-          }
-        )
-      } catch (e) {
-        if (e.name === 'NO_STUDENT') {
-          stan.publish(
-            'status',
-            JSON.stringify({ task: task.task, status: 'NO_STUDENT', active: false, priority }),
-            err => {
-              if (err) console.log('STATUS PUBLISH FAILED', err)
-            }
-          )
-          return
-        } else {
-          console.error('Task failed', task)
-          throw e
-        }
-      }
-    }
-  } catch (e) {
-    console.log(e)
-    logger.info('failure', { service: 'API' })
-  }
-}
-
-const handleMessage = priority => async msg => {
-  await fetchData(priority, msg)
-  msg.ack()
-}
-
-stan.on('connect', async () => {
-  const sub = stan.subscribe('UpdateApi', 'updater.workers', opts)
-  const prioSub = stan.subscribe('PriorityApi', 'updater.workers.prio', opts)
-
-  sub.on('message', handleMessage(false))
-  prioSub.on('message', handleMessage(true))
-})

old_oodikone_stuff/updater_api/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_api/package.json

@@ -1,34 +0,0 @@
-{
-  "name": "updater_api",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "axios": "^0.21.1",
-    "dotenv": "^7.0.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^5.1.1",
-    "moment": "^2.29.1",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_api/stan.js

@@ -1,13 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-
-const opts = stan.subscriptionOptions()
-opts.setManualAckMode(true)
-opts.setAckWait(10 * 60 * 1000)
-opts.setMaxInFlight(1)
-
-module.exports = {
-  stan,
-  opts
-}

old_oodikone_stuff/updater_api/test_producer.js

@@ -1,32 +0,0 @@
-var stan = require('node-nats-streaming').connect('updaterNATS', 'producer', process.env.NATS_URI)
-
-stan.on('connect', () => {
-  stan.publish('UpdateApi', 'meta', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '011120775', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014441008', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014272112', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-})

old_oodikone_stuff/updater_scheduler/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_scheduler/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_scheduler/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_scheduler/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_scheduler/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_scheduler/index.js

@@ -1,126 +0,0 @@
-const { stan } = require('./src/nats_connection')
-const { CronJob } = require('cron')
-const Schedule = require('./models')
-const logger = require('./logger')
-const {
-  scheduleDaily,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  scheduleMeta
-} = require('./src/schedule_students')
-const { isValidStudentId } = require('./src/util')
-require('./src/api')
-
-const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
-
-let updatedCount = 0
-let scheduledCount = 0
-let fetchedCount = 0
-
-const updateTask = async ({ task, status, type, updatetime, active }) => {
-  const doc = { task, status, type }
-  if (updatetime) doc.updatedAt = new Date()
-  if (active != null) doc.active = active
-  await Schedule.findOneAndUpdate({ task }, doc, { upsert: true })
-}
-
-stan.on('connect', async () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 0 * * 6', async () => {
-      // Every Friday-Saturday midnight
-      try {
-        console.log('SCHEDULING ALL STUDENTS')
-        await scheduleAllStudents()
-      } catch (err) {
-        console.log('SCHEDULING ALL STUDENTS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 6 * * 1-5', async () => {
-      // Every Monday through Friday at 06:00
-      try {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        await scheduleMeta()
-        await scheduleAttainmentUpdate()
-      } catch (err) {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 20 * * 1-4', async () => {
-      // Every Monday through Thursday at 20:00
-      try {
-        console.log('SCHEDULING DAILY TASKS')
-        await scheduleDaily()
-      } catch (err) {
-        console.log('SCHEDULING DAILY TASKS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-  }
-
-  schedule('0 * * * *', async () => {
-    // Every hour
-    console.log(
-      `${updatedCount} TASKS DONE IN LAST HOUR. ${scheduledCount} TASKS SCHEDULED IN LAST HOUR. ${fetchedCount} TASKS FETCHED FROM API IN LAST HOUR`
-    )
-    updatedCount = 0
-    fetchedCount = 0
-    scheduledCount = 0
-  })
-
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  // opts.setDeliverAllAvailable()
-  // opts.setDurableName('durable')
-  opts.setMaxInFlight(20)
-
-  const statusSub = stan.subscribe('status', opts)
-
-  const handleStatusMessage = async msg => {
-    const data = JSON.parse(msg.getData())
-    const { task, status, timems, active, priority } = data
-    let updatetime = false
-    switch (status) {
-      case 'DONE':
-      case 'NO_STUDENT':
-        updatedCount = updatedCount + 1
-        updatetime = true
-        break
-      case 'FETCHED':
-        fetchedCount = fetchedCount + 1
-        break
-      case 'SCHEDULED':
-        scheduledCount = scheduledCount + 1
-        break
-      default:
-        throw 'unknown status: ' + msg.getData()
-    }
-    const isStudent = !!isValidStudentId(task)
-    if (status === 'DONE')
-      logger.info(`Status changed for ${task} to ${status}`, {
-        task,
-        status,
-        student: isStudent,
-        timems,
-        active,
-        priority
-      })
-    await updateTask({ task, status, type: isStudent ? 'student' : 'other', updatetime, active })
-  }
-  statusSub.on('message', async msg => {
-    try {
-      await handleStatusMessage(msg)
-    } catch (err) {
-      console.log(err)
-      logger.info('failure', { service: 'SCHEDULER' })
-    }
-    msg.ack()
-  })
-})

old_oodikone_stuff/updater_scheduler/logger.js

@@ -1,29 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-transports.push(new winston.transports.Console({ level: 'debug' }))
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_scheduler/models.js

@@ -1,18 +0,0 @@
-const mongoose = require('mongoose')
-mongoose.connect(process.env.MONGO_URI, { useNewUrlParser: false, useFindAndModify: false })
-
-const scheduleSchema = new mongoose.Schema({
-  task: {
-    type: String,
-    unique: true
-  },
-  status: String,
-  type: String,
-  active: Boolean,
-  updatedAt: {
-    type: Date,
-    index: true
-  }
-})
-
-module.exports = mongoose.model('Schedule', scheduleSchema)

old_oodikone_stuff/updater_scheduler/package.json

@@ -1,35 +0,0 @@
-{
-  "name": "updater_scheduler",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node --max_old_space_size=4096 index.js",
-    "dev": "nodemon --max_old_space_size=4096 index.js",
-    "scheduleActiveStudents": "NATS_CLIENTID=manualActiveSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleActiveStudents()'",
-    "scheduleMeta": "NATS_CLIENTID=manualAllSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleMeta()'",
-    "getOldestTasks": "node --max_old_space_size=4096 -e 'require(\"./src/SchedulingStatistics.js\").getOldestTasks()'"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "cron": "^1.8.2",
-    "express": "^4.17.1",
-    "mongoose": "^5.12.3",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_scheduler/src/SchedulingStatistics.js

@@ -1,48 +0,0 @@
-const Schedule = require('../models')
-
-const getOldestTasks = async () => {
-  const oldestTask = (await Schedule.find({ type: 'student' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestActiveTask = (await Schedule.find({ type: 'student', active: true })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestMetaTask = (await Schedule.find({ type: 'other', task: 'meta' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestAttainmentTask = (await Schedule.find({ type: 'other', task: 'attainment' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-
-  return {
-    oldestChangedStatus: {
-      studentnumber: oldestTask ? oldestTask.task : null,
-      updatedAt: oldestTask ? oldestTask.updatedAt : null
-    },
-    oldestActiveStudentDone: {
-      studentnumber: oldestActiveTask ? oldestActiveTask.task : null,
-      updatedAt: oldestActiveTask ? oldestActiveTask.updatedAt : null
-    },
-    oldestMetaTask: {
-      updatedAt: oldestMetaTask ? oldestMetaTask.updatedAt : null,
-      status: oldestMetaTask ? oldestMetaTask.status : null
-    },
-    oldestAttainmentTask: {
-      updatedAt: oldestAttainmentTask ? oldestAttainmentTask.updatedAt : null,
-      status: oldestAttainmentTask ? oldestAttainmentTask.status : null
-    }
-  }
-}
-
-const getCurrentStatus = async () => {
-  const allTasksScheduled = await Schedule.count({ type: 'student', status: 'SCHEDULED' })
-  const allTasksFetched = await Schedule.count({ type: 'student', status: 'FETCHED' })
-  const allTasksDone = await Schedule.count({ type: 'student', status: 'DONE' })
-  const allTasksCreated = await Schedule.count({ type: 'student', status: 'CREATED' })
-  const allTasksNoStudent = await Schedule.count({ type: 'student', status: 'NO_STUDENT' })
-  const allTasksActive = await Schedule.count({ type: 'student', active: true })
-
-  return { allTasksScheduled, allTasksFetched, allTasksDone, allTasksCreated, allTasksActive, allTasksNoStudent }
-}
-
-module.exports = { getOldestTasks, getCurrentStatus }

old_oodikone_stuff/updater_scheduler/src/api.js

@@ -1,127 +0,0 @@
-const express = require('express')
-const app = express()
-const port = process.env.PORT
-const bodyParser = require('body-parser')
-const {
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleAllStudents,
-  scheduleActiveStudents,
-  scheduleMeta,
-  scheduleAttainmentUpdate,
-  scheduleStudentCheck,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily
-} = require('./schedule_students')
-const { getOldestTasks, getCurrentStatus } = require('./SchedulingStatistics')
-const { createTasks } = require('./student_list_updater')
-
-app.use(bodyParser.json())
-
-app.get('/ping', (req, res) => res.json({ message: 'pong ' }))
-
-app.post('/update', async (req, res) => {
-  await createTasks(req.body)
-  const scheduled = await scheduleStudentsByArray(req.body)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/oldest', async (req, res) => {
-  const { amount } = req.body
-  const scheduled = await scheduleOldestNStudents(amount)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/all', async (req, res) => {
-  const scheduled = await scheduleAllStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/active', async (req, res) => {
-  const scheduled = await scheduleActiveStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/attainment', async (req, res) => {
-  scheduleAttainmentUpdate()
-  res.json({ message: `scheduled attainments` })
-})
-
-app.post('/update/meta', async (req, res) => {
-  scheduleMeta()
-  res.json({ message: 'scheduled meta' })
-})
-
-app.post('/update/no_student', async (req, res) => {
-  const scheduled = await scheduleStudentCheck()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/studentlist', async (req, res) => {
-  createTasks()
-  res.json({ message: `tasks created` })
-})
-
-app.post('/update/daily', async (req, res) => {
-  const scheduled = await scheduleDaily()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/created', async (req, res) => {
-  const scheduled = await rescheduleCreated()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/scheduled', async (req, res) => {
-  const scheduled = await rescheduleScheduled()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/fetched', async (req, res) => {
-  const scheduled = await rescheduleFetched()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.get('/statuses', async (req, res) => {
-  const statuses = []
-
-  const {
-    oldestChangedStatus: { studentnumber, updatedAt },
-    oldestActiveStudentDone: { studentnumber: studentnumberActive, updatedAt: updatedAtActive },
-    oldestMetaTask,
-    oldestAttainmentTask
-  } = await getOldestTasks()
-
-  const {
-    allTasksScheduled,
-    allTasksFetched,
-    allTasksDone,
-    allTasksCreated,
-    allTasksActive,
-    allTasksNoStudent
-  } = await getCurrentStatus()
-
-  statuses.push(
-    { label: 'CREATED', value: allTasksCreated },
-    { label: 'SCHEDULED', value: allTasksScheduled },
-    { label: 'FETCHED', value: allTasksFetched },
-    { label: 'DONE', value: allTasksDone },
-    { label: 'NO_STUDENT', value: allTasksNoStudent },
-    { label: 'ACTIVE', value: allTasksActive },
-    { label: 'oldest student studentnumber', value: studentnumber },
-    { label: 'oldest student timestamp', value: updatedAt },
-    { label: 'oldest active student studentnumber', value: studentnumberActive },
-    { label: 'oldest active student timestamp', value: updatedAtActive },
-    { label: 'oldest metadata update', value: oldestMetaTask.updatedAt },
-    { label: 'current metadata status', value: oldestMetaTask.status },
-    { label: 'oldest attainment update', value: oldestAttainmentTask.updatedAt },
-    { label: 'current attainment status', value: oldestAttainmentTask.status }
-  )
-
-  res.json(statuses)
-})
-
-app.listen(port, () => console.log(`listening on port ${port}!`))
-module.exports = { app }

old_oodikone_stuff/updater_scheduler/src/nats_connection.js

@@ -1,4 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.NATS_CLIENTID || 'scheduler', {
-  url: process.env.NATS_URI
-})
-module.exports = { stan }

old_oodikone_stuff/updater_scheduler/src/schedule_students.js

@@ -1,181 +0,0 @@
-const { stan } = require('./nats_connection')
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-stan.on('connect', async () => {
-  const taskspermin = 300
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  opts.setMaxInFlight(taskspermin)
-
-  const scheduleSub = stan.subscribe('schedule', opts)
-  scheduleSub.on('message', async msg => {
-    try {
-      const { task, priority } = JSON.parse(msg.getData())
-      stan.publish('status', JSON.stringify({ task, status: 'SCHEDULED', priority }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      stan.publish(priority ? 'PriorityApi' : 'UpdateApi', JSON.stringify({ task }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      await sleep(60 * 1000)
-    } catch (err) {
-      console.log(err)
-    }
-    msg.ack()
-  })
-})
-
-const publishSingle = async ({ priority, task }) => {
-  const promise = new Promise(resolve => {
-    stan.publish('schedule', JSON.stringify({ task: task.task, priority }), async err => {
-      if (err) {
-        console.log('publish failed, waiting 10 seconds and republishing', err)
-        await sleep(10 * 1000)
-        resolve(await publishSingle({ priority, task }))
-      } else {
-        resolve(true)
-      }
-    })
-  })
-  return await promise
-}
-
-const publishAll = async (tasks, priority = false) => {
-  for (const task of tasks) {
-    await publishSingle({ task, priority })
-  }
-}
-
-const scheduleActiveStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleAllStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleMeta = async () => {
-  await publishAll([{ task: 'meta' }], true)
-}
-
-const scheduleAttainmentUpdate = async () => {
-  stan.publish('status', JSON.stringify({ task: 'attainment', status: 'SCHEDULED' }), err => {
-    if (err) {
-      console.log('publish failed')
-    }
-  })
-  stan.publish('UpdateAttainmentDates', null, err => {
-    if (err) {
-      console.log('publish failed', 'UpdateAttainmentDates')
-    }
-  })
-}
-
-const scheduleStudentCheck = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'NO_STUDENT' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleStudentsByArray = async studentNumbers => {
-  try {
-    const tasks = await Schedule.find({ type: 'student', task: { $in: studentNumbers } })
-    publishAll(tasks, true)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const scheduleOldestNStudents = async amount => {
-  try {
-    const tasks = [
-      ...(await Schedule.find({ type: 'student' })
-        .sort({ updatedAt: 1 })
-        .limit(Number(amount)))
-    ]
-    publishAll(tasks)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const rescheduleCreated = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'CREATED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleScheduled = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'SCHEDULED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleFetched = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'FETCHED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleDaily = async () => {
-  const findStudentsRange = 10000
-  const highestStudentNumber = (await Schedule.find({ type: 'student', status: { $in: ['DONE', 'FETCHED'] } })
-    .sort({ task: -1 })
-    .limit(1))[0]
-  const regexResult = highestStudentNumber ? highestStudentNumber.task.match(/0(\d{7})\d/) : null
-  const minstudentnumber = 1000000
-  const studentNumberMid = (regexResult && Number(regexResult[1])) || minstudentnumber + findStudentsRange
-  const min = Math.max(studentNumberMid - findStudentsRange, minstudentnumber)
-  const max = Math.min(studentNumberMid + findStudentsRange, 9999999)
-
-  console.log({ findStudentsRange, minstudentnumber, studentNumberMid, min, max })
-
-  const activeStudents = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  const studentnumbers = activeStudents.map(t => t.task)
-  for (let i = min; i <= max; ++i) {
-    studentnumbers.push(`0${i}${getStudentNumberChecksum(i)}`)
-  }
-  const oldestStudents = [
-    ...(await Schedule.find({ type: 'student' })
-      .sort({ updatedAt: 1 })
-      .limit(10000))
-  ]
-  studentnumbers.push(...oldestStudents.map(t => t.task))
-
-  const tasks = [...new Set(studentnumbers)].map(sn => ({ task: sn }))
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-module.exports = {
-  scheduleActiveStudents,
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleMeta,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily,
-  scheduleStudentCheck
-}

old_oodikone_stuff/updater_scheduler/src/student_list_updater.js

@@ -1,62 +0,0 @@
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-const createTasks = async (fromStudents = []) => {
-  console.log('Creating studentnumbers to DB...')
-
-  const writeStudents = async studentsToAdd => {
-    const tasks = studentsToAdd.map(student => ({ task: student, status: 'CREATED', type: 'student', active: true }))
-    const bulkOperation = Schedule.collection.initializeUnorderedBulkOp()
-    for (const task of tasks) {
-      bulkOperation.insert(task)
-    }
-    await bulkOperation.execute()
-  }
-
-  const writeToDB = async studentnumbers => {
-    try {
-      await writeStudents(studentnumbers)
-    } catch (err) {
-      const BulkWriteResult = err.result.result
-      const duplicateIndexError = 11000
-      const realErrors = BulkWriteResult.writeErrors.filter(e => e.err.code !== duplicateIndexError)
-      if (realErrors.length > 0 || BulkWriteResult.writeConcernErrors.length > 0) {
-        console.log(err)
-        return false
-      }
-    }
-    return true
-  }
-
-  const mongoWriteBatchSize = 1000
-  let studentnumbers = []
-  if (fromStudents.length) {
-    for (let i = 0; i < fromStudents.length; i++) {
-      studentnumbers.push(fromStudents[i])
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === fromStudents.length - 1) {
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        console.log(`Created tasks from ${studentnumbers}`)
-        studentnumbers = []
-      }
-    }
-  } else {
-    const minStudentNumber = 1010000
-    const maxStudentNumber = 1530000
-    for (let i = minStudentNumber; i <= maxStudentNumber; ++i) {
-      const studentNumber = '0' + i + getStudentNumberChecksum(i)
-      studentnumbers.push(studentNumber)
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === maxStudentNumber) {
-        console.log('current studentnumber', i)
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        studentnumbers = []
-      }
-    }
-  }
-  console.log('Studentnumbers created to DB')
-}
-
-module.exports = { createTasks }

old_oodikone_stuff/updater_scheduler/src/util.js

@@ -1,43 +0,0 @@
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-// Shuffle in place, returns same array
-const shuffleArray = a => {
-  for (let i = a.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1))
-    const x = a[i]
-    a[i] = a[j]
-    a[j] = x
-  }
-  return a
-}
-
-const isValidStudentId = id => {
-  if (/^0\d{8}$/.test(id)) {
-    // is a 9 digit number
-    const multipliers = [7, 1, 3, 7, 1, 3, 7]
-    const checksum = id
-      .substring(1, 8)
-      .split('')
-      .reduce((sum, curr, index) => {
-        return (sum + curr * multipliers[index]) % 10
-      }, 0)
-    return (10 - checksum) % 10 == id[8]
-  }
-  return false
-}
-
-const getStudentNumberChecksum = studentNumber => {
-  const studentNumberString = String(studentNumber)
-  let checksumNumbers = [7, 3, 1]
-  let checksum = 0
-
-  for (let i = 0; i < studentNumberString.length; i++) {
-    // go from end t start
-    let currentNumber = studentNumberString[studentNumberString.length - (i + 1)]
-    checksum += currentNumber * checksumNumbers[i % checksumNumbers.length]
-  }
-
-  return (10 - (checksum % 10)) % 10
-}
-
-module.exports = { sleep, isValidStudentId, getStudentNumberChecksum, shuffleArray }

old_oodikone_stuff/updater_writer/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_writer/.eslintignore

@@ -1,2 +0,0 @@
-*.sql
-updater/test_assets

old_oodikone_stuff/updater_writer/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_writer/.gitignore

@@ -1,4 +0,0 @@
-node_modules
-
-models
-migrations

old_oodikone_stuff/updater_writer/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_writer/Dockerfile

@@ -1,18 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-RUN apk add postgresql-client
-
-COPY ./updater_writer/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY ./updater_writer /usr/src/app
-COPY ./shared/models /usr/src/app/models
-COPY ./shared/migrations /usr/src/app/database/migrations
-COPY ./shared/migrations_kone /usr/src/app/database/migrations_kone

old_oodikone_stuff/updater_writer/conf-backend.js

@@ -1,77 +0,0 @@
-require('dotenv').config()
-
-const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
-let DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const isTest = NODE_ENV === 'test'
-if (isTest) {
-  DB_URL = process.env.TEST_DB
-  DB_SCHEMA = process.env.TEST_DB_SCHEMA
-} else if (NODE_ENV === 'anon') {
-  DB_URL = process.env.ANON_DB
-}
-const frontend_addr = process.env.FRONT_URL
-const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
-const CERT_PATH = process.env.CERT_PATH // production/staging only
-const KEY_PATH = process.env.KEY_PATH // production/staging only
-const USERSERVICE_URL = process.env.USERSERVICE_URL
-const ANALYTICS_URL = process.env.ANALYTICS_URL
-const PORT = isTest ? 8079 : 8080
-const OODI_SECRET = process.env.OODI_SECRET
-
-const FEATURES = {
-  ERROR_HANDLER: false
-}
-
-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-if (process.env.NODE_ENV === 'dev' && process.env.FEATURES) {
-  const toggled = process.env.FEATURES.split(',')
-  toggled
-    .map(toggle => toggle.trim())
-    .forEach(feature => {
-      if (FEATURES[feature] !== undefined) {
-        FEATURES[feature] = true
-      }
-    })
-}
-
-const OODI = {
-  test: 'http://localhost',
-  anon: process.env.OODI_ADDR_ANON
-}
-
-const OODI_ADDR = OODI[process.env.NODE_ENV] || process.env.OODI_ADDR
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-const OODI_SECRET_HEADER_KEY = 'x-oodi-secret'
-
-let requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
-if (process.env.NODE_ENV === 'staging') {
-  requiredGroup = ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-}
-if (process.env.NODE_ENV === 'dev' || isTest) {
-  requiredGroup = null
-}
-
-module.exports = {
-  frontend_addr,
-  DB_URL,
-  redis,
-  TOKEN_SECRET,
-  DB_SCHEMA,
-  OODI_ADDR,
-  CERT_PATH,
-  KEY_PATH,
-  FEATURES,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
-  PORT,
-  ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  requiredGroup,
-  OODI_SECRET,
-  OODI_SECRET_HEADER_KEY,
-  isTest
-}

old_oodikone_stuff/updater_writer/database/connection.js

@@ -1,69 +0,0 @@
-const Sequelize = require('sequelize')
-const Umzug = require('umzug')
-
-const conf = require('../conf-backend')
-
-const sequelize = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA,
-  searchPath: conf.DB_SCHEMA,
-  logging: false
-})
-sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
-
-const initializeDatabaseConnection = async () => {
-  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelize.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
-      }
-      console.log('.')
-      await sleep(1000)
-    }
-  }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelize,
-          tableName: 'migrations'
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelize.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/database/migrations`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA
-        }
-      })
-      const migrations = await migrator.up()
-
-      console.log('Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
-  }
-}
-
-const forceSyncDatabase = async () => {
-  try {
-    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
-  } catch (e) {
-    // console.log(e)
-  }
-  console.log('OODI SYNC')
-  await sequelize.sync({ force: true })
-}
-
-module.exports = {
-  sequelize,
-  initializeDatabaseConnection,
-  forceSyncDatabase
-}

old_oodikone_stuff/updater_writer/database/dump_database.js

@@ -1,27 +0,0 @@
-const spawn = require('child_process').spawn
-const fs = require('fs')
-const { DB_SCHEMA } = require('../conf-backend')
-
-const dumpDatabase = async () => {
-  const { DB_URL } = process.env
-  const dumpOptions = ['-Fc', `-n${DB_SCHEMA}`, `${DB_URL}`]
-  console.log(dumpOptions)
-  const dumpProcess = spawn('pg_dump', dumpOptions, {
-    stdio: ['ignore', 'pipe', 'inherit']
-  })
-  dumpProcess.on('error', err => {
-    console.log(' dump process error:', err)
-  })
-  const writeStream = fs.createWriteStream(`./dumps/oodidb_prod-${new Date().toISOString()}`)
-
-  dumpProcess.stdout.pipe(writeStream)
-
-  dumpProcess.on('exit', async code => {
-    if (code !== 0) {
-      throw new Error('pg_dump: Bad exit code (' + code + ')')
-    } else {
-      console.log('database dump created')
-    }
-  })
-}
-module.exports = { dumpDatabase }

old_oodikone_stuff/updater_writer/database/force_sync_database.js

@@ -1,17 +0,0 @@
-const { seedAllMigrations } = require('./seed_migrations')
-const { forceSyncDatabase } = require('./connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-//sync()
-module.exports = { sync }

old_oodikone_stuff/updater_writer/database/seed_migrations.js

@@ -1,28 +0,0 @@
-const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
-
-const DEFAULT_PATH = `${process.cwd()}/database/migrations`
-const DEFAULT_PATH_KONE = `${process.cwd()}/database/migrations_kone`
-
-const saveMigrationsToDatabase = Migration => async filenames =>
-  Promise.all(filenames.map(name => Migration.upsert({ name })))
-
-const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(Migration)(filenames)
-}
-
-const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(MigrationKone)(filenames)
-}
-
-const seedAllMigrations = async () => {
-  await seedMigrations()
-  await seedMigrationsKone()
-}
-
-module.exports = {
-  seedMigrations,
-  seedAllMigrations
-}

old_oodikone_stuff/updater_writer/index.js

@@ -1,69 +0,0 @@
-const natsStreaming = require('node-nats-streaming')
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./updater/database_updater')
-const logger = require('./logger')
-const { initializeDatabaseConnection } = require('./database/connection')
-
-initializeDatabaseConnection()
-  .then(() => {
-    const stan = natsStreaming.connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-    console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-    const opts = stan.subscriptionOptions()
-    opts.setManualAckMode(true)
-    opts.setAckWait(10 * 60 * 1000) // some students have taken over 5 min to write!
-    // opts.setDeliverAllAvailable()
-    // opts.setDurableName('durable')
-    opts.setMaxInFlight(1)
-
-    stan.on('connect', function() {
-      const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
-      const attSub = stan.subscribe('UpdateAttainmentDates', 'updater.workers.attainments', opts)
-      const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
-
-      const writeStudent = priority => async msg => {
-        let data = null
-        try {
-          const start = new Date()
-          data = JSON.parse(msg.getData())
-          if (data.task === 'meta') {
-            await updateMeta(data.data)
-          } else {
-            await updateStudent(data.data)
-          }
-          stan.publish(
-            'status',
-            JSON.stringify({ task: data.task, status: 'DONE', timems: new Date() - start }, priority),
-            err => {
-              if (err) console.log(err)
-            }
-          )
-        } catch (err) {
-          console.log('update failed', data.task, err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      }
-
-      sub.on('message', writeStudent(false))
-      prioSub.on('message', writeStudent(true))
-
-      attSub.on('message', async msg => {
-        try {
-          await updateAttainmentMeta()
-          stan.publish('status', JSON.stringify({ task: 'attainment', status: 'DONE' }), err => {
-            if (err) {
-              console.log('publish failed')
-            }
-          })
-        } catch (err) {
-          console.log('attainment meta update failed', err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      })
-    })
-  })
-  .catch(e => {
-    process.exitCode = 1
-    console.log(e)
-  })

old_oodikone_stuff/updater_writer/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_writer/package.json

@@ -1,39 +0,0 @@
-{
-  "name": "updater_writer",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run jest",
-    "jest": "NODE_ENV=test jest --runInBand --verbose",
-    "test_docker": "docker exec -t updater_writer /bin/bash -c 'npm test'",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js",
-    "force_sync": "node -e \"require('./database/force_sync_database').sync()\"",
-    "dump": "node -e \"require('./database/dump_database').dumpDatabase()\""
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "dotenv": "^8.0.0",
-    "jest": "^24.8.0",
-    "lodash": "^4.17.19",
-    "node-nats-streaming": "^0.2.6",
-    "pg": "^8.6.0",
-    "sequelize": "^6.6.2",
-    "umzug": "^2.2.0",
-    "winston": "^3.2.1",
-    "winston-log2gelf": "^2.3.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.0.3",
-    "eslint": "^6.6.0",
-    "eslint-config-prettier": "^6.3.0",
-    "eslint-plugin-prettier": "^3.1.0",
-    "nodemon": "^1.19.4",
-    "prettier": "^1.18.2",
-    "prettier-eslint": "^9.0.0"
-  }
-}

old_oodikone_stuff/updater_writer/updater/database_updater.js

@@ -1,217 +0,0 @@
-const { notIn } = require('sequelize').Op
-const { sequelize } = require('../database/connection')
-const { sortBy, flatMap, uniqBy, sortedUniqBy } = require('lodash')
-
-const {
-  Student,
-  Credit,
-  Course,
-  CreditTeacher,
-  Teacher,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { updateAttainmentDates } = require('./update_attainment_dates')
-
-const getColumnsToUpdate = arr => (arr[0] ? Object.keys(arr[0]) : [])
-
-const updateAttainments = async (studyAttainments, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const courses = sortedUniqBy(sortBy(studyAttainments.map(e => e.course), 'code'), 'code')
-  const disciplines = sortBy(flatMap(studyAttainments, e => e.course.disciplines || []), 'discipline_id', 'course_id')
-  const providers = sortedUniqBy(
-    sortBy(flatMap(studyAttainments, e => e.course.providers), 'providercode'),
-    'providercode'
-  )
-  const courseproviders = sortBy(flatMap(studyAttainments, e => e.course.courseproviders), 'coursecode', 'providercode')
-  const credits = sortedUniqBy(sortBy(studyAttainments.map(e => e.credit), 'id'), 'id')
-  const teachers = sortedUniqBy(sortBy(flatMap(studyAttainments, e => e.teachers || []), 'id'), 'id')
-  const creditTeachers = sortBy(flatMap(studyAttainments, e => e.creditTeachers), 'teacher_id', 'credit_id')
-
-  await Promise.all([
-    Course.bulkCreate(courses, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(courses)
-    }),
-    Provider.bulkCreate(providers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(providers)
-    }),
-    Credit.bulkCreate(credits, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(credits)
-    }),
-    Teacher.bulkCreate(teachers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(teachers)
-    })
-  ])
-
-  await Promise.all([
-    // must be after courses
-    CourseDisciplines.bulkCreate(disciplines, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after providers
-    CourseProvider.bulkCreate(courseproviders, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after teachers
-    CreditTeacher.bulkCreate(creditTeachers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudyRights = async (studyRights, studentnumber, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const studyRightExtents = sortedUniqBy(sortBy(studyRights.map(e => e.studyRightExtent), 'extentcode'), 'extentcode')
-  const studyrights = sortedUniqBy(sortBy(studyRights.map(e => e.studyright), 'studyrightid'), 'studyrightid')
-  const elementDetails = sortedUniqBy(sortBy(flatMap(studyRights, e => e.elementDetails), 'code'), 'code')
-  const studyRightElements = uniqBy(
-    sortBy(flatMap(studyRights, e => e.studyRightElements), 'startdate', 'enddate', 'studyrightid', 'code'),
-    s => `${s.startdate}${s.enddate}${s.studyrightid}${s.code}`
-  )
-  const transfers = sortBy(
-    flatMap(studyRights, e => e.transfers),
-    'transferdate',
-    'studentnumber',
-    'studyrightid',
-    'sourcecode',
-    'targetcode'
-  )
-
-  const studyrightids = studyrights.map(s => s.studyrightid)
-  await Studyright.destroy(
-    { where: { student_studentnumber: studentnumber, studyrightid: { [notIn]: studyrightids } } },
-    { transaction }
-  )
-  await StudyrightElement.destroy({ where: { studentnumber } }, { transaction })
-
-  await Promise.all([
-    StudyrightExtent.bulkCreate(studyRightExtents, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyRightExtents)
-    }),
-
-    Studyright.bulkCreate(studyrights, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyrights)
-    }),
-
-    ElementDetails.bulkCreate(elementDetails, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(elementDetails)
-    }),
-
-    StudyrightElement.bulkCreate(studyRightElements, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-
-    Transfers.bulkCreate(transfers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudent = async student => {
-  let { studentInfo, studyAttainments, semesterEnrollments, studyRights } = student
-
-  // sort data to avoid deadlocks
-  semesterEnrollments = uniqBy(
-    sortBy(semesterEnrollments, 'studentnumber', 'semestercode'),
-    sE => `${sE.semestercode}${sE.studentnumber}`
-  )
-
-  const transaction = await sequelize.transaction()
-  try {
-    await Student.upsert(studentInfo, { transaction })
-
-    // Change this to bulkCreate after https://github.com/sequelize/sequelize/issues/11569
-    // is merged and released
-    for (let i = 0; i < semesterEnrollments.length; i++) {
-      await SemesterEnrollment.upsert(semesterEnrollments[i], { transaction })
-    }
-
-    if (studyAttainments) await updateAttainments(studyAttainments, transaction)
-
-    if (studyRights) await updateStudyRights(studyRights, studentInfo.studentnumber, transaction)
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-const updateAttainmentMeta = async () => {
-  await updateAttainmentDates()
-}
-
-const updateMeta = async ({
-  faculties,
-  courseRealisationsTypes,
-  semesters,
-  creditTypeCodes,
-  courseTypeCodes,
-  disciplines
-}) => {
-  const transaction = await sequelize.transaction()
-
-  try {
-    await Promise.all([
-      CourseType.bulkCreate(uniqBy(courseTypeCodes, 'coursetypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseTypeCodes)
-      }),
-      Organisation.bulkCreate(uniqBy(faculties, 'code'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(faculties)
-      }),
-      CourseRealisationType.bulkCreate(uniqBy(courseRealisationsTypes, 'realisationtypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseRealisationsTypes)
-      }),
-      Semester.bulkCreate(uniqBy(semesters, 'semestercode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(semesters)
-      }),
-      CreditType.bulkCreate(uniqBy(creditTypeCodes, 'credittypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(creditTypeCodes)
-      }),
-      Discipline.bulkCreate(uniqBy(disciplines, 'discipline_id'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(disciplines)
-      })
-    ])
-
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-module.exports = {
-  updateStudent,
-  updateMeta,
-  updateAttainmentMeta
-}

old_oodikone_stuff/updater_writer/updater/database_updater.test.js

@@ -1,518 +0,0 @@
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./database_updater')
-const {
-  Student,
-  Credit,
-  Course,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  Teacher,
-  CreditTeacher,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { students } = require('./test_assets/test_students')
-const meta = require('./test_assets/meta')
-const { sequelize, forceSyncDatabase } = require('../database/connection')
-const { seedMigrations } = require('../database/seed_migrations')
-jest.setTimeout(10000)
-describe('Updater works', () => {
-  beforeAll(async () => {
-    await forceSyncDatabase()
-    await seedMigrations()
-
-    await updateMeta(meta)
-    await updateStudent(students[0])
-    await updateStudent(students[1])
-    await updateStudent(students[2])
-    await updateAttainmentMeta()
-  })
-
-  afterAll(async () => {
-    await sequelize.close()
-  })
-
-  describe('All metadata is written', () => {
-    test('Organisations are written to database', async () => {
-      const organizations = await Organisation.findAll()
-      expect(organizations.length).toBe(17)
-      expect(organizations.find(_ => _.code === 'H50').name.find(n => n.langcode === 'fi').text).toBe(
-        'Matemaattis-luonnontieteellinen tiedekunta'
-      )
-    })
-
-    test('Course realisation types are written to database', async () => {
-      const courseRealisationTypes = await CourseRealisationType.findAll()
-      expect(courseRealisationTypes.length).toBe(18)
-      expect(courseRealisationTypes.map(_ => _.realisationtypecode)).not.toContain(null)
-      expect(courseRealisationTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Semesters are written to database', async () => {
-      const semesters = await Semester.findAll()
-      expect(semesters.length).toBe(160)
-      expect(semesters.map(_ => _.semestercode)).toEqual(expect.arrayContaining([...Array(160).keys()].map(_ => ++_)))
-      expect(semesters.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-      expect(semesters.map(_ => _.startdate)).not.toContain(null)
-      expect(semesters.map(_ => _.enddate)).not.toContain(null)
-      expect(semesters.map(_ => _.createdAt)).not.toContain(null)
-      expect(semesters.map(_ => _.updatedAt)).not.toContain(null)
-      expect(new Set(semesters.map(_ => _.yearcode)).size).toBe(80)
-      expect(semesters.find(_ => _.yearcode === 1).yearname).toBe('1950-51')
-    })
-
-    test('Credit types are written to database', async () => {
-      const creditTypes = await CreditType.findAll()
-      expect(creditTypes.length).toBe(11)
-      expect(creditTypes.map(_ => _.credittypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(creditTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Course types are written to database', async () => {
-      const courseTypes = await CourseType.findAll()
-      expect(courseTypes.length).toBe(35)
-      expect(courseTypes.map(_ => _.coursetypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(courseTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Disciplines are written to database', async () => {
-      const disciplines = await Discipline.findAll()
-      expect(disciplines.length).toBe(430)
-      expect(disciplines.map(_ => _.discipline_id)).toEqual(expect.arrayContaining([expect.any(String)]))
-      expect(disciplines.map(_ => _.name)[0]).toMatchObject({
-        fi: expect.any(String)
-      })
-    })
-  })
-  describe('Updater writes students the right way', () => {
-    test('Student info is written to database', async () => {
-      const students = await Student.findAll({
-        order: [['studentnumber', 'DESC']]
-      })
-      expect(students.length).toBe(3)
-      const {
-        studentnumber,
-        lastname,
-        firstnames,
-        abbreviatedname,
-        birthdate,
-        communicationlanguage,
-        creditcount,
-        dateofuniversityenrollment,
-        city_fi,
-        city_sv,
-        gender_code,
-        gender_fi,
-        createdAt,
-        updatedAt
-      } = students[0]
-
-      expect(studentnumber).toBe('014441008')
-      expect(lastname).toBe('Orttenvuori')
-      expect(firstnames).toBe('Oona')
-      expect(abbreviatedname).toBe('Orttenvuori Oona')
-      expect(birthdate.toISOString()).toBe(new Date('1966-04-24 00:00:00+00').toISOString())
-      expect(communicationlanguage).toBe('Finnish')
-      expect(creditcount).toBe(34)
-      expect(dateofuniversityenrollment).toBe(null)
-      expect(city_fi).toBe('HELSINKI')
-      expect(city_sv).toBe('HELSINGFORS')
-      expect(gender_code).toBe(2)
-      expect(gender_fi).toBe('Nainen')
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-
-      const { creditcount: creditcount1, dateofuniversityenrollment: dofue } = students[1]
-
-      expect(creditcount1).toBe(380)
-      expect(dofue.toISOString()).toBe(new Date('2003-07-31 21:00:00+00').toISOString())
-    })
-
-    test('Courses are written to database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const {
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        startdate,
-        enddate,
-        max_attainment_date,
-        min_attainment_date,
-        coursetypecode
-      } = courses[0]
-
-      expect(courses.length).toBe(147)
-      expect(code).toBe('00031')
-      expect(name.en).toBe('Master of Science (Agriculture and Forestry)')
-      expect(latest_instance_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(is_study_module).toBeTruthy()
-      expect(startdate.toISOString()).toBe(new Date('1899-12-31 00:00:00+00').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2112-12-20 00:00:00+00').toISOString())
-      expect(max_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(min_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-
-      expect(coursetypecode).toBe(8)
-
-      const { code: code1, is_study_module: ism, coursetypecode: ctc } = courses[6]
-
-      expect(code1).toBe('402512')
-      expect(ism).toBe(false)
-      expect(ctc).toBe(2)
-    })
-
-    test('Credits are written to database', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const {
-        id,
-        grade,
-        student_studentnumber,
-        credits,
-        ordering,
-        attainment_date,
-        isStudyModule,
-        lastmodifieddate,
-        course_code,
-        credittypecode /*, semestercode*/
-      } = allCredits[0]
-
-      expect(allCredits.length).toBe(150)
-      expect(id).toBe('2340421')
-      expect(grade).toBe('4')
-      expect(allCredits.map(_ => _.grade)).not.toContain(null)
-      expect(student_studentnumber).toBe('014441008')
-      expect(credits).toBe(2)
-      expect(allCredits.map(_ => _.credits)).not.toContain(null)
-      expect(ordering).toBe('1997-10-21')
-      expect(attainment_date.toISOString()).toBe(new Date('1997-10-21 00:00:00+00').toISOString())
-      expect(isStudyModule).toBe(false)
-      expect(lastmodifieddate).not.toBe(null)
-      expect(course_code).toBe('65002')
-      expect(allCredits.map(_ => _.course_code)).not.toContain(null)
-      expect(credittypecode).toBe(4)
-      expect(allCredits.map(_ => _.semestercode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-    })
-
-    test('Teachers are written to database', async () => {
-      const teachers = await Teacher.findAll({ order: [['name', 'ASC']] })
-      const { id, code, name } = teachers[0]
-
-      expect(teachers.length).toBe(51)
-      expect(id).toBe('9016947')
-      expect(code).toBe('hpyrhone')
-      expect(name).toBe('Ahvenniemi Emilia')
-    })
-
-    test('Credit Teachers are written to database', async () => {
-      const creditteachers = await CreditTeacher.findAll()
-
-      expect(creditteachers.length).toBe(119)
-      expect(creditteachers.map(_ => _.credit_id)).not.toContain(null)
-      expect(creditteachers.map(_ => _.teacher_id)).not.toContain(null)
-    })
-
-    test('Semester enrollments are written to database', async () => {
-      const semesterEnrollments = await SemesterEnrollment.findAll({
-        order: [['semestercode', 'ASC']]
-      })
-      const { id, enrollmenttype, enrollment_date, studentnumber, semestercode } = semesterEnrollments[0]
-
-      expect(semesterEnrollments.length).toBe(28)
-      expect(id).toEqual(expect.any(String))
-      expect(enrollmenttype).toBe(1)
-      expect(enrollment_date.toISOString()).toBe(new Date('2003-08-17 00:00:00+00').toISOString())
-      expect(studentnumber).toBe('014272112')
-      expect(semestercode).toBe(107)
-    })
-
-    test('Providers are written to database', async () => {
-      const providers = await Provider.findAll({
-        order: [['providercode', 'DESC']]
-      })
-      const { providercode, name } = providers[0]
-
-      expect(providers.length).toBe(20)
-      expect(providercode).toBe('H906')
-      expect(name).toMatchObject({
-        en: 'Language Centre',
-        fi: 'Kielikeskus',
-        sv: 'Språkcentrum'
-      })
-      expect(providers.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course providers are written to database', async () => {
-      const courseproviders = await CourseProvider.findAll()
-
-      expect(courseproviders.length).toBe(123)
-      expect(courseproviders.map(_ => _.coursecode)).not.toContain(null)
-      expect(courseproviders.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course disciplines are written to database', async () => {
-      const courseDisciplines = await CourseDisciplines.findAll()
-      expect(courseDisciplines.length).toBe(40)
-      expect(courseDisciplines.map(_ => _.discipline_id)).not.toContain(null)
-      expect(courseDisciplines.map(_ => _.course_id)).not.toContain(null)
-    })
-
-    test('Study right extent are written to database', async () => {
-      const studyrightextent = await StudyrightExtent.findAll()
-      const { extentcode, name, createdAt, updatedAt } = studyrightextent[0]
-
-      expect(studyrightextent.length).toBe(2)
-      expect(extentcode).toBe(1)
-      expect(name.en).toBe("Bachelor's Degree")
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('Study rights are written to database', async () => {
-      const studyrights = await Studyright.findAll({
-        order: [['studyrightid', 'DESC']]
-      })
-      const {
-        studyrightid /*, canceldate*/,
-        cancelorganisation,
-        enddate,
-        givendate,
-        graduated,
-        prioritycode,
-        startdate,
-        studystartdate,
-        organization_code,
-        student_studentnumber,
-        extentcode
-      } = studyrights[0]
-
-      expect(studyrights.length).toBe(4)
-      expect(studyrightid).toBe('67628057')
-      expect(cancelorganisation).toBe('4325')
-      expect(enddate.toISOString()).toBe(new Date('2010-04-12 00:00:00+00').toISOString())
-      expect(givendate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(graduated).toBe(1)
-      expect(prioritycode).toBe(30)
-      expect(startdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(studystartdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(organization_code).toBe('4325')
-      expect(student_studentnumber).toBe('011120775')
-      expect(extentcode).toBe(1)
-    })
-
-    test('element details are written to database', async () => {
-      const elementDetails = await ElementDetails.findAll({
-        order: [['code', 'DESC']]
-      })
-      const { code, name, type, createdAt, updatedAt } = elementDetails[0]
-
-      expect(elementDetails.length).toBe(12)
-      expect(code).toBe('A2004')
-      expect(name.en).toBe('Government Decree (794/2004) on University Degrees')
-      expect(type).toBe(15)
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('study right elements are written to database', async () => {
-      const studyright_elements = await StudyrightElement.findAll({
-        order: ['startdate', 'enddate', 'studyrightid', 'code']
-      })
-      const { id, startdate, enddate, createdAt, updatedAt, studyrightid, code, studentnumber } = studyright_elements[0]
-
-      expect(studyright_elements.length).toBe(21)
-      expect(id).toBe(11)
-      expect(startdate.toISOString()).toBe(new Date('2003-07-31T21:00:00.000Z').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2007-08-24T21:00:00.000Z').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studyrightid).toBe('45853818')
-      expect(code).toBe('A1995MAAT')
-      expect(studentnumber).toBe('014272112')
-    })
-    test('transfers are written to database', async () => {
-      const transfers = await Transfers.findAll()
-      const {
-        id,
-        transferdate,
-        createdAt,
-        updatedAt,
-        studentnumber,
-        studyrightid,
-        sourcecode,
-        targetcode
-      } = transfers[0]
-
-      expect(transfers.length).toBe(1)
-      expect(id).toBe('1')
-      expect(transferdate.toISOString()).toBe(new Date('2009-07-30 21:00:00+00').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studentnumber).toBe('014272112')
-      expect(studyrightid).toBe('62346253')
-      expect(sourcecode).toBe('820016')
-      expect(targetcode).toBe('820042')
-    })
-  })
-  describe('Updater updates existing students correctly', () => {
-    beforeAll(async () => {
-      const { studentInfo, studyRights, studyAttainments } = students[0]
-      students[0].studentInfo = {
-        ...studentInfo,
-        email: 'Calgary.Flames@NHLonJätettä.fi',
-        mobile: '0400521981',
-        address: 'Yliopistonkatu 4',
-        creditcount: 322,
-        age: 41,
-        lastname: 'Tauriainen',
-        gender_code: 1,
-        abbreviatedname: 'Tauriainen Maria Helena'
-      }
-      const newCredit = {
-        id: 'ayyyyylmao',
-        grade: '5',
-        credits: 5,
-        ordering: '2019.05.04',
-        credittypecode: 1,
-        student_studentnumber: '011120775',
-        attainment_date: new Date(),
-        course_code: '420_666',
-        semestercode: 138,
-        isStudyModule: false
-      }
-      const newCourse = {
-        code: '420_666',
-        name: {
-          fi: 'Kaljan juonti',
-          en: 'Beer drinking',
-          sv: 'dryckade öl'
-        },
-        latest_instance_date: new Date(),
-        is_study_module: false,
-        coursetypecode: 1,
-        disciplines: [],
-        startdate: new Date(),
-        enddate: new Date(),
-        providers: [],
-        courseproviders: []
-      }
-
-      const newCreditTeachers = [
-        {
-          credit_id: 'ayyyyylmao',
-          teacher_id: '9016417'
-        }
-      ]
-      const newAttainment = {
-        credit: newCredit,
-        creditTeachers: newCreditTeachers,
-        course: newCourse
-      }
-      students[0].studyAttainments = [
-        ...studyAttainments,
-        newAttainment,
-        {
-          ...students[1].studyAttainments[2],
-          creditTeachers: [],
-          credit: {
-            ...students[1].studyAttainments[2].credit,
-            id: 'Kvas',
-            student_studentnumber: '011120775'
-          }
-        }
-      ]
-      students[0].studyAttainments[2] = {
-        ...students[0].studyAttainments[2],
-        credit: {
-          ...students[0].studyAttainments[2].credit,
-          grade: '0'
-        }
-      }
-      const newStudyRight = {
-        ...studyRights[0],
-        studyright: {
-          ...studyRights[0].studyright
-        }
-      }
-      students[0].studyRights = [newStudyRight]
-
-      await updateStudent(students[0])
-    })
-
-    test('StudentInfo for existing student is updated correclty', async () => {
-      const updatedStudent = await Student.findOne({
-        where: {
-          studentnumber: '011120775'
-        }
-      })
-      const {
-        studentnumber,
-        lastname,
-        abbreviatedname,
-        gender_code,
-        email,
-        age,
-        gender_fi,
-        address,
-        mobile
-      } = updatedStudent
-
-      expect(studentnumber).toBe('011120775')
-      expect(lastname).toBe('Tauriainen')
-      expect(abbreviatedname).toBe('Tauriainen Maria Helena')
-      expect(email).toBe('Calgary.Flames@NHLonJätettä.fi')
-      expect(gender_code).toBe(1)
-      expect(gender_fi).toBe('Nainen')
-      expect(age).toBe(41)
-      expect(address).toBe('Yliopistonkatu 4')
-      expect(mobile).toBe('0400521981')
-    })
-    test('Existing courses are updated (not duplicated) in database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const newCourse = courses.find(_ => _.code === '420_666')
-      expect(courses.length).toBe(148)
-
-      expect(newCourse).not.toBe(undefined)
-    })
-
-    test('Updating credits works', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const updatedCredit = allCredits.find(_ => _.id === students[0].studyAttainments[2].credit.id)
-      expect(allCredits.length).toBe(152)
-      expect(updatedCredit.grade).toBe('0')
-    })
-
-    test('Studyrights are rewritten', async () => {
-      const updatedStudyRights = await Studyright.findAll({
-        where: {
-          student_studentnumber: '011120775'
-        }
-      })
-      expect(updatedStudyRights.length).toBe(1)
-      // theres really not a lot to test here since everything related to studyrights are just rewritten
-    })
-  })
-})

old_oodikone_stuff/updater_writer/updater/update_attainment_dates.js

@@ -1,28 +0,0 @@
-const { sequelize } = require('../database/connection')
-
-const updateAttainmentDates = async () => {
-  const { schema } = sequelize.options
-  const transaction = await sequelize.transaction()
-  try {
-    await sequelize.query(
-      `
-      -- Block writing to table to avoid deadlocks:
-      LOCK TABLE ONLY ${schema}.course IN EXCLUSIVE MODE;
-
-      UPDATE  ${schema}.course
-      SET max_attainment_date = cr.max, min_attainment_date = cr.min
-      FROM (select course_code, max(attainment_date), min(attainment_date) from ${schema}.credit group by course_code) cr
-      WHERE course.code=cr.course_code;
-     `,
-      { transaction }
-    )
-    await transaction.commit()
-  } catch (e) {
-    await transaction.rollback()
-    throw e
-  }
-}
-
-module.exports = {
-  updateAttainmentDates
-}

package.json

@@ -2,10 +2,11 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "concurrently": "concurrently",
     "cypress": "cypress",
+    "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
@@ -13,29 +14,8 @@
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install",
-    "lint:docker-compose": "docker-compose config -q && :",
-    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
-    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
-    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
-  },
-  "lint-staged": {
-    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
-    "*.{js,json,md}": "prettier --write",
-    "*.css": "stylelint --fix",
-    ".github/workflows/*": "actionlint",
-    "Dockerfile": "hadolint",
-    "*.sh": "shellcheck -x",
-    "docker-compose.yml": "npm run lint:docker-compose",
-    "docker-compose.real.yml": "npm run lint:docker-compose:real",
-    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
-    "docker-compose.test.yml": "npm run lint:docker-compose:test"
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "prepare": "husky install"
   },
   "repository": {
     "type": "git",
@@ -50,24 +30,28 @@
     "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
     "concurrently": "^6.2.1",
-    "cypress": "^8.2.0",
+    "cypress": "^8.3.1",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.0",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-prettier": "^3.4.1",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.1",
+    "husky": "^7.0.2",
     "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",
     "stylelint-config-prettier": "^8.0.2",
     "stylelint-config-standard": "^22.0.0",
     "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
   }
 }

run.sh

@@ -64,8 +64,7 @@ parse_profiles() {
   return 0
 }
 
-# Set which docker-compose files to use based on version. In ci, profiles are also
-# emptied, since they're passed from github actions.
+# Set which docker-compose files to use based on version.
 parse_env() {
   env=""
   if [[ "$version" == "real" ]]; then
@@ -85,7 +84,7 @@ parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
 elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${compose_command}"
+  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

scripts/functions_for_setup.sh

@@ -18,15 +18,13 @@ SIS_DB_NAME="sis-db"
 SIS_IMPORTER_DB_NAME="sis-importer-db"
 USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
-OODI_DB_NAME="oodi-db" # TODO: Remove when oodi is removed
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
-OODI_DB_REAL_DUMP_URL="svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/latest-pg.sqz" # TODO: Remove when oodi is removed
 
 # Source utility functions
 source "$PROJECT_ROOT"/scripts/utils.sh
@@ -73,7 +71,7 @@ reset_databases() {
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
-  docker-compose down
+  npm run docker:down
   docker-compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
@@ -112,20 +110,54 @@ reset_all_real_data() {
   reset_databases "${DATABASES[@]}"
 }
 
-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
-}
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
 
-reset_old_oodi_data() {
-  infomsg "Downloading old oodi-db dump"
-  local database=$OODI_DB_NAME
-  local url=$OODI_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
+  )
+
+  local database=""
+  local url=""
+
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
 }
 
 set_up_oodikone_from_scratch() {
@@ -133,7 +165,6 @@ set_up_oodikone_from_scratch() {
   folders_to_set_up=(
     "$PROJECT_ROOT"
     "$PROJECT_ROOT/services/frontend"
-    "$PROJECT_ROOT/services/userservice"
     "$PROJECT_ROOT/services/backend"
     "$PROJECT_ROOT/updater/sis-updater-scheduler"
     "$PROJECT_ROOT/updater/sis-updater-worker"
@@ -162,10 +193,10 @@ linting and formatting"
   npm run prepare
 
   infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon pull
+  "$PROJECT_ROOT"/run.sh both anon pull
 
   infomsg "Building images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon build
 
   successmsg "Setup ready, oodikone can be started! See README for more info."
 }

services/backend/.dockerignore

@@ -1,6 +1,3 @@
 .dockerignore
 Dockerfile
 node_modules/
-requests/
-scripts/
-*.log

services/backend/.gitignore

@@ -1,3 +0,0 @@
-# Student diff output
-/student_diff_*
-/diff_partial_att_data.json

services/backend/index.js

@@ -1,5 +1,6 @@
 require('./src/app')
+const logger = require('./src/util/logger')
 
 process.on('unhandledRejection', reason => {
-  console.log(reason)
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
 })