Classic happy birthday to this one guy on the dataset -change

Not individual studytracks at first

.env.template

@@ -0,0 +1,3 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+PATE_TOKEN=

.eslintrc.json

@@ -8,17 +8,10 @@
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
   "rules": {
-    "no-console": "off",
+    "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error"

.github/workflows/release-oodikone-staging.yaml

@@ -1,138 +0,0 @@
-name: "Release oodikone to staging"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-oodikone-staging.yaml'
-      - 'services/**'
-
-jobs:
-
-  build_and_release_backend:
-    name: "Build and release backend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:staging
-          target: production
-
-  build_and_release_frontend:
-    name: "Build and release frontend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:staging
-          target: production
-      - name: "Create a container from built image "
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: "Copy assets from the container"
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:staging
-          target: production
-
-  report_success:
-    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: success()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -1,4 +1,4 @@
-name: "Release oodikone to production"
+name: 'Release oodikone to production'
 
 on:
   release:
@@ -6,30 +6,29 @@ on:
 
 jobs:
   build_and_release_backend:
-    name: "Build and release backend"
+    name: 'Build and release backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
@@ -39,30 +38,33 @@ jobs:
           target: production
 
   build_and_release_frontend:
-    name: "Build and release frontend"
+    name: 'Build and release frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Extract release version"
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
         run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
-      - name: "Build and push"
+      - name: 'Build and push'
         uses: docker/build-push-action@v2
         with:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
@@ -70,11 +72,11 @@ jobs:
           push: true
           tags: toska/oodikone2-frontend:latest
           target: production
-      - name: "Create a container from built image "
+      - name: 'Create a container from built image '
         run: docker create --name frontend toska/oodikone2-frontend:latest
-      - name: "Copy assets from the container"
+      - name: 'Copy assets from the container'
         run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
+      - name: 'Create Sentry release'
         uses: getsentry/action-release@v1
         env:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
@@ -85,43 +87,19 @@ jobs:
           environment: production
           set_commits: 'skip'
           sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:latest
-          target: production
+          url_prefix: '~/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
 
   report_success:
     runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
+    name: 'Report deployment success to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: success()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -130,15 +108,14 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
+    name: 'Report deployment failure to oodikone notifications'
     needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
+      - build_and_release_backend
+      - build_and_release_frontend
     if: failure()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/release-sis-updater-scheduler.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-scheduler"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-scheduler.yaml'
       - 'updater/sis-updater-scheduler/**'

.github/workflows/release-sis-updater-worker.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-worker"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-worker.yaml'
       - 'updater/sis-updater-worker/**'

.github/workflows/test-oodikone.yaml

@@ -1,454 +1,491 @@
-name: "Test oodikone"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
-    branches-ignore:
+    branches:
       - master
     paths:
-      - .github/workflows/test-oodikone.yaml
-      - cypress/**
-      - cypress.json
-      - services/**
-      - docker-compose.ci.yml
-      - nginx.conf
-      - package*.json
+      - '.github/workflows/test-oodikone.yaml'
+      - 'cypress/**'
+      - 'cypress.json'
+      - 'services/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'nginx.conf'
+      - 'package*.json'
 
 jobs:
   # === Static tests / linting ===
 
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
-      - name: "Run backend linter"
+      - name: 'Run backend linter'
         run: npm run eslint services/backend
 
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run concurrently \
-              'eslint --ext .jsx,.js services/frontend' \
-              'stylelint services/frontend/src/**/*.css'
-
-  lint_userservice:
-    name: "Lint userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: "Install dependencies"
-        run: npm ci && cd services/userservice && npm ci
-      - name: "Run frontend linter"
-        run: npm run eslint services/userservice
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
 
   # === Build images ===
-
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
           context: services/backend
           push: false
-          tags: toska/oodikone2-backend:trunk
+          tags: toska/oodikone2-backend:staging
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
 
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
           context: services/frontend
           push: false
-          tags: toska/oodikone2-frontend:trunk
+          tags: toska/oodikone2-frontend:staging
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
 
-  build_userservice:
-    name: "Build userservice"
+  build_and_push_anon_sis_db:
+    name: 'Build anon sis-db and push to toska hub if new migrations have been added'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
-        uses: docker/build-push-action@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
         with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: false
-          tags: toska/oodikone2-userservice:trunk
-          target: production
-          outputs: type=docker,dest=/tmp/userservice.tar
-      - name: "Upload image for other jobs"
-        uses: actions/upload-artifact@v2
+          filters: |
+            migrations:
+              - 'updater/sis-updater-worker/src/db/migrations/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
         with:
-          path: /tmp/userservice.tar
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
-  # === Integration tests ===
-  run_userservice_integration_tests:
-    name: "Run userservice integration tests"
+  build_and_push_anon_kone_db:
+    name: 'Build anon kone-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Run tests included in docker build"
-        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_kone/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
+
+  build_and_push_anon_user_db:
+    name: 'Build anon user-db and push to toska hub if new migrations have been added'
+    needs:
+      - build_backend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: check_migrations
+        with:
+          filters: |
+            migrations:
+              - 'services/backend/src/database/migrations_user/**'
+              - '.github/workflows/test-oodikone.yaml'
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built backend image'
+        run: docker load --input /tmp/artifact/backend.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Let backend run migrations, then push new version of image to toska hub'
+        if: steps.check_migrations.outputs.migrations == 'true'
+        run: |
+          docker-compose --file docker-compose.ci.yml up --detach backend
+          sleep 30 # wait for sis-updater-worker to start and run migrations
+          ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
   # === End-to-end tests ===
-
-  test_course_population:
-    name: "Test course population"
-    needs:
-      - build_backend
-      - build_frontend
-      - build_userservice
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: "Download images"
-        uses: actions/download-artifact@v2
-        with:
-          path: /tmp/
-      - name: "Load built images"
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Course_population.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-
   test_course_statistics:
-    name: "Test course statistics"
+    name: 'Test course statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
-    name: "Test population statistics"
+    name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Population_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
-    name: "Test filters"
+    name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
-    name: "Test student statistics"
+    name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Student_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
-    name: "Test studyprogramme overview"
+    name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Studyprogramme_overview.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
-    name: "Test teachers and users"
+    name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -457,111 +494,143 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
       - build_backend
       - build_frontend
-      - build_userservice
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Check that exit statuses are not errors"
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
-  push_to_dockerhub:
-    name: "Push images to docker hub for caching"
-    needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_population
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+  # === Release ===
+
+  release_images:
+    name: 'Release built images'
     runs-on: ubuntu-latest
+    needs:
+      - lint_backend
+      - lint_frontend
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Push to docker hub"
-        run: docker-compose --file docker-compose.ci.yml push
+      - name: 'Create a container from frontend'
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: 'Copy assets from the container'
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ github.sha }}
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+      - name: 'Send deployment success notification to slack'
+        if: success()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+      - name: 'Send deployment failure notification to slack'
+        if: failure()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
 
-  report_failure:
+  report_test_failure:
     runs-on: ubuntu-latest
-    name: "Report test failure to oodikone notifications"
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_population
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
-    - push_to_dockerhub
+      - lint_backend
+      - lint_frontend
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
       - uses: actions/checkout@v2
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure
-      - name: "Print logs on failure"
-        run: docker-compose --file docker-compose.ci.yml logs

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0 hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

README.md

@@ -7,17 +7,22 @@ An application for analyzing university data, running at [https://oodikone.cs.he
 To run Oodikone locally, you will need the following:
 
 1. Applications:
-
-- [Docker](https://docs.docker.com/install/) (version 20.10+),
-- [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
-- [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-
-2. Access to Toska Docker Hub, see Toska Gitlab for more information on how to login
-3. (Optional) For real data setup, access to Toskas production servers.
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
 ## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
@@ -28,26 +33,18 @@ _Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary
 What different CLI options do:
 
 1. Set up oodikone from scratch:
-
-- Cleans up any previous installations
-- Installs all needed npm packages locally
-- Sets up pre-commit linting hooks.
-- Pulls and builds all needed Docker images and sets up dockerized development environment
-- **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
-
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
 2. Reset all real data:
-
-- Cleans up any previous real data databases
-- Downloads needed database dumps from production servers.
-- Creates real data databases and populates them with downloaded dumps.
-
-3. Reset sis-importer data
-
-- Like option 2, but only for sis-importer database. Useful when developing updater microservice.
-
-4. Reset old oodi data:
-
-- Like option 2, but for old oodi data, which is not refreshed anymore. Is only needed when setting up real databases after running option 1.
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
+3. Reset single database
+   - Like option 2, but only for single database. Leaves other databases untouched.
 
 ## ⌨️ Development
 
@@ -57,7 +54,7 @@ The development environment is entirely configured in the `docker-compose.yml` f
 
 Running oodikone with real data requires separated databases and redis, which are defined in `docker-compose.real.yml` file. Otherwise real data development environment uses the configuration as anonymized data development environment.
 
-Some useful commands are predefined in `package.json` and can be run with `npm run <command>` as follows:
+Some useful commands are defined in `package.json` and can be run with `npm run <command>` as follows:
 
 - `npm run oodikone`: starts oodikone with anonymized data
 - `npm run oodikone:real`: starts oodikone with real data
@@ -67,11 +64,7 @@ Some useful commands are predefined in `package.json` and can be run with `npm r
 - `npm run both:real`: starts oodikone and updater with real data
 - `npm run docker:down`: stops the whole environment
 
-If you're first timer, just run `npm run oodikone` after setting up oodikone according to [installation](#installation). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
-
-### Anon / real data users
-
-Some notes about mluukkai / tkl user here.
+Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
 ### Run.sh script
 
@@ -86,51 +79,71 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker-compose ps`: view the containers in the running environment
 - `docker-compose logs frontend`: print logs for just frontend
 - `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
-- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it backend sh`: open bash terminal inside backend container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
-### Linting and tests
+### User types for development
 
-Linting and other stuff here
+By default, you're login as `mluukkai` dev user when running oodikone in development mode. If you want to debug with certain type of user, you can use mocking: go to "Users" -page, click on "edit" on user you want to use and then click icon on the right corner of user card. Another way is to mock user headers - see frontend's api configuration for how this is done.
 
-The test suite is run in CI on every push to `trunk`. Take advantage of this as running the tests can take upwards of 15 minutes.
+Anon data contains some preset user types, corresponding to most usual user types in real data. These are used in testing too (see "Testing" below for more info).
 
-### Updater stuff
+### Linting
 
-Stop with HTTP GET
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-```bash
-http://localhost:8082/v1/abort?token=dev
-```
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-Schedule students with HTTP POST
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
-```bash
-http://localhost:8082/v1/students?token=dev
-```
+However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-body
+## 🔨 Testing & CI
 
-```bash
-{
-    "studentnumbers": [
-        "014598456"
-    ]
-}
-```
+There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
-## 📖 Documentation
+- Cypress
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+- Jest
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
-Testing + CI docs link here.
+Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
-##❓FAQ
+- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- tests are run on every push.
+- after successful test run, oodikone is deployed to staging
+- after creating a release, oodikone is deployed to production
+- updater is deployed to production on every push, no tests included
+
+## ❓FAQ
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
+
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
+### I made a change to sis-db models aand created a migration, but tests are failing in CI!
+
+Migrations are handled by sis-importer-worker, since oodikone is using only replica. Since CI is testing only services included in oodikone, your new changes aren't migrated and thus are failing. Open anonyymioodi-folder in this repo to find more about how to to update the anon sis-db.
 
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
-## Maintainers and contribution
+## ✌🏼 Maintainers and contribution
+
+Toska of course.
+
+University of Helsinki
+
+## Tietosuoja / data protection
+
+See [tietosuoja.md](tietosuoja.md).

adminer-plugins/login-servers.php

@@ -8,7 +8,6 @@ $driver = "pgsql";
 return new AdminerLoginServers(
     $servers = array(
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
-        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),
         "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
         "user-db" => array("server" => "user-db", "driver" => $driver)

anonyymioodi/.gitignore

@@ -0,0 +1,2 @@
+*.sqz
+

anonyymioodi/README.md

@@ -0,0 +1,65 @@
+# Anonyymioodi - anonymous database setup for oodikone
+
+(name anonyymioodi for :culture: reasons)
+
+This repository contains a guide and scripts to create docker images from database dumps and/or push database images to Toska Hub docker registry. Images are mostly used by oodikone.
+
+## How to update database images
+
+Updating images varies a bit depending on the db you're trying to update. Complete steps as defined below and run `./commit_and_push_to_toska_hub.sh <dbname>` to publish the image you've created.
+
+### Sis-importer-db
+
+The following steps are for building a clean sis-importer-db image from scratch. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- See [sis-importer/importer-db-staging-sampletaker](https://github.com/UniversityOfHelsinkiCS/sis-importer/tree/master/importer-db-staging-sampletaker) and create `sis-importer-db.sqz` dump
+- Move `sis-importer-db.sqz` to this folder
+- Run `./create_container_from_dump.sh sis-importer-db`
+- Enrich data according to what's in `enrich_data.sql` file
+
+### Sis-db:
+
+The following steps are for building a clean sis-importer-db image from scratch using the sis-updater. If you want to just modify the existing image a bit, follow the steps defined for user-db / kone-db.
+
+- Temporarily replace sis-db -service in oodikone's docker-compose.yml with sis-db -service from this repo's docker-compose.yml
+- Ensure the newest sis-importer-db image is pulled from toska hub in oodikone repo
+- Modify sis scheduler limit in config.js by setting: `module.exports.DEV_SCHEDULE_COUNT = null`
+- Start updater, see oodikone readme for more info.
+- Populate sis-db from sis-importer-db:
+
+```
+curl --request GET --url 'http://localhost:8082/v1/meta?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/programmes?token=dev' && \
+curl --request GET --url 'http://localhost:8082/v1/students?token=dev'
+```
+
+(It might be a good idea to follow sis-updater-worker logs while updater is running)
+
+- Clean up any extra stuff you don't want to keep in sis-db.
+- (Bonus: check that cypress tests run okay)
+
+### User-db / kone-db
+
+- Temporarily replace user-db/kone-db -service in oodikone's docker-compose.yml with the same service from this repo's docker-compose.yml
+- Ensure the newest user-db/kone-db image is pulled from the registry with `docker pull`
+- Start oodikone, let it run migrations
+- Modify db to wanted state. Tips:
+  - Easy way to create new users is to modify dev user details in the frontend API config. A new user is created during login if the given uid is not present in user-db.
+
+By default, previous user-db/kone-db images are used as a base. If you want to build the image from a clean Postgres image, just comment out toska hub image and uncomment the clean Postgres image.
+
+## How to modify database images
+
+If you need to clean up / modify / do whatever with dbs, either use `psql` directly in docker by running `docker exec -it <dbname> psql -U postgres <dbname>` or start adminer with oodikone's docker-compose.
+
+## How to update Postgres versions
+
+When Postgres is updated on the server, anon db images need to be updated to match server versions. Do following:
+
+- Ensure you're using toska hub image in docker-compose and that you've pulled the newest image from toska hub: `docker-compose pull <dbname>`.
+- Start database with docker-compose
+- Create dump from database by running `docker exec -i <dbname> pg_dump -Fc -U postgres <dbname> > <dbname>.sqz`
+- Close container with docker-compose down.
+- Update the Postgres version in docker-compose.yml in this folder.
+- Uncomment row with Postgres image and comment row with Toskahub image.
+- Run `./create_container_from_dump.sh <dbname>`, followed by `./commit_and_push_to_toska_hub.sh <dbname>`

anonyymioodi/commit_and_push_to_toska_hub.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+infomsg "Committing and pushing to registry"
+database=${args[0]}
+docker commit "$database" "registry.toska.cs.helsinki.fi/$database"
+docker push "registry.toska.cs.helsinki.fi/$database"
+successmsg "Succesfully pushed"

anonyymioodi/create_container_from_dump.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# === Config ===
+
+# Quit if failing
+set -euo pipefail
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE=''
+fi
+
+# === Create images ===
+
+retry () {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+args=("$@")
+if [[ ${#args[@]} -ne 1 ]]; then
+  errormsg "No database given, exiting"
+  exit 1
+fi
+
+
+infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+database=${args[0]}
+docker-compose down --remove-orphans
+docker-compose up -d "$database"
+
+database_dump="$database.sqz"
+infomsg "Attempting to restore database $database from dump $database_dump"
+retry docker-compose exec "$database" pg_isready --dbname="$database"
+docker exec -i "$database" /bin/bash -c "pg_restore --username=postgres \
+--format=custom --dbname=$database --no-owner" < "$database_dump"
+successmsg "Database $database succesfully created"

anonyymioodi/docker-compose.yml

@@ -0,0 +1,50 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/kone-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-db
+    image: postgres:13.2
+    shm_size: 2gb
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-importer-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    # image: registry.toska.cs.helsinki.fi/sis-importer-db
+    image: postgres:13.1
+    profiles:
+      - updater
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
+    # image: postgres:13.4-alpine
+    profiles:
+      - oodikone

anonyymioodi/enrich_sampledata.sql

@@ -0,0 +1,133 @@
+-- add these to sis-importer db by running:
+-- docker exec -i sis-importer-db psql -U postgres sis-importer-db < enrich_sampledata.sql
+
+-- if you need to run updater for these students only, see studentnumberlist in the end
+-- this file
+
+-- Add admission types (valintaperusteet) to studyrights for some of the computer
+-- science 2018-2019 population students. See person_ids / student_number below 
+-- (in case you need to update dumps). 
+
+-- TODO: this should be moved into the sampledata creator script in
+-- importer repo, but since this file contains studentnumbers and person ids, it's 
+-- better to keep this here
+
+-- KP (koepisteet)
+-- hy-hlo-1412620   | 013683126
+-- hy-hlo-1485246   | 014340523
+-- hy-hlo-32230770  | 010589388
+-- hy-hlo-35626233  | 010775888
+-- hy-hlo-40150144  | 011065995
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:kp' 
+WHERE person_id IN (
+'hy-hlo-1412620',
+'hy-hlo-1485246',
+'hy-hlo-32230770',
+'hy-hlo-35626233',
+'hy-hlo-40150144'
+);
+
+-- TV (todistusvalinta)
+-- hy-hlo-45783433  | 011321150
+-- hy-hlo-48537946  | 010764653
+-- hy-hlo-49801905  | 011194400
+-- hy-hlo-53321302  | 011338871
+-- hy-hlo-51223300  | 013951984
+-- hy-hlo-53310137  | 012520688
+-- hy-hlo-53621583  | 011196796
+-- hy-hlo-53863064  | 011736884
+-- hy-hlo-57711592  | 010262566
+-- hy-hlo-57732872  | 013670647
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:tv' 
+WHERE person_id IN (
+'hy-hlo-45783433',
+'hy-hlo-48537946',
+'hy-hlo-49801905',
+'hy-hlo-53321302',
+'hy-hlo-51223300',
+'hy-hlo-53310137',
+'hy-hlo-53621583',
+'hy-hlo-53863064',
+'hy-hlo-57711592',
+'hy-hlo-57732872'
+);
+
+-- YP:
+-- hy-hlo-67768022  | 011952325
+-- hy-hlo-73740025  | 011736606
+-- hy-hlo-74780917  | 012775433
+-- hy-hlo-79615371  | 014113941
+-- hy-hlo-78118455  | 014300615
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:yp' 
+WHERE person_id IN (
+'hy-hlo-67768022',
+'hy-hlo-73740025',
+'hy-hlo-74780917',
+'hy-hlo-79615371',
+'hy-hlo-78118455'
+);
+-- AV:
+-- hy-hlo-78777265  | 010779392
+-- hy-hlo-81909550  | 011894153
+-- hy-hlo-84571890  | 013897158
+-- hy-hlo-80330834  | 015085926
+-- hy-hlo-84775722  | 011791272
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:av' 
+WHERE person_id IN (
+'hy-hlo-78777265',
+'hy-hlo-81909550',
+'hy-hlo-84571890',
+'hy-hlo-80330834',
+'hy-hlo-84775722'
+);
+-- M:
+-- hy-hlo-87439827  | 011928456
+-- hy-hlo-96612371  | 014418361
+-- hy-hlo-84840261  | 010975895
+-- hy-hlo-90330681  | 010522688
+-- hy-hlo-96594603  | 013951573
+UPDATE studyrights 
+SET admission_type_urn = 'urn:code:admission-type:m' 
+WHERE person_id IN (
+'hy-hlo-87439827',
+'hy-hlo-96612371',
+'hy-hlo-84840261',
+'hy-hlo-90330681',
+'hy-hlo-96594603'
+);
+
+-- all studentnumbers for running updater:
+013683126
+014340523
+010589388
+010775888
+011065995
+011321150
+010764653
+011194400
+011338871
+013951984
+012520688
+011196796
+011736884
+010262566
+013670647
+011952325
+011736606
+012775433
+014113941
+014300615
+010779392
+011894153
+013897158
+015085926
+011791272
+011928456
+014418361
+010975895
+010522688
+013951573

cli.sh

@@ -44,21 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
   msg ""
 }
 
-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "
 
-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Reset old oodi data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}
 
-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt
 
 while true; do
   select opt in "${options[@]}"; do
@@ -67,10 +68,10 @@ while true; do
         set_up_oodikone_from_scratch;;
       "Reset all real data.")
         reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
-      "Reset old oodi data.")
-        reset_old_oodi_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress.json

@@ -1,9 +1,7 @@
 {
   "projectId": "c3jsph",
   "baseUrl": "http://localhost:3000",
-  "defaultCommandTimeout": 60000,
-  "requestTimeout": 60000,
-  "responseTimeout": 60000,
-  "viewportHeight": 1080,
-  "viewportWidth": 1920
-}
+  "defaultCommandTimeout": 30000,
+  "requestTimeout": 30000,
+  "videoCompression": false
+}

cypress/fixtures/example.json

@@ -1,5 +0,0 @@
-{
-  "name": "Using fixtures to represent data",
-  "email": "hello@cypress.io",
-  "body": "Fixtures are a great way to mock data for responses to routes"
-}

cypress/integration/Course_population.js

@@ -1,29 +0,0 @@
-/// <reference types="Cypress" />
-
-describe('Course population tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Course statistics').click()
-    cy.contains('Search for courses')
-  })
-
-  it('Can find course population', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains('tr', 'TKT20003').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-
-    cy.contains('Käyttöjärjestelmät')
-    cy.contains('TKT20003')
-
-    cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-    cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
-    cy.contains('TKT20003')
-    // cy.contains("Students (all=24)");
-
-    cy.contains('Students (127)').click()
-    cy.contains('010135486')
-    cy.contains('010431753')
-  })
-})

cypress/integration/Course_statistics.js

@@ -1,695 +1,340 @@
 /// <reference types="Cypress" />
 
 describe('Course Statistics tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Course statistics').click()
-    cy.contains('Search for courses')
-  })
-
-  it('Searching single course having substitution mappings shows course statistics', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-    // Click uni course, not avoin
-    cy.contains('td', /^TKT20001/).click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-
-    cy.contains('Tietorakenteet ja algoritmit')
-    cy.contains('TKT20001')
-    cy.contains('58131') // old mapped code
-
-    cy.contains('.tabular.menu a', 'Table').click()
-    cy.contains('All')
-    cy.contains('.modeSelectorRow a', 'Attempts').click()
-    cy.contains('.modeSelectorRow a', 'Students').click()
-
-    cy.contains('.tabular.menu a', 'Pass rate chart').click()
-    cy.get('div.modeSelectorContainer').click()
-    cy.contains('svg', 'Pass rate chart')
-
-    cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-    cy.get('div.modeSelectorContainer').click()
-    cy.contains('svg', 'Grades')
-
-    cy.contains('a', 'New query').click()
-    cy.contains('Search for courses')
-  })
-
-  it('Searching multiple courses having substitution mappings shows course statistics', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT')
-    cy.contains('td', /^TKT20001/).click()
-    cy.contains('td', /^TKT10002/).click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-
-    cy.contains('.courseNameCell', 'Tietorakenteet ja algoritmit').contains('TKT20001').click()
-    cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').should('not.exist')
-    cy.contains('TKT20001')
-    cy.contains('58131') // old mapped code
-    cy.contains('Summary').click()
-
-    cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').contains('TKT10002').click()
-    cy.contains('.courseNameCell', 'Käyttöjärjestelmät').should('not.exist')
-    cy.contains('TKT10002')
-    cy.contains('581325') // old mapped code
-    cy.contains('Summary').click()
-
-    cy.contains('a', 'New query').click()
-    cy.contains('Search for courses')
-  })
-
-  it('On consecutive searches should not crash and search should work', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains('td', 'TKT20003').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Show population')
-    cy.contains('TKT20003')
-
-    cy.contains('Course statistics').click()
-    cy.contains('Search for courses')
-    //cy.wait(500)
-    cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-    cy.contains('td', 'TKT20001').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Show population')
-    cy.contains('TKT20001')
-  })
-
-  it('Shows statistics when separating by semester', () => {
-    const courseCode = 'DATA11002'
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type(courseCode)
-    cy.contains('td', courseCode).click()
-    cy.contains('Separate statistics for Spring and Fall semesters').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Syksy 2019')
-    cy.contains('Syksy 2017')
-    cy.get('tbody').contains('19')
-    cy.contains(courseCode)
-  })
-
-  it('Searching course by name displays right courses', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-
-    cy.contains('Tietokantojen perusteet')
-    cy.contains('Avoin yo: Tietokantojen perusteet')
-    cy.contains('A581328')
-    cy.contains('AYTKT10004')
-    cy.contains('TKT10004, 581328')
-    cy.contains('td', /^TKT10004/).click()
-
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-
-    cy.contains('TKT10004, 581328 Tietokantojen perusteet')
-    cy.get('.right').click()
-    cy.contains('No results')
-
-    cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-    cy.contains('td', /^AYTKT10004/).click()
-
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-    cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
-  })
-
-  describe('When searching unified course stats', () => {
+  describe('when using basic user', () => {
     beforeEach(() => {
+      cy.init('/coursestatistics')
+    })
+    it('Searching single course having substitution mappings shows course statistics', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
-      cy.get("input[name='unifyOpenUniCourses']").parent().click()
-      cy.get("input[placeholder='Search by a course code']").type('TKT10002')
-      cy.contains('td', 'TKT10002, AYTKT10002').click()
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      // Click uni course, not avoin
+      cy.contains('td', /^TKT20001/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
-      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001')
+      cy.contains('58131') // old mapped code
+
+      cy.contains('.tabular.menu a', 'Table').click()
+      cy.contains('All')
+      cy.contains('.modeSelectorRow a', 'Attempts').click()
+      cy.contains('.modeSelectorRow a', 'Students').click()
+
+      cy.contains('.tabular.menu a', 'Pass rate chart').click()
+      cy.get('div.modeSelectorContainer').click()
+      cy.contains('svg', 'Pass rate chart')
+
+      cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+      cy.get('div.modeSelectorContainer').click()
+      cy.contains('svg', 'Grades')
+
+      cy.contains('a', 'New query').click()
+      cy.contains('Search for courses')
     })
 
-    // Statistics
-    const yearRange = { from: '2000-2001', to: '2020-2021' }
-    const attemptsTableContents = [
-      // [time, passed, failed, passrate]
-      ['Total', 511, 486, 25],
-    ]
-    /*
-    const studentTableContents = [
-      // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-      ['Total', 289, 254, 13, '92.39 %', 22, '7.61 %'],
-      ['2017-18', 157, 132, 5, '87.26 %', 20, '12.74 %'],
-      ['2016-17', 62, 60, 1, '98.39 %', 1, '1.61 %'],
-      ['2015-16', 27, 25, 2, '100.00 %', 0, '0.00 %'],
-      ['2014-15', 16, 16, 0, '100.00 %', 0, '0.00 %'],
-      ['2013-14', 5, 3, 2, '100.00 %', 0, '0.00 %'],
-    ]
-    */
+    it('Searching multiple courses having substitution mappings shows course statistics', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.contains('td', /^TKT20001/).click()
+      cy.contains('td', /^TKT10002/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
 
-    const gradesTableContents = [
-      // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-      ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
-      ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
-      ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
-      ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
-      ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
-      ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
-      ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
-    ]
+      cy.contains('.courseNameCell', 'Tietorakenteet ja algoritmit').contains('TKT20001').click()
+      cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').should('not.exist')
+      cy.contains('TKT20001')
+      cy.contains('58131') // old mapped code
+      cy.contains('Summary').click()
 
-    it('shows stats', () => {
-      // Time range
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', yearRange.to)
-        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', '2000-2001')
-        cy.get("div[role='option']").should('have.length', 21)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', '2020-2021')
-        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', yearRange.from)
-        cy.get("div[role='option']").should('have.length', 21)
-      })
-      cy.contains('Show population').should('not.be.enabled')
+      cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').contains('TKT10002').click()
+      cy.contains('.courseNameCell', 'Käyttöjärjestelmät').should('not.exist')
+      cy.contains('TKT10002')
+      cy.contains('581325') // old mapped code
+      cy.contains('Summary').click()
 
-      /*
-      const timeRangeFilter =
-        timeRange =>
-        ([time]) => {
-          return (timeRange.from <= time && time <= timeRange.to) || time === 'Total'
-        }
-*/
-      cy.contains('#CourseStatPanes a.item', 'Table').click()
-      cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-      cy.get('#CourseStatPanes table>tbody').within(() => {
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get('tr')
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get('td').eq(tdIndex).contains(value)
-              })
-            })
-        })
-        cy.get('tr').should('have.length', 17)
-      })
-      cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
-      cy.get('#CourseStatPanes table>tbody').within(() => {
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get('tr')
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get('td').eq(tdIndex).contains(value)
-              })
-            })
-        })
-        cy.get('tr').should('have.length', 17)
-      })
+      cy.contains('a', 'New query').click()
+      cy.contains('Search for courses')
     })
 
-    it('after changing time range shows same stats', () => {
-      const newYearRange = { from: '2016-2017', to: '2019-2020' }
-      cy.get("div[name='fromYear']")
-        .click()
-        .within(() => {
-          cy.contains(newYearRange.from).click()
-        })
-      cy.get("div[name='toYear']")
-        .click()
-        .within(() => {
-          cy.contains(newYearRange.to).click()
-        })
+    it('On consecutive searches should not crash and search should work', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains('td', 'TKT20003').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Show population')
+      cy.contains('TKT20003')
 
-      // Time range
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', newYearRange.to)
-        cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', '2000-2001')
-        cy.get("div[role='option']").should('have.length', 20)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', '2020-2021')
-        cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', newYearRange.from)
-        cy.get("div[role='option']").should('have.length', 5)
-      })
-      cy.contains('Show population').should('be.enabled')
+      cy.contains('Course statistics').click()
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      cy.contains('td', 'TKT20001').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Show population')
+      cy.contains('TKT20001')
     })
-    /*
-      // Filters
-      // Data currently has only "Muu" so we can't test these yet! See
-      // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3058
-      // const primaryContents = [
-      //   // [studentcount, label, code]
-      //   [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-      //   [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-      //   [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-      //   [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-      //   [9, "Matematiikan koulutusohjelma", "520070"],
-      // ];
-      // const totalStudyProgrammes = 20;
-      // cy.get("div[name='primary']").within(() => {
-      //   primaryContents.forEach(([count, label, code], i) => {
-      //     cy.get("div[role='option']")
-      //       .eq(i)
-      //       .within(() => {
-      //         cy.contains(label);
-      //         cy.contains("div>div:last-child>div.label", code);
-      //         cy.contains("div>div:first-child>div.label", count);
-      //       });
-      //   });
-      //   cy.get("div[role='option']")
-      //     .eq(10)
-      //     .within(() => {
-      //       cy.contains("Muu");
-      //       cy.contains("div>div:last-child>div.label", "OTHER");
-      //       cy.contains("div>div:first-child>div.label", "2");
-      //     });
-      //   cy.get("div[role='option']").should(
-      //     "have.length",
-      //     totalStudyProgrammes
-      //   );
-      // });
-      // const comparisonContents = [[289, "All", "ALL"], ...primaryContents];
-      // cy.get("div[name='comparison']").within(() => {
-      //   comparisonContents.forEach(([count, label, code], i) => {
-      //     cy.get("div[role='option']")
-      //       .eq(i)
-      //       .within(() => {
-      //         cy.contains(label);
-      //         cy.contains("div>div:last-child>div.label", code);
-      //         cy.contains("div>div:first-child>div.label", count);
-      //       });
-      //   });
-      //   cy.get("div[role='option']")
-      //     .eq(11)
-      //     .within(() => {
-      //       cy.contains("Muu");
-      //       cy.contains("div>div:last-child>div.label", "OTHER");
-      //       cy.contains("div>div:first-child>div.label", "2");
-      //     });
-      //   cy.get("div[role='option']").should(
-      //     "have.length",
-      //     totalStudyProgrammes + 1
-      //   ); // +1 comes from already selected option
-      // });
-      // cy.contains("Select excluded study programmes").should("be.disabled");
 
-        
+    it('Shows statistics when separating by semester', () => {
+      const courseCode = 'DATA11002'
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type(courseCode)
+      cy.contains('td', courseCode).click()
+      cy.contains('Separate statistics for Spring and Fall semesters').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Syksy 2019')
+      cy.contains('Syksy 2017')
+      cy.get('tbody').contains('19')
+      cy.contains(courseCode)
+    })
 
-      // Filters
-      // Data currently has only "Muu" so we can't test these yet! See
-      // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3058
-      //const primaryContentsAfterTimeChange = [
-      //  // [studentcount, label, code]
-      //  [79, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-      //  [15, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-      //  [10, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-      //  [7, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-      //];
-      //const totalStudyProgrammesAfterTimeChange = 13;
-      //cy.get("div[name='primary']").within(() => {
-      //  primaryContentsAfterTimeChange.forEach(([count, label, code], i) => {
-      //    cy.get("div[role='option']")
-      //      .eq(i)
-      //      .within(() => {
-      //        cy.contains(label);
-      //        cy.contains("div>div:last-child>div.label", code);
-      //        cy.contains("div>div:first-child>div.label", count);
-      //      });
-      //  });
-      //  cy.get("div[role='option']")
-      //    .eq(5)
-      //    .within(() => {
-      //      cy.contains("Muu");
-      //      cy.contains("div>div:last-child>div.label", "OTHER");
-      //      cy.contains("div>div:first-child>div.label", "2");
-      //    });
-      //  cy.get("div[role='option']").should(
-      //    "have.length",
-      //    totalStudyProgrammesAfterTimeChange
-      //  );
-      //});
-      //const comparisonContentsAfterTimeChange = [
-      //  [107, "All", "ALL"],
-      //  ...primaryContentsAfterTimeChange,
-      //];
-      //cy.get("div[name='comparison']").within(() => {
-      //  comparisonContentsAfterTimeChange.forEach(([count, label, code], i) => {
-      //    cy.get("div[role='option']")
-      //      .eq(i)
-      //      .within(() => {
-      //        cy.contains(label);
-      //        cy.contains("div>div:last-child>div.label", code);
-      //        cy.contains("div>div:first-child>div.label", count);
-      //      });
-      //  });
-      //  cy.get("div[role='option']")
-      //    .eq(6)
-      //    .within(() => {
-      //      cy.contains("Muu");
-      //      cy.contains("div>div:last-child>div.label", "OTHER");
-      //      cy.contains("div>div:first-child>div.label", "2");
-      //    });
-      //  cy.get("div[role='option']").should(
-      //    "have.length",
-      //    totalStudyProgrammesAfterTimeChange + 1
-      //  ); // +1 comes from already selected option
-      //});
-      //cy.contains("Select excluded study programmes").should("be.disabled");
+    it('Searching course by name displays right courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
-      const timeAttemptsTableContents = [
-        // [time, passed, failed, passrate]
-        ["Total", 113, 109, 4, "96.46 %"],
-        ["2017-18", 204, 175, 29, "85.78 %"],
-        ["2016-17", 68, 66, 2, "97.06 %"],
-        ["2015-16", 29, 27, 2, "93.10 %"],
-        ["2014-15", 16, 16, 0, "100.00 %"],
-        ["2013-14", 8, 6, 2, "75.00 %"],
-      ];
+      cy.contains('Tietokantojen perusteet')
+      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('A581328')
+      cy.contains('AYTKT10004')
+      cy.contains('TKT10004, 581328')
+      cy.contains('td', /^TKT10004/).click()
 
-      const timeStudentTableContents = [
-        // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-        ["Total", 105, 101, 3, "99.05 %", 1, "0.95 %"],
-        ["2017-18", 195, 131, 43, "89.23 %", 1, "10.77 %"],
-        ["2016-17", 62, 60, 1, "98.39 %", 1, "1.61 %"],
-        ["2015-16", 27, 25, 2, "100.00 %", 0, "0.00 %"],
-        ["2014-15", 16, 16, 0, "100.00 %", 0, "0.00 %"],
-        ["2013-14", 5, 2, 3, "100.00 %", 0, "0.00 %"],
-      ];
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
 
-      const timeGradesTableContents = [
-        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ["Total", 113, 4, 1, 2, 7, 15, 76, 8],
-        ["2017-18", 204, 29, 5, 8, 10, 16, 127, 9],
-        ["2016-17", 68, 2, 0, 1, 6, 10, 49, 0],
-        ["2015-16", 29, 2, 1, 1, 0, 5, 16, 4],
-        ["2014-15", 16, 0, 0, 0, 1, 0, 11, 4],
-        ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
-      ];
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.get('.right').click()
+      cy.contains('No results')
 
-    
+      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
+      cy.contains('td', /^AYTKT10004/).click()
 
-    // Not working since only "Muu" students
-    it("after changing filters shows different stats", () => {
-      cy.get("div[name='primary']")
-        .click()
-        .within(() => {
-          cy.contains(
-            "div[role='option']",
-            "Tietojenkäsittelytieteen kandiohjelma"
-          ).click();
-        });
-      cy.get("body").type("{esc}"); // close dropdown
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+    })
 
-      // Time range (should not have changed)
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", yearRange.to);
-        cy.contains("div[role='option']", yearRange.from).should(
-          "have.class",
-          "selected"
-        );
-        cy.get("div[role='option']").last().should("have.text", "2000-2001");
-        cy.get("div[role='option']").should("have.length", 21);
-      });
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", "2020-2021");
-        cy.contains("div[role='option']", yearRange.to).should(
-          "have.class",
-          "selected"
-        );
-        cy.get("div[role='option']").last().should("have.text", yearRange.from);
-        cy.get("div[role='option']").should("have.length", );
-      });
-      cy.contains("Show population").should("be.enabled");
+    it('Searching course by name displays right courses, 10 credit courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
 
-      // Filters
-      // Data currently has only "Muu" so we can't test these yet! See
-      // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3058
-      const primaryContentsAfterFilterChange = [
-        // [studentcount, label, code]
-        [289, "All", "ALL"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ];
-      const totalStudyProgrammesAfterFilterChange = 20;
-      cy.get("div[name='primary']").within(() => {
-        primaryContentsAfterFilterChange.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']")
-            .eq(i)
-            .within(() => {
-              cy.contains(label);
-              cy.contains("div>div:last-child>div.label", code);
-              cy.contains("div>div:first-child>div.label", count);
-            });
-        });
-        cy.get("div[role='option']").should(
-          "have.length",
-          totalStudyProgrammesAfterFilterChange
-        );
-      });
-      const comparisonContentsAfterFilterChange = [
-        // [studentcount, label, code]
-        [289, "All", "ALL"],
-        [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-        [122, "Excluded", "EXCLUDED"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ];
-      cy.get("div[name='comparison']").within(() => {
-        comparisonContentsAfterFilterChange.forEach(
-          ([count, label, code], i) => {
-            cy.get("div[role='option']")
-              .eq(i)
-              .within(() => {
-                cy.contains(label);
-                cy.contains("div>div:last-child>div.label", code);
-                cy.contains("div>div:first-child>div.label", count);
-              });
-          }
-        );
-        cy.get("div[role='option']").should(
-          "have.length",
-          totalStudyProgrammesAfterFilterChange + 2
-        ); // +2 comes from EXCLUDED and the filtered programme
-      });
-      cy.contains("Select excluded study programmes")
-        .should("be.enabled")
-        .click();
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
+      cy.contains('AYTKT20001')
+      cy.contains('TKT20001, 58131')
+      cy.contains('td', /^TKT20001/).click()
 
-      // Statistics
-      cy.contains("#CourseStatPanes a.item", "Table").click();
-      cy.contains("#CourseStatPanes a.item", "Attempts").click();
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const attemptsTableContents = [
-          // [time, passed, failed, passrate]
-          ["Total", 227, 215, 12, "94.71 %"],
-          ["2017-18", 136, 126, 10, "92.65 %"],
-          ["2016-17", 57, 56, 1, "98.25 %"],
-          ["2015-16", 21, 20, 1, "95.24 %"],
-          ["2014-15", 6, 6, 0, "100.00 %"],
-        ];
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const attemptsTableContents = [
-          // [time, passed, failed, passrate]
-          ["Total", 140, 110, 30, "78.57 %"],
-          ["2017-18", 71, 52, 19, "73.24 %"],
-          ["2016-17", 16, 15, 1, "93.75 %"],
-          ["2015-16", 10, 9, 1, "90.00 %"],
-          ["2014-15", 11, 11, 0, "100.00 %"],
-        ];
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
 
-      cy.contains("#CourseStatPanes a.item", "Table").click();
-      cy.contains("#CourseStatPanes a.item", "Students").click();
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const studentTableContents = [
-          // time, students, passedfirsttry, eventuallypassed, pass%, neverpassed, fail%]
-          ["Total", 179, 168, 6, "97.21 %", 5, "2.79 %"],
-          ["2017-18", 95, 86, 4, "94.74 %", 5, "5.26 %"],
-          ["2016-17", 53, 52, 1, "100.00 %", 0, "0.00 %"],
-          ["2015-16", 19, 18, 1, "100.00 %", 0, "0.00 %"],
-          ["2014-15", 6, 6, 0, "100.00 %", 0, "0.00 %"],
-        ];
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const studentTableContents = [
-          // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-          ["Total", 122, 98, 7, "86.07 %", 17, "13.93 %"],
-          ["2017-18", 65, 49, 1, "76.92 %", 15, "23.08 %"],
-          ["2016-17", 13, 12, 0, "92.31 %", 1, "7.69 %"],
-          ["2015-16", 10, 9, 1, "100.00 %", 0, "0.00 %"],
-          ["2014-15", 11, 11, 0, "100.00 %", 0, "0.00 %"],
-        ];
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get('.right').click()
+      cy.contains('No results')
 
-      cy.contains("#CourseStatPanes a.item", "Table").click();
-      cy.contains("#CourseStatPanes a.item", "Attempts").click();
-      cy.get("[data-cy=gradeToggle]", { force: true }).click({ force: true });
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', /^AYTKT20001/).click()
 
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const gradesTableContents = [
-          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-          ["Total", 227, 12, 4, 7, 11, 18, 161, 14],
-          ["2017-18", 136, 10, 3, 4, 4, 4, 102, 9],
-          ["2016-17", 57, 1, 0, 1, 6, 8, 41, 0],
-          ["2015-16", 21, 1, 1, 1, 0, 5, 11, 2],
-          ["2014-15", 6, 0, 0, 0, 0, 0, 3, 3],
-          ["2013-14", 1, 0, 0, 0, 0, 0, 1, 0],
-        ];
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const gradesTableContents = [
-          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-          ["Total", 140, 30, 2, 5, 13, 20, 67, 3],
-          ["2017-18", 71, 19, 2, 4, 6, 13, 27, 0],
-          ["2016-17", 16, 1, 0, 0, 0, 3, 12, 0],
-          ["2015-16", 10, 1, 0, 0, 0, 0, 7, 2],
-          ["2014-15", 11, 0, 0, 0, 1, 0, 9, 1],
-          ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
-        ];
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-    });
-  });
-*/
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
 
-    it('Some features of Course Statistics are hidden for courseStatistics-users', () => {
-      cy.contains('Users').click()
-      cy.contains('Oodikone users')
-      cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'User manager').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
+      cy.get('.right').click()
+      cy.contains('No results')
+      cy.get("input[name='unifyOpenUniCourses']").parent().click()
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
 
-      cy.get('[data-cy=access-groups-form]').click().contains('courseStatistics').click()
-      cy.get('[data-cy=access-groups-save]').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+    })
 
-      cy.route('POST', '/api/superlogin/usermk').as('superlogin')
-      cy.get('i.spy').click()
-      cy.wait('@superlogin')
-      cy.contains('mocking as usermk')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(1000)
-
-      cy.get('[data-cy=navbar-courseStatistics]').click()
-      cy.get('[data-cy=course-code-input]').type('TKT20003')
+    it('Can find course population', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('tr', 'TKT20003').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('Filter statistics by study programmes').should('not.exist')
-      cy.contains('Faculty statistics').should('not.exist')
-      cy.contains('Show population').should('not.exist')
 
+      cy.contains('Käyttöjärjestelmät')
+      cy.contains('TKT20003')
+
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('TKT20003')
+
+      cy.contains('Students (127)').click()
+      cy.contains('010135486')
+      cy.contains('010431753')
+    })
+
+    it('Population of course shows grades for each student', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      cy.contains(/^TKT20001$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
+      cy.contains('Students (197)').click()
+      cy.contains('td', '010262566').siblings().contains('4')
+      cy.contains('td', '010674989').siblings().contains('1')
+    })
+
+    it.skip('Language distribution is correct', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains(/^TKT20003$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20003, 582219 Käyttöjärjestelmät')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('Language distribution').click()
+      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'english').siblings().contains('5')
+    })
+
+    describe('When searching unified course stats', () => {
+      beforeEach(() => {
+        cy.url().should('include', '/coursestatistics')
+        cy.contains('Search for courses')
+        cy.get("input[name='unifyOpenUniCourses']").parent().click()
+        cy.get("input[placeholder='Search by a course code']").type('TKT10002')
+        cy.contains('td', 'TKT10002, AYTKT10002').click()
+        cy.contains('Fetch statistics').should('be.enabled').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+      })
+
+      // Statistics
+      const yearRange = { from: '2000-2001', to: '2020-2021' }
       const attemptsTableContents = [
         // [time, passed, failed, passrate]
-        ['Total*', 288, 213, 75, '73.96 %'],
-        ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-        ['2019-2020', 164, 121, 43, '73.78 %'],
-        ['2018-2019', 85, 60, 25, '70.59 %'],
-        ['2017-2018', 39, 32, 7, '82.05 %'],
+        ['Total', 511, 486, 25],
       ]
 
-      cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-      cy.get('#CourseStatPanes table>tbody').within(() => {
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get('tr')
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get('td').eq(tdIndex).contains(value)
+      const gradesTableContents = [
+        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
+      ]
+
+      it('shows stats', () => {
+        // Time range
+        cy.get("div[name='fromYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', yearRange.to)
+          cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', '2000-2001')
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('Show population').should('not.be.enabled')
+
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContents.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
               })
+          })
+          cy.get('tr').should('have.length', 17)
+        })
+        cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          gradesTableContents.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+          cy.get('tr').should('have.length', 17)
+        })
+      })
+
+      it('after changing time range shows same stats', () => {
+        const newYearRange = { from: '2016-2017', to: '2019-2020' }
+        cy.get("div[name='fromYear']")
+          .click()
+          .within(() => {
+            cy.contains(newYearRange.from).click()
+          })
+        cy.get("div[name='toYear']")
+          .click()
+          .within(() => {
+            cy.contains(newYearRange.to).click()
+          })
+
+        // Time range
+        cy.get("div[name='fromYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', newYearRange.to)
+          cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', '2000-2001')
+          cy.get("div[role='option']").should('have.length', 20)
+        })
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', newYearRange.from)
+          cy.get("div[role='option']").should('have.length', 5)
+        })
+        cy.contains('Show population').should('be.enabled')
+      })
+    })
+  })
+
+  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+    cy.init('/coursestatistics', 'onlycoursestatistics')
+    cy.get('[data-cy=navbar-courseStatistics]').click()
+    cy.get('[data-cy=course-code-input]').type('TKT20003')
+    cy.contains('tr', 'TKT20003').click()
+    cy.contains('Fetch statistics').should('be.enabled').click()
+    cy.contains('Filter statistics by study programmes').should('not.exist')
+    cy.contains('Faculty statistics').should('not.exist')
+    cy.contains('Show population').should('not.exist')
+
+    const attemptsTableContents = [
+      // [time, passed, failed, passrate]
+      ['Total*', 288, 213, 75, '73.96 %'],
+      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      ['2019-2020', 164, 121, 43, '73.78 %'],
+      ['2018-2019', 85, 60, 25, '70.59 %'],
+      ['2017-2018', 39, 32, 7, '82.05 %'],
+    ]
+
+    cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+    cy.get('#CourseStatPanes table>tbody').within(() => {
+      attemptsTableContents.forEach((values, trIndex) => {
+        cy.get('tr')
+          .eq(trIndex)
+          .within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get('td').eq(tdIndex).contains(value)
             })
-        })
-        cy.get('tr').should('have.length', 7)
+          })
       })
-
-      cy.contains('Stop mocking').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(1000)
-
-      cy.contains('Users').click()
-      cy.contains('Oodikone users')
-      cy.contains('tr', 'User manager').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
-      cy.contains('Access Groups')
-        .siblings()
-        .within(() => {
-          cy.get("i[class='dropdown icon clear']").click()
-        })
-      cy.get('[data-cy=access-groups-save]').click()
+      cy.get('tr').should('have.length', 7)
     })
   })
 })

cypress/integration/Custom_population.js

@@ -74,15 +74,6 @@ const containsSpecificStudents = (studentnumbers = []) => {
   studentnumbers.forEach(s => cy.contains(s))
 }
 
-const loginAs = username => {
-  cy.contains('Users').click()
-  cy.contains(username).siblings().contains('Edit').click()
-  cy.get('.spy').click()
-  cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
-  cy.url().should('include', '/custompopulation')
-  cy.contains('Custom population')
-}
-
 describe('Custom population tests', () => {
   const nonExistentStudentNumbers = ['123', 'X', '-', ' ']
   beforeEach(() => {
@@ -128,22 +119,16 @@ describe('Custom population tests', () => {
       cy.get('button').contains('Custom population')
     })
 
-    // Check login as when anon user database rights have been corrected
-    // it("Doesn't return students to whom the user doesn't have rights to", () => {
-    //   const kasvatusStudents = ["010102273", "010112616", "010116531"];
-    //   const kapistelyStudents = ["010623419", "010942404", "010484975"];
-    //   loginAs("Normaalikäyttäjä");
-
-    //   searchFor(kasvatusStudents);
-    //   cy.contains("Credit accumulation").should("not.exist");
-    //   cy.contains("Programme distribution").should("not.exist");
-    //   cy.contains("Courses of population").should("not.exist");
-
-    //   searchFor(kapistelyStudents.concat(kasvatusStudents));
-    //   hasLanded();
-    //   containsAmountOfStudents(3);
-    //   containsSpecificStudents(kapistelyStudents);
-    // });
+    it("Doesn't return students user has no right to", () => {
+      // TODO: check that this doesn't create false positives, since this test goes
+      // through even if following students are not in anon sis-db.
+      const studentsForEduBachStudents = ['014990067', '013069465', '014853890']
+      searchFor(studentsForEduBachStudents)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
   })
 
   describe('Custom population search saving', () => {

cypress/integration/Filters.js

@@ -1,6 +1,6 @@
 /// <reference types="Cypress" />
 
-const baseUrl = Cypress.config().baseUrl
+// const baseUrl = Cypress.config().baseUrl
 
 const checkFilteringResult = studentCount => {
   cy.contains(`Students (${studentCount})`)
@@ -9,23 +9,24 @@ const checkFilteringResult = studentCount => {
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.delete').click()
+  cy.cs(dataCyAttribute).find('i.delete').click({ force: true })
+  cy.cs(dataCyAttribute).click()
 }
 
 const clearSingleDropdownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find('i.clear').click()
+  cy.cs(dataCyAttribute).find('i.clear').click({ force: true })
 }
 
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
 // to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
-  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  return (filterName, testStepFunctionToRun) => {
+    cy.get(`[data-cy="${filterName}-header"]`).click()
     checkFilteringResult(amountWithoutFiltering)
     testStepFunctionToRun()
     checkFilteringResult(amountWithoutFiltering)
-    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    cy.get(`[data-cy="${filterName}-header"]`).click({ force: true })
   }
 }
 
@@ -36,116 +37,136 @@ describe('Population Statistics', () => {
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
-    cy.init()
-    cy.visit(baseUrl.concat(pathToCSBach2018))
+    cy.init(pathToCSBach2018)
   })
 
   it('Transfer filter is on not transferred by default', () => {
     cy.contains('By default only students who have not transferred to this study programme are shown.')
-    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+    const card = cy.cs('TransferredToProgramme-filter-card')
+    card.cs('TransferredToProgramme-header').click()
+    card.get('[data-cy="option-havenot"] input').should('be.checked')
+    card.cs('TransferredToProgramme-header').click()
   })
 
   it('Graduation filter works', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      const card = cy.cs('GraduatedFromProgramme-filter-card')
+
       const graduated = 11
-      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      card.cs('option-graduated-true').click()
       checkFilteringResult(graduated)
-      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      card.cs('option-graduated-false').click()
       checkFilteringResult(defaultAmountOfStudents - graduated)
-      cy.cs('graduatedFromProgrammeFilter-all').click()
+      card.get('[data-cy="GraduatedFromProgramme-filter-card"] [data-cy="option-all"]').click()
     })
   })
 
   it('Transfer filter works', () => {
-    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+    runTestStepWithPreAndPostParts('TransferredToProgramme', () => {
       const transferred = 1
-      cy.cs('transferredToProgrammeFilter-have').click()
+      cy.cs('option-have').click()
       checkFilteringResult(transferred)
-      cy.cs('transferredToProgrammeFilter-all').click()
+      cy.cs('option-all').click()
       checkFilteringResult(defaultAmountOfStudents + transferred)
-      cy.cs('transferredToProgrammeFilter-havenot').click()
+      cy.cs('option-havenot').click()
     })
   })
 
   it('Enrollment filter works', () => {
-    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
-      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
-      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+    runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
+      cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
       checkFilteringResult(137)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
 
   it('Credit filter works', () => {
-    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+    runTestStepWithPreAndPostParts('CreditsEarned', () => {
       cy.cs('credit-filter-min').type('50')
       cy.cs('credit-filter-max').type('150')
       checkFilteringResult(116)
-      cy.cs('credit-filter-min').find('input').clear()
-      cy.cs('credit-filter-max').find('input').clear()
+      cy.cs('credit-filter-min').clear()
+      cy.cs('credit-filter-max').clear()
     })
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(41)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(36)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(112)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
       checkFilteringResult(121)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
-  it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('AdmissionTypeFilter', () => {
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(5)
+      cy.cs('admissionTypeFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(10)
+      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
+    })
+  })
+
+  // plz fix
+  it.skip('Courses filter works', () => {
+    // courses takes some time to load, wait for it to complete
+    cy.wait(10000)
+    runTestStepWithPreAndPostParts('Courses', () => {
+      cy.cs('courseFilter-course-dropdown').click()
       const courses = ['TKT20001', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
       checkFilteringResult(140)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
       checkFilteringResult(131)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(56)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(2)
       checkFilteringResult(2)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
-        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+    runTestStepWithPreAndPostParts('GraduatedFromProgramme', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
+        const card = cy.cs('GraduatedFromProgramme-filter-card')
+        card.cs('option-graduated-true').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
         checkFilteringResult(11)
         cy.cs('ageFilter-min').find('input').clear()
         cy.cs('ageFilter-max').find('input').clear()
-        cy.cs('graduatedFromProgrammeFilter-all').click()
+        card.cs('option-all').click()
       })
     })
   })
@@ -158,12 +179,11 @@ describe('Course Statistics', () => {
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
-    cy.init()
-    cy.visit(baseUrl.concat(pathToDSAndAlgoSpring2019))
+    cy.init(pathToDSAndAlgoSpring2019)
   })
 
   it('Grade filter works', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
       cy.cs('gradeFilter-5').click()
       checkFilteringResult(20)
       cy.cs('gradeFilter-3').click()
@@ -174,43 +194,43 @@ describe('Course Statistics', () => {
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(32)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
   })
 
   it('Gender filter works', () => {
-    runTestStepWithPreAndPostParts('genderFilter-header', () => {
-      cy.selectFromDropdown('genderFilter-dropdown', 0)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
-      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
       checkFilteringResult(91)
-      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
       checkFilteringResult(0)
-      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(3)
       checkFilteringResult(0)
       clearSingleDropdownSelection('genderFilter-dropdown')
     })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
       checkFilteringResult(29)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
 
   it('Filter combinations work', () => {
-    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
-      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Grade', () => {
+      runTestStepWithPreAndPostParts('Age', () => {
         cy.cs('gradeFilter-3').click()
         cy.cs('ageFilter-min').type('20')
         cy.cs('ageFilter-max').type('30')
@@ -232,13 +252,14 @@ describe('Custom Population Statistics', () => {
     cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs('student-no-input').click().type(students.join('/n'))
+      cy.cs('student-no-input').click().type(students.join('\n'))
+      cy.wait(10000)
     })
     cy.cs('search-button').click()
   })
 
   it('Age filter works', () => {
-    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+    runTestStepWithPreAndPostParts('Age', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('30')
       checkFilteringResult(1)
@@ -248,35 +269,43 @@ describe('Custom Population Statistics', () => {
   })
 
   it('Gender filter works', () => {
-    cy.cs('genderFilter-header').click()
-    cy.selectFromDropdown('genderFilter-dropdown', 0)
-    checkFilteringResult(3)
-    cy.selectFromDropdown('genderFilter-dropdown', 1)
-    checkFilteringResult(2)
-    cy.selectFromDropdown('genderFilter-dropdown', 2)
-    checkFilteringResult(0)
-    clearSingleDropdownSelection('genderFilter-dropdown')
-    checkFilteringResult(5, true)
+    runTestStepWithPreAndPostParts('Gender', () => {
+      cy.cs('genderFilter-dropdown').selectFromDropdown(0)
+      checkFilteringResult(3)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(1)
+      checkFilteringResult(2)
+      cy.cs('genderFilter-dropdown').selectFromDropdown(2)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+      checkFilteringResult(5, true)
+    })
   })
 
   it('Starting year filter works', () => {
-    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
-      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+    runTestStepWithPreAndPostParts('StartYearAtUni', () => {
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.cs('startYearAtUni-dropdown').click()
     })
   })
 
   it('Courses filter works', () => {
-    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+    runTestStepWithPreAndPostParts('Courses', () => {
       const courses = ['MAT11001', 'TKT20004']
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(4)
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(4)
-      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[0] + ' ')
+        .click()
+      checkFilteringResult(5)
+      cy.cs(`courseFilter-${courses[0]}-dropdown`).selectFromDropdown(1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown')
+        .click()
+        .contains(courses[1] + ' ')
+        .click()
       checkFilteringResult(3)
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      cy.cs(`courseFilter-${courses[1]}-dropdown`).selectFromDropdown(3)
       checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })

cypress/integration/Population_statistics.js

@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />
 
+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
   includeSettings.forEach(setting => {
@@ -11,167 +14,205 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 }
 
 describe('Population Statistics tests', () => {
-  beforeEach(() => {
-    cy.init()
-  })
+  const pathToCSBach2017 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
+  describe('when using basic user', () => {
+    beforeEach(() => {
+      cy.init('/populations')
+    })
 
-  /*
-  const checkAmountOfStudents = assertion => {
-    let students = 0
-    cy.contains('Credit accumulation')
-      .click()
-      .invoke('text')
-      .then(text => {
-        students = Number(text.match(/\d+/g)[0])
-        expect(students).to.equal(assertion)
-      })
-  }
-  */
+    it('Population statistics search form is usable', () => {
+      cy.contains('See population').should('be.disabled')
+      cy.contains('Search for population')
+      cy.contains('Class of')
+        .parent()
+        .within(() => {
+          cy.get('.form-control').as('enrollmentSelect')
+        })
 
-  it('Population statistics search form is usable', () => {
-    cy.cs('navbar-studyProgramme').click()
-    cy.cs('navbar-class').click()
-    cy.contains('See population').should('be.disabled')
-    cy.url().should('include', '/populations')
-    cy.contains('Search for population')
-    cy.contains('Class of')
-      .parent()
-      .within(() => {
-        cy.get('.form-control').as('enrollmentSelect')
-      })
+      cy.get('@enrollmentSelect')
+        .its(`${[0]}.value`)
+        .then(beforeVal => {
+          cy.get('@enrollmentSelect').click()
+          cy.get('.yearSelectInput .rdtPrev').click({ force: true })
+          cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
+          cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
+        })
 
-    cy.get('@enrollmentSelect')
-      .its(`${[0]}.value`)
-      .then(beforeVal => {
-        cy.get('@enrollmentSelect').click()
-        cy.get('.yearSelectInput .rdtPrev').click({ force: true })
-        cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
-        cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
-      })
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen maisteriohjelma')
+        .click()
+      cy.contains('See population').should('be.enabled')
+    })
 
-    cy.contains('Select study programme')
-    cy.get('[data-cy=select-study-programme]')
-      .click()
-      .children()
-      .contains('Tietojenkäsittelytieteen maisteriohjelma')
-      .click()
-  })
-
-  it('Population statistics is usable on general level', () => {
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    setPopStatsUntil('toukokuu 2020')
-
-    cy.get('.card').within(() => {
+    it('Searching for population really shows population', () => {
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen kandiohjelma')
+        .click()
+      cy.contains('See population').click()
+      cy.contains('Population statistics')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-      cy.contains('Sample size: 149 students')
-      cy.contains('Excludes exchange students')
-      cy.contains("Excludes students who haven't enrolled present nor absent")
-      cy.contains('Excludes students with non-degree study right')
-      cy.contains('Excludes students who have transferred out of this programme')
     })
-    cy.contains('Courses of population').click({ force: true })
 
-    cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
-    // eslint-disable-next-line cypress/no-unnecessary-waiting
-    cy.wait(3000) // a bit hacky way, wait until ui is ready
-    cy.cs('expand-TKT1').click()
-    cy.cs('coursestats-link-TKT10002').click()
-    cy.wait('@coursePage')
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
-  })
+    it('Population statistics is usable on general level', () => {
+      cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
+      setPopStatsUntil('toukokuu 2020')
 
-  it('Student list checking works as intended', () => {
-    const existing = '010113437'
-    const nonExisting = '66666666'
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    cy.contains('Students (149)').click()
-    cy.contains(existing)
-    cy.contains(nonExisting).should('not.exist')
-    cy.contains('button', 'Check studentnumbers').click()
-    cy.contains('Check for studentnumbers')
-    cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
-    cy.contains('button', 'check students').click()
-    cy.contains('#checkstudentsresults', 'Results').within(() => {
-      cy.contains('Student numbers in list and in oodi').click()
-      cy.contains('#found', existing)
-      cy.contains('Student numbers in list but not in oodi').click()
-      cy.contains('#notfound', nonExisting)
-      cy.contains('Student numbers in oodi but not in list').click()
-      cy.contains('#notsearched', '010614509')
+      cy.get('.card').within(() => {
+        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.contains('Sample size: 149 students')
+        cy.contains('Excludes exchange students')
+        cy.contains("Excludes students who haven't enrolled present nor absent")
+        cy.contains('Excludes students with non-degree study right')
+        cy.contains('Excludes students who have transferred out of this programme')
+      })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()
+
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=expand-TKT1]').click()
+      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+    })
+
+    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
+      cy.contains('No tags defined. You can define them here.').find('a').click()
+      cy.contains('Tags').click()
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Create new tag')
+    })
+
+    // TODO: this fails in CI, but not locally. Investigate and fix
+    it.skip('Advanced settings work', () => {
+      cy.visit(pathToCSBach2017)
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.contains('Statistics until')
+      // only spring
+      cy.cs('toggle-fall').click()
+      cy.contains('Fetch population').click()
+
+      // moar waiting hack
+      cy.wait(10000)
+
+      cy.contains('No statistics found for the given query.')
+
+      // only fall
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-fall').click()
+      cy.cs('toggle-spring').click()
+
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 149 students)')
+
+      // spring + fall and include cancelled
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-spring').click()
+      cy.cs('toggle-cancelled').click()
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 170 students)')
+    })
+
+    it('Credit Statistics, Credits Gained tab works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
+      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
+
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })
 
-  it("Empty 'tags' tab has a link to the page where tags can be created", () => {
-    cy.cs('navbar-studyProgramme').click()
-    cy.cs('navbar-class').click()
-    cy.contains('See population').should('be.disabled')
-    cy.url().should('include', '/populations')
-    cy.contains('Search for population')
-    cy.contains('Class of')
-      .parent()
-      .within(() => {
-        cy.get('.form-control').as('enrollmentSelect')
+  describe('when using admin', () => {
+    beforeEach(() => {
+      cy.init(pathToCSBach2017, 'admin')
+    })
+    it('Student list checking works as intended', () => {
+      const existing = '010113437'
+      const nonExisting = '66666666'
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.contains(existing)
+      cy.contains(nonExisting).should('not.exist')
+      cy.contains('button', 'Check studentnumbers').click()
+      cy.contains('Check for studentnumbers')
+      cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
+      cy.contains('button', 'check students').click()
+      cy.contains('#checkstudentsresults', 'Results').within(() => {
+        cy.contains('Student numbers in list and in oodi').click()
+        cy.contains('#found', existing)
+        cy.contains('Student numbers in list but not in oodi').click()
+        cy.contains('#notfound', nonExisting)
+        cy.contains('Student numbers in oodi but not in list').click()
+        cy.contains('#notsearched', '010614509')
       })
-
-    cy.get('@enrollmentSelect')
-      .its(`${[0]}.value`)
-      .then(beforeVal => {
-        cy.get('@enrollmentSelect').click()
-        cy.get('.yearSelectInput .rdtPrev').click({ force: true })
-        cy.get('.yearSelectInput table').contains('2019-2020').click({ force: true })
-        cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
-      })
-    cy.selectStudyProgramme('Datatieteen maisteriohjelma')
-    cy.contains('Students (13)').click()
-    cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
-    cy.contains('No tags defined. You can define them here.').find('a').click()
-    cy.contains('Tags').click()
-    cy.contains('Datatieteen maisteriohjelma')
-    cy.contains('Create new tag')
-  })
-
-  it.skip('Advanced settings work', () => {
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.contains('Statistics until')
-    // only spring
-    cy.cs('toggle-fall').click()
-    cy.contains('Fetch population').click()
-
-    cy.contains('Credit accumulation (for 0 students)')
-
-    // only fall
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.cs('toggle-fall').click()
-    cy.cs('toggle-spring').click()
-
-    cy.contains('Fetch population').click()
-
-    // FIXME: This fails randomly both locally and on CI.
-    cy.contains('Credit accumulation (for 194 students)')
-
-    // spring + fall and include cancelled
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.cs('toggle-spring').click()
-    cy.cs('toggle-cancelled').click()
-    cy.contains('Fetch population').click()
-
-    cy.contains('Credit accumulation (for 202 students)')
-  })
-
-  it('Credit Statistics, Statistics pane works', () => {
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    cy.contains('Credit statistics').click()
-    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
-
-    cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
-    cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
-    cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
-    cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-    cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-    cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-    cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
-    cy.get("[data-cy='credit-stats-max']").should('contain', '314')
+    })
   })
 })

cypress/integration/Student_statistics.js

@@ -13,7 +13,7 @@ describe('Student Statistics tests', () => {
     studentnumber: '010654019',
   }
 
-  it.only('Student statistics search form is usable', () => {
+  it('Student statistics search form is usable', () => {
     cy.contains('Show student names')
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.lastname)
@@ -80,7 +80,7 @@ describe('Student Statistics tests', () => {
     cy.contains('Student number').should('not.exist')
   })
 
-  it.only('Can jump to population page', () => {
+  it('Can jump to population page', () => {
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
@@ -88,3 +88,13 @@ describe('Student Statistics tests', () => {
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
   })
 })
+
+// Use admin to see all students
+describe('Testing with admin rights', () => {
+  it('Does not crash if student has no studyright or courses', () => {
+    const studentNumber = '011825096'
+    cy.init(`/students/${studentNumber}`, 'admin') // use admin to see all students
+    cy.contains('Started: Unavailable')
+    cy.contains('Credits: 0')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -8,53 +8,51 @@ const deleteTag = name => {
 }
 
 describe('Studyprogramme overview', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Study programme').click().siblings().contains('Overview').click()
-    cy.contains('Study Programme', { timeout: 100000 })
-  })
+  // check is used by couple of tests
+  const checkProgressAndProductivity = (user = 'basic') => {
+    // Column amount depends on user. Note that with dev user, you'll see even more
+    // stuff than with admin or basic.
+    const CSBachPopulationProgress2018ForAdmin = [
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
+      '1',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
+      '16',
+    ]
+    const CSBachPopulationProgress2018ForBasic = [
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
+      '1',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
+      '16',
+    ] // cancelled and transferred from are missing
 
-  // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-  const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-  let originalProgressCalculatedText
-  let originalProductivityCalculatedText
-
-  // Following function used by two tests
-  const testProgressAndProductivity = () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.contains('Admin').click()
-    cy.contains('productivity').click()
-    cy.contains('throughput').click()
-
-    cy.wait(1000)
-    cy.get('.attached > :nth-child(1)').click()
-    cy.get('table').should('have.length', 3)
+    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
     cy.contains('Population progress')
     cy.contains('Yearly productivity')
 
-    // Graduation feature is shown for only for devs, these are the values for normal user
-    const populationprogress2018 = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
-      '11',
-      '1',
-      '0',
-      '150',
-      '123',
-      '76',
-      '41',
-      '16',
-    ]
     cy.contains('2018-2019')
       .siblings()
       .each((elem, index) => {
-        cy.wrap(elem).contains(populationprogress2018[index])
+        cy.wrap(elem).contains(population[index])
       })
-    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
     cy.get('table')
       .eq(1)
       .contains('2019')
@@ -64,130 +62,167 @@ describe('Studyprogramme overview', () => {
       })
   }
 
-  it('progress should not be recalculating when opened for the first time', () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.wait(1000)
-    cy.contains('Recalculating').should('not.exist')
-  })
-
-  it('renders progress and productivity tables with calculated status', () => {
-    testProgressAndProductivity()
-    // Wait to "recalculating" to disappear
-    cy.wait(1000)
-    // Grab update dates to be compared later
-    cy.cs('throughputUpdateStatus')
-      .invoke('text')
-      .then(text => {
-        originalProgressCalculatedText = normalizeText(text)
-        expect(originalProgressCalculatedText).not.to.contain('recalculating')
-        expect(originalProgressCalculatedText).not.to.contain('refresh')
-      })
-
-    cy.cs('productivityUpdateStatus')
-      .invoke('text')
-      .then(text => {
-        originalProductivityCalculatedText = normalizeText(text)
-        expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-        expect(originalProgressCalculatedText).not.to.contain('refresh')
-      })
-  })
-
-  it('can open Thesis page', () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.contains('Thesis Courses').click()
-    cy.contains('Add thesis course').click()
-    cy.contains('No results')
-  })
-
-  it('can move to Population statistics page by clickin', () => {
-    cy.contains('Lääketieteen koulutusohjelma').click()
-    cy.get('i.level.up.alternate.icon').eq(0).click()
-    cy.contains('Students (121)')
-  })
-
-  it('can create and delete tags for population', () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached').contains('Tags').click()
-    cy.get('.tagNameSelectInput > .ui > input').type(name)
-    cy.get('.yearSelectInput').type('2018')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
-    cy.contains('2018')
-    deleteTag(name)
-  })
-
-  it('can create personal tags', () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached').contains('Tags').click()
-    cy.get('.tagNameSelectInput > .ui > input').type(name)
-    cy.get('.yearSelectInput').type('2018')
-
-    cy.get('.ui > label').click()
-    cy.contains('Create new tag').click()
-    cy.get('.purple')
-    cy.contains(name)
-    deleteTag(name)
-  })
-
-  it('can add tags to students', () => {
-    const name = `tag-${new Date().getTime()}`
-
-    const student = '010113437'
-
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached').contains('Tags').click()
-    cy.get('.tagNameSelectInput > .ui > input').type(name)
-    cy.get('.yearSelectInput').type('2018')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
-
-    cy.contains('Add tags to students').click()
-    cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
-
-    cy.get('.form > .field > .dropdown > .visible').contains(name).click()
-
-    cy.get('textarea').type('010113437')
-    cy.get('.positive').click()
-
-    cy.contains('Student statistics').click()
-    cy.get('.prompt').type(student)
-    cy.contains(student).click()
-    cy.contains(name)
-
-    cy.go('back')
-    cy.go('back')
-
-    deleteTag(name)
-
-    cy.contains('Student statistics').click()
-    cy.get('.prompt').type(student)
-    cy.contains(student).click()
-    cy.contains(name).should('not.exist')
-  })
-
-  it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-    testProgressAndProductivity()
-
-    // Wait to "recalculating" to disappear
-    cy.wait(1000)
-    // Check new calculation statuses are reported
-    const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-    const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
-
-    newProgressCalculatedTextElement.should(text => {
-      const newProgressCalculatedText = normalizeText(text)
-      expect(newProgressCalculatedText).not.to.contain('recalculating')
-      expect(newProgressCalculatedText).not.to.contain('refresh')
-      expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+  describe('when opening programme page with admin user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005', 'admin')
     })
 
-    newProductivityCalculatedTextElement.should(text => {
-      const newProductivityCalculatedText = normalizeText(text)
-      expect(newProductivityCalculatedText).not.to.contain('recalculating')
-      expect(newProductivityCalculatedText).not.to.contain('refresh')
-      expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+    it('progress should not be recalculating when opened for the first time', () => {
+      cy.contains('Recalculating').should('not.exist')
+      cy.contains('Last updated').should('exist')
+    })
+
+    // Following vars are used to compare values before and after stats calculation
+    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
+    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
+    let originalProgressCalculatedText
+    let originalProductivityCalculatedText
+    const doRecalculation = () => {
+      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
+      cy.wait(5000)
+      cy.get('.attached').contains('Admin').click()
+      cy.contains('recalculate productivity').click()
+      cy.contains('recalculate throughput').click()
+      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
+      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
+      cy.wait(5000)
+      cy.reload()
+    }
+
+    it('renders progress and productivity tables with calculated status', () => {
+      doRecalculation()
+
+      // check contains also study programme options for admin
+      cy.contains('Study programme options')
+      cy.get('table').should('have.length', 3)
+
+      checkProgressAndProductivity('admin')
+
+      // Grab update dates to be compared later
+      cy.cs('throughputUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProgressCalculatedText = normalizeText(text)
+          expect(originalProgressCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+
+      cy.cs('productivityUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProductivityCalculatedText = normalizeText(text)
+          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+    })
+
+    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
+      doRecalculation()
+      checkProgressAndProductivity('admin')
+
+      // Check new calculation statuses are reported
+      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
+      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+
+      newProgressCalculatedTextElement.should(text => {
+        const newProgressCalculatedText = normalizeText(text)
+        expect(newProgressCalculatedText).not.to.contain('recalculating')
+        expect(newProgressCalculatedText).not.to.contain('refresh')
+        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+
+      newProductivityCalculatedTextElement.should(text => {
+        const newProductivityCalculatedText = normalizeText(text)
+        expect(newProductivityCalculatedText).not.to.contain('recalculating')
+        expect(newProductivityCalculatedText).not.to.contain('refresh')
+        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+    })
+  })
+
+  describe('when opening programme page with basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005')
+    })
+
+    it('renders progress and productivity tables with previosly calculated values', () => {
+      // check doesn't contain study programme options for admin
+      cy.contains('Study programme options').should('not.exist')
+      cy.get('table').should('have.length', 2)
+      checkProgressAndProductivity()
+    })
+
+    it('can open Thesis page', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('Thesis Courses').click()
+      cy.contains('Add thesis course').click()
+      cy.contains('No results')
+    })
+
+    it('can move to Population statistics page by clickin', () => {
+      cy.get('i.level.up.alternate.icon').eq(0).click()
+      cy.contains('Students (10)')
+    })
+
+    it('can create and delete tags for population', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      cy.contains('2018')
+      deleteTag(name)
+    })
+
+    it('can create personal tags', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+
+      cy.get('.ui > label').click()
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      deleteTag(name)
+    })
+
+    it('can add tags to students', () => {
+      const name = `tag-${new Date().getTime()}`
+
+      const student = '010113437'
+
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+
+      cy.contains('Add tags to students').click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+
+      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+
+      cy.get('textarea').type('010113437')
+      cy.get('.positive').click()
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name)
+
+      cy.go('back')
+      cy.go('back')
+
+      deleteTag(name)
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name).should('not.exist')
     })
   })
 })

cypress/integration/Teachers.js

@@ -2,9 +2,8 @@
 
 describe('Teachers page tests', () => {
   beforeEach(() => {
-    cy.init()
-    cy.contains('Teachers').click()
-    cy.contains('Teacher statistics by course providers')
+    // login as admin = has teacher rights
+    cy.init('/teachers', 'admin')
   })
 
   const teacher1 = 'Landgraf Leo'

cypress/integration/Users.js

@@ -1,70 +1,57 @@
 /// <reference types="Cypress" />
 
 describe('Users tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Users').click()
-    cy.contains('Oodikone users')
+  describe('Using user with just grp-oodikone-user, no other rights', () => {
+    it('shows only frontpage, trends feedback', () => {
+      cy.init('', 'norights') // login with norights user
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
+
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
+    })
   })
 
-  it("Admin mocking normal user shows only the mocked user's programmes", () => {
-    cy.contains('mocking').should('not.exist')
-    cy.contains('tr', 'Normaalikäyttäjä').within($row => {
-      cy.contains('.button', 'Edit').click()
-    })
-    cy.contains('Access rights')
-      .siblings()
-      .within($row => {
-        cy.get("div[role='listitem']").should('have.length', 1).contains('Tietojenkäsittelytieteen kandiohjelma')
-      })
+  // TODO: Add user right checks for onlycoursestats user (should be like above + should
+  // show course stats
+  //
+  // TODO: Add user right checks for basic user (should show also study programme,
+  // student stats, course stats
 
-    cy.route('POST', '/api/superlogin/normk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains('mocking as normk')
-    cy.wait(1000)
-    cy.contains('Study programme').click().siblings().contains('Search by class').click()
-    cy.contains('label', 'Study programme')
-    cy.contains('label', 'Study programme')
-      .siblings()
-      .within($row => {
-        cy.get("div[role='option']").should('have.length', 1).contains('Tietojenkäsittelytieteen kandiohjelma')
-      })
-  })
-
-  it('Admin mocking courseStatistics-user can see only Trends and Course Statistics', () => {
-    cy.contains('mocking').should('not.exist')
-    cy.contains('tr', 'User manager').within($row => {
-      cy.contains('.button', 'Edit').click()
+  describe('Using as admin', () => {
+    beforeEach(() => {
+      cy.init('/users', 'admin')
     })
 
-    cy.get('[data-cy=access-groups-form]').click().contains('courseStatistics').click()
-    cy.get('[data-cy=access-groups-save]').click()
-
-    cy.route('POST', '/api/superlogin/usermk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains('mocking as usermk')
-    cy.wait(1000)
-
-    cy.get('[data-cy=navbar-studyProgramme').should('not.exist')
-    cy.get('[data-cy=navbar-students]').should('not.exist')
-    cy.get('[data-cy=navbar-teachers]').should('not.exist')
-    cy.get('[data-cy=navbar-users').should('not.exist')
-
-    cy.contains('Stop mocking').click()
-    cy.wait(1000)
-
-    cy.contains('Users').click()
-    cy.contains('Oodikone users')
-    cy.contains('tr', 'User manager').within($row => {
-      cy.contains('.button', 'Edit').click()
+    it('should see more stuff than others', () => {
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
     })
-    cy.contains('Access Groups')
-      .siblings()
-      .within($row => {
-        cy.get("i[class='dropdown icon clear']").click()
+
+    it("mocking normal user shows only the mocked user's programmes", () => {
+      cy.contains('mocking').should('not.exist')
+      cy.contains('tr', 'basic').within(() => {
+        cy.contains('.button', 'Edit').click()
       })
-    cy.get('[data-cy=access-groups-save]').click()
+      cy.get('i.spy').click()
+      cy.contains('mocking as basic')
+      cy.wait(1000)
+      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('label', 'Study programme')
+      cy.contains('label', 'Study programme')
+        .siblings()
+        .within(() => {
+          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
+        })
+    })
   })
 })

cypress/support/commands.js

@@ -1,25 +1,59 @@
 /// <reference types="Cypress" />
+//
+// Note: here we need to set keys to be all lowercase, since
+// we're replacing headers after they've left browser / frontend.
+// All of these users are available in anon user-db
+const adminUserHeaders = {
+  uid: 'admin',
+  displayname: 'Admin User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member;employee;faculty',
+  mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
+}
+
+const basicUserHeaders = {
+  uid: 'basic',
+  displayname: 'Basic User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
+}
+
+const onlycoursestatisticsUserHeaders = {
+  uid: 'onlycoursestatistics',
+  displayName: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-session',
+  hyGroupCn: 'grp-oodikone-basic-users',
+  eduPersonAffiliation: 'member',
+  mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
+}
+
+const norightsUserHeaders = {
+  uid: 'norights',
+  displayname: 'Norights User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
+}
+
+const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
 
 /**
- * Initialize headers and load the base URL or optional path.
+ Set up headers to login, set up correct user (admin / basic / etc.) and open given path.
  */
-Cypress.Commands.add('init', (path = '') => {
-  cy.server({
-    onAnyRequest: function (route, proxy) {
-      proxy.xhr.setRequestHeader('uid', 'tktl')
-      proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
-      proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
-      proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
-    },
+Cypress.Commands.add('init', (path = '', userId = 'basic') => {
+  cy.intercept('', req => {
+    const headersToUse = userHeaders.find(({ uid }) => uid === userId)
+    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
+    req.headers = headersToUse
   })
-
-  // Babel throws an error probably because of markdown files. I'm sorry about this :lul:.
-  cy.on('uncaught:exception', (err, runnable) => {
-    expect(err.message).to.include("Cannot read property 'helpers' of undefined")
-    done()
-    return false
-  })
-
   const baseUrl = Cypress.config().baseUrl
   cy.visit(baseUrl.concat(path))
 })
@@ -27,24 +61,28 @@ Cypress.Commands.add('init', (path = '') => {
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add('cs', name => {
-  return cy.get(`[data-cy='${name}']`)
+Cypress.Commands.add('cs', { prevSubject: 'optional' }, (subject, name) => {
+  const p = subject === undefined ? cy : cy.wrap(subject)
+
+  return p.get(`[data-cy='${name}']`)
 })
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add('selectFromDropdown', (name, index) => {
+Cypress.Commands.add('selectFromDropdown', { prevSubject: true }, (s, index) => {
+  const subject = cy.wrap(s)
   const indexes = Array.isArray(index) ? index : [index]
 
   indexes.forEach(i => {
-    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+    subject.click()
+    subject.children('.menu').children().eq(i).click({ force: true })
   })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children('.icon').click({ force: true })
+    cy.wrap(s).children('.icon').click({ force: true })
   }
 })
 

docker-compose.ci.yml

@@ -1,51 +1,43 @@
-version: "3.9"
-
 x-defaults: &defaults
   restart: unless-stopped
 
 services:
-
   backend:
     <<: *defaults
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
+      - user-db
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
+      CI: 'true'
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:trunk
+    image: toska/oodikone2-backend:staging
 
   frontend:
     <<: *defaults
     container_name: frontend
-    environment:
-      BACKEND_ADDR: backend
-    image: toska/oodikone2-frontend:trunk
+    image: toska/oodikone2-frontend:staging
 
   kone-db:
     <<: *defaults
     container_name: kone-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: kone-db
+      POSTGRES_HOST_AUTH_METHOD: trust
     image: registry.toska.cs.helsinki.fi/kone-db
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
@@ -55,22 +47,20 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
+    environment:
+      PGDATA: /data
+      POSTGRES_DB: sis-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    shm_size: 2gb
 
   user-db:
     <<: *defaults
     container_name: user-db
-    image: registry.toska.cs.helsinki.fi/user-db
-
-  userservice:
-    <<: *defaults
-    container_name: userservice
-    depends_on:
-      - user-db
     environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:trunk
+      PGDATA: /data
+      POSTGRES_DB: user-db
+      POSTGRES_HOST_AUTH_METHOD: trust
+    image: registry.toska.cs.helsinki.fi/user-db
 
   nginx:
     image: nginx:latest
@@ -78,8 +68,45 @@ services:
     depends_on:
       - frontend
       - backend
-      - userservice
     ports:
-      -  1337:80
+      - 1337:80
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
+
+  # Following services are used to run sis-updater-worker when anon sis-db needs to be updated. Could probably also use production version, but this certainly works.
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+
+  sis-updater-nats:
+    <<: *defaults
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater

docker-compose.real.yml

@@ -1,8 +1,7 @@
 services:
-
   backend:
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db-real
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 
@@ -19,9 +18,5 @@ services:
       DB_URL: postgres://postgres@sis-db:5432/sis-db-real
       SIS_IMPORTER_DATABASE: sis-importer-db-real
 
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db-real
-
 volumes:
   redis-data-real:

docker-compose.test.yml

@@ -1,10 +0,0 @@
-services:
-
-  userservice-test:
-    build:
-      context: services/userservice
-      target: test
-    container_name: userservice-test
-    environment:
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -5,7 +5,6 @@ x-build_defaults: &build_defaults
   target: development
 
 services:
-
   adminer:
     <<: *defaults
     container_name: adminer
@@ -25,20 +24,19 @@ services:
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
+      - user-db
+    env_file:
+      - .env
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      DB_URL_USER: postgres://postgres@user-db:5432/user-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
       SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USERSERVICE_URL: http://userservice:4567
     ports:
       - 8080:8080
     profiles:
@@ -53,13 +51,6 @@ services:
       <<: *build_defaults
       context: services/frontend
     container_name: frontend
-    environment:
-      ADMINER_URL: http://localhost:5050/?pgsql=oodi-db&username=postgres
-      BACKEND_ADDR: backend
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=kone-db&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=sis-db&username=postgres
-      SIS_IMPORTER_ADMINER_URL: http://localhost:5050/?pgsql=sis-importer-db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user-db&username=postgres
     ports:
       - 3000:3000
     profiles:
@@ -77,22 +68,13 @@ services:
     volumes:
       - kone-db-data:/data
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    profiles:
-      - oodikone
-    volumes:
-      - oodi-db-data:/data
-
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -102,6 +84,7 @@ services:
     image: registry.toska.cs.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
+    shm_size: 2gb
 
   sis-importer-db:
     <<: *defaults
@@ -114,7 +97,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -181,35 +164,13 @@ services:
     volumes:
       - user-db-data:/data
 
-  userservice:
-    <<: *defaults
-    build:
-      <<: *build_defaults
-      context: services/userservice
-    container_name: userservice
-    depends_on:
-      - user-db
-    environment:
-      DB_URL: postgres://postgres@user-db:5432/user-db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    ports:
-      - 4567:4567
-    profiles:
-      - oodikone
-    volumes:
-      - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/userservice/:/usr/src/app
-
 volumes:
   backend-nodemod:
   frontend-nodemod:
   kone-db-data:
-  oodi-db-data: # TODO: Remove when oodi stuff has been removed from oodikone
   redis-data:
   sis-db-data:
   sis-importer-db-data:
   sis-updater-scheduler-nodemod:
   sis-updater-worker-nodemod:
   user-db-data:
-  userservice-nodemod:

old_oodikone_stuff/old-backend-tests/course_stats_counter.test.js

@@ -1,197 +0,0 @@
-const { CourseStatsCounter } = require('./course_stats_counter')
-
-describe('CourseStatsCounter tests', () => {
-  const studentnumbers = ['1', '2', '3', '4', '5']
-  const coursecode = 'COURSE_01'
-  const coursename = 'COURSE NAME'
-
-  const createCounter = () => new CourseStatsCounter(coursecode, coursename, studentnumbers.length)
-
-  describe('counter constructor should set course values correctly', () => {
-    const counter = createCounter()
-
-    test('course name should be set after creation', () => {
-      expect(counter.course.name).toBe(coursename)
-    })
-
-    test('course code should be set after creation', () => {
-      expect(counter.course.code).toBe(coursecode)
-    })
-  })
-
-  describe('mark passed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '5'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, true, false, false)
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '0'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, false, true, false)
-
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('failed students should have studentnumber as key', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber as key', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then passed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '5', true, false, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber as key', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have student', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { failed, passed, failedMany } = counter.students
-
-    test('failed students should have studentnumber', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-
-    test('failed many students should have students', () => {
-      expect(failedMany).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then failed then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, failedMany, retryPassed } = counter.students
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('failedMany should not have studentnumber', () => {
-      expect(failedMany).not.toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then improved grade for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', true, false, false)
-    counter.markCredit(student, '5', false, false, true)
-
-    const { passed, improvedPassedGrade } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should have student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-  })
-
-  describe('mark improved grade then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', false, false, true)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, improvedPassedGrade, failed } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should contain student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-
-    test('failed should not contain student', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/globals.js

@@ -1,17 +0,0 @@
-const { sequelize } = require('../../src/models/index')
-const { sequelizeKone } = require('../../src/models/models_kone')
-const { redisClient } = require('../../src/services/redis')
-const { forceSyncDatabase } = require('../../src/database/connection')
-const { seedAllMigrations } = require('../../src/database/seed_migrations')
-
-beforeAll(async () => {
-  await forceSyncDatabase()
-  await seedAllMigrations()
-})
-
-afterAll(async () => {
-  await sequelize.close()
-  await sequelizeKone.close()
-  // https://stackoverflow.com/a/54560610
-  await new Promise(res => redisClient.quit(() => setImmediate(res)))
-})

old_oodikone_stuff/old-backend-tests/services/population.test.js

@@ -1,227 +0,0 @@
-const {
-  Student,
-  Course,
-  ElementDetails,
-  StudyrightElement,
-  Studyright,
-  Credit,
-  StudyrightExtent,
-  Semester,
-  SemesterEnrollment
-} = require('../../../src/models/index')
-const { optimizedStatisticsOf } = require('../../../src/services/populations')
-jest.mock('../../../src/services/semesters')
-const semesterService = require('../../../src/services/semesters')
-jest.setTimeout(10000)
-
-const langify = name => ({
-  en: `${name}_en`,
-  fi: `${name}_fi`,
-  sv: `${name}_sv`
-})
-
-const SEMESTER = { SPRING: 'SPRING', FALL: 'FALL' }
-
-const elementdetails = {
-  bsc: {
-    code: 'Element_BSC',
-    type: 10,
-    name: langify('Bachelor of Science')
-  },
-  maths: {
-    code: 'Element_MATH',
-    type: 20,
-    name: langify('Mathematics')
-  },
-  cs: {
-    code: 'ELEMENT_CS',
-    type: 20,
-    name: langify('Computer Science')
-  }
-}
-
-const courses = {
-  elements_of_ai: {
-    code: 'COURSE-1',
-    name: langify('ELEMENTS_OF_AI')
-  }
-}
-
-const semesters = {
-  fall: {
-    semestercode: 123,
-    name: { en: 'Autumn 2011', fi: 'Syksy 2011', sv: 'Hösten 2011' },
-    startdate: new Date('2011-07-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  spring: {
-    semestercode: 124,
-    name: { en: 'Spring 2012', fi: 'Kevät 2012', sv: 'Våren 2012' },
-    startdate: new Date('2011-12-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  // used for mocking current
-  current: {
-    semestercode: 125,
-    name: { en: 'Autumn 2012', fi: 'Syksy 2012', sv: 'Hösten 2012' },
-    startdate: new Date('2012-08-01'),
-    enddate: new Date('2012-12-31'),
-    yearcode: 63,
-    yearname: '2012-13'
-  }
-}
-
-const createQueryObject = (year, semester, codes, months) => ({
-  studyRights: codes,
-  year,
-  semesters: [semester],
-  months
-})
-
-afterAll(async () => {})
-
-describe('optimizedStatisticsOf tests', () => {
-  const courseinstanceFall = {
-    id: 1,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2011-09-31 21:00:00+00')
-  }
-
-  const courseinstanceSpring = {
-    id: 2,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2012-02-31 21:00:00+00')
-  }
-
-  const student = {
-    studentnumber: '1234'
-  }
-
-  const studyrightextents = {
-    bachelors: {
-      extentcode: 1,
-      name: langify("bachelor's degree extent")
-    }
-  }
-
-  const studyright = {
-    studyrightid: 1,
-    studystartdate: new Date('2011-07-31 21:00:00+00'),
-    student_studentnumber: student.studentnumber,
-    extentcode: studyrightextents.bachelors.extentcode
-  }
-
-  const creditFall = {
-    id: 'CREDIT-1',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceFall.course_code,
-    attainment_date: courseinstanceFall.coursedate,
-    semestercode: 123
-  }
-
-  const creditSpring = {
-    id: 'CREDIT-2',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceSpring.course_code,
-    attainment_date: courseinstanceSpring.coursedate,
-    semestercode: 124
-  }
-
-  const semesterEnrollments = {
-    fall: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.fall.semestercode,
-      enrollment_date: semesters.fall.startdate
-    },
-    spring: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.spring.semestercode,
-      enrollment_date: semesters.spring.startdate
-    },
-    current: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.current.semestercode,
-      enrollment_date: semesters.current.startdate
-    }
-  }
-
-  const studyrightelements = {
-    bsc: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.bsc.code,
-      studentnumber: student.studentnumber
-    },
-    maths: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.maths.code,
-      studentnumber: student.studentnumber
-    }
-  }
-
-  describe(`
-    Student with:
-    - BSc studyright (2011-07-31 – 2016-12-20),
-    - Mathematics studyright (2011-07-31 – 2016-12-20),
-    - Two credits in 2011-09-31 and 2012-02-31.
-    `, () => {
-    beforeAll(async () => {
-      await Semester.bulkCreate([semesters.fall, semesters.spring, semesters.current])
-      await Student.create(student)
-      await Course.create(courses.elements_of_ai)
-      await Credit.bulkCreate([creditFall, creditSpring])
-      await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
-      await StudyrightExtent.create(studyrightextents.bachelors)
-      await Studyright.create(studyright)
-      await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
-      await SemesterEnrollment.bulkCreate([
-        semesterEnrollments.fall,
-        semesterEnrollments.spring,
-        semesterEnrollments.current
-      ])
-      // Mock current semester implementation
-      semesterService.getCurrentSemester.mockImplementation(async () => {
-        const currentSemester = await Semester.findOne({
-          where: {
-            semestercode: semesters.current.semestercode
-          }
-        })
-        return currentSemester
-      })
-    })
-
-    test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
-      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/services/test_data.js

@@ -1,42 +0,0 @@
-const faculties = [
-  {
-    code: 'FAC1',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 1 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 1 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 1 SV'
-      }
-    ]
-  },
-  {
-    code: 'FAC2',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 2 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 2 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 3 SV'
-      }
-    ]
-  }
-]
-
-module.exports = {
-  faculties
-}

old_oodikone_stuff/old-backend-tests/studyprogramme.test.js

@@ -1,264 +0,0 @@
-const {
-  getCreditsForProvider,
-  productivityStatsFromCredits,
-  productivityStatsForProvider,
-  findGraduated,
-  graduatedStatsFromStudyrights,
-  combineStatistics,
-  productivityStatsForCode,
-  findProgrammeThesisCredits,
-  thesisProductivityFromCredits,
-  thesisProductivityForStudytrack
-} = require('./studyprogramme')
-const { sequelize } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { readFileSync } = require('fs')
-const { mapToProviders } = require('../util/utils')
-
-const { MASTER, BACHELOR } = ThesisTypeEnums
-
-const provider = '500-M010'
-const studytrack = 'MH50_010'
-
-const thesisCourse = {
-  // from studyprogramme.test.sql
-  programmeCode: 'MH50_010',
-  courseCode: 'THESIS_01',
-  thesisType: MASTER
-}
-
-beforeAll(async () => {
-  const query = readFileSync('./src/services/studyprogramme.test.sql', 'utf8')
-  await sequelize.query(query)
-  await ThesisCourse.create(thesisCourse)
-})
-
-test('Credits for provider should contain passed course credit', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({ id: 'CREDIT_01' })
-})
-
-test('Credits for provider should return object in correct format', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({
-    credits: 5,
-    year: 2016,
-    id: 'CREDIT_01'
-  })
-})
-
-test('Credits for provider should not contain passed credits from other providers', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_04')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit with failed credit typecode 10', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_02')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit for passed study module', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_03')
-  expect(credit).toBeFalsy()
-})
-
-test('KH50_005 track code maps to 500-K005 provider code', () => {
-  const track = 'KH50_005'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-K005')
-})
-
-test('MH50_010 track code maps to 500-M010 provider code', () => {
-  const track = 'MH50_010'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-M010')
-})
-
-test('productivityStatsFromCredits calculates stats correctly', () => {
-  const credits = [
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 10 }
-  ]
-  const stats = productivityStatsFromCredits(credits)
-  expect(stats).toMatchObject({
-    2016: { year: 2016, credits: 15 },
-    2015: { year: 2015, credits: 20 }
-  })
-})
-
-test('productivityStats integrates correctly', async () => {
-  const stats = await productivityStatsForProvider(provider, '2001-01-01')
-  expect(stats).toMatchObject({
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 }
-  })
-})
-
-test('findGraduated finds graduated studyright', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.studyrightid === '10')
-  expect(match).toBeTruthy()
-})
-
-test('findGraduated formats graduated studyrights correctly', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  expect(studyrights).toContainEqual({ studyrightid: '10', year: 2016, timeToGraduation: NaN })
-})
-
-test('findGraduated does not return studyrights that are not graduated', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '11')
-  expect(match).toBeFalsy()
-})
-
-test('findGraduated does return graduated studyrights from other studytracks', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '12')
-  expect(match).toBeFalsy()
-})
-
-test('graduatedStatsFromStudyrights calculates stats correctly', () => {
-  const studyrights = [{ year: 2015 }, { year: 2015 }, { year: 2014 }]
-  const stats = graduatedStatsFromStudyrights(studyrights)
-  expect(stats).toMatchObject({
-    2015: { graduated: 2 },
-    2014: { graduated: 1 }
-  })
-})
-
-test('combineStatistics returns correctly formatted array', () => {
-  const creditStats = {
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 },
-    2014: { year: 2014, credits: 20 }
-  }
-  const studyrightStats = {
-    2015: { graduated: 2 },
-    2016: { graduated: 1 }
-  }
-  const thesisStats = {
-    2014: { mThesis: 1 },
-    2015: { mThesis: 2, bThesis: 1 }
-  }
-  const creditsForMajors = {
-    2014: 10,
-    2015: 22
-  }
-  const transferredCredits = {
-    2014: 2,
-    2015: 4
-  }
-
-  const stats = combineStatistics(creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits)
-  expect(stats).toContainEqual({
-    year: 2015,
-    mThesis: 2,
-    bThesis: 1,
-    credits: 40,
-    graduated: 2,
-    creditsForMajors: 22,
-    transferredCredits: 4
-  })
-  expect(stats).toContainEqual({
-    year: 2014,
-    mThesis: 1,
-    bThesis: 0,
-    credits: 20,
-    graduated: 0,
-    creditsForMajors: 10,
-    transferredCredits: 2
-  })
-  expect(stats).toContainEqual({
-    year: 2016,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    graduated: 1,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('productivityStatsForCode integrates', async () => {
-  const stats = await productivityStatsForCode(studytrack, '2001-01-01')
-  expect(stats.data).toContainEqual({
-    year: 2015,
-    graduated: 0,
-    bThesis: 0,
-    mThesis: 1,
-    credits: 40,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-  expect(stats.data).toContainEqual({
-    year: 2016,
-    graduated: 1,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('findProgrammeThesisCredits returns correct credit', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toBeTruthy()
-  expect(credits.length).toBe(1)
-  expect(credits.find(c => c.id === 'CREDIT_07')).toBeTruthy()
-})
-
-test('findProgrammeThesisCredits format credit correctly', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toContainEqual({
-    id: 'CREDIT_07',
-    code: 'THESIS_01',
-    type: MASTER,
-    year: 2015
-  })
-})
-
-test('thesisProductivityFromCredits', async () => {
-  const credits = [
-    { type: MASTER, year: 2015 },
-    { type: MASTER, year: 2015 },
-    { type: BACHELOR, year: 2015 },
-    { type: BACHELOR, year: 2014 },
-    { type: MASTER, year: 2013 }
-  ]
-  const stats = thesisProductivityFromCredits(credits)
-  expect(stats).toHaveProperty('2015', '2014', '2013')
-  expect(stats).toMatchObject({
-    '2015': {
-      mThesis: 2,
-      bThesis: 1
-    },
-    '2014': {
-      mThesis: 0,
-      bThesis: 1
-    },
-    '2013': {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})
-
-test('thesisProductivityForStudytrack integrates', async () => {
-  const stats = await thesisProductivityForStudytrack(studytrack)
-  expect(stats).toMatchObject({
-    2015: {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})

old_oodikone_stuff/old-backend-tests/studyprogramme.test.sql

@@ -1,93 +0,0 @@
-DO $$
-DECLARE
-  ts TIMESTAMP = CURRENT_TIMESTAMP;
-  studytrk_01 VARCHAR(50) = 'MH50_010';
-  studytrk_02 VARCHAR(50) = 'KH50_005';
-  provider_01 VARCHAR(50) = '500-M010';
-  provider_02 VARCHAR(50) = '500-K005';
-  course_01 VARCHAR(50) = 'M010_COURSE';
-  module_01 VARCHAR(50) = 'M010_MODULE';
-  course_02 VARCHAR(50) = 'K005_COURSE';
-  thesis VARCHAR(50) = 'THESIS_01';
-  date_2016 TIMESTAMPTZ = '2016-09-28 00:00:00+00';
-  date_2015 TIMESTAMPTZ = '2015-09-28 00:00:00+00';
-BEGIN
-  INSERT INTO semesters ("semestercode", "createdAt", "updatedAt")
-  VALUES
-  (5, ts, ts)
-  ;
-
-  INSERT INTO course
-  ("code", "name", "is_study_module")
-  VALUES
-  (course_01, '{ "en": "COURSE_EN" }', 'false'),
-  (module_01, '{ "en": "MODULE_EN" }', 'true'),
-  (course_02, '{ "en": "COURSE_EN" }', 'false'),
-  (thesis, '{ "en": "Bachelor thesis" }', 'false')
-  ;
-
-  INSERT INTO student
-  ("studentnumber", "createdAt", "updatedAt")
-  VALUES
-  ('012111517', ts, ts)
-  ;
-
-  INSERT INTO credit_types
-  ("credittypecode", "createdAt", "updatedAt")
-  VALUES
-  (10, ts, ts),
-  (4, ts, ts)
-  ;
-
-  INSERT INTO credit
-  ("id", "createddate", "lastmodifieddate", "semestercode", "course_code", "credittypecode", "attainment_date", "credits", "student_studentnumber")
-  VALUES
-  ('CREDIT_01', ts, ts, 5, course_01, 4, date_2016, 5, '012111517'),
-  ('CREDIT_02', ts, ts, 5, course_01, 10, date_2016, 5, '012111517'),
-  ('CREDIT_03', ts, ts, 5, module_01, 4, date_2016, 20, '012111517'),
-  ('CREDIT_04', ts, ts, 5, course_02, 4, date_2016, 5, '012111517'),
-  ('CREDIT_05', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_06', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_07', ts, ts, 5, thesis, 4, date_2015, 30, '012111517')
-  ;
-
-  INSERT INTO providers
-  ("providercode", "createdAt", "updatedAt")
-  VALUES
-  (provider_01, ts, ts),
-  (provider_02, ts, ts)
-  ;
-
-  INSERT INTO course_providers
-  ("coursecode", "providercode", "createdAt", "updatedAt")
-  VALUES
-  (course_01, provider_01, ts, ts),
-  (module_01, provider_01, ts, ts),
-  (course_02, provider_02, ts, ts),
-  (thesis, provider_01, ts, ts)
-  ;
-
-  INSERT INTO element_details
-  ("code", "createdAt", "updatedAt")
-  VALUES
-  (studytrk_01, ts, ts),
-  (studytrk_02, ts, ts)
-  ;
-
-  INSERT INTO studyright
-  ("studyrightid", "graduated", "enddate")
-  VALUES
-  (10, 1, date_2016),
-  (11, 0, date_2016),
-  (12, 1, date_2016)
-  ;
-
-  INSERT INTO studyright_elements
-  ("studyrightid", "code", "createdAt", "updatedAt", "studentnumber")
-  VALUES
-  (10, studytrk_01, ts, ts, '012111517'),
-  (11, studytrk_01, ts, ts, '012111517'),
-  (12, studytrk_02, ts, ts, '012111517')
-  ;
-
-END $$;

old_oodikone_stuff/old-backend-tests/teachers.test.js

@@ -1,53 +0,0 @@
-const { Teacher } = require('../models/index')
-const service = require('./teachers')
-
-const teacher = {
-  id: 'TID_001',
-  code: 'usercode',
-  name: 'Lastname Firstname Middlename'
-}
-
-beforeAll(async () => {
-  await Teacher.create(teacher)
-})
-
-describe('teacher service tests', () => {
-  describe('teacher by search term tests', () => {
-    const teacherIsInResults = (id, matches) => matches.some(teacher => teacher.id === 'TID_001')
-
-    test('Search by `usercode` finds teacher', async () => {
-      const matches = await service.bySearchTerm('usercode')
-      expect(teacherIsInResults('TID_001', matches)).toBe(true)
-    })
-
-    test('Search by `usercodeEXTRA` does not find teacher', async () => {
-      const matches = await service.bySearchTerm('usercodeEXTRA')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(false)
-    })
-
-    test('Search by just `Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Lastname Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `id` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('TID_001')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/thesis.test.js

@@ -1,85 +0,0 @@
-const { forceSyncDatabase } = require('../database/connection')
-const { Course, ElementDetails } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { createThesisCourse, deleteThesisCourse, findProgrammeTheses } = require('./thesis')
-
-const courseOne = {
-  code: 'COURSE',
-  name: { en: 'Course_EN' }
-}
-
-const courseTwo = {
-  code: 'COURSE_2',
-  name: { en: 'Course_EN_2' }
-}
-
-const progOne = {
-  code: 'CS_BSC',
-  name: { en: 'CS_BSC' }
-}
-
-const progTwo = {
-  code: 'CS_MSC',
-  name: { en: 'CS_MSC' }
-}
-
-beforeEach(async () => {
-  await forceSyncDatabase()
-  await Course.create(courseOne)
-  await Course.create(courseTwo)
-  await ElementDetails.create(progOne)
-  await ElementDetails.create(progTwo)
-})
-
-test('createThesisCourse should save thesis correctly', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-})
-
-test('createThesisCourse should create separate bachelor and thesis for same programme and course', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  const master = courses.find(c => c.thesisType === 'MASTER')
-  const bachelor = courses.find(c => c.thesisType === 'BACHELOR')
-  expect(master).toBeTruthy()
-  expect(master).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-  expect(bachelor).toBeTruthy()
-  expect(bachelor).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE_2', thesisType: 'BACHELOR' })
-})
-
-test('createThesisCourse should error for creating a thesis for existing (course, prog) pair', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const promise = createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.BACHELOR)
-  await expect(promise).rejects.toThrow('Validation error')
-})
-
-test('deleteThesisCourse removes thesis from table', async () => {
-  let courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(0)
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  await deleteThesisCourse(progOne.code, courseOne.code)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ thesisType: 'BACHELOR' })
-})
-
-test('findProgrammeTheses returns correct courses', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  await createThesisCourse(progTwo.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const theses = await findProgrammeTheses(progOne.code)
-  expect(theses.length).toBe(2)
-})
-
-test('createThesisCourse should return created thesis', async () => {
-  const thesis = await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  expect(thesis).toBeTruthy()
-  expect(thesis).toMatchObject({ programmeCode: progOne.code, courseCode: courseOne.code })
-})

old_oodikone_stuff/old-backend-tests/utils.js

@@ -1,248 +0,0 @@
-const faker = require('faker')
-
-const numberFromTo = (from, to) => Math.round(Math.random() * (to - from)) + from
-const daysAgo = daysAgo => new Date(new Date() - 1000 * 60 * 60 * 24 * daysAgo)
-const idFromTwoIds = (a, b) => (1 / 2) * (a + b) * (a + b + 1) + b
-
-const gradeArray = [
-  '2',
-  'CL',
-  'Eisa',
-  'L',
-  'Hyv.',
-  'TT',
-  'LUB',
-  'HT',
-  'MCLA',
-  '4',
-  '0',
-  'A',
-  'NSLA',
-  'Luop',
-  '5',
-  '3',
-  'Hyl.',
-  'ECLA',
-  '1',
-  'KH'
-]
-const statusArray = [
-  { status: 'Suoritettu', statuscode: 4 },
-  { status: 'Korotettu', statuscode: 7 },
-  { status: 'Hyväksiluettu', statuscode: 9 },
-  { status: 'Hylätty', statuscode: 10 }
-]
-const teacherRoleArray = ['Responsible', 'Approver', 'Teacher']
-
-const generateCourses = async amount => {
-  const number = amount || numberFromTo(10, 100)
-  const courses = []
-  for (let i = 0; i < number; i++) {
-    courses.push({
-      code: i.toString(10),
-      name: faker.lorem.words(3)
-    })
-  }
-  return courses
-}
-
-const generateCourseInstances = async (courses, amount) => {
-  const instances = []
-  let id = 0
-  courses.forEach(course => {
-    const number = amount || numberFromTo(5, 40)
-    for (let i = 0; i < number; i++) {
-      instances.push({
-        id,
-        coursedate: daysAgo(numberFromTo(10, 365)).toUTCString(),
-        course_code: course.code
-      })
-      id++
-    }
-  })
-  return instances
-}
-
-const generateStudents = async amount => {
-  const students = []
-  const number = amount || numberFromTo(5, 40)
-
-  const now = new Date()
-
-  for (let i = 0; i < number; i++) {
-    const birthdate = daysAgo(numberFromTo(365 * 18, 365 * 70))
-    const dateoffirstcredit = faker.date.between(birthdate, now)
-    const dateoflastcredit = faker.date.between(dateoffirstcredit, now)
-    const dateofuniversityenrollment = faker.date.between(birthdate, dateoflastcredit)
-    const lastname = faker.name.lastName()
-    const firstnames = faker.name.firstName(2)
-    students.push({
-      studentnumber: `0${numberFromTo(10000000, 15000000)}`,
-      lastname,
-      firstnames,
-      abbreviatedname: `${lastname} ${firstnames}`,
-      birthdate,
-      communicationlanguage: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      country: faker.address.country(),
-      creditcount: numberFromTo(0, 600),
-      dateoffirstcredit,
-      dateoflastcredit,
-      dateofuniversityenrollment,
-      gradestudent: `${numberFromTo(0, 1)}`,
-      nationalities: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      semesterenrollmenttypecode: [null, '1.0', '2.0'][numberFromTo(0, 2)],
-      sex: ['female', 'male'][numberFromTo(0, 1)],
-      studentstatuscode: numberFromTo(2, 8)
-    })
-  }
-
-  return students
-}
-
-const generateCredits = async (courseInstances, students, amount) => {
-  const credits = []
-  students.forEach((student, studentIdx) => {
-    courseInstances.forEach(courseInstance => {
-      const number = amount || numberFromTo(5, 40)
-      for (let i = 0; i < number; i++) {
-        const statusObject = statusArray[numberFromTo(0, statusArray.length - 1)]
-        credits.push({
-          id: idFromTwoIds(idFromTwoIds(studentIdx, courseInstance.id), i),
-          grade: gradeArray[numberFromTo(0, gradeArray.length - 1)],
-          student_studentnumber: student.studentnumber,
-          credits: [5, 10][numberFromTo(0, 1)],
-          ordering: faker.date.between(student.dateoffirstcredit, student.dateoflastcredit),
-          status: statusObject.status,
-          statuscode: statusObject.statuscode,
-          courseinstance_id: courseInstance.id
-        })
-      }
-    })
-  })
-  return credits
-}
-
-const generateTeachers = async amount => {
-  const teachers = []
-  const number = amount || numberFromTo(5, 40)
-
-  for (let i = 0; i < number; i++) {
-    teachers.push({
-      id: i,
-      code: faker.lorem.word(),
-      name: `${faker.name.lastName()} ${faker.name.firstName()}`
-    })
-  }
-
-  return teachers
-}
-
-const generateCourseTeachers = async (courseInstances, teachers, maximumAmount) => {
-  const courseTeachers = []
-  courseInstances.forEach((courseInstance, index) => {
-    const number = numberFromTo(0, maximumAmount)
-    for (let i = 0; i < number; i++) {
-      const teacher = teachers[numberFromTo(0, teachers.length - 1)]
-      courseTeachers.push({
-        id: idFromTwoIds(index, i),
-        teacherrole: teacherRoleArray[numberFromTo(0, teacherRoleArray.length - 1)],
-        courseinstance_id: courseInstance.id,
-        teacher_id: teacher.id
-      })
-    }
-  })
-
-  return courseTeachers
-}
-
-const generateOrganizations = async amount => {
-  const organizations = []
-
-  const number = amount || numberFromTo(5, 40)
-  for (let i = 0; i < number; i++) {
-    organizations.push({
-      code: faker.lorem.words(3),
-      name: faker.company.companyName()
-    })
-  }
-  return organizations
-}
-
-const generateStudyrights = async (students, organization, amount, date, prioritycode) => {
-  const studyrights = []
-  const possibleStudyrightNames = []
-  if (amount) {
-    for (let i = 0; i < amount; i++) {
-      possibleStudyrightNames.push(faker.company.catchPhrase())
-    }
-  }
-  students.forEach(student => {
-    studyrights.push({
-      studyrightid: student.studentnumber,
-      canceldate: null,
-      cancelorganisation: null,
-      enddate: new Date().toUTCString(),
-      extentcode: numberFromTo(2, 99),
-      givendate: daysAgo(365 * numberFromTo(1, 100)),
-      graduated: null,
-      prioritycode: prioritycode || numberFromTo(1, 30),
-      startdate: date,
-      studystartdate: date,
-      organization_code: organization.code,
-      student_studentnumber: student.studentnumber
-    })
-  })
-  return studyrights
-}
-
-const generateUsers = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const users = []
-  for (let i = 0; i < number; i++) {
-    users.push({
-      full_name: `${faker.name.lastName()}, ${faker.name.firstName(2)}`,
-      username: faker.lorem.word(),
-      is_enabled: numberFromTo(0, 1) === 1
-    })
-  }
-  return users
-}
-
-const generateTags = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const tags = []
-  for (let i = 0; i < number; i++) {
-    tags.push({
-      tagname: `${faker.company.catchPhraseAdjective()} ${i}` //tagname should be unique, add i just in case
-    })
-  }
-  return tags
-}
-
-// adds all tags to all students
-const generateTagStudents = (students, tags) => {
-  let tag_students = []
-  students.forEach(student => {
-    tags.forEach(tag => {
-      tag_students.push({
-        taggedstudents_studentnumber: student.studentnumber,
-        tags_tagname: tag.tagname
-      })
-    })
-  })
-  return tag_students
-}
-
-module.exports = {
-  generateCourses,
-  generateCourseInstances,
-  generateStudents,
-  generateStudyrights,
-  generateCredits,
-  generateTeachers,
-  generateCourseTeachers,
-  generateOrganizations,
-  generateUsers,
-  generateTags,
-  generateTagStudents
-}

old_oodikone_stuff/old-backend-tests/utils/semester.test.js

@@ -1,14 +0,0 @@
-const moment = require('moment')
-const semesterUtils = require('../../../src/util/semester')
-
-describe('Semester utils tests', () => {
-  test('Get passing semester', () => {
-    expect(semesterUtils.getPassingSemester(2018, moment('2017-09-09'))).toBe('BEFORE')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-09-09'))).toBe('0-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-05-01'))).toBe('0-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-09-01'))).toBe('1-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2020-05-01'))).toBe('1-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2026-05-01'))).toBe('LATER')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-05-01'))).toBe('BEFORE')
-  })
-})

old_oodikone_stuff/updater_api/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_api/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_api/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_api/.gitignore

@@ -1 +0,0 @@
-node_modules/

old_oodikone_stuff/updater_api/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_api/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_api/doo_api_database_updater/db_studentnumbers_to_file.js

@@ -1,15 +0,0 @@
-const fs = require('fs')
-const { Student } = require('../../models/index')
-
-const getStudentNumbers = async (writefilepath = './studentnumbers.txt') => {
-  const file = fs.openSync(writefilepath, 'w')
-  const students = await Student.findAll()
-  for (let student of students) {
-    const data = `${student.studentnumber} \n`
-    fs.appendFileSync(file, data)
-  }
-  fs.closeSync(file)
-  process.exit(0)
-}
-
-getStudentNumbers()

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.js

@@ -1,350 +0,0 @@
-const moment = require('moment')
-const _ = require('lodash')
-
-const getStudyRightIdStrings = data => data['data'].map(elements => elements[0])
-
-const getTextsByLanguage = texts => {
-  const names = {}
-  texts.forEach(text => (names[text.langcode] = text.text))
-  return { fi: null, sv: null, en: null, ...names }
-}
-
-const defaultNameFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names.fi || names.en || names.sv
-}
-
-const jsonNamesFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names
-}
-
-const universityEnrollmentDateFromStudyRights = studyRightArray => {
-  return _.sortBy(studyRightArray.map(s => s.start_date), n => moment(n).valueOf())[0]
-}
-
-const getStudentFromData = (student, studyrights) => {
-  const city = getTextsByLanguage(student.city)
-  const language = getTextsByLanguage(student.language)
-  const country = getTextsByLanguage(student.country)
-  const gender = getTextsByLanguage(student.gender)
-  const home_country = getTextsByLanguage(student.home_country)
-  return {
-    studentnumber: student.student_number,
-    email: student.email,
-    phone: student.phone,
-    city_fi: city.fi,
-    city_sv: city.sv,
-    national_student_number: student.national_student_number,
-    zipcode: student.zipcode,
-    address: student.address1,
-    creditcount: student.studyattainments,
-    address2: student.address2,
-    birthdate: getDate(student.birth_date),
-    language_fi: language.fi,
-    language_sv: language.sv,
-    language_en: language.en,
-    age: student.age_years,
-    lastname: student.last_name,
-    mobile: student.mobile_phone,
-    home_county_id: student.home_county_id,
-    country_fi: country.fi,
-    country_sv: country.sv,
-    country_en: country.en,
-    home_country_fi: home_country.fi,
-    home_country_sv: home_country.sv,
-    home_country_en: home_country.en,
-    firstnames: student.first_names,
-    communicationlanguage: language.en || language.fi || language.sv,
-    dateofuniversityenrollment: universityEnrollmentDateFromStudyRights(studyrights),
-    abbreviatedname: [student.last_name, student.first_names].join(' '),
-    gender_code: student.gender_code,
-    gender_fi: gender.fi,
-    gender_sv: gender.sv,
-    gender_en: gender.en
-  }
-}
-
-const getSemesterCode = attainment_date => moment(attainment_date).diff(moment('1950', 'YYYY'), 'months') / 6
-
-const getDate = (date, format = 'DD.MM.YYYY') => {
-  if (!date) return null
-  return moment(date, format).format('YYYY-MM-DD')
-}
-
-const getOrganisationFromData = ({ name, code }) => {
-  return {
-    code,
-    name: jsonNamesFromTexts(name)
-  }
-}
-
-const attainmentDataToCredit = (attainment, studentnumber) => {
-  return {
-    id: String(attainment.studyattainment_id),
-    grade: defaultNameFromTexts(attainment.grade),
-    credits: attainment.credits,
-    ordering: getDate(attainment.attainment_date, null),
-    credittypecode: attainment.attainment_status_code,
-    student_studentnumber: studentnumber,
-    attainment_date: parseDate(attainment.attainment_date),
-    course_code: `${attainment.learningopportunity_id}`,
-    semestercode: attainment.semester_code,
-    isStudyModule: attainment.is_leaf === 'false'
-  }
-}
-
-const attainmentDataToCourse = attainment => {
-  const { learningopportunity_name, attainment_date } = attainment
-  return {
-    code: attainment.learningopportunity_id,
-    name: jsonNamesFromTexts(learningopportunity_name),
-    latest_instance_date: parseDate(attainment_date),
-    is_study_module: attainment.is_leaf === 'false'
-  }
-}
-
-const getTeacherFromData = teacher => {
-  if (teacher) {
-    return {
-      id: teacher.teacher_id,
-      code: teacher.userid,
-      name: teacher.full_name
-    }
-  }
-}
-
-const parseDate = (date, format = 'YYYY-MM-DD HH:mm') => date && moment.utc(date, format).toDate()
-
-const getStudyRightFromData = (data, studentNumber) => {
-  return {
-    studyrightid: `${data.studyright_id}`,
-    canceldate: data.cancel_date,
-    cancelorganisation: data.organisation_code,
-    enddate: parseDate(data.degree_date) || parseDate(data.end_date),
-    extentcode: data.extent_code,
-    givendate: parseDate(data.admission_date),
-    graduated: Number(!!data.degree_date),
-    graduation_date: data.degree_date,
-    prioritycode: data.priority,
-    startdate: parseDate(data.start_date),
-    studystartdate: parseDate(data.study_start_date),
-    organization_code: data.organisation_code,
-    student_studentnumber: studentNumber,
-    faculty_code: data.faculty_code
-  }
-}
-
-const elementDetailFromData = element => {
-  const { code, name, element_id } = element
-  const names = getTextsByLanguage(name)
-  return {
-    code,
-    name: names,
-    type: element_id
-  }
-}
-
-const studyrightElementFromData = (element, studyrightid, studentnumber, degree_date) => {
-  const { start_date, end_date, code } = element
-  return {
-    code,
-    studyrightid,
-    studentnumber,
-    startdate: start_date,
-    enddate: degree_date || end_date
-  }
-}
-
-const attainmentDataToTeachers = data =>
-  data.teachers.map(teacherdata => ({
-    id: teacherdata.teacher_id
-  }))
-
-const studyrightDataToExtent = data => ({
-  extentcode: data.extent_code,
-  name: getTextsByLanguage(data.extent)
-})
-
-const courseTypeFromData = data => ({
-  coursetypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const disciplineFromData = data => ({
-  discipline_id: data.discipline_id,
-  name: getTextsByLanguage(data.name)
-})
-
-const learningOpportunityDataToCourseDisciplines = data => {
-  if (data) {
-    return data.disciplines.map(discipline => ({
-      discipline_id: discipline.discipline_id,
-      course_id: data.learningopportunity_id
-    }))
-  }
-}
-
-const learningOpportunityDataToCourse = data => {
-  if (data) {
-    return {
-      code: data.learningopportunity_id,
-      coursetypecode: data.learningopportunity_type_code,
-      disciplines: data.disciplines,
-      name: getTextsByLanguage(data.names),
-      startdate: parseDate(data.start_date),
-      enddate: parseDate(data.end_date)
-    }
-  }
-}
-
-const learningOpportunityDataToCourseProviders = data => {
-  if (!data) {
-    return { providers: [], courseproviders: [] }
-  }
-  const providers = data.organisations.map(organisation => ({
-    providercode: organisation.code,
-    name: getTextsByLanguage(organisation.name)
-  }))
-  const courseproviders = providers.map(provider => ({
-    providercode: provider.providercode,
-    coursecode: data.learningopportunity_id
-  }))
-  return {
-    providers,
-    courseproviders
-  }
-}
-
-const studyattainmentStatusCodeToCreditType = data => ({
-  credittypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const semesterFromData = data => ({
-  semestercode: data.semester_code,
-  name: getTextsByLanguage(data.name),
-  startdate: parseDate(data.start_date),
-  enddate: parseDate(data.end_date),
-  yearcode: data.year_code,
-  yearname: data.year_name
-})
-
-const semesterEnrollmentFromData = (data, studentnumber) => ({
-  enrollmenttype: data.semester_enrollment_type_code,
-  semestercode: data.semester_code,
-  enrollment_date: parseDate(data.semester_enrollment_date),
-  studentnumber
-})
-
-const getTransfersFromData = (data, studentnumber) => {
-  const studytracks = data.elements.filter(element => element.element_id === 20)
-  const sorted = _.sortBy(studytracks, 'end_date')
-  let transfers = []
-  let i = 0
-  while (i < sorted.length) {
-    if (i === 0) {
-      i++
-      continue
-    }
-    let target = sorted[i].code
-    let source = sorted[i - 1].code
-    let transferdate = sorted[i - 1].end_date
-
-    transfers = transfers.concat({
-      sourcecode: source,
-      targetcode: target,
-      transferdate,
-      studentnumber,
-      studyrightid: `${data.studyright_id}`
-    })
-    i++
-  }
-  return transfers
-}
-
-const courseRealisationTypeFromData = data => ({
-  realisationtypecode: `${data.code}`,
-  name: getTextsByLanguage(data.name)
-})
-
-const studentEnrollmentToModels = (data, studentnumber) => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.name),
-    realisationtypecode: `${data.type_code}`,
-    parent: `${data.parent_id}`,
-    root: `${data.root_id}`
-  }
-  const courseenrollment = {
-    studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }
-  const course = {
-    code: data.learningopportunity_id
-  }
-  return {
-    courserealisation,
-    courseenrollment,
-    course
-  }
-}
-
-const courseUnitRealisationDataToModels = data => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.realisation_name),
-    realisationtypecode: `${data.realisation_type_code}`,
-    parent: `${data.parent_id}`
-  }
-  const course = {
-    code: courserealisation.coursecode
-  }
-  const students = data.students.map(({ student_number }) => ({
-    studentnumber: student_number
-  }))
-  const courseenrollments = students.map(student => ({
-    studentnumber: student.studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }))
-  return {
-    courserealisation,
-    course,
-    students,
-    courseenrollments
-  }
-}
-
-module.exports = {
-  getStudentFromData,
-  getStudyRightIdStrings,
-  getStudyRightFromData,
-  getDate,
-  getTeacherFromData,
-  elementDetailFromData,
-  studyrightElementFromData,
-  attainmentDataToCourse,
-  attainmentDataToCredit,
-  getOrganisationFromData,
-  attainmentDataToTeachers,
-  studyrightDataToExtent,
-  courseTypeFromData,
-  learningOpportunityDataToCourse,
-  studyattainmentStatusCodeToCreditType,
-  disciplineFromData,
-  learningOpportunityDataToCourseDisciplines,
-  semesterFromData,
-  semesterEnrollmentFromData,
-  learningOpportunityDataToCourseProviders,
-  getTransfersFromData,
-  courseRealisationTypeFromData,
-  studentEnrollmentToModels,
-  courseUnitRealisationDataToModels,
-  getSemesterCode
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.test.js

@@ -1,199 +0,0 @@
-const mapper = require('./oodi_data_mapper')
-
-describe('studyattainment data mapping tests', () => {
-  const data = {
-    attainment_status_code: 4,
-    attainment_date: '2018-05-30T21:00:00.000Z',
-    studyattainment_id: 123456789,
-    attainment_lang: null,
-    person_id: 1000002,
-    textual_assessment: null,
-    teachers: [
-      {
-        teacher_id: 'TID_01',
-        person_id: 1000001,
-        short_name: 'Matti Meikäläinen',
-        last_names: 'Meikäläinen',
-        first_names: 'Matti Teppo'
-      }
-    ],
-    studyright_id: null,
-    semester_code: 136,
-    grade: [
-      {
-        langcode: 'fi',
-        text: 'Hyv.'
-      },
-      {
-        langcode: 'sv',
-        text: 'Godk'
-      },
-      {
-        langcode: 'en',
-        text: 'Pass'
-      }
-    ],
-    credits: 10,
-    expiry_date: null,
-    learningopportunity_name: [
-      {
-        langcode: 'fi',
-        text: 'learningopportunity_name FI'
-      },
-      {
-        langcode: 'sv',
-        text: 'learningopportunity_name SV'
-      },
-      {
-        langcode: 'en',
-        text: 'learningopportunity_name EN'
-      }
-    ],
-    learningopportunity_id: 'LID_001'
-  }
-
-  describe('credit mapping tests', () => {
-    const studentnumber = '0876654321'
-    const credit = mapper.attainmentDataToCredit(data, studentnumber)
-
-    describe('studyattainment date tests', () => {
-      const { attainment_date } = credit
-      const MAY = 4
-
-      test('attainment date is truthy', () => {
-        expect(attainment_date).toBeTruthy()
-      })
-
-      test('attainment date is instance of Date', () => {
-        expect(attainment_date instanceof Date).toBe(true)
-      })
-
-      test('attainment date has correct month, year and date', () => {
-        expect(attainment_date.getMonth()).toBe(MAY)
-        expect(attainment_date.getFullYear()).toBe(2018)
-        expect(attainment_date.getDate()).toBe(30)
-      })
-    })
-
-    test('course code matches learningopportunity_id', () => {
-      expect(credit.course_code).toBe('LID_001')
-    })
-
-    test('credit id should attainment id as string', () => {
-      expect(credit.id).toBe('123456789')
-    })
-
-    test('studentnumber is correct', () => {
-      expect(credit.student_studentnumber).toBe(studentnumber)
-    })
-
-    test('semestercode is correct', () => {
-      expect(credit.semestercode).toBe(136)
-    })
-  })
-
-  describe('teacher mapping tests', () => {
-    const teachers = mapper.attainmentDataToTeachers(data)
-
-    test('amount of mapped teachers is correct (1)', () => {
-      expect(teachers.length).toBe(1)
-    })
-
-    test('teacher should have correct id', () => {
-      expect(teachers[0].id).toBe('TID_01')
-    })
-  })
-})
-
-describe('semester enrollment data mapping tests', () => {
-  const studentnumber = '12345678'
-
-  const data = {
-    full_time_student: 'true',
-    yths_code: null,
-    semester_enrollment_type_code: 1,
-    absence_reason_code: null,
-    payment_amount: null,
-    yths_description: null,
-    student_union_member: 'true',
-    semester_enrollment_date: '2011-08-30T21:00:00.000Z',
-    payment_date: null,
-    semester_code: 123,
-    doctoral_current_year_estimate: null,
-    doctoral_previous_year: null
-  }
-
-  describe('mapping of enrolled data', () => {
-    const mapping = mapper.semesterEnrollmentFromData(data, studentnumber)
-
-    test('mapped semestercode should be integer 123', () => {
-      expect(mapping.semestercode).toBe(123)
-    })
-
-    test('mapped enrollmenttype code should be integer 1', () => {
-      expect(mapping.enrollmenttype).toBe(1)
-    })
-
-    test('mapped studentnumber should be correct', () => {
-      expect(mapping.studentnumber).toBe(studentnumber)
-    })
-
-    test('mapped enrollment_date should be instance of Date', () => {
-      expect(mapping.enrollment_date instanceof Date).toBe(true)
-    })
-
-    test('mapped enrollment_date should be equal to Date object with same day, month and year', () => {
-      expect(mapping.enrollment_date).toEqual(new Date('2011-08-30'))
-    })
-  })
-})
-
-describe('semester code data mapping tests', () => {
-  const data = {
-    teaching_start_date: '1950-09-14T22:00:00.000Z',
-    name: [
-      {
-        langcode: 'fi',
-        text: 'Syksy 1950'
-      },
-      {
-        langcode: 'sv',
-        text: 'Hösten 1950'
-      },
-      {
-        langcode: 'en',
-        text: 'Autumn 1950'
-      }
-    ],
-    year_name: '1950-51',
-    enroll_end_date: '1950-09-29T22:00:00.000Z',
-    enroll_start_date: '1950-05-31T22:00:00.000Z',
-    end_date: '1950-12-30T22:00:00.000Z',
-    semester_code: 1,
-    start_date: '1900-07-31T22:20:11.000Z',
-    teaching_end_date: '1950-12-20T22:00:00.000Z',
-    year_code: 1
-  }
-
-  const semester = mapper.semesterFromData(data)
-
-  test('semester code should be correct', () => {
-    expect(semester.semestercode).toBe(1)
-  })
-
-  test('semester name should have all language keys set as properties', () => {
-    expect(semester.name).toHaveProperty('fi')
-    expect(semester.name).toHaveProperty('en')
-    expect(semester.name).toHaveProperty('sv')
-  })
-
-  test('semester start and end date should be instances of Date', () => {
-    expect(semester.startdate instanceof Date).toBe(true)
-    expect(semester.enddate instanceof Date).toBe(true)
-  })
-
-  test('semester yearcode and yearname should be correct', () => {
-    expect(semester.yearcode).toBe(1)
-    expect(semester.yearname).toBe('1950-51')
-  })
-})

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_interface.js

@@ -1,253 +0,0 @@
-require('dotenv').config()
-const axios = require('axios')
-const LRU = require('lru-cache')
-const https = require('https')
-const fs = require('fs')
-
-const { OODI_ADDR, KEY_PATH, CERT_PATH } = process.env
-const base_url = OODI_ADDR
-
-const requestCache = new LRU({
-  max: 25000,
-  length: () => 1,
-  maxAge: 60 * 60 * 1000
-})
-
-const agent =
-  KEY_PATH && CERT_PATH
-    ? new https.Agent({
-        cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-        key: fs.readFileSync(process.env.KEY_PATH, 'utf8')
-      })
-    : new https.Agent({
-        rejectUnauthorized: false
-      })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = async url => {
-  const cacheHit = requestCache.get(url)
-  if (cacheHit) return cacheHit
-
-  const res = await instance.get(url)
-
-  // Sometimes oodi seems to randomly return a response with status 200
-  // that in reality contains an error. Check this out:
-  // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550167612
-  if (res.data.exception) {
-    throw new Error(res.data.exception.message)
-  }
-
-  if (!url.includes('/students/')) requestCache.set(url, res.data.data)
-  return res.data.data
-}
-
-const getOodiApi = async relative => {
-  const route = base_url + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-const attemptGetFor = async (url, attempts = 6) => {
-  for (let attempt = 1; attempt <= attempts; ++attempt) {
-    try {
-      return await getUrl(url)
-    } catch (error) {
-      if (attempt === attempts) {
-        console.log('ATTEMPT GET FOR FAILURE')
-        throw error
-      }
-      await sleep(attempt * 5000)
-    }
-  }
-}
-
-if (process.env.NODE_ENV === 'development') {
-  // axios.defaults.auth = {
-  //   username: 'tktl',
-  //   password: process.env.OODI_PW
-  // }
-  instance.interceptors.request.use(config => {
-    config.params = { token: process.env.TOKEN }
-    return config
-  })
-}
-console.log(process.env.NODE_ENV)
-
-const getStudent = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDENT FAILED')
-    throw e
-  }
-}
-
-const getStudentStudyRights = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyrights`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDYRIGHT FAIELD')
-    throw e
-  }
-}
-
-const getFaculties = async () => {
-  const url = `${base_url}/codes/faculties`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET FACULTIE S FAIELD')
-    throw e
-  }
-}
-
-const getStudyAttainments = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyattainments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STSDU ATTAINEMNT FIALE')
-    throw e
-  }
-}
-
-const getTeacherInfo = async id => {
-  const url = `${base_url}/teachers/${id}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET TCHR FIAIELd', id)
-    throw e
-  }
-}
-
-const getCourseTypeCodes = async () => {
-  const url = `${base_url}/codes/learningopportunities/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('CORUSE TTYPE CODE FAIEL')
-    throw e
-  }
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${base_url}/learningopportunities/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET LEARNIGN OEPPRITUNCITY FAIELD', id)
-    throw e
-  }
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${base_url}/codes/studyattainments/statuses`
-  return await attemptGetFor(url)
-}
-
-const getCourseDisciplines = async () => {
-  const url = `${base_url}/codes/learningopportunities/disciplines`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET COURS EDISCIIPLINES FIALED')
-    throw e
-  }
-}
-const getSemesters = async () => {
-  const url = `${base_url}/codes/semesters`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET SEMESTERS FAIELD')
-    throw e
-  }
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/semesterenrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GEST EMESTER ENROLLMENTS FAIELD')
-    throw e
-  }
-}
-
-const getCourseEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/enrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EENREOLMETNS FAIELD')
-    throw e
-  }
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${base_url}/codes/courseunitrealisations/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURSE REALSITASITON TYPES')
-    throw e
-  }
-}
-
-const courseUnitRealisationsSince = async sinceDate => {
-  const url = `${base_url}/courseunitrealisations/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('COURSEU UNTI REALISTIATION')
-    throw e
-  }
-}
-
-const learningOpportunitiesSince = async (sinceDate = '0000-01-01') => {
-  const url = `${base_url}/learningopportunities/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('LRNG OPRRTN SINCE')
-    throw e
-  }
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${base_url}/courseunitrealisations/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EUNTI GREALSISTAITOn', id)
-    throw e
-  }
-}
-
-module.exports = {
-  getStudentStudyRights,
-  getStudent,
-  getFaculties,
-  getStudyAttainments,
-  getTeacherInfo,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getCourseDisciplines,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisationsSince,
-  getCourseUnitRealisation,
-  learningOpportunitiesSince,
-  getOodiApi
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/updater_formatter.js

@@ -1,158 +0,0 @@
-const Oodi = require('./oodi_interface')
-const mapper = require('./oodi_data_mapper')
-
-const getAllStudentInformationFromApi = async studentnumber => {
-  const [student, studyrights, studyattainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-    Oodi.getStudent(studentnumber),
-    Oodi.getStudentStudyRights(studentnumber),
-    Oodi.getStudyAttainments(studentnumber),
-    Oodi.getSemesterEnrollments(studentnumber),
-    Oodi.getCourseEnrollments(studentnumber)
-  ])
-  return {
-    student,
-    studyrights,
-    studyattainments,
-    studentnumber,
-    semesterEnrollments,
-    courseEnrollments
-  }
-}
-
-const formatStudyrights = async (api, studentnumber) => {
-  if (api.studyrights.length === 0) {
-    console.log(`No studyrights for ${studentnumber}`)
-    return []
-  }
-  return api.studyrights.map(studyrightData => {
-    const studyRightExtent = mapper.studyrightDataToExtent(studyrightData)
-    const studyright = mapper.getStudyRightFromData(studyrightData, studentnumber)
-
-    const elementDetails = studyrightData.elements.map(element => mapper.elementDetailFromData(element))
-    const studyRightElements = studyrightData.elements.map(element =>
-      mapper.studyrightElementFromData(element, studyright.studyrightid, studentnumber, studyrightData.degree_date)
-    )
-
-    const transfers = mapper.getTransfersFromData(studyrightData, studentnumber)
-    return { studyRightExtent, studyright, elementDetails, studyRightElements, transfers }
-  })
-}
-
-const formatCourseEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.courseEnrollments.map(enrollment => mapper.studentEnrollmentToModels(enrollment, studentnumber))
-  )
-
-const parseAttainmentData = (data, studentnumber) => {
-  return {
-    credit: mapper.attainmentDataToCredit(data, studentnumber),
-    teachers: mapper.attainmentDataToTeachers(data),
-    course: mapper.attainmentDataToCourse(data)
-  }
-}
-
-const createCreditTeachers = async (credit, teachers) => {
-  const creditTeachers = teachers.map(teacher => ({
-    credit_id: credit.id,
-    teacher_id: teacher.id
-  }))
-  return creditTeachers
-}
-
-const formatStudyattainments = async (api, studentnumber) => {
-  let studyAttainments = []
-  for (let data of api.studyattainments) {
-    const { credit, teachers, course } = parseAttainmentData(data, studentnumber)
-    let learningOpportunity
-    // For some learning opportunities oodi returns an SQL error.
-    // F.ex. /learningopportunities/590202B. Therefore, until the
-    // bug is fixed on Oodi's side, this is what we need to do.
-    try {
-      learningOpportunity = await Oodi.getLearningOpportunity(course.code)
-    } catch (e) {
-      if (e.message && e.message.includes('PreparedStatementCallback')) continue
-      throw e
-    }
-    const { providers, courseproviders } = mapper.learningOpportunityDataToCourseProviders(learningOpportunity)
-
-    studyAttainments = [
-      ...studyAttainments,
-      {
-        credit: credit.semestercode
-          ? credit
-          : { ...credit, semestercode: mapper.getSemesterCode(credit.attainment_date) },
-        creditTeachers: await createCreditTeachers(credit, teachers),
-        teachers: (await Promise.all(
-          teachers.map(async t => {
-            const teacher = await Oodi.getTeacherInfo(t.id)
-            // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550184131
-            // Sometimes Oodi returns a null teacher... We need to figure out why.
-            if (!teacher) {
-              console.log('TEACHER MISSING!', t.id)
-              return null
-            }
-            return mapper.getTeacherFromData(teacher)
-          })
-        )).filter(t => t),
-        course: {
-          ...course,
-          ...mapper.learningOpportunityDataToCourse(learningOpportunity),
-          disciplines: mapper.learningOpportunityDataToCourseDisciplines(learningOpportunity),
-          providers,
-          courseproviders
-        }
-      }
-    ]
-  }
-  return studyAttainments
-}
-const formatSemesterEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.semesterEnrollments.map(apiEnrollment => mapper.semesterEnrollmentFromData(apiEnrollment, studentnumber))
-  )
-
-const getStudent = async studentnumber => {
-  const api = await getAllStudentInformationFromApi(studentnumber)
-  if (api.student == null) {
-    const error = new Error(`API returned ${api.student} for studentnumber ${studentnumber}.`)
-    error.name = 'NO_STUDENT'
-    throw error
-  }
-  try {
-    const studentInfo = await mapper.getStudentFromData(api.student, api.studyrights)
-    const [studyRights, studyAttainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-      formatStudyrights(api, studentnumber),
-      formatStudyattainments(api, studentnumber),
-      formatSemesterEnrollments(api, studentnumber),
-      formatCourseEnrollments(api, studentnumber)
-    ])
-    return { studentInfo, studyRights, studyAttainments, semesterEnrollments, courseEnrollments }
-  } catch (e) {
-    console.error('getStudent failed', api)
-    throw e
-  }
-}
-
-const getMeta = async () => {
-  const [
-    faculties,
-    courseRealisationsTypes,
-    semesters,
-    creditTypeCodes,
-    courseTypeCodes,
-    disciplines
-  ] = await Promise.all([
-    (await Oodi.getFaculties()).map(mapper.getOrganisationFromData),
-    (await Oodi.getCourseRealisationTypes()).map(mapper.courseRealisationTypeFromData),
-    (await Oodi.getSemesters()).map(mapper.semesterFromData),
-    (await Oodi.getStudyattainmentStatusCodes()).map(mapper.studyattainmentStatusCodeToCreditType),
-    (await Oodi.getCourseTypeCodes()).map(mapper.courseTypeFromData),
-    (await Oodi.getCourseDisciplines()).map(mapper.disciplineFromData)
-  ])
-  return { faculties, courseRealisationsTypes, semesters, creditTypeCodes, courseTypeCodes, disciplines }
-}
-
-module.exports = {
-  getStudent,
-  getMeta
-}

old_oodikone_stuff/updater_api/index.js

@@ -1,72 +0,0 @@
-const { stan, opts } = require('./stan')
-const { getStudent, getMeta } = require('./doo_api_database_updater/updater_formatter')
-const logger = require('./logger')
-const moment = require('moment')
-
-const hasActiveEnrollment = semesterEnrollments => {
-  if (semesterEnrollments) {
-    const currentSemester = Math.floor(moment().diff(moment('1950', 'YYYY'), 'months') / 6)
-    const active = semesterEnrollments.find(e => e.semestercode === currentSemester && e.enrollmenttype === 1)
-    return active ? true : false
-  }
-  return false
-}
-
-const fetchData = async (priority, msg) => {
-  try {
-    const task = JSON.parse(msg.getData())
-    if (task.task === 'meta') {
-      const data = await getMeta()
-      stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', priority }), err => {
-        if (err) console.log('STATUS PUBLISH FAILED', err)
-      })
-      stan.publish(priority ? 'PriorityWrite' : 'UpdateWrite', JSON.stringify({ task: task.task, data }))
-    } else {
-      try {
-        const data = await getStudent(task.task)
-        // TODO: check that data is properly structured(?)
-        const active = hasActiveEnrollment(data.semesterEnrollments)
-        stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', active, priority }), err => {
-          if (err) console.log('STATUS PUBLISH FAIELD', err)
-        })
-        stan.publish(
-          priority ? 'PriorityWrite' : 'UpdateWrite',
-          JSON.stringify({ task: task.task, data, active }),
-          err => {
-            if (err) console.log('STATUS PUBLISH FAILED', err)
-          }
-        )
-      } catch (e) {
-        if (e.name === 'NO_STUDENT') {
-          stan.publish(
-            'status',
-            JSON.stringify({ task: task.task, status: 'NO_STUDENT', active: false, priority }),
-            err => {
-              if (err) console.log('STATUS PUBLISH FAILED', err)
-            }
-          )
-          return
-        } else {
-          console.error('Task failed', task)
-          throw e
-        }
-      }
-    }
-  } catch (e) {
-    console.log(e)
-    logger.info('failure', { service: 'API' })
-  }
-}
-
-const handleMessage = priority => async msg => {
-  await fetchData(priority, msg)
-  msg.ack()
-}
-
-stan.on('connect', async () => {
-  const sub = stan.subscribe('UpdateApi', 'updater.workers', opts)
-  const prioSub = stan.subscribe('PriorityApi', 'updater.workers.prio', opts)
-
-  sub.on('message', handleMessage(false))
-  prioSub.on('message', handleMessage(true))
-})

old_oodikone_stuff/updater_api/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_api/package.json

@@ -1,34 +0,0 @@
-{
-  "name": "updater_api",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "axios": "^0.21.1",
-    "dotenv": "^7.0.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^5.1.1",
-    "moment": "^2.29.1",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_api/stan.js

@@ -1,13 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-
-const opts = stan.subscriptionOptions()
-opts.setManualAckMode(true)
-opts.setAckWait(10 * 60 * 1000)
-opts.setMaxInFlight(1)
-
-module.exports = {
-  stan,
-  opts
-}

old_oodikone_stuff/updater_api/test_producer.js

@@ -1,32 +0,0 @@
-var stan = require('node-nats-streaming').connect('updaterNATS', 'producer', process.env.NATS_URI)
-
-stan.on('connect', () => {
-  stan.publish('UpdateApi', 'meta', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '011120775', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014441008', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014272112', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-})

old_oodikone_stuff/updater_scheduler/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_scheduler/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_scheduler/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_scheduler/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_scheduler/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_scheduler/index.js

@@ -1,126 +0,0 @@
-const { stan } = require('./src/nats_connection')
-const { CronJob } = require('cron')
-const Schedule = require('./models')
-const logger = require('./logger')
-const {
-  scheduleDaily,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  scheduleMeta
-} = require('./src/schedule_students')
-const { isValidStudentId } = require('./src/util')
-require('./src/api')
-
-const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
-
-let updatedCount = 0
-let scheduledCount = 0
-let fetchedCount = 0
-
-const updateTask = async ({ task, status, type, updatetime, active }) => {
-  const doc = { task, status, type }
-  if (updatetime) doc.updatedAt = new Date()
-  if (active != null) doc.active = active
-  await Schedule.findOneAndUpdate({ task }, doc, { upsert: true })
-}
-
-stan.on('connect', async () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 0 * * 6', async () => {
-      // Every Friday-Saturday midnight
-      try {
-        console.log('SCHEDULING ALL STUDENTS')
-        await scheduleAllStudents()
-      } catch (err) {
-        console.log('SCHEDULING ALL STUDENTS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 6 * * 1-5', async () => {
-      // Every Monday through Friday at 06:00
-      try {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        await scheduleMeta()
-        await scheduleAttainmentUpdate()
-      } catch (err) {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 20 * * 1-4', async () => {
-      // Every Monday through Thursday at 20:00
-      try {
-        console.log('SCHEDULING DAILY TASKS')
-        await scheduleDaily()
-      } catch (err) {
-        console.log('SCHEDULING DAILY TASKS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-  }
-
-  schedule('0 * * * *', async () => {
-    // Every hour
-    console.log(
-      `${updatedCount} TASKS DONE IN LAST HOUR. ${scheduledCount} TASKS SCHEDULED IN LAST HOUR. ${fetchedCount} TASKS FETCHED FROM API IN LAST HOUR`
-    )
-    updatedCount = 0
-    fetchedCount = 0
-    scheduledCount = 0
-  })
-
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  // opts.setDeliverAllAvailable()
-  // opts.setDurableName('durable')
-  opts.setMaxInFlight(20)
-
-  const statusSub = stan.subscribe('status', opts)
-
-  const handleStatusMessage = async msg => {
-    const data = JSON.parse(msg.getData())
-    const { task, status, timems, active, priority } = data
-    let updatetime = false
-    switch (status) {
-      case 'DONE':
-      case 'NO_STUDENT':
-        updatedCount = updatedCount + 1
-        updatetime = true
-        break
-      case 'FETCHED':
-        fetchedCount = fetchedCount + 1
-        break
-      case 'SCHEDULED':
-        scheduledCount = scheduledCount + 1
-        break
-      default:
-        throw 'unknown status: ' + msg.getData()
-    }
-    const isStudent = !!isValidStudentId(task)
-    if (status === 'DONE')
-      logger.info(`Status changed for ${task} to ${status}`, {
-        task,
-        status,
-        student: isStudent,
-        timems,
-        active,
-        priority
-      })
-    await updateTask({ task, status, type: isStudent ? 'student' : 'other', updatetime, active })
-  }
-  statusSub.on('message', async msg => {
-    try {
-      await handleStatusMessage(msg)
-    } catch (err) {
-      console.log(err)
-      logger.info('failure', { service: 'SCHEDULER' })
-    }
-    msg.ack()
-  })
-})

old_oodikone_stuff/updater_scheduler/logger.js

@@ -1,29 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-transports.push(new winston.transports.Console({ level: 'debug' }))
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_scheduler/models.js

@@ -1,18 +0,0 @@
-const mongoose = require('mongoose')
-mongoose.connect(process.env.MONGO_URI, { useNewUrlParser: false, useFindAndModify: false })
-
-const scheduleSchema = new mongoose.Schema({
-  task: {
-    type: String,
-    unique: true
-  },
-  status: String,
-  type: String,
-  active: Boolean,
-  updatedAt: {
-    type: Date,
-    index: true
-  }
-})
-
-module.exports = mongoose.model('Schedule', scheduleSchema)

old_oodikone_stuff/updater_scheduler/package.json

@@ -1,35 +0,0 @@
-{
-  "name": "updater_scheduler",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node --max_old_space_size=4096 index.js",
-    "dev": "nodemon --max_old_space_size=4096 index.js",
-    "scheduleActiveStudents": "NATS_CLIENTID=manualActiveSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleActiveStudents()'",
-    "scheduleMeta": "NATS_CLIENTID=manualAllSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleMeta()'",
-    "getOldestTasks": "node --max_old_space_size=4096 -e 'require(\"./src/SchedulingStatistics.js\").getOldestTasks()'"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "cron": "^1.8.2",
-    "express": "^4.17.1",
-    "mongoose": "^5.12.3",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_scheduler/src/SchedulingStatistics.js

@@ -1,48 +0,0 @@
-const Schedule = require('../models')
-
-const getOldestTasks = async () => {
-  const oldestTask = (await Schedule.find({ type: 'student' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestActiveTask = (await Schedule.find({ type: 'student', active: true })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestMetaTask = (await Schedule.find({ type: 'other', task: 'meta' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestAttainmentTask = (await Schedule.find({ type: 'other', task: 'attainment' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-
-  return {
-    oldestChangedStatus: {
-      studentnumber: oldestTask ? oldestTask.task : null,
-      updatedAt: oldestTask ? oldestTask.updatedAt : null
-    },
-    oldestActiveStudentDone: {
-      studentnumber: oldestActiveTask ? oldestActiveTask.task : null,
-      updatedAt: oldestActiveTask ? oldestActiveTask.updatedAt : null
-    },
-    oldestMetaTask: {
-      updatedAt: oldestMetaTask ? oldestMetaTask.updatedAt : null,
-      status: oldestMetaTask ? oldestMetaTask.status : null
-    },
-    oldestAttainmentTask: {
-      updatedAt: oldestAttainmentTask ? oldestAttainmentTask.updatedAt : null,
-      status: oldestAttainmentTask ? oldestAttainmentTask.status : null
-    }
-  }
-}
-
-const getCurrentStatus = async () => {
-  const allTasksScheduled = await Schedule.count({ type: 'student', status: 'SCHEDULED' })
-  const allTasksFetched = await Schedule.count({ type: 'student', status: 'FETCHED' })
-  const allTasksDone = await Schedule.count({ type: 'student', status: 'DONE' })
-  const allTasksCreated = await Schedule.count({ type: 'student', status: 'CREATED' })
-  const allTasksNoStudent = await Schedule.count({ type: 'student', status: 'NO_STUDENT' })
-  const allTasksActive = await Schedule.count({ type: 'student', active: true })
-
-  return { allTasksScheduled, allTasksFetched, allTasksDone, allTasksCreated, allTasksActive, allTasksNoStudent }
-}
-
-module.exports = { getOldestTasks, getCurrentStatus }

old_oodikone_stuff/updater_scheduler/src/api.js

@@ -1,127 +0,0 @@
-const express = require('express')
-const app = express()
-const port = process.env.PORT
-const bodyParser = require('body-parser')
-const {
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleAllStudents,
-  scheduleActiveStudents,
-  scheduleMeta,
-  scheduleAttainmentUpdate,
-  scheduleStudentCheck,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily
-} = require('./schedule_students')
-const { getOldestTasks, getCurrentStatus } = require('./SchedulingStatistics')
-const { createTasks } = require('./student_list_updater')
-
-app.use(bodyParser.json())
-
-app.get('/ping', (req, res) => res.json({ message: 'pong ' }))
-
-app.post('/update', async (req, res) => {
-  await createTasks(req.body)
-  const scheduled = await scheduleStudentsByArray(req.body)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/oldest', async (req, res) => {
-  const { amount } = req.body
-  const scheduled = await scheduleOldestNStudents(amount)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/all', async (req, res) => {
-  const scheduled = await scheduleAllStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/active', async (req, res) => {
-  const scheduled = await scheduleActiveStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/attainment', async (req, res) => {
-  scheduleAttainmentUpdate()
-  res.json({ message: `scheduled attainments` })
-})
-
-app.post('/update/meta', async (req, res) => {
-  scheduleMeta()
-  res.json({ message: 'scheduled meta' })
-})
-
-app.post('/update/no_student', async (req, res) => {
-  const scheduled = await scheduleStudentCheck()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/studentlist', async (req, res) => {
-  createTasks()
-  res.json({ message: `tasks created` })
-})
-
-app.post('/update/daily', async (req, res) => {
-  const scheduled = await scheduleDaily()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/created', async (req, res) => {
-  const scheduled = await rescheduleCreated()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/scheduled', async (req, res) => {
-  const scheduled = await rescheduleScheduled()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/fetched', async (req, res) => {
-  const scheduled = await rescheduleFetched()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.get('/statuses', async (req, res) => {
-  const statuses = []
-
-  const {
-    oldestChangedStatus: { studentnumber, updatedAt },
-    oldestActiveStudentDone: { studentnumber: studentnumberActive, updatedAt: updatedAtActive },
-    oldestMetaTask,
-    oldestAttainmentTask
-  } = await getOldestTasks()
-
-  const {
-    allTasksScheduled,
-    allTasksFetched,
-    allTasksDone,
-    allTasksCreated,
-    allTasksActive,
-    allTasksNoStudent
-  } = await getCurrentStatus()
-
-  statuses.push(
-    { label: 'CREATED', value: allTasksCreated },
-    { label: 'SCHEDULED', value: allTasksScheduled },
-    { label: 'FETCHED', value: allTasksFetched },
-    { label: 'DONE', value: allTasksDone },
-    { label: 'NO_STUDENT', value: allTasksNoStudent },
-    { label: 'ACTIVE', value: allTasksActive },
-    { label: 'oldest student studentnumber', value: studentnumber },
-    { label: 'oldest student timestamp', value: updatedAt },
-    { label: 'oldest active student studentnumber', value: studentnumberActive },
-    { label: 'oldest active student timestamp', value: updatedAtActive },
-    { label: 'oldest metadata update', value: oldestMetaTask.updatedAt },
-    { label: 'current metadata status', value: oldestMetaTask.status },
-    { label: 'oldest attainment update', value: oldestAttainmentTask.updatedAt },
-    { label: 'current attainment status', value: oldestAttainmentTask.status }
-  )
-
-  res.json(statuses)
-})
-
-app.listen(port, () => console.log(`listening on port ${port}!`))
-module.exports = { app }

old_oodikone_stuff/updater_scheduler/src/nats_connection.js

@@ -1,4 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.NATS_CLIENTID || 'scheduler', {
-  url: process.env.NATS_URI
-})
-module.exports = { stan }

old_oodikone_stuff/updater_scheduler/src/schedule_students.js

@@ -1,181 +0,0 @@
-const { stan } = require('./nats_connection')
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-stan.on('connect', async () => {
-  const taskspermin = 300
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  opts.setMaxInFlight(taskspermin)
-
-  const scheduleSub = stan.subscribe('schedule', opts)
-  scheduleSub.on('message', async msg => {
-    try {
-      const { task, priority } = JSON.parse(msg.getData())
-      stan.publish('status', JSON.stringify({ task, status: 'SCHEDULED', priority }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      stan.publish(priority ? 'PriorityApi' : 'UpdateApi', JSON.stringify({ task }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      await sleep(60 * 1000)
-    } catch (err) {
-      console.log(err)
-    }
-    msg.ack()
-  })
-})
-
-const publishSingle = async ({ priority, task }) => {
-  const promise = new Promise(resolve => {
-    stan.publish('schedule', JSON.stringify({ task: task.task, priority }), async err => {
-      if (err) {
-        console.log('publish failed, waiting 10 seconds and republishing', err)
-        await sleep(10 * 1000)
-        resolve(await publishSingle({ priority, task }))
-      } else {
-        resolve(true)
-      }
-    })
-  })
-  return await promise
-}
-
-const publishAll = async (tasks, priority = false) => {
-  for (const task of tasks) {
-    await publishSingle({ task, priority })
-  }
-}
-
-const scheduleActiveStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleAllStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleMeta = async () => {
-  await publishAll([{ task: 'meta' }], true)
-}
-
-const scheduleAttainmentUpdate = async () => {
-  stan.publish('status', JSON.stringify({ task: 'attainment', status: 'SCHEDULED' }), err => {
-    if (err) {
-      console.log('publish failed')
-    }
-  })
-  stan.publish('UpdateAttainmentDates', null, err => {
-    if (err) {
-      console.log('publish failed', 'UpdateAttainmentDates')
-    }
-  })
-}
-
-const scheduleStudentCheck = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'NO_STUDENT' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleStudentsByArray = async studentNumbers => {
-  try {
-    const tasks = await Schedule.find({ type: 'student', task: { $in: studentNumbers } })
-    publishAll(tasks, true)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const scheduleOldestNStudents = async amount => {
-  try {
-    const tasks = [
-      ...(await Schedule.find({ type: 'student' })
-        .sort({ updatedAt: 1 })
-        .limit(Number(amount)))
-    ]
-    publishAll(tasks)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const rescheduleCreated = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'CREATED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleScheduled = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'SCHEDULED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleFetched = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'FETCHED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleDaily = async () => {
-  const findStudentsRange = 10000
-  const highestStudentNumber = (await Schedule.find({ type: 'student', status: { $in: ['DONE', 'FETCHED'] } })
-    .sort({ task: -1 })
-    .limit(1))[0]
-  const regexResult = highestStudentNumber ? highestStudentNumber.task.match(/0(\d{7})\d/) : null
-  const minstudentnumber = 1000000
-  const studentNumberMid = (regexResult && Number(regexResult[1])) || minstudentnumber + findStudentsRange
-  const min = Math.max(studentNumberMid - findStudentsRange, minstudentnumber)
-  const max = Math.min(studentNumberMid + findStudentsRange, 9999999)
-
-  console.log({ findStudentsRange, minstudentnumber, studentNumberMid, min, max })
-
-  const activeStudents = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  const studentnumbers = activeStudents.map(t => t.task)
-  for (let i = min; i <= max; ++i) {
-    studentnumbers.push(`0${i}${getStudentNumberChecksum(i)}`)
-  }
-  const oldestStudents = [
-    ...(await Schedule.find({ type: 'student' })
-      .sort({ updatedAt: 1 })
-      .limit(10000))
-  ]
-  studentnumbers.push(...oldestStudents.map(t => t.task))
-
-  const tasks = [...new Set(studentnumbers)].map(sn => ({ task: sn }))
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-module.exports = {
-  scheduleActiveStudents,
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleMeta,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily,
-  scheduleStudentCheck
-}

old_oodikone_stuff/updater_scheduler/src/student_list_updater.js

@@ -1,62 +0,0 @@
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-const createTasks = async (fromStudents = []) => {
-  console.log('Creating studentnumbers to DB...')
-
-  const writeStudents = async studentsToAdd => {
-    const tasks = studentsToAdd.map(student => ({ task: student, status: 'CREATED', type: 'student', active: true }))
-    const bulkOperation = Schedule.collection.initializeUnorderedBulkOp()
-    for (const task of tasks) {
-      bulkOperation.insert(task)
-    }
-    await bulkOperation.execute()
-  }
-
-  const writeToDB = async studentnumbers => {
-    try {
-      await writeStudents(studentnumbers)
-    } catch (err) {
-      const BulkWriteResult = err.result.result
-      const duplicateIndexError = 11000
-      const realErrors = BulkWriteResult.writeErrors.filter(e => e.err.code !== duplicateIndexError)
-      if (realErrors.length > 0 || BulkWriteResult.writeConcernErrors.length > 0) {
-        console.log(err)
-        return false
-      }
-    }
-    return true
-  }
-
-  const mongoWriteBatchSize = 1000
-  let studentnumbers = []
-  if (fromStudents.length) {
-    for (let i = 0; i < fromStudents.length; i++) {
-      studentnumbers.push(fromStudents[i])
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === fromStudents.length - 1) {
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        console.log(`Created tasks from ${studentnumbers}`)
-        studentnumbers = []
-      }
-    }
-  } else {
-    const minStudentNumber = 1010000
-    const maxStudentNumber = 1530000
-    for (let i = minStudentNumber; i <= maxStudentNumber; ++i) {
-      const studentNumber = '0' + i + getStudentNumberChecksum(i)
-      studentnumbers.push(studentNumber)
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === maxStudentNumber) {
-        console.log('current studentnumber', i)
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        studentnumbers = []
-      }
-    }
-  }
-  console.log('Studentnumbers created to DB')
-}
-
-module.exports = { createTasks }

old_oodikone_stuff/updater_scheduler/src/util.js

@@ -1,43 +0,0 @@
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-// Shuffle in place, returns same array
-const shuffleArray = a => {
-  for (let i = a.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1))
-    const x = a[i]
-    a[i] = a[j]
-    a[j] = x
-  }
-  return a
-}
-
-const isValidStudentId = id => {
-  if (/^0\d{8}$/.test(id)) {
-    // is a 9 digit number
-    const multipliers = [7, 1, 3, 7, 1, 3, 7]
-    const checksum = id
-      .substring(1, 8)
-      .split('')
-      .reduce((sum, curr, index) => {
-        return (sum + curr * multipliers[index]) % 10
-      }, 0)
-    return (10 - checksum) % 10 == id[8]
-  }
-  return false
-}
-
-const getStudentNumberChecksum = studentNumber => {
-  const studentNumberString = String(studentNumber)
-  let checksumNumbers = [7, 3, 1]
-  let checksum = 0
-
-  for (let i = 0; i < studentNumberString.length; i++) {
-    // go from end t start
-    let currentNumber = studentNumberString[studentNumberString.length - (i + 1)]
-    checksum += currentNumber * checksumNumbers[i % checksumNumbers.length]
-  }
-
-  return (10 - (checksum % 10)) % 10
-}
-
-module.exports = { sleep, isValidStudentId, getStudentNumberChecksum, shuffleArray }

old_oodikone_stuff/updater_writer/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_writer/.eslintignore

@@ -1,2 +0,0 @@
-*.sql
-updater/test_assets

old_oodikone_stuff/updater_writer/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_writer/.gitignore

@@ -1,4 +0,0 @@
-node_modules
-
-models
-migrations

old_oodikone_stuff/updater_writer/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_writer/Dockerfile

@@ -1,18 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-RUN apk add postgresql-client
-
-COPY ./updater_writer/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY ./updater_writer /usr/src/app
-COPY ./shared/models /usr/src/app/models
-COPY ./shared/migrations /usr/src/app/database/migrations
-COPY ./shared/migrations_kone /usr/src/app/database/migrations_kone

old_oodikone_stuff/updater_writer/conf-backend.js

@@ -1,77 +0,0 @@
-require('dotenv').config()
-
-const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
-let DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const isTest = NODE_ENV === 'test'
-if (isTest) {
-  DB_URL = process.env.TEST_DB
-  DB_SCHEMA = process.env.TEST_DB_SCHEMA
-} else if (NODE_ENV === 'anon') {
-  DB_URL = process.env.ANON_DB
-}
-const frontend_addr = process.env.FRONT_URL
-const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
-const CERT_PATH = process.env.CERT_PATH // production/staging only
-const KEY_PATH = process.env.KEY_PATH // production/staging only
-const USERSERVICE_URL = process.env.USERSERVICE_URL
-const ANALYTICS_URL = process.env.ANALYTICS_URL
-const PORT = isTest ? 8079 : 8080
-const OODI_SECRET = process.env.OODI_SECRET
-
-const FEATURES = {
-  ERROR_HANDLER: false
-}
-
-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-if (process.env.NODE_ENV === 'dev' && process.env.FEATURES) {
-  const toggled = process.env.FEATURES.split(',')
-  toggled
-    .map(toggle => toggle.trim())
-    .forEach(feature => {
-      if (FEATURES[feature] !== undefined) {
-        FEATURES[feature] = true
-      }
-    })
-}
-
-const OODI = {
-  test: 'http://localhost',
-  anon: process.env.OODI_ADDR_ANON
-}
-
-const OODI_ADDR = OODI[process.env.NODE_ENV] || process.env.OODI_ADDR
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-const OODI_SECRET_HEADER_KEY = 'x-oodi-secret'
-
-let requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
-if (process.env.NODE_ENV === 'staging') {
-  requiredGroup = ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-}
-if (process.env.NODE_ENV === 'dev' || isTest) {
-  requiredGroup = null
-}
-
-module.exports = {
-  frontend_addr,
-  DB_URL,
-  redis,
-  TOKEN_SECRET,
-  DB_SCHEMA,
-  OODI_ADDR,
-  CERT_PATH,
-  KEY_PATH,
-  FEATURES,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
-  PORT,
-  ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  requiredGroup,
-  OODI_SECRET,
-  OODI_SECRET_HEADER_KEY,
-  isTest
-}

old_oodikone_stuff/updater_writer/database/connection.js

@@ -1,69 +0,0 @@
-const Sequelize = require('sequelize')
-const Umzug = require('umzug')
-
-const conf = require('../conf-backend')
-
-const sequelize = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA,
-  searchPath: conf.DB_SCHEMA,
-  logging: false
-})
-sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
-
-const initializeDatabaseConnection = async () => {
-  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelize.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
-      }
-      console.log('.')
-      await sleep(1000)
-    }
-  }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelize,
-          tableName: 'migrations'
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelize.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/database/migrations`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA
-        }
-      })
-      const migrations = await migrator.up()
-
-      console.log('Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
-  }
-}
-
-const forceSyncDatabase = async () => {
-  try {
-    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
-  } catch (e) {
-    // console.log(e)
-  }
-  console.log('OODI SYNC')
-  await sequelize.sync({ force: true })
-}
-
-module.exports = {
-  sequelize,
-  initializeDatabaseConnection,
-  forceSyncDatabase
-}

old_oodikone_stuff/updater_writer/database/dump_database.js

@@ -1,27 +0,0 @@
-const spawn = require('child_process').spawn
-const fs = require('fs')
-const { DB_SCHEMA } = require('../conf-backend')
-
-const dumpDatabase = async () => {
-  const { DB_URL } = process.env
-  const dumpOptions = ['-Fc', `-n${DB_SCHEMA}`, `${DB_URL}`]
-  console.log(dumpOptions)
-  const dumpProcess = spawn('pg_dump', dumpOptions, {
-    stdio: ['ignore', 'pipe', 'inherit']
-  })
-  dumpProcess.on('error', err => {
-    console.log(' dump process error:', err)
-  })
-  const writeStream = fs.createWriteStream(`./dumps/oodidb_prod-${new Date().toISOString()}`)
-
-  dumpProcess.stdout.pipe(writeStream)
-
-  dumpProcess.on('exit', async code => {
-    if (code !== 0) {
-      throw new Error('pg_dump: Bad exit code (' + code + ')')
-    } else {
-      console.log('database dump created')
-    }
-  })
-}
-module.exports = { dumpDatabase }

old_oodikone_stuff/updater_writer/database/force_sync_database.js

@@ -1,17 +0,0 @@
-const { seedAllMigrations } = require('./seed_migrations')
-const { forceSyncDatabase } = require('./connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-//sync()
-module.exports = { sync }

old_oodikone_stuff/updater_writer/database/seed_migrations.js

@@ -1,28 +0,0 @@
-const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
-
-const DEFAULT_PATH = `${process.cwd()}/database/migrations`
-const DEFAULT_PATH_KONE = `${process.cwd()}/database/migrations_kone`
-
-const saveMigrationsToDatabase = Migration => async filenames =>
-  Promise.all(filenames.map(name => Migration.upsert({ name })))
-
-const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(Migration)(filenames)
-}
-
-const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(MigrationKone)(filenames)
-}
-
-const seedAllMigrations = async () => {
-  await seedMigrations()
-  await seedMigrationsKone()
-}
-
-module.exports = {
-  seedMigrations,
-  seedAllMigrations
-}

old_oodikone_stuff/updater_writer/index.js

@@ -1,69 +0,0 @@
-const natsStreaming = require('node-nats-streaming')
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./updater/database_updater')
-const logger = require('./logger')
-const { initializeDatabaseConnection } = require('./database/connection')
-
-initializeDatabaseConnection()
-  .then(() => {
-    const stan = natsStreaming.connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-    console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-    const opts = stan.subscriptionOptions()
-    opts.setManualAckMode(true)
-    opts.setAckWait(10 * 60 * 1000) // some students have taken over 5 min to write!
-    // opts.setDeliverAllAvailable()
-    // opts.setDurableName('durable')
-    opts.setMaxInFlight(1)
-
-    stan.on('connect', function() {
-      const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
-      const attSub = stan.subscribe('UpdateAttainmentDates', 'updater.workers.attainments', opts)
-      const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
-
-      const writeStudent = priority => async msg => {
-        let data = null
-        try {
-          const start = new Date()
-          data = JSON.parse(msg.getData())
-          if (data.task === 'meta') {
-            await updateMeta(data.data)
-          } else {
-            await updateStudent(data.data)
-          }
-          stan.publish(
-            'status',
-            JSON.stringify({ task: data.task, status: 'DONE', timems: new Date() - start }, priority),
-            err => {
-              if (err) console.log(err)
-            }
-          )
-        } catch (err) {
-          console.log('update failed', data.task, err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      }
-
-      sub.on('message', writeStudent(false))
-      prioSub.on('message', writeStudent(true))
-
-      attSub.on('message', async msg => {
-        try {
-          await updateAttainmentMeta()
-          stan.publish('status', JSON.stringify({ task: 'attainment', status: 'DONE' }), err => {
-            if (err) {
-              console.log('publish failed')
-            }
-          })
-        } catch (err) {
-          console.log('attainment meta update failed', err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      })
-    })
-  })
-  .catch(e => {
-    process.exitCode = 1
-    console.log(e)
-  })

old_oodikone_stuff/updater_writer/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_writer/package.json

@@ -1,39 +0,0 @@
-{
-  "name": "updater_writer",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run jest",
-    "jest": "NODE_ENV=test jest --runInBand --verbose",
-    "test_docker": "docker exec -t updater_writer /bin/bash -c 'npm test'",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js",
-    "force_sync": "node -e \"require('./database/force_sync_database').sync()\"",
-    "dump": "node -e \"require('./database/dump_database').dumpDatabase()\""
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "dotenv": "^8.0.0",
-    "jest": "^24.8.0",
-    "lodash": "^4.17.19",
-    "node-nats-streaming": "^0.2.6",
-    "pg": "^8.6.0",
-    "sequelize": "^6.6.2",
-    "umzug": "^2.2.0",
-    "winston": "^3.2.1",
-    "winston-log2gelf": "^2.3.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.0.3",
-    "eslint": "^6.6.0",
-    "eslint-config-prettier": "^6.3.0",
-    "eslint-plugin-prettier": "^3.1.0",
-    "nodemon": "^1.19.4",
-    "prettier": "^1.18.2",
-    "prettier-eslint": "^9.0.0"
-  }
-}

old_oodikone_stuff/updater_writer/updater/database_updater.js

@@ -1,217 +0,0 @@
-const { notIn } = require('sequelize').Op
-const { sequelize } = require('../database/connection')
-const { sortBy, flatMap, uniqBy, sortedUniqBy } = require('lodash')
-
-const {
-  Student,
-  Credit,
-  Course,
-  CreditTeacher,
-  Teacher,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { updateAttainmentDates } = require('./update_attainment_dates')
-
-const getColumnsToUpdate = arr => (arr[0] ? Object.keys(arr[0]) : [])
-
-const updateAttainments = async (studyAttainments, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const courses = sortedUniqBy(sortBy(studyAttainments.map(e => e.course), 'code'), 'code')
-  const disciplines = sortBy(flatMap(studyAttainments, e => e.course.disciplines || []), 'discipline_id', 'course_id')
-  const providers = sortedUniqBy(
-    sortBy(flatMap(studyAttainments, e => e.course.providers), 'providercode'),
-    'providercode'
-  )
-  const courseproviders = sortBy(flatMap(studyAttainments, e => e.course.courseproviders), 'coursecode', 'providercode')
-  const credits = sortedUniqBy(sortBy(studyAttainments.map(e => e.credit), 'id'), 'id')
-  const teachers = sortedUniqBy(sortBy(flatMap(studyAttainments, e => e.teachers || []), 'id'), 'id')
-  const creditTeachers = sortBy(flatMap(studyAttainments, e => e.creditTeachers), 'teacher_id', 'credit_id')
-
-  await Promise.all([
-    Course.bulkCreate(courses, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(courses)
-    }),
-    Provider.bulkCreate(providers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(providers)
-    }),
-    Credit.bulkCreate(credits, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(credits)
-    }),
-    Teacher.bulkCreate(teachers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(teachers)
-    })
-  ])
-
-  await Promise.all([
-    // must be after courses
-    CourseDisciplines.bulkCreate(disciplines, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after providers
-    CourseProvider.bulkCreate(courseproviders, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after teachers
-    CreditTeacher.bulkCreate(creditTeachers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudyRights = async (studyRights, studentnumber, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const studyRightExtents = sortedUniqBy(sortBy(studyRights.map(e => e.studyRightExtent), 'extentcode'), 'extentcode')
-  const studyrights = sortedUniqBy(sortBy(studyRights.map(e => e.studyright), 'studyrightid'), 'studyrightid')
-  const elementDetails = sortedUniqBy(sortBy(flatMap(studyRights, e => e.elementDetails), 'code'), 'code')
-  const studyRightElements = uniqBy(
-    sortBy(flatMap(studyRights, e => e.studyRightElements), 'startdate', 'enddate', 'studyrightid', 'code'),
-    s => `${s.startdate}${s.enddate}${s.studyrightid}${s.code}`
-  )
-  const transfers = sortBy(
-    flatMap(studyRights, e => e.transfers),
-    'transferdate',
-    'studentnumber',
-    'studyrightid',
-    'sourcecode',
-    'targetcode'
-  )
-
-  const studyrightids = studyrights.map(s => s.studyrightid)
-  await Studyright.destroy(
-    { where: { student_studentnumber: studentnumber, studyrightid: { [notIn]: studyrightids } } },
-    { transaction }
-  )
-  await StudyrightElement.destroy({ where: { studentnumber } }, { transaction })
-
-  await Promise.all([
-    StudyrightExtent.bulkCreate(studyRightExtents, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyRightExtents)
-    }),
-
-    Studyright.bulkCreate(studyrights, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyrights)
-    }),
-
-    ElementDetails.bulkCreate(elementDetails, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(elementDetails)
-    }),
-
-    StudyrightElement.bulkCreate(studyRightElements, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-
-    Transfers.bulkCreate(transfers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudent = async student => {
-  let { studentInfo, studyAttainments, semesterEnrollments, studyRights } = student
-
-  // sort data to avoid deadlocks
-  semesterEnrollments = uniqBy(
-    sortBy(semesterEnrollments, 'studentnumber', 'semestercode'),
-    sE => `${sE.semestercode}${sE.studentnumber}`
-  )
-
-  const transaction = await sequelize.transaction()
-  try {
-    await Student.upsert(studentInfo, { transaction })
-
-    // Change this to bulkCreate after https://github.com/sequelize/sequelize/issues/11569
-    // is merged and released
-    for (let i = 0; i < semesterEnrollments.length; i++) {
-      await SemesterEnrollment.upsert(semesterEnrollments[i], { transaction })
-    }
-
-    if (studyAttainments) await updateAttainments(studyAttainments, transaction)
-
-    if (studyRights) await updateStudyRights(studyRights, studentInfo.studentnumber, transaction)
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-const updateAttainmentMeta = async () => {
-  await updateAttainmentDates()
-}
-
-const updateMeta = async ({
-  faculties,
-  courseRealisationsTypes,
-  semesters,
-  creditTypeCodes,
-  courseTypeCodes,
-  disciplines
-}) => {
-  const transaction = await sequelize.transaction()
-
-  try {
-    await Promise.all([
-      CourseType.bulkCreate(uniqBy(courseTypeCodes, 'coursetypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseTypeCodes)
-      }),
-      Organisation.bulkCreate(uniqBy(faculties, 'code'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(faculties)
-      }),
-      CourseRealisationType.bulkCreate(uniqBy(courseRealisationsTypes, 'realisationtypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseRealisationsTypes)
-      }),
-      Semester.bulkCreate(uniqBy(semesters, 'semestercode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(semesters)
-      }),
-      CreditType.bulkCreate(uniqBy(creditTypeCodes, 'credittypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(creditTypeCodes)
-      }),
-      Discipline.bulkCreate(uniqBy(disciplines, 'discipline_id'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(disciplines)
-      })
-    ])
-
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-module.exports = {
-  updateStudent,
-  updateMeta,
-  updateAttainmentMeta
-}

old_oodikone_stuff/updater_writer/updater/database_updater.test.js

@@ -1,518 +0,0 @@
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./database_updater')
-const {
-  Student,
-  Credit,
-  Course,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  Teacher,
-  CreditTeacher,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { students } = require('./test_assets/test_students')
-const meta = require('./test_assets/meta')
-const { sequelize, forceSyncDatabase } = require('../database/connection')
-const { seedMigrations } = require('../database/seed_migrations')
-jest.setTimeout(10000)
-describe('Updater works', () => {
-  beforeAll(async () => {
-    await forceSyncDatabase()
-    await seedMigrations()
-
-    await updateMeta(meta)
-    await updateStudent(students[0])
-    await updateStudent(students[1])
-    await updateStudent(students[2])
-    await updateAttainmentMeta()
-  })
-
-  afterAll(async () => {
-    await sequelize.close()
-  })
-
-  describe('All metadata is written', () => {
-    test('Organisations are written to database', async () => {
-      const organizations = await Organisation.findAll()
-      expect(organizations.length).toBe(17)
-      expect(organizations.find(_ => _.code === 'H50').name.find(n => n.langcode === 'fi').text).toBe(
-        'Matemaattis-luonnontieteellinen tiedekunta'
-      )
-    })
-
-    test('Course realisation types are written to database', async () => {
-      const courseRealisationTypes = await CourseRealisationType.findAll()
-      expect(courseRealisationTypes.length).toBe(18)
-      expect(courseRealisationTypes.map(_ => _.realisationtypecode)).not.toContain(null)
-      expect(courseRealisationTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Semesters are written to database', async () => {
-      const semesters = await Semester.findAll()
-      expect(semesters.length).toBe(160)
-      expect(semesters.map(_ => _.semestercode)).toEqual(expect.arrayContaining([...Array(160).keys()].map(_ => ++_)))
-      expect(semesters.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-      expect(semesters.map(_ => _.startdate)).not.toContain(null)
-      expect(semesters.map(_ => _.enddate)).not.toContain(null)
-      expect(semesters.map(_ => _.createdAt)).not.toContain(null)
-      expect(semesters.map(_ => _.updatedAt)).not.toContain(null)
-      expect(new Set(semesters.map(_ => _.yearcode)).size).toBe(80)
-      expect(semesters.find(_ => _.yearcode === 1).yearname).toBe('1950-51')
-    })
-
-    test('Credit types are written to database', async () => {
-      const creditTypes = await CreditType.findAll()
-      expect(creditTypes.length).toBe(11)
-      expect(creditTypes.map(_ => _.credittypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(creditTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Course types are written to database', async () => {
-      const courseTypes = await CourseType.findAll()
-      expect(courseTypes.length).toBe(35)
-      expect(courseTypes.map(_ => _.coursetypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(courseTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Disciplines are written to database', async () => {
-      const disciplines = await Discipline.findAll()
-      expect(disciplines.length).toBe(430)
-      expect(disciplines.map(_ => _.discipline_id)).toEqual(expect.arrayContaining([expect.any(String)]))
-      expect(disciplines.map(_ => _.name)[0]).toMatchObject({
-        fi: expect.any(String)
-      })
-    })
-  })
-  describe('Updater writes students the right way', () => {
-    test('Student info is written to database', async () => {
-      const students = await Student.findAll({
-        order: [['studentnumber', 'DESC']]
-      })
-      expect(students.length).toBe(3)
-      const {
-        studentnumber,
-        lastname,
-        firstnames,
-        abbreviatedname,
-        birthdate,
-        communicationlanguage,
-        creditcount,
-        dateofuniversityenrollment,
-        city_fi,
-        city_sv,
-        gender_code,
-        gender_fi,
-        createdAt,
-        updatedAt
-      } = students[0]
-
-      expect(studentnumber).toBe('014441008')
-      expect(lastname).toBe('Orttenvuori')
-      expect(firstnames).toBe('Oona')
-      expect(abbreviatedname).toBe('Orttenvuori Oona')
-      expect(birthdate.toISOString()).toBe(new Date('1966-04-24 00:00:00+00').toISOString())
-      expect(communicationlanguage).toBe('Finnish')
-      expect(creditcount).toBe(34)
-      expect(dateofuniversityenrollment).toBe(null)
-      expect(city_fi).toBe('HELSINKI')
-      expect(city_sv).toBe('HELSINGFORS')
-      expect(gender_code).toBe(2)
-      expect(gender_fi).toBe('Nainen')
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-
-      const { creditcount: creditcount1, dateofuniversityenrollment: dofue } = students[1]
-
-      expect(creditcount1).toBe(380)
-      expect(dofue.toISOString()).toBe(new Date('2003-07-31 21:00:00+00').toISOString())
-    })
-
-    test('Courses are written to database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const {
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        startdate,
-        enddate,
-        max_attainment_date,
-        min_attainment_date,
-        coursetypecode
-      } = courses[0]
-
-      expect(courses.length).toBe(147)
-      expect(code).toBe('00031')
-      expect(name.en).toBe('Master of Science (Agriculture and Forestry)')
-      expect(latest_instance_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(is_study_module).toBeTruthy()
-      expect(startdate.toISOString()).toBe(new Date('1899-12-31 00:00:00+00').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2112-12-20 00:00:00+00').toISOString())
-      expect(max_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(min_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-
-      expect(coursetypecode).toBe(8)
-
-      const { code: code1, is_study_module: ism, coursetypecode: ctc } = courses[6]
-
-      expect(code1).toBe('402512')
-      expect(ism).toBe(false)
-      expect(ctc).toBe(2)
-    })
-
-    test('Credits are written to database', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const {
-        id,
-        grade,
-        student_studentnumber,
-        credits,
-        ordering,
-        attainment_date,
-        isStudyModule,
-        lastmodifieddate,
-        course_code,
-        credittypecode /*, semestercode*/
-      } = allCredits[0]
-
-      expect(allCredits.length).toBe(150)
-      expect(id).toBe('2340421')
-      expect(grade).toBe('4')
-      expect(allCredits.map(_ => _.grade)).not.toContain(null)
-      expect(student_studentnumber).toBe('014441008')
-      expect(credits).toBe(2)
-      expect(allCredits.map(_ => _.credits)).not.toContain(null)
-      expect(ordering).toBe('1997-10-21')
-      expect(attainment_date.toISOString()).toBe(new Date('1997-10-21 00:00:00+00').toISOString())
-      expect(isStudyModule).toBe(false)
-      expect(lastmodifieddate).not.toBe(null)
-      expect(course_code).toBe('65002')
-      expect(allCredits.map(_ => _.course_code)).not.toContain(null)
-      expect(credittypecode).toBe(4)
-      expect(allCredits.map(_ => _.semestercode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-    })
-
-    test('Teachers are written to database', async () => {
-      const teachers = await Teacher.findAll({ order: [['name', 'ASC']] })
-      const { id, code, name } = teachers[0]
-
-      expect(teachers.length).toBe(51)
-      expect(id).toBe('9016947')
-      expect(code).toBe('hpyrhone')
-      expect(name).toBe('Ahvenniemi Emilia')
-    })
-
-    test('Credit Teachers are written to database', async () => {
-      const creditteachers = await CreditTeacher.findAll()
-
-      expect(creditteachers.length).toBe(119)
-      expect(creditteachers.map(_ => _.credit_id)).not.toContain(null)
-      expect(creditteachers.map(_ => _.teacher_id)).not.toContain(null)
-    })
-
-    test('Semester enrollments are written to database', async () => {
-      const semesterEnrollments = await SemesterEnrollment.findAll({
-        order: [['semestercode', 'ASC']]
-      })
-      const { id, enrollmenttype, enrollment_date, studentnumber, semestercode } = semesterEnrollments[0]
-
-      expect(semesterEnrollments.length).toBe(28)
-      expect(id).toEqual(expect.any(String))
-      expect(enrollmenttype).toBe(1)
-      expect(enrollment_date.toISOString()).toBe(new Date('2003-08-17 00:00:00+00').toISOString())
-      expect(studentnumber).toBe('014272112')
-      expect(semestercode).toBe(107)
-    })
-
-    test('Providers are written to database', async () => {
-      const providers = await Provider.findAll({
-        order: [['providercode', 'DESC']]
-      })
-      const { providercode, name } = providers[0]
-
-      expect(providers.length).toBe(20)
-      expect(providercode).toBe('H906')
-      expect(name).toMatchObject({
-        en: 'Language Centre',
-        fi: 'Kielikeskus',
-        sv: 'Språkcentrum'
-      })
-      expect(providers.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course providers are written to database', async () => {
-      const courseproviders = await CourseProvider.findAll()
-
-      expect(courseproviders.length).toBe(123)
-      expect(courseproviders.map(_ => _.coursecode)).not.toContain(null)
-      expect(courseproviders.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course disciplines are written to database', async () => {
-      const courseDisciplines = await CourseDisciplines.findAll()
-      expect(courseDisciplines.length).toBe(40)
-      expect(courseDisciplines.map(_ => _.discipline_id)).not.toContain(null)
-      expect(courseDisciplines.map(_ => _.course_id)).not.toContain(null)
-    })
-
-    test('Study right extent are written to database', async () => {
-      const studyrightextent = await StudyrightExtent.findAll()
-      const { extentcode, name, createdAt, updatedAt } = studyrightextent[0]
-
-      expect(studyrightextent.length).toBe(2)
-      expect(extentcode).toBe(1)
-      expect(name.en).toBe("Bachelor's Degree")
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('Study rights are written to database', async () => {
-      const studyrights = await Studyright.findAll({
-        order: [['studyrightid', 'DESC']]
-      })
-      const {
-        studyrightid /*, canceldate*/,
-        cancelorganisation,
-        enddate,
-        givendate,
-        graduated,
-        prioritycode,
-        startdate,
-        studystartdate,
-        organization_code,
-        student_studentnumber,
-        extentcode
-      } = studyrights[0]
-
-      expect(studyrights.length).toBe(4)
-      expect(studyrightid).toBe('67628057')
-      expect(cancelorganisation).toBe('4325')
-      expect(enddate.toISOString()).toBe(new Date('2010-04-12 00:00:00+00').toISOString())
-      expect(givendate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(graduated).toBe(1)
-      expect(prioritycode).toBe(30)
-      expect(startdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(studystartdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(organization_code).toBe('4325')
-      expect(student_studentnumber).toBe('011120775')
-      expect(extentcode).toBe(1)
-    })
-
-    test('element details are written to database', async () => {
-      const elementDetails = await ElementDetails.findAll({
-        order: [['code', 'DESC']]
-      })
-      const { code, name, type, createdAt, updatedAt } = elementDetails[0]
-
-      expect(elementDetails.length).toBe(12)
-      expect(code).toBe('A2004')
-      expect(name.en).toBe('Government Decree (794/2004) on University Degrees')
-      expect(type).toBe(15)
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('study right elements are written to database', async () => {
-      const studyright_elements = await StudyrightElement.findAll({
-        order: ['startdate', 'enddate', 'studyrightid', 'code']
-      })
-      const { id, startdate, enddate, createdAt, updatedAt, studyrightid, code, studentnumber } = studyright_elements[0]
-
-      expect(studyright_elements.length).toBe(21)
-      expect(id).toBe(11)
-      expect(startdate.toISOString()).toBe(new Date('2003-07-31T21:00:00.000Z').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2007-08-24T21:00:00.000Z').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studyrightid).toBe('45853818')
-      expect(code).toBe('A1995MAAT')
-      expect(studentnumber).toBe('014272112')
-    })
-    test('transfers are written to database', async () => {
-      const transfers = await Transfers.findAll()
-      const {
-        id,
-        transferdate,
-        createdAt,
-        updatedAt,
-        studentnumber,
-        studyrightid,
-        sourcecode,
-        targetcode
-      } = transfers[0]
-
-      expect(transfers.length).toBe(1)
-      expect(id).toBe('1')
-      expect(transferdate.toISOString()).toBe(new Date('2009-07-30 21:00:00+00').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studentnumber).toBe('014272112')
-      expect(studyrightid).toBe('62346253')
-      expect(sourcecode).toBe('820016')
-      expect(targetcode).toBe('820042')
-    })
-  })
-  describe('Updater updates existing students correctly', () => {
-    beforeAll(async () => {
-      const { studentInfo, studyRights, studyAttainments } = students[0]
-      students[0].studentInfo = {
-        ...studentInfo,
-        email: 'Calgary.Flames@NHLonJätettä.fi',
-        mobile: '0400521981',
-        address: 'Yliopistonkatu 4',
-        creditcount: 322,
-        age: 41,
-        lastname: 'Tauriainen',
-        gender_code: 1,
-        abbreviatedname: 'Tauriainen Maria Helena'
-      }
-      const newCredit = {
-        id: 'ayyyyylmao',
-        grade: '5',
-        credits: 5,
-        ordering: '2019.05.04',
-        credittypecode: 1,
-        student_studentnumber: '011120775',
-        attainment_date: new Date(),
-        course_code: '420_666',
-        semestercode: 138,
-        isStudyModule: false
-      }
-      const newCourse = {
-        code: '420_666',
-        name: {
-          fi: 'Kaljan juonti',
-          en: 'Beer drinking',
-          sv: 'dryckade öl'
-        },
-        latest_instance_date: new Date(),
-        is_study_module: false,
-        coursetypecode: 1,
-        disciplines: [],
-        startdate: new Date(),
-        enddate: new Date(),
-        providers: [],
-        courseproviders: []
-      }
-
-      const newCreditTeachers = [
-        {
-          credit_id: 'ayyyyylmao',
-          teacher_id: '9016417'
-        }
-      ]
-      const newAttainment = {
-        credit: newCredit,
-        creditTeachers: newCreditTeachers,
-        course: newCourse
-      }
-      students[0].studyAttainments = [
-        ...studyAttainments,
-        newAttainment,
-        {
-          ...students[1].studyAttainments[2],
-          creditTeachers: [],
-          credit: {
-            ...students[1].studyAttainments[2].credit,
-            id: 'Kvas',
-            student_studentnumber: '011120775'
-          }
-        }
-      ]
-      students[0].studyAttainments[2] = {
-        ...students[0].studyAttainments[2],
-        credit: {
-          ...students[0].studyAttainments[2].credit,
-          grade: '0'
-        }
-      }
-      const newStudyRight = {
-        ...studyRights[0],
-        studyright: {
-          ...studyRights[0].studyright
-        }
-      }
-      students[0].studyRights = [newStudyRight]
-
-      await updateStudent(students[0])
-    })
-
-    test('StudentInfo for existing student is updated correclty', async () => {
-      const updatedStudent = await Student.findOne({
-        where: {
-          studentnumber: '011120775'
-        }
-      })
-      const {
-        studentnumber,
-        lastname,
-        abbreviatedname,
-        gender_code,
-        email,
-        age,
-        gender_fi,
-        address,
-        mobile
-      } = updatedStudent
-
-      expect(studentnumber).toBe('011120775')
-      expect(lastname).toBe('Tauriainen')
-      expect(abbreviatedname).toBe('Tauriainen Maria Helena')
-      expect(email).toBe('Calgary.Flames@NHLonJätettä.fi')
-      expect(gender_code).toBe(1)
-      expect(gender_fi).toBe('Nainen')
-      expect(age).toBe(41)
-      expect(address).toBe('Yliopistonkatu 4')
-      expect(mobile).toBe('0400521981')
-    })
-    test('Existing courses are updated (not duplicated) in database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const newCourse = courses.find(_ => _.code === '420_666')
-      expect(courses.length).toBe(148)
-
-      expect(newCourse).not.toBe(undefined)
-    })
-
-    test('Updating credits works', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const updatedCredit = allCredits.find(_ => _.id === students[0].studyAttainments[2].credit.id)
-      expect(allCredits.length).toBe(152)
-      expect(updatedCredit.grade).toBe('0')
-    })
-
-    test('Studyrights are rewritten', async () => {
-      const updatedStudyRights = await Studyright.findAll({
-        where: {
-          student_studentnumber: '011120775'
-        }
-      })
-      expect(updatedStudyRights.length).toBe(1)
-      // theres really not a lot to test here since everything related to studyrights are just rewritten
-    })
-  })
-})

old_oodikone_stuff/updater_writer/updater/update_attainment_dates.js

@@ -1,28 +0,0 @@
-const { sequelize } = require('../database/connection')
-
-const updateAttainmentDates = async () => {
-  const { schema } = sequelize.options
-  const transaction = await sequelize.transaction()
-  try {
-    await sequelize.query(
-      `
-      -- Block writing to table to avoid deadlocks:
-      LOCK TABLE ONLY ${schema}.course IN EXCLUSIVE MODE;
-
-      UPDATE  ${schema}.course
-      SET max_attainment_date = cr.max, min_attainment_date = cr.min
-      FROM (select course_code, max(attainment_date), min(attainment_date) from ${schema}.credit group by course_code) cr
-      WHERE course.code=cr.course_code;
-     `,
-      { transaction }
-    )
-    await transaction.commit()
-  } catch (e) {
-    await transaction.rollback()
-    throw e
-  }
-}
-
-module.exports = {
-  updateAttainmentDates
-}

package.json

@@ -2,10 +2,11 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "concurrently": "concurrently",
     "cypress": "cypress",
+    "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
@@ -13,26 +14,9 @@
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
     "prepare": "husky install"
   },
-  "lint-staged": {
-    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
-    "*.{js,json,md}": "prettier --write",
-    "*.css": "stylelint --fix",
-    ".github/workflows/*": "actionlint",
-    "Dockerfile": "hadolint",
-    "*.sh": "shellcheck -x",
-    "docker-compose.yml": "docker-compose config -q",
-    "docker-compose.real.yml": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q",
-    "docker-compose.ci.yml": "docker-compose -f docker-compose.ci.yml config -q",
-    "docker-compose.test.yml": "docker-compose -f docker-compose.test.yml config -q"
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/UniversityOfHelsinkiCS/oodikone.git"
@@ -43,27 +27,31 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/eslint-parser": "^7.14.7",
+    "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
-    "concurrently": "^6.2.0",
-    "cypress": "^7.7.0",
-    "eslint": "^7.30.0",
+    "concurrently": "^6.2.1",
+    "cypress": "^8.3.1",
+    "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.23.4",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-prettier": "^3.4.1",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.1",
-    "lint-staged": "^11.0.1",
+    "husky": "^7.0.2",
+    "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",
     "stylelint-config-prettier": "^8.0.2",
     "stylelint-config-standard": "^22.0.0",
     "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
   }
 }