Fix Statistics-tab crashing if stat is undefined

body-parser now included in express, see:
https://expressjs.com/en/changelog/4x.html#4.16.0

.env.template

@@ -0,0 +1,4 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+IMPORTER_DB_API_URL=
+PATE_TOKEN=

.eslintrc.json

@@ -8,17 +8,10 @@
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
   "rules": {
-    "no-console": "off",
+    "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error"

.github/workflows/release-oodikone-staging.yaml

@@ -1,143 +0,0 @@
-name: "Release oodikone to staging"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-oodikone-staging.yaml'
-      - 'services/**'
-
-jobs:
-
-  build_and_release_backend:
-    name: "Build and release backend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-          cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
-            type=registry,ref=toska/oodikone2-backend:staging
-            type=registry,ref=toska/oodikone2-backend:latest
-          cache-to: type=inline
-          context: services/backend
-          push: true
-          tags: toska/oodikone2-backend:staging
-          target: production
-
-  build_and_release_frontend:
-    name: "Build and release frontend"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Get current time"
-        id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          build-args: |
-            SENTRY_RELEASE=${{ github.sha }}
-            SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
-          cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
-            type=registry,ref=toska/oodikone2-frontend:staging
-            type=registry,ref=toska/oodikone2-frontend:latest
-          cache-to: type=inline
-          context: services/frontend
-          push: true
-          tags: toska/oodikone2-frontend:staging
-          target: production
-      - name: "Create a container from built image "
-        run: docker create --name frontend toska/oodikone2-frontend:staging
-      - name: "Copy assets from the container"
-        run: docker cp frontend:/usr/src/app/build ./build
-      - name: "Create Sentry release"
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: toska
-          SENTRY_PROJECT: oodikone
-        with:
-          environment: staging
-          set_commits: 'skip'
-          sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
-          version: ${{ github.sha }}
-
-  build_and_release_userservice:
-    name: "Build and release userservice"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build and push"
-        uses: docker/build-push-action@v2
-        with:
-          cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
-            type=registry,ref=toska/oodikone2-userservice:staging
-            type=registry,ref=toska/oodikone2-userservice:latest
-          cache-to: type=inline
-          context: services/userservice
-          push: true
-          tags: toska/oodikone2-userservice:staging
-          target: production
-
-  report_success:
-    runs-on: ubuntu-latest
-    name: "Report deployment success to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: success()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release_backend
-    - build_and_release_frontend
-    - build_and_release_userservice
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -29,7 +29,6 @@ jobs:
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
@@ -66,7 +65,6 @@ jobs:
             SENTRY_ENVIRONMENT=production
             BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
@@ -103,11 +101,19 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
       - name: "Build and push"
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
           cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
             type=registry,ref=toska/oodikone2-userservice:staging
             type=registry,ref=toska/oodikone2-userservice:latest
           cache-to: type=inline

.github/workflows/release-sis-updater-scheduler.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-scheduler"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-scheduler.yaml'
       - 'updater/sis-updater-scheduler/**'

.github/workflows/release-sis-updater-worker.yaml

@@ -3,7 +3,7 @@ name: "Release sis-updater-worker"
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-worker.yaml'
       - 'updater/sis-updater-worker/**'

.github/workflows/test-oodikone.yaml

@@ -1,149 +1,161 @@
-name: "Test oodikone"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
-    branches-ignore:
+    branches:
       - master
     paths:
-      - .github/workflows/test-oodikone.yaml
-      - cypress/**
-      - cypress.json
-      - services/**
-      - docker-compose.ci.yml
-      - nginx.conf
-      - package*.json
+      - '.github/workflows/test-oodikone.yaml'
+      - 'cypress/**'
+      - 'cypress.json'
+      - 'services/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'nginx.conf'
+      - 'package*.json'
 
 jobs:
   # === Static tests / linting ===
 
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/backend && npm ci
-      - name: "Run backend linter"
+      - name: 'Run backend linter'
         run: npm run eslint services/backend
 
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/frontend && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run concurrently \
-              'eslint --ext .jsx,.js services/frontend' \
-              'stylelint services/frontend/src/**/*.css'
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
 
   lint_userservice:
-    name: "Lint userservice"
+    name: 'Lint userservice'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: "Install dependencies"
+      - name: 'Install dependencies'
         run: npm ci && cd services/userservice && npm ci
-      - name: "Run frontend linter"
+      - name: 'Run frontend linter'
         run: npm run eslint services/userservice
 
   # === Build images ===
 
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-backend:trunk
             type=registry,ref=toska/oodikone2-backend:staging
             type=registry,ref=toska/oodikone2-backend:latest
           cache-to: type=inline
           context: services/backend
           push: false
-          tags: toska/oodikone2-backend:trunk
+          tags: toska/oodikone2-backend:staging
           target: production
           outputs: type=docker,dest=/tmp/backend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
 
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
-            type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
             type=registry,ref=toska/oodikone2-frontend:latest
           cache-to: type=inline
           context: services/frontend
           push: false
-          tags: toska/oodikone2-frontend:trunk
+          tags: toska/oodikone2-frontend:staging
           target: production
           outputs: type=docker,dest=/tmp/frontend.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
 
   build_userservice:
-    name: "Build userservice"
+    name: 'Build userservice'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: docker/setup-buildx-action@v1
-      - name: "Build image and export it"
+      - name: 'Build image and export it'
         uses: docker/build-push-action@v2
         with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
           cache-from: |
-            type=registry,ref=toska/oodikone2-userservice:trunk
             type=registry,ref=toska/oodikone2-userservice:staging
             type=registry,ref=toska/oodikone2-userservice:latest
           cache-to: type=inline
           context: services/userservice
           push: false
-          tags: toska/oodikone2-userservice:trunk
+          tags: toska/oodikone2-userservice:staging
           target: production
           outputs: type=docker,dest=/tmp/userservice.tar
-      - name: "Upload image for other jobs"
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/userservice.tar
 
   # === Integration tests ===
+  #
   run_userservice_integration_tests:
-    name: "Run userservice integration tests"
+    name: 'Run userservice integration tests'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Run tests included in docker build"
+      - name: 'Run tests included in docker build'
         run: docker-compose --file docker-compose.test.yml up --build userservice-test
 
   # === End-to-end tests ===
 
   test_course_statistics:
-    name: "Test course statistics"
+    name: 'Test course statistics'
     needs:
       - build_backend
       - build_frontend
@@ -151,42 +163,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
       - build_backend
       - build_frontend
@@ -194,42 +206,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
-    name: "Test population statistics"
+    name: 'Test population statistics'
     needs:
       - build_backend
       - build_frontend
@@ -237,42 +249,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Population_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
-    name: "Test filters"
+    name: 'Test filters'
     needs:
       - build_backend
       - build_frontend
@@ -280,42 +292,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
-    name: "Test student statistics"
+    name: 'Test student statistics'
     needs:
       - build_backend
       - build_frontend
@@ -323,42 +335,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Student_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
-    name: "Test studyprogramme overview"
+    name: 'Test studyprogramme overview'
     needs:
       - build_backend
       - build_frontend
@@ -366,42 +378,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Studyprogramme_overview.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
-    name: "Test teachers and users"
+    name: 'Test teachers and users'
     needs:
       - build_backend
       - build_frontend
@@ -409,24 +421,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -435,20 +447,20 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
       - build_backend
       - build_frontend
@@ -456,91 +468,127 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
+      - name: 'Run services, pull rest of the needed images'
         run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Check that exit statuses are not errors"
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
-      - name: "Print logs on failure"
+      - name: 'Print logs on failure'
         if: failure()
         run: docker-compose --file docker-compose.ci.yml logs
 
-  push_to_dockerhub:
-    name: "Push images to docker hub for caching"
-    needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
+  # === Release ===
+
+  release_images:
+    name: 'Release built images'
     runs-on: ubuntu-latest
+    needs:
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v2
-      - name: "Download images"
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: "Load built images"
+      - name: 'Load built images'
         run: |
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to DockerHub"
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Push to docker hub"
-        run: docker-compose --file docker-compose.ci.yml push
+      - name: 'Create a container from frontend'
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: 'Copy assets from the container'
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+          docker push toska/oodikone2-userservice:staging
+      - name: 'Send deployment success notification to slack'
+        if: success()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+      - name: 'Send deployment failure notification to slack'
+        if: failure()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
 
-  report_failure:
+  report_test_failure:
     runs-on: ubuntu-latest
-    name: "Report test failure to oodikone notifications"
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - run_userservice_integration_tests
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_population_statistics
-    - test_student_statistics
-    - test_studyprogramme_overview
-    - test_teachers_and_users
-    - test_service_exit_statuses
-    - push_to_dockerhub
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
       - uses: actions/checkout@v2
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

README.md

@@ -7,15 +7,22 @@ An application for analyzing university data, running at [https://oodikone.cs.he
 To run Oodikone locally, you will need the following:
 
 1. Applications:
-    - [Docker](https://docs.docker.com/install/) (version 20.10+),
-    - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
-    - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
-3. (Optional) For real data setup, access to Toskas production servers.
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
 ## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
@@ -26,20 +33,18 @@ _Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary
 What different CLI options do:
 
 1. Set up oodikone from scratch:
-    - Cleans up any previous installations
-    - Installs all needed npm packages locally
-    - Sets up pre-commit linting hooks.
-    - Pulls and builds all needed Docker images and sets up dockerized development environment. 
-    - Setup happens with anonymized data, which developers should use by default.
-    - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
 2. Reset all real data:
-    - Cleans up any previous real data databases
-    - Downloads needed database dumps from production servers.
-    - Creates real data databases and populates them with downloaded dumps.
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
 3. Reset sis-importer data
-    - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
-4. Reset old oodi data:
-    - Like option 2, but for old oodi data, which is not refreshed anymore. Is only needed when setting up real databases after running option 1.
+   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
 
 ## ⌨️ Development
 
@@ -85,34 +90,35 @@ Anon data contains some preset user types, corresponding to most usual user type
 
 ### Linting
 
-Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`. 
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question. 
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually. 
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
 However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-## 🔨 Testing & CI 
+## 🔨 Testing & CI
 
 There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
 - Cypress
-    - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
-    - are defined in cypress -folder and cypress.json
-    - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
-    - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
 - Jest
-    - are ran with `docker-compose.test.yml`
-    - test command and environment themselves are in Dockerfile for service in question
-    - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
 Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
+
 - oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
-- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
-- after successful merge to `master`, oodikone is deployed to staging
+- tests are run on every push.
+- after successful test run, oodikone is deployed to staging
 - after creating a release, oodikone is deployed to production
-- updater is deployed to production on every push to `trunk`.
+- updater is deployed to production on every push, no tests included
 
 ## ❓FAQ
 
@@ -120,12 +126,16 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
 
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
 ## ✌🏼 Maintainers and contribution
 
-Toska of course. 
+Toska of course.
 
 University of Helsinki

adminer-plugins/login-servers.php

@@ -8,7 +8,6 @@ $driver = "pgsql";
 return new AdminerLoginServers(
     $servers = array(
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
-        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),
         "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
         "user-db" => array("server" => "user-db", "driver" => $driver)

cli.sh

@@ -51,7 +51,6 @@ options=(
   "Set up oodikone from scratch."
   "Reset all real data."
   "Reset sis-importer data."
-  "Reset old oodi data."
   "Quit."
 )
 
@@ -69,8 +68,6 @@ while true; do
         reset_all_real_data;;
       "Reset sis-importer data.")
         reset_sis_importer_data;;
-      "Reset old oodi data.")
-        reset_old_oodi_data;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress/integration/Filters.js

@@ -283,9 +283,9 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('courseFilter-header', () => {
       const courses = ['MAT11001', 'TKT20004']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
-      checkFilteringResult(4)
+      checkFilteringResult(5)
       cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
       checkFilteringResult(3)
       cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)

cypress/integration/Studyprogramme_overview.js

@@ -13,33 +13,33 @@ describe('Studyprogramme overview', () => {
     // Column amount depends on user. Note that with dev user, you'll see even more
     // stuff than with admin or basic.
     const CSBachPopulationProgress2018ForAdmin = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
-      '11',
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
       '1',
-      '0',
-      '150',
-      '123',
-      '76',
-      '41',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
       '16',
     ]
     const CSBachPopulationProgress2018ForBasic = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
       '1',
-      '150',
-      '123',
-      '76',
-      '41',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
       '16',
     ] // cancelled and transferred from are missing
 
@@ -52,7 +52,7 @@ describe('Studyprogramme overview', () => {
       .each((elem, index) => {
         cy.wrap(elem).contains(population[index])
       })
-    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
     cy.get('table')
       .eq(1)
       .contains('2019')

cypress/integration/Users.js

@@ -14,7 +14,6 @@ describe('Users tests', () => {
       cy.get('.navBar').contains('Users').should('not.exist')
       cy.get('.navBar').contains('Faculty').should('not.exist')
       cy.get('.navBar').contains('Updater').should('not.exist')
-      cy.get('.navBar').contains('Sandbox').should('not.exist')
     })
   })
 

cypress/support/commands.js

@@ -10,6 +10,7 @@ const adminUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
 }
 
 const basicUserHeaders = {
@@ -19,6 +20,7 @@ const basicUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
@@ -28,6 +30,7 @@ const onlycoursestatisticsUserHeaders = {
   hyGroupCn: 'grp-oodikone-basic-users',
   eduPersonAffiliation: 'member',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
 }
 
 const norightsUserHeaders = {
@@ -37,6 +40,7 @@ const norightsUserHeaders = {
   hygroupcn: 'grp-oodikone-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
 }
 
 const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]

docker-compose.ci.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -10,11 +8,10 @@ services:
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
+      CI: "true"
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
@@ -24,23 +21,18 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:trunk
+    image: toska/oodikone2-backend:staging
 
   frontend:
     <<: *defaults
     container_name: frontend
-    image: toska/oodikone2-frontend:trunk
+    image: toska/oodikone2-frontend:staging
 
   kone-db:
     <<: *defaults
     container_name: kone-db
     image: registry.toska.cs.helsinki.fi/kone-db
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-
   redis:
     <<: *defaults
     command: "redis-server --appendonly yes"
@@ -65,10 +57,11 @@ services:
     depends_on:
       - user-db
     environment:
+      CI: "true"
       DB_URL: postgres://postgres@user-db:5432/user-db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:trunk
+    image: toska/oodikone2-userservice:staging
 
   nginx:
     image: nginx:latest

docker-compose.real.yml

@@ -2,7 +2,6 @@ services:
 
   backend:
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 

docker-compose.yml

@@ -5,7 +5,6 @@ x-build_defaults: &build_defaults
   target: development
 
 services:
-
   adminer:
     <<: *defaults
     container_name: adminer
@@ -25,11 +24,11 @@ services:
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
+    env_file:
+      - .env
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
@@ -70,22 +69,13 @@ services:
     volumes:
       - kone-db-data:/data
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    profiles:
-      - oodikone
-    volumes:
-      - oodi-db-data:/data
-
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -95,6 +85,7 @@ services:
     image: registry.toska.cs.helsinki.fi/sis-db
     volumes:
       - sis-db-data:/data
+    shm_size: 2gb
 
   sis-importer-db:
     <<: *defaults
@@ -107,7 +98,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -198,7 +189,6 @@ volumes:
   backend-nodemod:
   frontend-nodemod:
   kone-db-data:
-  oodi-db-data: # TODO: Remove when oodi stuff has been removed from oodikone
   redis-data:
   sis-db-data:
   sis-importer-db-data:

old_oodikone_stuff/old-backend-tests/course_stats_counter.test.js

@@ -1,197 +0,0 @@
-const { CourseStatsCounter } = require('./course_stats_counter')
-
-describe('CourseStatsCounter tests', () => {
-  const studentnumbers = ['1', '2', '3', '4', '5']
-  const coursecode = 'COURSE_01'
-  const coursename = 'COURSE NAME'
-
-  const createCounter = () => new CourseStatsCounter(coursecode, coursename, studentnumbers.length)
-
-  describe('counter constructor should set course values correctly', () => {
-    const counter = createCounter()
-
-    test('course name should be set after creation', () => {
-      expect(counter.course.name).toBe(coursename)
-    })
-
-    test('course code should be set after creation', () => {
-      expect(counter.course.code).toBe(coursecode)
-    })
-  })
-
-  describe('mark passed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '5'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, true, false, false)
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '0'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, false, true, false)
-
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('failed students should have studentnumber as key', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber as key', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then passed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '5', true, false, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber as key', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have student', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { failed, passed, failedMany } = counter.students
-
-    test('failed students should have studentnumber', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-
-    test('failed many students should have students', () => {
-      expect(failedMany).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then failed then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, failedMany, retryPassed } = counter.students
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('failedMany should not have studentnumber', () => {
-      expect(failedMany).not.toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then improved grade for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', true, false, false)
-    counter.markCredit(student, '5', false, false, true)
-
-    const { passed, improvedPassedGrade } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should have student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-  })
-
-  describe('mark improved grade then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', false, false, true)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, improvedPassedGrade, failed } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should contain student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-
-    test('failed should not contain student', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/globals.js

@@ -1,17 +0,0 @@
-const { sequelize } = require('../../src/models/index')
-const { sequelizeKone } = require('../../src/models/models_kone')
-const { redisClient } = require('../../src/services/redis')
-const { forceSyncDatabase } = require('../../src/database/connection')
-const { seedAllMigrations } = require('../../src/database/seed_migrations')
-
-beforeAll(async () => {
-  await forceSyncDatabase()
-  await seedAllMigrations()
-})
-
-afterAll(async () => {
-  await sequelize.close()
-  await sequelizeKone.close()
-  // https://stackoverflow.com/a/54560610
-  await new Promise(res => redisClient.quit(() => setImmediate(res)))
-})

old_oodikone_stuff/old-backend-tests/services/population.test.js

@@ -1,227 +0,0 @@
-const {
-  Student,
-  Course,
-  ElementDetails,
-  StudyrightElement,
-  Studyright,
-  Credit,
-  StudyrightExtent,
-  Semester,
-  SemesterEnrollment
-} = require('../../../src/models/index')
-const { optimizedStatisticsOf } = require('../../../src/services/populations')
-jest.mock('../../../src/services/semesters')
-const semesterService = require('../../../src/services/semesters')
-jest.setTimeout(10000)
-
-const langify = name => ({
-  en: `${name}_en`,
-  fi: `${name}_fi`,
-  sv: `${name}_sv`
-})
-
-const SEMESTER = { SPRING: 'SPRING', FALL: 'FALL' }
-
-const elementdetails = {
-  bsc: {
-    code: 'Element_BSC',
-    type: 10,
-    name: langify('Bachelor of Science')
-  },
-  maths: {
-    code: 'Element_MATH',
-    type: 20,
-    name: langify('Mathematics')
-  },
-  cs: {
-    code: 'ELEMENT_CS',
-    type: 20,
-    name: langify('Computer Science')
-  }
-}
-
-const courses = {
-  elements_of_ai: {
-    code: 'COURSE-1',
-    name: langify('ELEMENTS_OF_AI')
-  }
-}
-
-const semesters = {
-  fall: {
-    semestercode: 123,
-    name: { en: 'Autumn 2011', fi: 'Syksy 2011', sv: 'Hösten 2011' },
-    startdate: new Date('2011-07-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  spring: {
-    semestercode: 124,
-    name: { en: 'Spring 2012', fi: 'Kevät 2012', sv: 'Våren 2012' },
-    startdate: new Date('2011-12-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  // used for mocking current
-  current: {
-    semestercode: 125,
-    name: { en: 'Autumn 2012', fi: 'Syksy 2012', sv: 'Hösten 2012' },
-    startdate: new Date('2012-08-01'),
-    enddate: new Date('2012-12-31'),
-    yearcode: 63,
-    yearname: '2012-13'
-  }
-}
-
-const createQueryObject = (year, semester, codes, months) => ({
-  studyRights: codes,
-  year,
-  semesters: [semester],
-  months
-})
-
-afterAll(async () => {})
-
-describe('optimizedStatisticsOf tests', () => {
-  const courseinstanceFall = {
-    id: 1,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2011-09-31 21:00:00+00')
-  }
-
-  const courseinstanceSpring = {
-    id: 2,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2012-02-31 21:00:00+00')
-  }
-
-  const student = {
-    studentnumber: '1234'
-  }
-
-  const studyrightextents = {
-    bachelors: {
-      extentcode: 1,
-      name: langify("bachelor's degree extent")
-    }
-  }
-
-  const studyright = {
-    studyrightid: 1,
-    studystartdate: new Date('2011-07-31 21:00:00+00'),
-    student_studentnumber: student.studentnumber,
-    extentcode: studyrightextents.bachelors.extentcode
-  }
-
-  const creditFall = {
-    id: 'CREDIT-1',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceFall.course_code,
-    attainment_date: courseinstanceFall.coursedate,
-    semestercode: 123
-  }
-
-  const creditSpring = {
-    id: 'CREDIT-2',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceSpring.course_code,
-    attainment_date: courseinstanceSpring.coursedate,
-    semestercode: 124
-  }
-
-  const semesterEnrollments = {
-    fall: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.fall.semestercode,
-      enrollment_date: semesters.fall.startdate
-    },
-    spring: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.spring.semestercode,
-      enrollment_date: semesters.spring.startdate
-    },
-    current: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.current.semestercode,
-      enrollment_date: semesters.current.startdate
-    }
-  }
-
-  const studyrightelements = {
-    bsc: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.bsc.code,
-      studentnumber: student.studentnumber
-    },
-    maths: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.maths.code,
-      studentnumber: student.studentnumber
-    }
-  }
-
-  describe(`
-    Student with:
-    - BSc studyright (2011-07-31 – 2016-12-20),
-    - Mathematics studyright (2011-07-31 – 2016-12-20),
-    - Two credits in 2011-09-31 and 2012-02-31.
-    `, () => {
-    beforeAll(async () => {
-      await Semester.bulkCreate([semesters.fall, semesters.spring, semesters.current])
-      await Student.create(student)
-      await Course.create(courses.elements_of_ai)
-      await Credit.bulkCreate([creditFall, creditSpring])
-      await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
-      await StudyrightExtent.create(studyrightextents.bachelors)
-      await Studyright.create(studyright)
-      await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
-      await SemesterEnrollment.bulkCreate([
-        semesterEnrollments.fall,
-        semesterEnrollments.spring,
-        semesterEnrollments.current
-      ])
-      // Mock current semester implementation
-      semesterService.getCurrentSemester.mockImplementation(async () => {
-        const currentSemester = await Semester.findOne({
-          where: {
-            semestercode: semesters.current.semestercode
-          }
-        })
-        return currentSemester
-      })
-    })
-
-    test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
-      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/services/test_data.js

@@ -1,42 +0,0 @@
-const faculties = [
-  {
-    code: 'FAC1',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 1 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 1 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 1 SV'
-      }
-    ]
-  },
-  {
-    code: 'FAC2',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 2 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 2 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 3 SV'
-      }
-    ]
-  }
-]
-
-module.exports = {
-  faculties
-}

old_oodikone_stuff/old-backend-tests/studyprogramme.test.js

@@ -1,264 +0,0 @@
-const {
-  getCreditsForProvider,
-  productivityStatsFromCredits,
-  productivityStatsForProvider,
-  findGraduated,
-  graduatedStatsFromStudyrights,
-  combineStatistics,
-  productivityStatsForCode,
-  findProgrammeThesisCredits,
-  thesisProductivityFromCredits,
-  thesisProductivityForStudytrack
-} = require('./studyprogramme')
-const { sequelize } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { readFileSync } = require('fs')
-const { mapToProviders } = require('../util/utils')
-
-const { MASTER, BACHELOR } = ThesisTypeEnums
-
-const provider = '500-M010'
-const studytrack = 'MH50_010'
-
-const thesisCourse = {
-  // from studyprogramme.test.sql
-  programmeCode: 'MH50_010',
-  courseCode: 'THESIS_01',
-  thesisType: MASTER
-}
-
-beforeAll(async () => {
-  const query = readFileSync('./src/services/studyprogramme.test.sql', 'utf8')
-  await sequelize.query(query)
-  await ThesisCourse.create(thesisCourse)
-})
-
-test('Credits for provider should contain passed course credit', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({ id: 'CREDIT_01' })
-})
-
-test('Credits for provider should return object in correct format', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({
-    credits: 5,
-    year: 2016,
-    id: 'CREDIT_01'
-  })
-})
-
-test('Credits for provider should not contain passed credits from other providers', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_04')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit with failed credit typecode 10', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_02')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit for passed study module', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_03')
-  expect(credit).toBeFalsy()
-})
-
-test('KH50_005 track code maps to 500-K005 provider code', () => {
-  const track = 'KH50_005'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-K005')
-})
-
-test('MH50_010 track code maps to 500-M010 provider code', () => {
-  const track = 'MH50_010'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-M010')
-})
-
-test('productivityStatsFromCredits calculates stats correctly', () => {
-  const credits = [
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 10 }
-  ]
-  const stats = productivityStatsFromCredits(credits)
-  expect(stats).toMatchObject({
-    2016: { year: 2016, credits: 15 },
-    2015: { year: 2015, credits: 20 }
-  })
-})
-
-test('productivityStats integrates correctly', async () => {
-  const stats = await productivityStatsForProvider(provider, '2001-01-01')
-  expect(stats).toMatchObject({
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 }
-  })
-})
-
-test('findGraduated finds graduated studyright', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.studyrightid === '10')
-  expect(match).toBeTruthy()
-})
-
-test('findGraduated formats graduated studyrights correctly', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  expect(studyrights).toContainEqual({ studyrightid: '10', year: 2016, timeToGraduation: NaN })
-})
-
-test('findGraduated does not return studyrights that are not graduated', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '11')
-  expect(match).toBeFalsy()
-})
-
-test('findGraduated does return graduated studyrights from other studytracks', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '12')
-  expect(match).toBeFalsy()
-})
-
-test('graduatedStatsFromStudyrights calculates stats correctly', () => {
-  const studyrights = [{ year: 2015 }, { year: 2015 }, { year: 2014 }]
-  const stats = graduatedStatsFromStudyrights(studyrights)
-  expect(stats).toMatchObject({
-    2015: { graduated: 2 },
-    2014: { graduated: 1 }
-  })
-})
-
-test('combineStatistics returns correctly formatted array', () => {
-  const creditStats = {
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 },
-    2014: { year: 2014, credits: 20 }
-  }
-  const studyrightStats = {
-    2015: { graduated: 2 },
-    2016: { graduated: 1 }
-  }
-  const thesisStats = {
-    2014: { mThesis: 1 },
-    2015: { mThesis: 2, bThesis: 1 }
-  }
-  const creditsForMajors = {
-    2014: 10,
-    2015: 22
-  }
-  const transferredCredits = {
-    2014: 2,
-    2015: 4
-  }
-
-  const stats = combineStatistics(creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits)
-  expect(stats).toContainEqual({
-    year: 2015,
-    mThesis: 2,
-    bThesis: 1,
-    credits: 40,
-    graduated: 2,
-    creditsForMajors: 22,
-    transferredCredits: 4
-  })
-  expect(stats).toContainEqual({
-    year: 2014,
-    mThesis: 1,
-    bThesis: 0,
-    credits: 20,
-    graduated: 0,
-    creditsForMajors: 10,
-    transferredCredits: 2
-  })
-  expect(stats).toContainEqual({
-    year: 2016,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    graduated: 1,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('productivityStatsForCode integrates', async () => {
-  const stats = await productivityStatsForCode(studytrack, '2001-01-01')
-  expect(stats.data).toContainEqual({
-    year: 2015,
-    graduated: 0,
-    bThesis: 0,
-    mThesis: 1,
-    credits: 40,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-  expect(stats.data).toContainEqual({
-    year: 2016,
-    graduated: 1,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('findProgrammeThesisCredits returns correct credit', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toBeTruthy()
-  expect(credits.length).toBe(1)
-  expect(credits.find(c => c.id === 'CREDIT_07')).toBeTruthy()
-})
-
-test('findProgrammeThesisCredits format credit correctly', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toContainEqual({
-    id: 'CREDIT_07',
-    code: 'THESIS_01',
-    type: MASTER,
-    year: 2015
-  })
-})
-
-test('thesisProductivityFromCredits', async () => {
-  const credits = [
-    { type: MASTER, year: 2015 },
-    { type: MASTER, year: 2015 },
-    { type: BACHELOR, year: 2015 },
-    { type: BACHELOR, year: 2014 },
-    { type: MASTER, year: 2013 }
-  ]
-  const stats = thesisProductivityFromCredits(credits)
-  expect(stats).toHaveProperty('2015', '2014', '2013')
-  expect(stats).toMatchObject({
-    '2015': {
-      mThesis: 2,
-      bThesis: 1
-    },
-    '2014': {
-      mThesis: 0,
-      bThesis: 1
-    },
-    '2013': {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})
-
-test('thesisProductivityForStudytrack integrates', async () => {
-  const stats = await thesisProductivityForStudytrack(studytrack)
-  expect(stats).toMatchObject({
-    2015: {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})

old_oodikone_stuff/old-backend-tests/studyprogramme.test.sql

@@ -1,93 +0,0 @@
-DO $$
-DECLARE
-  ts TIMESTAMP = CURRENT_TIMESTAMP;
-  studytrk_01 VARCHAR(50) = 'MH50_010';
-  studytrk_02 VARCHAR(50) = 'KH50_005';
-  provider_01 VARCHAR(50) = '500-M010';
-  provider_02 VARCHAR(50) = '500-K005';
-  course_01 VARCHAR(50) = 'M010_COURSE';
-  module_01 VARCHAR(50) = 'M010_MODULE';
-  course_02 VARCHAR(50) = 'K005_COURSE';
-  thesis VARCHAR(50) = 'THESIS_01';
-  date_2016 TIMESTAMPTZ = '2016-09-28 00:00:00+00';
-  date_2015 TIMESTAMPTZ = '2015-09-28 00:00:00+00';
-BEGIN
-  INSERT INTO semesters ("semestercode", "createdAt", "updatedAt")
-  VALUES
-  (5, ts, ts)
-  ;
-
-  INSERT INTO course
-  ("code", "name", "is_study_module")
-  VALUES
-  (course_01, '{ "en": "COURSE_EN" }', 'false'),
-  (module_01, '{ "en": "MODULE_EN" }', 'true'),
-  (course_02, '{ "en": "COURSE_EN" }', 'false'),
-  (thesis, '{ "en": "Bachelor thesis" }', 'false')
-  ;
-
-  INSERT INTO student
-  ("studentnumber", "createdAt", "updatedAt")
-  VALUES
-  ('012111517', ts, ts)
-  ;
-
-  INSERT INTO credit_types
-  ("credittypecode", "createdAt", "updatedAt")
-  VALUES
-  (10, ts, ts),
-  (4, ts, ts)
-  ;
-
-  INSERT INTO credit
-  ("id", "createddate", "lastmodifieddate", "semestercode", "course_code", "credittypecode", "attainment_date", "credits", "student_studentnumber")
-  VALUES
-  ('CREDIT_01', ts, ts, 5, course_01, 4, date_2016, 5, '012111517'),
-  ('CREDIT_02', ts, ts, 5, course_01, 10, date_2016, 5, '012111517'),
-  ('CREDIT_03', ts, ts, 5, module_01, 4, date_2016, 20, '012111517'),
-  ('CREDIT_04', ts, ts, 5, course_02, 4, date_2016, 5, '012111517'),
-  ('CREDIT_05', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_06', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_07', ts, ts, 5, thesis, 4, date_2015, 30, '012111517')
-  ;
-
-  INSERT INTO providers
-  ("providercode", "createdAt", "updatedAt")
-  VALUES
-  (provider_01, ts, ts),
-  (provider_02, ts, ts)
-  ;
-
-  INSERT INTO course_providers
-  ("coursecode", "providercode", "createdAt", "updatedAt")
-  VALUES
-  (course_01, provider_01, ts, ts),
-  (module_01, provider_01, ts, ts),
-  (course_02, provider_02, ts, ts),
-  (thesis, provider_01, ts, ts)
-  ;
-
-  INSERT INTO element_details
-  ("code", "createdAt", "updatedAt")
-  VALUES
-  (studytrk_01, ts, ts),
-  (studytrk_02, ts, ts)
-  ;
-
-  INSERT INTO studyright
-  ("studyrightid", "graduated", "enddate")
-  VALUES
-  (10, 1, date_2016),
-  (11, 0, date_2016),
-  (12, 1, date_2016)
-  ;
-
-  INSERT INTO studyright_elements
-  ("studyrightid", "code", "createdAt", "updatedAt", "studentnumber")
-  VALUES
-  (10, studytrk_01, ts, ts, '012111517'),
-  (11, studytrk_01, ts, ts, '012111517'),
-  (12, studytrk_02, ts, ts, '012111517')
-  ;
-
-END $$;

old_oodikone_stuff/old-backend-tests/teachers.test.js

@@ -1,53 +0,0 @@
-const { Teacher } = require('../models/index')
-const service = require('./teachers')
-
-const teacher = {
-  id: 'TID_001',
-  code: 'usercode',
-  name: 'Lastname Firstname Middlename'
-}
-
-beforeAll(async () => {
-  await Teacher.create(teacher)
-})
-
-describe('teacher service tests', () => {
-  describe('teacher by search term tests', () => {
-    const teacherIsInResults = (id, matches) => matches.some(teacher => teacher.id === 'TID_001')
-
-    test('Search by `usercode` finds teacher', async () => {
-      const matches = await service.bySearchTerm('usercode')
-      expect(teacherIsInResults('TID_001', matches)).toBe(true)
-    })
-
-    test('Search by `usercodeEXTRA` does not find teacher', async () => {
-      const matches = await service.bySearchTerm('usercodeEXTRA')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(false)
-    })
-
-    test('Search by just `Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Lastname Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `id` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('TID_001')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/thesis.test.js

@@ -1,85 +0,0 @@
-const { forceSyncDatabase } = require('../database/connection')
-const { Course, ElementDetails } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { createThesisCourse, deleteThesisCourse, findProgrammeTheses } = require('./thesis')
-
-const courseOne = {
-  code: 'COURSE',
-  name: { en: 'Course_EN' }
-}
-
-const courseTwo = {
-  code: 'COURSE_2',
-  name: { en: 'Course_EN_2' }
-}
-
-const progOne = {
-  code: 'CS_BSC',
-  name: { en: 'CS_BSC' }
-}
-
-const progTwo = {
-  code: 'CS_MSC',
-  name: { en: 'CS_MSC' }
-}
-
-beforeEach(async () => {
-  await forceSyncDatabase()
-  await Course.create(courseOne)
-  await Course.create(courseTwo)
-  await ElementDetails.create(progOne)
-  await ElementDetails.create(progTwo)
-})
-
-test('createThesisCourse should save thesis correctly', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-})
-
-test('createThesisCourse should create separate bachelor and thesis for same programme and course', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  const master = courses.find(c => c.thesisType === 'MASTER')
-  const bachelor = courses.find(c => c.thesisType === 'BACHELOR')
-  expect(master).toBeTruthy()
-  expect(master).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-  expect(bachelor).toBeTruthy()
-  expect(bachelor).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE_2', thesisType: 'BACHELOR' })
-})
-
-test('createThesisCourse should error for creating a thesis for existing (course, prog) pair', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const promise = createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.BACHELOR)
-  await expect(promise).rejects.toThrow('Validation error')
-})
-
-test('deleteThesisCourse removes thesis from table', async () => {
-  let courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(0)
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  await deleteThesisCourse(progOne.code, courseOne.code)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ thesisType: 'BACHELOR' })
-})
-
-test('findProgrammeTheses returns correct courses', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  await createThesisCourse(progTwo.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const theses = await findProgrammeTheses(progOne.code)
-  expect(theses.length).toBe(2)
-})
-
-test('createThesisCourse should return created thesis', async () => {
-  const thesis = await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  expect(thesis).toBeTruthy()
-  expect(thesis).toMatchObject({ programmeCode: progOne.code, courseCode: courseOne.code })
-})

old_oodikone_stuff/old-backend-tests/utils.js

@@ -1,248 +0,0 @@
-const faker = require('faker')
-
-const numberFromTo = (from, to) => Math.round(Math.random() * (to - from)) + from
-const daysAgo = daysAgo => new Date(new Date() - 1000 * 60 * 60 * 24 * daysAgo)
-const idFromTwoIds = (a, b) => (1 / 2) * (a + b) * (a + b + 1) + b
-
-const gradeArray = [
-  '2',
-  'CL',
-  'Eisa',
-  'L',
-  'Hyv.',
-  'TT',
-  'LUB',
-  'HT',
-  'MCLA',
-  '4',
-  '0',
-  'A',
-  'NSLA',
-  'Luop',
-  '5',
-  '3',
-  'Hyl.',
-  'ECLA',
-  '1',
-  'KH'
-]
-const statusArray = [
-  { status: 'Suoritettu', statuscode: 4 },
-  { status: 'Korotettu', statuscode: 7 },
-  { status: 'Hyväksiluettu', statuscode: 9 },
-  { status: 'Hylätty', statuscode: 10 }
-]
-const teacherRoleArray = ['Responsible', 'Approver', 'Teacher']
-
-const generateCourses = async amount => {
-  const number = amount || numberFromTo(10, 100)
-  const courses = []
-  for (let i = 0; i < number; i++) {
-    courses.push({
-      code: i.toString(10),
-      name: faker.lorem.words(3)
-    })
-  }
-  return courses
-}
-
-const generateCourseInstances = async (courses, amount) => {
-  const instances = []
-  let id = 0
-  courses.forEach(course => {
-    const number = amount || numberFromTo(5, 40)
-    for (let i = 0; i < number; i++) {
-      instances.push({
-        id,
-        coursedate: daysAgo(numberFromTo(10, 365)).toUTCString(),
-        course_code: course.code
-      })
-      id++
-    }
-  })
-  return instances
-}
-
-const generateStudents = async amount => {
-  const students = []
-  const number = amount || numberFromTo(5, 40)
-
-  const now = new Date()
-
-  for (let i = 0; i < number; i++) {
-    const birthdate = daysAgo(numberFromTo(365 * 18, 365 * 70))
-    const dateoffirstcredit = faker.date.between(birthdate, now)
-    const dateoflastcredit = faker.date.between(dateoffirstcredit, now)
-    const dateofuniversityenrollment = faker.date.between(birthdate, dateoflastcredit)
-    const lastname = faker.name.lastName()
-    const firstnames = faker.name.firstName(2)
-    students.push({
-      studentnumber: `0${numberFromTo(10000000, 15000000)}`,
-      lastname,
-      firstnames,
-      abbreviatedname: `${lastname} ${firstnames}`,
-      birthdate,
-      communicationlanguage: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      country: faker.address.country(),
-      creditcount: numberFromTo(0, 600),
-      dateoffirstcredit,
-      dateoflastcredit,
-      dateofuniversityenrollment,
-      gradestudent: `${numberFromTo(0, 1)}`,
-      nationalities: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      semesterenrollmenttypecode: [null, '1.0', '2.0'][numberFromTo(0, 2)],
-      sex: ['female', 'male'][numberFromTo(0, 1)],
-      studentstatuscode: numberFromTo(2, 8)
-    })
-  }
-
-  return students
-}
-
-const generateCredits = async (courseInstances, students, amount) => {
-  const credits = []
-  students.forEach((student, studentIdx) => {
-    courseInstances.forEach(courseInstance => {
-      const number = amount || numberFromTo(5, 40)
-      for (let i = 0; i < number; i++) {
-        const statusObject = statusArray[numberFromTo(0, statusArray.length - 1)]
-        credits.push({
-          id: idFromTwoIds(idFromTwoIds(studentIdx, courseInstance.id), i),
-          grade: gradeArray[numberFromTo(0, gradeArray.length - 1)],
-          student_studentnumber: student.studentnumber,
-          credits: [5, 10][numberFromTo(0, 1)],
-          ordering: faker.date.between(student.dateoffirstcredit, student.dateoflastcredit),
-          status: statusObject.status,
-          statuscode: statusObject.statuscode,
-          courseinstance_id: courseInstance.id
-        })
-      }
-    })
-  })
-  return credits
-}
-
-const generateTeachers = async amount => {
-  const teachers = []
-  const number = amount || numberFromTo(5, 40)
-
-  for (let i = 0; i < number; i++) {
-    teachers.push({
-      id: i,
-      code: faker.lorem.word(),
-      name: `${faker.name.lastName()} ${faker.name.firstName()}`
-    })
-  }
-
-  return teachers
-}
-
-const generateCourseTeachers = async (courseInstances, teachers, maximumAmount) => {
-  const courseTeachers = []
-  courseInstances.forEach((courseInstance, index) => {
-    const number = numberFromTo(0, maximumAmount)
-    for (let i = 0; i < number; i++) {
-      const teacher = teachers[numberFromTo(0, teachers.length - 1)]
-      courseTeachers.push({
-        id: idFromTwoIds(index, i),
-        teacherrole: teacherRoleArray[numberFromTo(0, teacherRoleArray.length - 1)],
-        courseinstance_id: courseInstance.id,
-        teacher_id: teacher.id
-      })
-    }
-  })
-
-  return courseTeachers
-}
-
-const generateOrganizations = async amount => {
-  const organizations = []
-
-  const number = amount || numberFromTo(5, 40)
-  for (let i = 0; i < number; i++) {
-    organizations.push({
-      code: faker.lorem.words(3),
-      name: faker.company.companyName()
-    })
-  }
-  return organizations
-}
-
-const generateStudyrights = async (students, organization, amount, date, prioritycode) => {
-  const studyrights = []
-  const possibleStudyrightNames = []
-  if (amount) {
-    for (let i = 0; i < amount; i++) {
-      possibleStudyrightNames.push(faker.company.catchPhrase())
-    }
-  }
-  students.forEach(student => {
-    studyrights.push({
-      studyrightid: student.studentnumber,
-      canceldate: null,
-      cancelorganisation: null,
-      enddate: new Date().toUTCString(),
-      extentcode: numberFromTo(2, 99),
-      givendate: daysAgo(365 * numberFromTo(1, 100)),
-      graduated: null,
-      prioritycode: prioritycode || numberFromTo(1, 30),
-      startdate: date,
-      studystartdate: date,
-      organization_code: organization.code,
-      student_studentnumber: student.studentnumber
-    })
-  })
-  return studyrights
-}
-
-const generateUsers = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const users = []
-  for (let i = 0; i < number; i++) {
-    users.push({
-      full_name: `${faker.name.lastName()}, ${faker.name.firstName(2)}`,
-      username: faker.lorem.word(),
-      is_enabled: numberFromTo(0, 1) === 1
-    })
-  }
-  return users
-}
-
-const generateTags = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const tags = []
-  for (let i = 0; i < number; i++) {
-    tags.push({
-      tagname: `${faker.company.catchPhraseAdjective()} ${i}` //tagname should be unique, add i just in case
-    })
-  }
-  return tags
-}
-
-// adds all tags to all students
-const generateTagStudents = (students, tags) => {
-  let tag_students = []
-  students.forEach(student => {
-    tags.forEach(tag => {
-      tag_students.push({
-        taggedstudents_studentnumber: student.studentnumber,
-        tags_tagname: tag.tagname
-      })
-    })
-  })
-  return tag_students
-}
-
-module.exports = {
-  generateCourses,
-  generateCourseInstances,
-  generateStudents,
-  generateStudyrights,
-  generateCredits,
-  generateTeachers,
-  generateCourseTeachers,
-  generateOrganizations,
-  generateUsers,
-  generateTags,
-  generateTagStudents
-}

old_oodikone_stuff/old-backend-tests/utils/semester.test.js

@@ -1,14 +0,0 @@
-const moment = require('moment')
-const semesterUtils = require('../../../src/util/semester')
-
-describe('Semester utils tests', () => {
-  test('Get passing semester', () => {
-    expect(semesterUtils.getPassingSemester(2018, moment('2017-09-09'))).toBe('BEFORE')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-09-09'))).toBe('0-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-05-01'))).toBe('0-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-09-01'))).toBe('1-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2020-05-01'))).toBe('1-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2026-05-01'))).toBe('LATER')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-05-01'))).toBe('BEFORE')
-  })
-})

old_oodikone_stuff/updater_api/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_api/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_api/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_api/.gitignore

@@ -1 +0,0 @@
-node_modules/

old_oodikone_stuff/updater_api/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_api/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_api/doo_api_database_updater/db_studentnumbers_to_file.js

@@ -1,15 +0,0 @@
-const fs = require('fs')
-const { Student } = require('../../models/index')
-
-const getStudentNumbers = async (writefilepath = './studentnumbers.txt') => {
-  const file = fs.openSync(writefilepath, 'w')
-  const students = await Student.findAll()
-  for (let student of students) {
-    const data = `${student.studentnumber} \n`
-    fs.appendFileSync(file, data)
-  }
-  fs.closeSync(file)
-  process.exit(0)
-}
-
-getStudentNumbers()

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.js

@@ -1,350 +0,0 @@
-const moment = require('moment')
-const _ = require('lodash')
-
-const getStudyRightIdStrings = data => data['data'].map(elements => elements[0])
-
-const getTextsByLanguage = texts => {
-  const names = {}
-  texts.forEach(text => (names[text.langcode] = text.text))
-  return { fi: null, sv: null, en: null, ...names }
-}
-
-const defaultNameFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names.fi || names.en || names.sv
-}
-
-const jsonNamesFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names
-}
-
-const universityEnrollmentDateFromStudyRights = studyRightArray => {
-  return _.sortBy(studyRightArray.map(s => s.start_date), n => moment(n).valueOf())[0]
-}
-
-const getStudentFromData = (student, studyrights) => {
-  const city = getTextsByLanguage(student.city)
-  const language = getTextsByLanguage(student.language)
-  const country = getTextsByLanguage(student.country)
-  const gender = getTextsByLanguage(student.gender)
-  const home_country = getTextsByLanguage(student.home_country)
-  return {
-    studentnumber: student.student_number,
-    email: student.email,
-    phone: student.phone,
-    city_fi: city.fi,
-    city_sv: city.sv,
-    national_student_number: student.national_student_number,
-    zipcode: student.zipcode,
-    address: student.address1,
-    creditcount: student.studyattainments,
-    address2: student.address2,
-    birthdate: getDate(student.birth_date),
-    language_fi: language.fi,
-    language_sv: language.sv,
-    language_en: language.en,
-    age: student.age_years,
-    lastname: student.last_name,
-    mobile: student.mobile_phone,
-    home_county_id: student.home_county_id,
-    country_fi: country.fi,
-    country_sv: country.sv,
-    country_en: country.en,
-    home_country_fi: home_country.fi,
-    home_country_sv: home_country.sv,
-    home_country_en: home_country.en,
-    firstnames: student.first_names,
-    communicationlanguage: language.en || language.fi || language.sv,
-    dateofuniversityenrollment: universityEnrollmentDateFromStudyRights(studyrights),
-    abbreviatedname: [student.last_name, student.first_names].join(' '),
-    gender_code: student.gender_code,
-    gender_fi: gender.fi,
-    gender_sv: gender.sv,
-    gender_en: gender.en
-  }
-}
-
-const getSemesterCode = attainment_date => moment(attainment_date).diff(moment('1950', 'YYYY'), 'months') / 6
-
-const getDate = (date, format = 'DD.MM.YYYY') => {
-  if (!date) return null
-  return moment(date, format).format('YYYY-MM-DD')
-}
-
-const getOrganisationFromData = ({ name, code }) => {
-  return {
-    code,
-    name: jsonNamesFromTexts(name)
-  }
-}
-
-const attainmentDataToCredit = (attainment, studentnumber) => {
-  return {
-    id: String(attainment.studyattainment_id),
-    grade: defaultNameFromTexts(attainment.grade),
-    credits: attainment.credits,
-    ordering: getDate(attainment.attainment_date, null),
-    credittypecode: attainment.attainment_status_code,
-    student_studentnumber: studentnumber,
-    attainment_date: parseDate(attainment.attainment_date),
-    course_code: `${attainment.learningopportunity_id}`,
-    semestercode: attainment.semester_code,
-    isStudyModule: attainment.is_leaf === 'false'
-  }
-}
-
-const attainmentDataToCourse = attainment => {
-  const { learningopportunity_name, attainment_date } = attainment
-  return {
-    code: attainment.learningopportunity_id,
-    name: jsonNamesFromTexts(learningopportunity_name),
-    latest_instance_date: parseDate(attainment_date),
-    is_study_module: attainment.is_leaf === 'false'
-  }
-}
-
-const getTeacherFromData = teacher => {
-  if (teacher) {
-    return {
-      id: teacher.teacher_id,
-      code: teacher.userid,
-      name: teacher.full_name
-    }
-  }
-}
-
-const parseDate = (date, format = 'YYYY-MM-DD HH:mm') => date && moment.utc(date, format).toDate()
-
-const getStudyRightFromData = (data, studentNumber) => {
-  return {
-    studyrightid: `${data.studyright_id}`,
-    canceldate: data.cancel_date,
-    cancelorganisation: data.organisation_code,
-    enddate: parseDate(data.degree_date) || parseDate(data.end_date),
-    extentcode: data.extent_code,
-    givendate: parseDate(data.admission_date),
-    graduated: Number(!!data.degree_date),
-    graduation_date: data.degree_date,
-    prioritycode: data.priority,
-    startdate: parseDate(data.start_date),
-    studystartdate: parseDate(data.study_start_date),
-    organization_code: data.organisation_code,
-    student_studentnumber: studentNumber,
-    faculty_code: data.faculty_code
-  }
-}
-
-const elementDetailFromData = element => {
-  const { code, name, element_id } = element
-  const names = getTextsByLanguage(name)
-  return {
-    code,
-    name: names,
-    type: element_id
-  }
-}
-
-const studyrightElementFromData = (element, studyrightid, studentnumber, degree_date) => {
-  const { start_date, end_date, code } = element
-  return {
-    code,
-    studyrightid,
-    studentnumber,
-    startdate: start_date,
-    enddate: degree_date || end_date
-  }
-}
-
-const attainmentDataToTeachers = data =>
-  data.teachers.map(teacherdata => ({
-    id: teacherdata.teacher_id
-  }))
-
-const studyrightDataToExtent = data => ({
-  extentcode: data.extent_code,
-  name: getTextsByLanguage(data.extent)
-})
-
-const courseTypeFromData = data => ({
-  coursetypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const disciplineFromData = data => ({
-  discipline_id: data.discipline_id,
-  name: getTextsByLanguage(data.name)
-})
-
-const learningOpportunityDataToCourseDisciplines = data => {
-  if (data) {
-    return data.disciplines.map(discipline => ({
-      discipline_id: discipline.discipline_id,
-      course_id: data.learningopportunity_id
-    }))
-  }
-}
-
-const learningOpportunityDataToCourse = data => {
-  if (data) {
-    return {
-      code: data.learningopportunity_id,
-      coursetypecode: data.learningopportunity_type_code,
-      disciplines: data.disciplines,
-      name: getTextsByLanguage(data.names),
-      startdate: parseDate(data.start_date),
-      enddate: parseDate(data.end_date)
-    }
-  }
-}
-
-const learningOpportunityDataToCourseProviders = data => {
-  if (!data) {
-    return { providers: [], courseproviders: [] }
-  }
-  const providers = data.organisations.map(organisation => ({
-    providercode: organisation.code,
-    name: getTextsByLanguage(organisation.name)
-  }))
-  const courseproviders = providers.map(provider => ({
-    providercode: provider.providercode,
-    coursecode: data.learningopportunity_id
-  }))
-  return {
-    providers,
-    courseproviders
-  }
-}
-
-const studyattainmentStatusCodeToCreditType = data => ({
-  credittypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const semesterFromData = data => ({
-  semestercode: data.semester_code,
-  name: getTextsByLanguage(data.name),
-  startdate: parseDate(data.start_date),
-  enddate: parseDate(data.end_date),
-  yearcode: data.year_code,
-  yearname: data.year_name
-})
-
-const semesterEnrollmentFromData = (data, studentnumber) => ({
-  enrollmenttype: data.semester_enrollment_type_code,
-  semestercode: data.semester_code,
-  enrollment_date: parseDate(data.semester_enrollment_date),
-  studentnumber
-})
-
-const getTransfersFromData = (data, studentnumber) => {
-  const studytracks = data.elements.filter(element => element.element_id === 20)
-  const sorted = _.sortBy(studytracks, 'end_date')
-  let transfers = []
-  let i = 0
-  while (i < sorted.length) {
-    if (i === 0) {
-      i++
-      continue
-    }
-    let target = sorted[i].code
-    let source = sorted[i - 1].code
-    let transferdate = sorted[i - 1].end_date
-
-    transfers = transfers.concat({
-      sourcecode: source,
-      targetcode: target,
-      transferdate,
-      studentnumber,
-      studyrightid: `${data.studyright_id}`
-    })
-    i++
-  }
-  return transfers
-}
-
-const courseRealisationTypeFromData = data => ({
-  realisationtypecode: `${data.code}`,
-  name: getTextsByLanguage(data.name)
-})
-
-const studentEnrollmentToModels = (data, studentnumber) => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.name),
-    realisationtypecode: `${data.type_code}`,
-    parent: `${data.parent_id}`,
-    root: `${data.root_id}`
-  }
-  const courseenrollment = {
-    studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }
-  const course = {
-    code: data.learningopportunity_id
-  }
-  return {
-    courserealisation,
-    courseenrollment,
-    course
-  }
-}
-
-const courseUnitRealisationDataToModels = data => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.realisation_name),
-    realisationtypecode: `${data.realisation_type_code}`,
-    parent: `${data.parent_id}`
-  }
-  const course = {
-    code: courserealisation.coursecode
-  }
-  const students = data.students.map(({ student_number }) => ({
-    studentnumber: student_number
-  }))
-  const courseenrollments = students.map(student => ({
-    studentnumber: student.studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }))
-  return {
-    courserealisation,
-    course,
-    students,
-    courseenrollments
-  }
-}
-
-module.exports = {
-  getStudentFromData,
-  getStudyRightIdStrings,
-  getStudyRightFromData,
-  getDate,
-  getTeacherFromData,
-  elementDetailFromData,
-  studyrightElementFromData,
-  attainmentDataToCourse,
-  attainmentDataToCredit,
-  getOrganisationFromData,
-  attainmentDataToTeachers,
-  studyrightDataToExtent,
-  courseTypeFromData,
-  learningOpportunityDataToCourse,
-  studyattainmentStatusCodeToCreditType,
-  disciplineFromData,
-  learningOpportunityDataToCourseDisciplines,
-  semesterFromData,
-  semesterEnrollmentFromData,
-  learningOpportunityDataToCourseProviders,
-  getTransfersFromData,
-  courseRealisationTypeFromData,
-  studentEnrollmentToModels,
-  courseUnitRealisationDataToModels,
-  getSemesterCode
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.test.js

@@ -1,199 +0,0 @@
-const mapper = require('./oodi_data_mapper')
-
-describe('studyattainment data mapping tests', () => {
-  const data = {
-    attainment_status_code: 4,
-    attainment_date: '2018-05-30T21:00:00.000Z',
-    studyattainment_id: 123456789,
-    attainment_lang: null,
-    person_id: 1000002,
-    textual_assessment: null,
-    teachers: [
-      {
-        teacher_id: 'TID_01',
-        person_id: 1000001,
-        short_name: 'Matti Meikäläinen',
-        last_names: 'Meikäläinen',
-        first_names: 'Matti Teppo'
-      }
-    ],
-    studyright_id: null,
-    semester_code: 136,
-    grade: [
-      {
-        langcode: 'fi',
-        text: 'Hyv.'
-      },
-      {
-        langcode: 'sv',
-        text: 'Godk'
-      },
-      {
-        langcode: 'en',
-        text: 'Pass'
-      }
-    ],
-    credits: 10,
-    expiry_date: null,
-    learningopportunity_name: [
-      {
-        langcode: 'fi',
-        text: 'learningopportunity_name FI'
-      },
-      {
-        langcode: 'sv',
-        text: 'learningopportunity_name SV'
-      },
-      {
-        langcode: 'en',
-        text: 'learningopportunity_name EN'
-      }
-    ],
-    learningopportunity_id: 'LID_001'
-  }
-
-  describe('credit mapping tests', () => {
-    const studentnumber = '0876654321'
-    const credit = mapper.attainmentDataToCredit(data, studentnumber)
-
-    describe('studyattainment date tests', () => {
-      const { attainment_date } = credit
-      const MAY = 4
-
-      test('attainment date is truthy', () => {
-        expect(attainment_date).toBeTruthy()
-      })
-
-      test('attainment date is instance of Date', () => {
-        expect(attainment_date instanceof Date).toBe(true)
-      })
-
-      test('attainment date has correct month, year and date', () => {
-        expect(attainment_date.getMonth()).toBe(MAY)
-        expect(attainment_date.getFullYear()).toBe(2018)
-        expect(attainment_date.getDate()).toBe(30)
-      })
-    })
-
-    test('course code matches learningopportunity_id', () => {
-      expect(credit.course_code).toBe('LID_001')
-    })
-
-    test('credit id should attainment id as string', () => {
-      expect(credit.id).toBe('123456789')
-    })
-
-    test('studentnumber is correct', () => {
-      expect(credit.student_studentnumber).toBe(studentnumber)
-    })
-
-    test('semestercode is correct', () => {
-      expect(credit.semestercode).toBe(136)
-    })
-  })
-
-  describe('teacher mapping tests', () => {
-    const teachers = mapper.attainmentDataToTeachers(data)
-
-    test('amount of mapped teachers is correct (1)', () => {
-      expect(teachers.length).toBe(1)
-    })
-
-    test('teacher should have correct id', () => {
-      expect(teachers[0].id).toBe('TID_01')
-    })
-  })
-})
-
-describe('semester enrollment data mapping tests', () => {
-  const studentnumber = '12345678'
-
-  const data = {
-    full_time_student: 'true',
-    yths_code: null,
-    semester_enrollment_type_code: 1,
-    absence_reason_code: null,
-    payment_amount: null,
-    yths_description: null,
-    student_union_member: 'true',
-    semester_enrollment_date: '2011-08-30T21:00:00.000Z',
-    payment_date: null,
-    semester_code: 123,
-    doctoral_current_year_estimate: null,
-    doctoral_previous_year: null
-  }
-
-  describe('mapping of enrolled data', () => {
-    const mapping = mapper.semesterEnrollmentFromData(data, studentnumber)
-
-    test('mapped semestercode should be integer 123', () => {
-      expect(mapping.semestercode).toBe(123)
-    })
-
-    test('mapped enrollmenttype code should be integer 1', () => {
-      expect(mapping.enrollmenttype).toBe(1)
-    })
-
-    test('mapped studentnumber should be correct', () => {
-      expect(mapping.studentnumber).toBe(studentnumber)
-    })
-
-    test('mapped enrollment_date should be instance of Date', () => {
-      expect(mapping.enrollment_date instanceof Date).toBe(true)
-    })
-
-    test('mapped enrollment_date should be equal to Date object with same day, month and year', () => {
-      expect(mapping.enrollment_date).toEqual(new Date('2011-08-30'))
-    })
-  })
-})
-
-describe('semester code data mapping tests', () => {
-  const data = {
-    teaching_start_date: '1950-09-14T22:00:00.000Z',
-    name: [
-      {
-        langcode: 'fi',
-        text: 'Syksy 1950'
-      },
-      {
-        langcode: 'sv',
-        text: 'Hösten 1950'
-      },
-      {
-        langcode: 'en',
-        text: 'Autumn 1950'
-      }
-    ],
-    year_name: '1950-51',
-    enroll_end_date: '1950-09-29T22:00:00.000Z',
-    enroll_start_date: '1950-05-31T22:00:00.000Z',
-    end_date: '1950-12-30T22:00:00.000Z',
-    semester_code: 1,
-    start_date: '1900-07-31T22:20:11.000Z',
-    teaching_end_date: '1950-12-20T22:00:00.000Z',
-    year_code: 1
-  }
-
-  const semester = mapper.semesterFromData(data)
-
-  test('semester code should be correct', () => {
-    expect(semester.semestercode).toBe(1)
-  })
-
-  test('semester name should have all language keys set as properties', () => {
-    expect(semester.name).toHaveProperty('fi')
-    expect(semester.name).toHaveProperty('en')
-    expect(semester.name).toHaveProperty('sv')
-  })
-
-  test('semester start and end date should be instances of Date', () => {
-    expect(semester.startdate instanceof Date).toBe(true)
-    expect(semester.enddate instanceof Date).toBe(true)
-  })
-
-  test('semester yearcode and yearname should be correct', () => {
-    expect(semester.yearcode).toBe(1)
-    expect(semester.yearname).toBe('1950-51')
-  })
-})

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_interface.js

@@ -1,253 +0,0 @@
-require('dotenv').config()
-const axios = require('axios')
-const LRU = require('lru-cache')
-const https = require('https')
-const fs = require('fs')
-
-const { OODI_ADDR, KEY_PATH, CERT_PATH } = process.env
-const base_url = OODI_ADDR
-
-const requestCache = new LRU({
-  max: 25000,
-  length: () => 1,
-  maxAge: 60 * 60 * 1000
-})
-
-const agent =
-  KEY_PATH && CERT_PATH
-    ? new https.Agent({
-        cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-        key: fs.readFileSync(process.env.KEY_PATH, 'utf8')
-      })
-    : new https.Agent({
-        rejectUnauthorized: false
-      })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = async url => {
-  const cacheHit = requestCache.get(url)
-  if (cacheHit) return cacheHit
-
-  const res = await instance.get(url)
-
-  // Sometimes oodi seems to randomly return a response with status 200
-  // that in reality contains an error. Check this out:
-  // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550167612
-  if (res.data.exception) {
-    throw new Error(res.data.exception.message)
-  }
-
-  if (!url.includes('/students/')) requestCache.set(url, res.data.data)
-  return res.data.data
-}
-
-const getOodiApi = async relative => {
-  const route = base_url + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-const attemptGetFor = async (url, attempts = 6) => {
-  for (let attempt = 1; attempt <= attempts; ++attempt) {
-    try {
-      return await getUrl(url)
-    } catch (error) {
-      if (attempt === attempts) {
-        console.log('ATTEMPT GET FOR FAILURE')
-        throw error
-      }
-      await sleep(attempt * 5000)
-    }
-  }
-}
-
-if (process.env.NODE_ENV === 'development') {
-  // axios.defaults.auth = {
-  //   username: 'tktl',
-  //   password: process.env.OODI_PW
-  // }
-  instance.interceptors.request.use(config => {
-    config.params = { token: process.env.TOKEN }
-    return config
-  })
-}
-console.log(process.env.NODE_ENV)
-
-const getStudent = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDENT FAILED')
-    throw e
-  }
-}
-
-const getStudentStudyRights = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyrights`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDYRIGHT FAIELD')
-    throw e
-  }
-}
-
-const getFaculties = async () => {
-  const url = `${base_url}/codes/faculties`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET FACULTIE S FAIELD')
-    throw e
-  }
-}
-
-const getStudyAttainments = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyattainments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STSDU ATTAINEMNT FIALE')
-    throw e
-  }
-}
-
-const getTeacherInfo = async id => {
-  const url = `${base_url}/teachers/${id}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET TCHR FIAIELd', id)
-    throw e
-  }
-}
-
-const getCourseTypeCodes = async () => {
-  const url = `${base_url}/codes/learningopportunities/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('CORUSE TTYPE CODE FAIEL')
-    throw e
-  }
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${base_url}/learningopportunities/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET LEARNIGN OEPPRITUNCITY FAIELD', id)
-    throw e
-  }
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${base_url}/codes/studyattainments/statuses`
-  return await attemptGetFor(url)
-}
-
-const getCourseDisciplines = async () => {
-  const url = `${base_url}/codes/learningopportunities/disciplines`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET COURS EDISCIIPLINES FIALED')
-    throw e
-  }
-}
-const getSemesters = async () => {
-  const url = `${base_url}/codes/semesters`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET SEMESTERS FAIELD')
-    throw e
-  }
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/semesterenrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GEST EMESTER ENROLLMENTS FAIELD')
-    throw e
-  }
-}
-
-const getCourseEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/enrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EENREOLMETNS FAIELD')
-    throw e
-  }
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${base_url}/codes/courseunitrealisations/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURSE REALSITASITON TYPES')
-    throw e
-  }
-}
-
-const courseUnitRealisationsSince = async sinceDate => {
-  const url = `${base_url}/courseunitrealisations/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('COURSEU UNTI REALISTIATION')
-    throw e
-  }
-}
-
-const learningOpportunitiesSince = async (sinceDate = '0000-01-01') => {
-  const url = `${base_url}/learningopportunities/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('LRNG OPRRTN SINCE')
-    throw e
-  }
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${base_url}/courseunitrealisations/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EUNTI GREALSISTAITOn', id)
-    throw e
-  }
-}
-
-module.exports = {
-  getStudentStudyRights,
-  getStudent,
-  getFaculties,
-  getStudyAttainments,
-  getTeacherInfo,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getCourseDisciplines,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisationsSince,
-  getCourseUnitRealisation,
-  learningOpportunitiesSince,
-  getOodiApi
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/updater_formatter.js

@@ -1,158 +0,0 @@
-const Oodi = require('./oodi_interface')
-const mapper = require('./oodi_data_mapper')
-
-const getAllStudentInformationFromApi = async studentnumber => {
-  const [student, studyrights, studyattainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-    Oodi.getStudent(studentnumber),
-    Oodi.getStudentStudyRights(studentnumber),
-    Oodi.getStudyAttainments(studentnumber),
-    Oodi.getSemesterEnrollments(studentnumber),
-    Oodi.getCourseEnrollments(studentnumber)
-  ])
-  return {
-    student,
-    studyrights,
-    studyattainments,
-    studentnumber,
-    semesterEnrollments,
-    courseEnrollments
-  }
-}
-
-const formatStudyrights = async (api, studentnumber) => {
-  if (api.studyrights.length === 0) {
-    console.log(`No studyrights for ${studentnumber}`)
-    return []
-  }
-  return api.studyrights.map(studyrightData => {
-    const studyRightExtent = mapper.studyrightDataToExtent(studyrightData)
-    const studyright = mapper.getStudyRightFromData(studyrightData, studentnumber)
-
-    const elementDetails = studyrightData.elements.map(element => mapper.elementDetailFromData(element))
-    const studyRightElements = studyrightData.elements.map(element =>
-      mapper.studyrightElementFromData(element, studyright.studyrightid, studentnumber, studyrightData.degree_date)
-    )
-
-    const transfers = mapper.getTransfersFromData(studyrightData, studentnumber)
-    return { studyRightExtent, studyright, elementDetails, studyRightElements, transfers }
-  })
-}
-
-const formatCourseEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.courseEnrollments.map(enrollment => mapper.studentEnrollmentToModels(enrollment, studentnumber))
-  )
-
-const parseAttainmentData = (data, studentnumber) => {
-  return {
-    credit: mapper.attainmentDataToCredit(data, studentnumber),
-    teachers: mapper.attainmentDataToTeachers(data),
-    course: mapper.attainmentDataToCourse(data)
-  }
-}
-
-const createCreditTeachers = async (credit, teachers) => {
-  const creditTeachers = teachers.map(teacher => ({
-    credit_id: credit.id,
-    teacher_id: teacher.id
-  }))
-  return creditTeachers
-}
-
-const formatStudyattainments = async (api, studentnumber) => {
-  let studyAttainments = []
-  for (let data of api.studyattainments) {
-    const { credit, teachers, course } = parseAttainmentData(data, studentnumber)
-    let learningOpportunity
-    // For some learning opportunities oodi returns an SQL error.
-    // F.ex. /learningopportunities/590202B. Therefore, until the
-    // bug is fixed on Oodi's side, this is what we need to do.
-    try {
-      learningOpportunity = await Oodi.getLearningOpportunity(course.code)
-    } catch (e) {
-      if (e.message && e.message.includes('PreparedStatementCallback')) continue
-      throw e
-    }
-    const { providers, courseproviders } = mapper.learningOpportunityDataToCourseProviders(learningOpportunity)
-
-    studyAttainments = [
-      ...studyAttainments,
-      {
-        credit: credit.semestercode
-          ? credit
-          : { ...credit, semestercode: mapper.getSemesterCode(credit.attainment_date) },
-        creditTeachers: await createCreditTeachers(credit, teachers),
-        teachers: (await Promise.all(
-          teachers.map(async t => {
-            const teacher = await Oodi.getTeacherInfo(t.id)
-            // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550184131
-            // Sometimes Oodi returns a null teacher... We need to figure out why.
-            if (!teacher) {
-              console.log('TEACHER MISSING!', t.id)
-              return null
-            }
-            return mapper.getTeacherFromData(teacher)
-          })
-        )).filter(t => t),
-        course: {
-          ...course,
-          ...mapper.learningOpportunityDataToCourse(learningOpportunity),
-          disciplines: mapper.learningOpportunityDataToCourseDisciplines(learningOpportunity),
-          providers,
-          courseproviders
-        }
-      }
-    ]
-  }
-  return studyAttainments
-}
-const formatSemesterEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.semesterEnrollments.map(apiEnrollment => mapper.semesterEnrollmentFromData(apiEnrollment, studentnumber))
-  )
-
-const getStudent = async studentnumber => {
-  const api = await getAllStudentInformationFromApi(studentnumber)
-  if (api.student == null) {
-    const error = new Error(`API returned ${api.student} for studentnumber ${studentnumber}.`)
-    error.name = 'NO_STUDENT'
-    throw error
-  }
-  try {
-    const studentInfo = await mapper.getStudentFromData(api.student, api.studyrights)
-    const [studyRights, studyAttainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-      formatStudyrights(api, studentnumber),
-      formatStudyattainments(api, studentnumber),
-      formatSemesterEnrollments(api, studentnumber),
-      formatCourseEnrollments(api, studentnumber)
-    ])
-    return { studentInfo, studyRights, studyAttainments, semesterEnrollments, courseEnrollments }
-  } catch (e) {
-    console.error('getStudent failed', api)
-    throw e
-  }
-}
-
-const getMeta = async () => {
-  const [
-    faculties,
-    courseRealisationsTypes,
-    semesters,
-    creditTypeCodes,
-    courseTypeCodes,
-    disciplines
-  ] = await Promise.all([
-    (await Oodi.getFaculties()).map(mapper.getOrganisationFromData),
-    (await Oodi.getCourseRealisationTypes()).map(mapper.courseRealisationTypeFromData),
-    (await Oodi.getSemesters()).map(mapper.semesterFromData),
-    (await Oodi.getStudyattainmentStatusCodes()).map(mapper.studyattainmentStatusCodeToCreditType),
-    (await Oodi.getCourseTypeCodes()).map(mapper.courseTypeFromData),
-    (await Oodi.getCourseDisciplines()).map(mapper.disciplineFromData)
-  ])
-  return { faculties, courseRealisationsTypes, semesters, creditTypeCodes, courseTypeCodes, disciplines }
-}
-
-module.exports = {
-  getStudent,
-  getMeta
-}

old_oodikone_stuff/updater_api/index.js

@@ -1,72 +0,0 @@
-const { stan, opts } = require('./stan')
-const { getStudent, getMeta } = require('./doo_api_database_updater/updater_formatter')
-const logger = require('./logger')
-const moment = require('moment')
-
-const hasActiveEnrollment = semesterEnrollments => {
-  if (semesterEnrollments) {
-    const currentSemester = Math.floor(moment().diff(moment('1950', 'YYYY'), 'months') / 6)
-    const active = semesterEnrollments.find(e => e.semestercode === currentSemester && e.enrollmenttype === 1)
-    return active ? true : false
-  }
-  return false
-}
-
-const fetchData = async (priority, msg) => {
-  try {
-    const task = JSON.parse(msg.getData())
-    if (task.task === 'meta') {
-      const data = await getMeta()
-      stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', priority }), err => {
-        if (err) console.log('STATUS PUBLISH FAILED', err)
-      })
-      stan.publish(priority ? 'PriorityWrite' : 'UpdateWrite', JSON.stringify({ task: task.task, data }))
-    } else {
-      try {
-        const data = await getStudent(task.task)
-        // TODO: check that data is properly structured(?)
-        const active = hasActiveEnrollment(data.semesterEnrollments)
-        stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', active, priority }), err => {
-          if (err) console.log('STATUS PUBLISH FAIELD', err)
-        })
-        stan.publish(
-          priority ? 'PriorityWrite' : 'UpdateWrite',
-          JSON.stringify({ task: task.task, data, active }),
-          err => {
-            if (err) console.log('STATUS PUBLISH FAILED', err)
-          }
-        )
-      } catch (e) {
-        if (e.name === 'NO_STUDENT') {
-          stan.publish(
-            'status',
-            JSON.stringify({ task: task.task, status: 'NO_STUDENT', active: false, priority }),
-            err => {
-              if (err) console.log('STATUS PUBLISH FAILED', err)
-            }
-          )
-          return
-        } else {
-          console.error('Task failed', task)
-          throw e
-        }
-      }
-    }
-  } catch (e) {
-    console.log(e)
-    logger.info('failure', { service: 'API' })
-  }
-}
-
-const handleMessage = priority => async msg => {
-  await fetchData(priority, msg)
-  msg.ack()
-}
-
-stan.on('connect', async () => {
-  const sub = stan.subscribe('UpdateApi', 'updater.workers', opts)
-  const prioSub = stan.subscribe('PriorityApi', 'updater.workers.prio', opts)
-
-  sub.on('message', handleMessage(false))
-  prioSub.on('message', handleMessage(true))
-})

old_oodikone_stuff/updater_api/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_api/stan.js

@@ -1,13 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-
-const opts = stan.subscriptionOptions()
-opts.setManualAckMode(true)
-opts.setAckWait(10 * 60 * 1000)
-opts.setMaxInFlight(1)
-
-module.exports = {
-  stan,
-  opts
-}

old_oodikone_stuff/updater_api/test_producer.js

@@ -1,32 +0,0 @@
-var stan = require('node-nats-streaming').connect('updaterNATS', 'producer', process.env.NATS_URI)
-
-stan.on('connect', () => {
-  stan.publish('UpdateApi', 'meta', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '011120775', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014441008', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014272112', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-})

old_oodikone_stuff/updater_scheduler/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_scheduler/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_scheduler/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_scheduler/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_scheduler/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_scheduler/index.js

@@ -1,126 +0,0 @@
-const { stan } = require('./src/nats_connection')
-const { CronJob } = require('cron')
-const Schedule = require('./models')
-const logger = require('./logger')
-const {
-  scheduleDaily,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  scheduleMeta
-} = require('./src/schedule_students')
-const { isValidStudentId } = require('./src/util')
-require('./src/api')
-
-const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
-
-let updatedCount = 0
-let scheduledCount = 0
-let fetchedCount = 0
-
-const updateTask = async ({ task, status, type, updatetime, active }) => {
-  const doc = { task, status, type }
-  if (updatetime) doc.updatedAt = new Date()
-  if (active != null) doc.active = active
-  await Schedule.findOneAndUpdate({ task }, doc, { upsert: true })
-}
-
-stan.on('connect', async () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 0 * * 6', async () => {
-      // Every Friday-Saturday midnight
-      try {
-        console.log('SCHEDULING ALL STUDENTS')
-        await scheduleAllStudents()
-      } catch (err) {
-        console.log('SCHEDULING ALL STUDENTS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 6 * * 1-5', async () => {
-      // Every Monday through Friday at 06:00
-      try {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        await scheduleMeta()
-        await scheduleAttainmentUpdate()
-      } catch (err) {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 20 * * 1-4', async () => {
-      // Every Monday through Thursday at 20:00
-      try {
-        console.log('SCHEDULING DAILY TASKS')
-        await scheduleDaily()
-      } catch (err) {
-        console.log('SCHEDULING DAILY TASKS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-  }
-
-  schedule('0 * * * *', async () => {
-    // Every hour
-    console.log(
-      `${updatedCount} TASKS DONE IN LAST HOUR. ${scheduledCount} TASKS SCHEDULED IN LAST HOUR. ${fetchedCount} TASKS FETCHED FROM API IN LAST HOUR`
-    )
-    updatedCount = 0
-    fetchedCount = 0
-    scheduledCount = 0
-  })
-
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  // opts.setDeliverAllAvailable()
-  // opts.setDurableName('durable')
-  opts.setMaxInFlight(20)
-
-  const statusSub = stan.subscribe('status', opts)
-
-  const handleStatusMessage = async msg => {
-    const data = JSON.parse(msg.getData())
-    const { task, status, timems, active, priority } = data
-    let updatetime = false
-    switch (status) {
-      case 'DONE':
-      case 'NO_STUDENT':
-        updatedCount = updatedCount + 1
-        updatetime = true
-        break
-      case 'FETCHED':
-        fetchedCount = fetchedCount + 1
-        break
-      case 'SCHEDULED':
-        scheduledCount = scheduledCount + 1
-        break
-      default:
-        throw 'unknown status: ' + msg.getData()
-    }
-    const isStudent = !!isValidStudentId(task)
-    if (status === 'DONE')
-      logger.info(`Status changed for ${task} to ${status}`, {
-        task,
-        status,
-        student: isStudent,
-        timems,
-        active,
-        priority
-      })
-    await updateTask({ task, status, type: isStudent ? 'student' : 'other', updatetime, active })
-  }
-  statusSub.on('message', async msg => {
-    try {
-      await handleStatusMessage(msg)
-    } catch (err) {
-      console.log(err)
-      logger.info('failure', { service: 'SCHEDULER' })
-    }
-    msg.ack()
-  })
-})

old_oodikone_stuff/updater_scheduler/logger.js

@@ -1,29 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-transports.push(new winston.transports.Console({ level: 'debug' }))
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_scheduler/models.js

@@ -1,18 +0,0 @@
-const mongoose = require('mongoose')
-mongoose.connect(process.env.MONGO_URI, { useNewUrlParser: false, useFindAndModify: false })
-
-const scheduleSchema = new mongoose.Schema({
-  task: {
-    type: String,
-    unique: true
-  },
-  status: String,
-  type: String,
-  active: Boolean,
-  updatedAt: {
-    type: Date,
-    index: true
-  }
-})
-
-module.exports = mongoose.model('Schedule', scheduleSchema)

old_oodikone_stuff/updater_scheduler/src/SchedulingStatistics.js

@@ -1,48 +0,0 @@
-const Schedule = require('../models')
-
-const getOldestTasks = async () => {
-  const oldestTask = (await Schedule.find({ type: 'student' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestActiveTask = (await Schedule.find({ type: 'student', active: true })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestMetaTask = (await Schedule.find({ type: 'other', task: 'meta' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestAttainmentTask = (await Schedule.find({ type: 'other', task: 'attainment' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-
-  return {
-    oldestChangedStatus: {
-      studentnumber: oldestTask ? oldestTask.task : null,
-      updatedAt: oldestTask ? oldestTask.updatedAt : null
-    },
-    oldestActiveStudentDone: {
-      studentnumber: oldestActiveTask ? oldestActiveTask.task : null,
-      updatedAt: oldestActiveTask ? oldestActiveTask.updatedAt : null
-    },
-    oldestMetaTask: {
-      updatedAt: oldestMetaTask ? oldestMetaTask.updatedAt : null,
-      status: oldestMetaTask ? oldestMetaTask.status : null
-    },
-    oldestAttainmentTask: {
-      updatedAt: oldestAttainmentTask ? oldestAttainmentTask.updatedAt : null,
-      status: oldestAttainmentTask ? oldestAttainmentTask.status : null
-    }
-  }
-}
-
-const getCurrentStatus = async () => {
-  const allTasksScheduled = await Schedule.count({ type: 'student', status: 'SCHEDULED' })
-  const allTasksFetched = await Schedule.count({ type: 'student', status: 'FETCHED' })
-  const allTasksDone = await Schedule.count({ type: 'student', status: 'DONE' })
-  const allTasksCreated = await Schedule.count({ type: 'student', status: 'CREATED' })
-  const allTasksNoStudent = await Schedule.count({ type: 'student', status: 'NO_STUDENT' })
-  const allTasksActive = await Schedule.count({ type: 'student', active: true })
-
-  return { allTasksScheduled, allTasksFetched, allTasksDone, allTasksCreated, allTasksActive, allTasksNoStudent }
-}
-
-module.exports = { getOldestTasks, getCurrentStatus }

old_oodikone_stuff/updater_scheduler/src/api.js

@@ -1,127 +0,0 @@
-const express = require('express')
-const app = express()
-const port = process.env.PORT
-const bodyParser = require('body-parser')
-const {
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleAllStudents,
-  scheduleActiveStudents,
-  scheduleMeta,
-  scheduleAttainmentUpdate,
-  scheduleStudentCheck,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily
-} = require('./schedule_students')
-const { getOldestTasks, getCurrentStatus } = require('./SchedulingStatistics')
-const { createTasks } = require('./student_list_updater')
-
-app.use(bodyParser.json())
-
-app.get('/ping', (req, res) => res.json({ message: 'pong ' }))
-
-app.post('/update', async (req, res) => {
-  await createTasks(req.body)
-  const scheduled = await scheduleStudentsByArray(req.body)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/oldest', async (req, res) => {
-  const { amount } = req.body
-  const scheduled = await scheduleOldestNStudents(amount)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/all', async (req, res) => {
-  const scheduled = await scheduleAllStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/active', async (req, res) => {
-  const scheduled = await scheduleActiveStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/attainment', async (req, res) => {
-  scheduleAttainmentUpdate()
-  res.json({ message: `scheduled attainments` })
-})
-
-app.post('/update/meta', async (req, res) => {
-  scheduleMeta()
-  res.json({ message: 'scheduled meta' })
-})
-
-app.post('/update/no_student', async (req, res) => {
-  const scheduled = await scheduleStudentCheck()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/studentlist', async (req, res) => {
-  createTasks()
-  res.json({ message: `tasks created` })
-})
-
-app.post('/update/daily', async (req, res) => {
-  const scheduled = await scheduleDaily()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/created', async (req, res) => {
-  const scheduled = await rescheduleCreated()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/scheduled', async (req, res) => {
-  const scheduled = await rescheduleScheduled()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/fetched', async (req, res) => {
-  const scheduled = await rescheduleFetched()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.get('/statuses', async (req, res) => {
-  const statuses = []
-
-  const {
-    oldestChangedStatus: { studentnumber, updatedAt },
-    oldestActiveStudentDone: { studentnumber: studentnumberActive, updatedAt: updatedAtActive },
-    oldestMetaTask,
-    oldestAttainmentTask
-  } = await getOldestTasks()
-
-  const {
-    allTasksScheduled,
-    allTasksFetched,
-    allTasksDone,
-    allTasksCreated,
-    allTasksActive,
-    allTasksNoStudent
-  } = await getCurrentStatus()
-
-  statuses.push(
-    { label: 'CREATED', value: allTasksCreated },
-    { label: 'SCHEDULED', value: allTasksScheduled },
-    { label: 'FETCHED', value: allTasksFetched },
-    { label: 'DONE', value: allTasksDone },
-    { label: 'NO_STUDENT', value: allTasksNoStudent },
-    { label: 'ACTIVE', value: allTasksActive },
-    { label: 'oldest student studentnumber', value: studentnumber },
-    { label: 'oldest student timestamp', value: updatedAt },
-    { label: 'oldest active student studentnumber', value: studentnumberActive },
-    { label: 'oldest active student timestamp', value: updatedAtActive },
-    { label: 'oldest metadata update', value: oldestMetaTask.updatedAt },
-    { label: 'current metadata status', value: oldestMetaTask.status },
-    { label: 'oldest attainment update', value: oldestAttainmentTask.updatedAt },
-    { label: 'current attainment status', value: oldestAttainmentTask.status }
-  )
-
-  res.json(statuses)
-})
-
-app.listen(port, () => console.log(`listening on port ${port}!`))
-module.exports = { app }

old_oodikone_stuff/updater_scheduler/src/nats_connection.js

@@ -1,4 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.NATS_CLIENTID || 'scheduler', {
-  url: process.env.NATS_URI
-})
-module.exports = { stan }

old_oodikone_stuff/updater_scheduler/src/schedule_students.js

@@ -1,181 +0,0 @@
-const { stan } = require('./nats_connection')
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-stan.on('connect', async () => {
-  const taskspermin = 300
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  opts.setMaxInFlight(taskspermin)
-
-  const scheduleSub = stan.subscribe('schedule', opts)
-  scheduleSub.on('message', async msg => {
-    try {
-      const { task, priority } = JSON.parse(msg.getData())
-      stan.publish('status', JSON.stringify({ task, status: 'SCHEDULED', priority }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      stan.publish(priority ? 'PriorityApi' : 'UpdateApi', JSON.stringify({ task }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      await sleep(60 * 1000)
-    } catch (err) {
-      console.log(err)
-    }
-    msg.ack()
-  })
-})
-
-const publishSingle = async ({ priority, task }) => {
-  const promise = new Promise(resolve => {
-    stan.publish('schedule', JSON.stringify({ task: task.task, priority }), async err => {
-      if (err) {
-        console.log('publish failed, waiting 10 seconds and republishing', err)
-        await sleep(10 * 1000)
-        resolve(await publishSingle({ priority, task }))
-      } else {
-        resolve(true)
-      }
-    })
-  })
-  return await promise
-}
-
-const publishAll = async (tasks, priority = false) => {
-  for (const task of tasks) {
-    await publishSingle({ task, priority })
-  }
-}
-
-const scheduleActiveStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleAllStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleMeta = async () => {
-  await publishAll([{ task: 'meta' }], true)
-}
-
-const scheduleAttainmentUpdate = async () => {
-  stan.publish('status', JSON.stringify({ task: 'attainment', status: 'SCHEDULED' }), err => {
-    if (err) {
-      console.log('publish failed')
-    }
-  })
-  stan.publish('UpdateAttainmentDates', null, err => {
-    if (err) {
-      console.log('publish failed', 'UpdateAttainmentDates')
-    }
-  })
-}
-
-const scheduleStudentCheck = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'NO_STUDENT' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleStudentsByArray = async studentNumbers => {
-  try {
-    const tasks = await Schedule.find({ type: 'student', task: { $in: studentNumbers } })
-    publishAll(tasks, true)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const scheduleOldestNStudents = async amount => {
-  try {
-    const tasks = [
-      ...(await Schedule.find({ type: 'student' })
-        .sort({ updatedAt: 1 })
-        .limit(Number(amount)))
-    ]
-    publishAll(tasks)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const rescheduleCreated = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'CREATED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleScheduled = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'SCHEDULED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleFetched = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'FETCHED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleDaily = async () => {
-  const findStudentsRange = 10000
-  const highestStudentNumber = (await Schedule.find({ type: 'student', status: { $in: ['DONE', 'FETCHED'] } })
-    .sort({ task: -1 })
-    .limit(1))[0]
-  const regexResult = highestStudentNumber ? highestStudentNumber.task.match(/0(\d{7})\d/) : null
-  const minstudentnumber = 1000000
-  const studentNumberMid = (regexResult && Number(regexResult[1])) || minstudentnumber + findStudentsRange
-  const min = Math.max(studentNumberMid - findStudentsRange, minstudentnumber)
-  const max = Math.min(studentNumberMid + findStudentsRange, 9999999)
-
-  console.log({ findStudentsRange, minstudentnumber, studentNumberMid, min, max })
-
-  const activeStudents = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  const studentnumbers = activeStudents.map(t => t.task)
-  for (let i = min; i <= max; ++i) {
-    studentnumbers.push(`0${i}${getStudentNumberChecksum(i)}`)
-  }
-  const oldestStudents = [
-    ...(await Schedule.find({ type: 'student' })
-      .sort({ updatedAt: 1 })
-      .limit(10000))
-  ]
-  studentnumbers.push(...oldestStudents.map(t => t.task))
-
-  const tasks = [...new Set(studentnumbers)].map(sn => ({ task: sn }))
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-module.exports = {
-  scheduleActiveStudents,
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleMeta,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily,
-  scheduleStudentCheck
-}

old_oodikone_stuff/updater_scheduler/src/student_list_updater.js

@@ -1,62 +0,0 @@
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-const createTasks = async (fromStudents = []) => {
-  console.log('Creating studentnumbers to DB...')
-
-  const writeStudents = async studentsToAdd => {
-    const tasks = studentsToAdd.map(student => ({ task: student, status: 'CREATED', type: 'student', active: true }))
-    const bulkOperation = Schedule.collection.initializeUnorderedBulkOp()
-    for (const task of tasks) {
-      bulkOperation.insert(task)
-    }
-    await bulkOperation.execute()
-  }
-
-  const writeToDB = async studentnumbers => {
-    try {
-      await writeStudents(studentnumbers)
-    } catch (err) {
-      const BulkWriteResult = err.result.result
-      const duplicateIndexError = 11000
-      const realErrors = BulkWriteResult.writeErrors.filter(e => e.err.code !== duplicateIndexError)
-      if (realErrors.length > 0 || BulkWriteResult.writeConcernErrors.length > 0) {
-        console.log(err)
-        return false
-      }
-    }
-    return true
-  }
-
-  const mongoWriteBatchSize = 1000
-  let studentnumbers = []
-  if (fromStudents.length) {
-    for (let i = 0; i < fromStudents.length; i++) {
-      studentnumbers.push(fromStudents[i])
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === fromStudents.length - 1) {
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        console.log(`Created tasks from ${studentnumbers}`)
-        studentnumbers = []
-      }
-    }
-  } else {
-    const minStudentNumber = 1010000
-    const maxStudentNumber = 1530000
-    for (let i = minStudentNumber; i <= maxStudentNumber; ++i) {
-      const studentNumber = '0' + i + getStudentNumberChecksum(i)
-      studentnumbers.push(studentNumber)
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === maxStudentNumber) {
-        console.log('current studentnumber', i)
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        studentnumbers = []
-      }
-    }
-  }
-  console.log('Studentnumbers created to DB')
-}
-
-module.exports = { createTasks }

old_oodikone_stuff/updater_scheduler/src/util.js

@@ -1,43 +0,0 @@
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-// Shuffle in place, returns same array
-const shuffleArray = a => {
-  for (let i = a.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1))
-    const x = a[i]
-    a[i] = a[j]
-    a[j] = x
-  }
-  return a
-}
-
-const isValidStudentId = id => {
-  if (/^0\d{8}$/.test(id)) {
-    // is a 9 digit number
-    const multipliers = [7, 1, 3, 7, 1, 3, 7]
-    const checksum = id
-      .substring(1, 8)
-      .split('')
-      .reduce((sum, curr, index) => {
-        return (sum + curr * multipliers[index]) % 10
-      }, 0)
-    return (10 - checksum) % 10 == id[8]
-  }
-  return false
-}
-
-const getStudentNumberChecksum = studentNumber => {
-  const studentNumberString = String(studentNumber)
-  let checksumNumbers = [7, 3, 1]
-  let checksum = 0
-
-  for (let i = 0; i < studentNumberString.length; i++) {
-    // go from end t start
-    let currentNumber = studentNumberString[studentNumberString.length - (i + 1)]
-    checksum += currentNumber * checksumNumbers[i % checksumNumbers.length]
-  }
-
-  return (10 - (checksum % 10)) % 10
-}
-
-module.exports = { sleep, isValidStudentId, getStudentNumberChecksum, shuffleArray }

old_oodikone_stuff/updater_writer/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_writer/.eslintignore

@@ -1,2 +0,0 @@
-*.sql
-updater/test_assets

old_oodikone_stuff/updater_writer/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_writer/.gitignore

@@ -1,4 +0,0 @@
-node_modules
-
-models
-migrations

old_oodikone_stuff/updater_writer/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_writer/Dockerfile

@@ -1,18 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-RUN apk add postgresql-client
-
-COPY ./updater_writer/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY ./updater_writer /usr/src/app
-COPY ./shared/models /usr/src/app/models
-COPY ./shared/migrations /usr/src/app/database/migrations
-COPY ./shared/migrations_kone /usr/src/app/database/migrations_kone

old_oodikone_stuff/updater_writer/conf-backend.js

@@ -1,77 +0,0 @@
-require('dotenv').config()
-
-const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
-let DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const isTest = NODE_ENV === 'test'
-if (isTest) {
-  DB_URL = process.env.TEST_DB
-  DB_SCHEMA = process.env.TEST_DB_SCHEMA
-} else if (NODE_ENV === 'anon') {
-  DB_URL = process.env.ANON_DB
-}
-const frontend_addr = process.env.FRONT_URL
-const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
-const CERT_PATH = process.env.CERT_PATH // production/staging only
-const KEY_PATH = process.env.KEY_PATH // production/staging only
-const USERSERVICE_URL = process.env.USERSERVICE_URL
-const ANALYTICS_URL = process.env.ANALYTICS_URL
-const PORT = isTest ? 8079 : 8080
-const OODI_SECRET = process.env.OODI_SECRET
-
-const FEATURES = {
-  ERROR_HANDLER: false
-}
-
-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-if (process.env.NODE_ENV === 'dev' && process.env.FEATURES) {
-  const toggled = process.env.FEATURES.split(',')
-  toggled
-    .map(toggle => toggle.trim())
-    .forEach(feature => {
-      if (FEATURES[feature] !== undefined) {
-        FEATURES[feature] = true
-      }
-    })
-}
-
-const OODI = {
-  test: 'http://localhost',
-  anon: process.env.OODI_ADDR_ANON
-}
-
-const OODI_ADDR = OODI[process.env.NODE_ENV] || process.env.OODI_ADDR
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-const OODI_SECRET_HEADER_KEY = 'x-oodi-secret'
-
-let requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
-if (process.env.NODE_ENV === 'staging') {
-  requiredGroup = ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-}
-if (process.env.NODE_ENV === 'dev' || isTest) {
-  requiredGroup = null
-}
-
-module.exports = {
-  frontend_addr,
-  DB_URL,
-  redis,
-  TOKEN_SECRET,
-  DB_SCHEMA,
-  OODI_ADDR,
-  CERT_PATH,
-  KEY_PATH,
-  FEATURES,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
-  PORT,
-  ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  requiredGroup,
-  OODI_SECRET,
-  OODI_SECRET_HEADER_KEY,
-  isTest
-}

old_oodikone_stuff/updater_writer/database/connection.js

@@ -1,69 +0,0 @@
-const Sequelize = require('sequelize')
-const Umzug = require('umzug')
-
-const conf = require('../conf-backend')
-
-const sequelize = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA,
-  searchPath: conf.DB_SCHEMA,
-  logging: false
-})
-sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
-
-const initializeDatabaseConnection = async () => {
-  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelize.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
-      }
-      console.log('.')
-      await sleep(1000)
-    }
-  }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelize,
-          tableName: 'migrations'
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelize.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/database/migrations`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA
-        }
-      })
-      const migrations = await migrator.up()
-
-      console.log('Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
-  }
-}
-
-const forceSyncDatabase = async () => {
-  try {
-    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
-  } catch (e) {
-    // console.log(e)
-  }
-  console.log('OODI SYNC')
-  await sequelize.sync({ force: true })
-}
-
-module.exports = {
-  sequelize,
-  initializeDatabaseConnection,
-  forceSyncDatabase
-}

old_oodikone_stuff/updater_writer/database/dump_database.js

@@ -1,27 +0,0 @@
-const spawn = require('child_process').spawn
-const fs = require('fs')
-const { DB_SCHEMA } = require('../conf-backend')
-
-const dumpDatabase = async () => {
-  const { DB_URL } = process.env
-  const dumpOptions = ['-Fc', `-n${DB_SCHEMA}`, `${DB_URL}`]
-  console.log(dumpOptions)
-  const dumpProcess = spawn('pg_dump', dumpOptions, {
-    stdio: ['ignore', 'pipe', 'inherit']
-  })
-  dumpProcess.on('error', err => {
-    console.log(' dump process error:', err)
-  })
-  const writeStream = fs.createWriteStream(`./dumps/oodidb_prod-${new Date().toISOString()}`)
-
-  dumpProcess.stdout.pipe(writeStream)
-
-  dumpProcess.on('exit', async code => {
-    if (code !== 0) {
-      throw new Error('pg_dump: Bad exit code (' + code + ')')
-    } else {
-      console.log('database dump created')
-    }
-  })
-}
-module.exports = { dumpDatabase }

old_oodikone_stuff/updater_writer/database/force_sync_database.js

@@ -1,17 +0,0 @@
-const { seedAllMigrations } = require('./seed_migrations')
-const { forceSyncDatabase } = require('./connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-//sync()
-module.exports = { sync }

old_oodikone_stuff/updater_writer/database/seed_migrations.js

@@ -1,28 +0,0 @@
-const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
-
-const DEFAULT_PATH = `${process.cwd()}/database/migrations`
-const DEFAULT_PATH_KONE = `${process.cwd()}/database/migrations_kone`
-
-const saveMigrationsToDatabase = Migration => async filenames =>
-  Promise.all(filenames.map(name => Migration.upsert({ name })))
-
-const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(Migration)(filenames)
-}
-
-const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(MigrationKone)(filenames)
-}
-
-const seedAllMigrations = async () => {
-  await seedMigrations()
-  await seedMigrationsKone()
-}
-
-module.exports = {
-  seedMigrations,
-  seedAllMigrations
-}

old_oodikone_stuff/updater_writer/index.js

@@ -1,69 +0,0 @@
-const natsStreaming = require('node-nats-streaming')
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./updater/database_updater')
-const logger = require('./logger')
-const { initializeDatabaseConnection } = require('./database/connection')
-
-initializeDatabaseConnection()
-  .then(() => {
-    const stan = natsStreaming.connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-    console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-    const opts = stan.subscriptionOptions()
-    opts.setManualAckMode(true)
-    opts.setAckWait(10 * 60 * 1000) // some students have taken over 5 min to write!
-    // opts.setDeliverAllAvailable()
-    // opts.setDurableName('durable')
-    opts.setMaxInFlight(1)
-
-    stan.on('connect', function() {
-      const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
-      const attSub = stan.subscribe('UpdateAttainmentDates', 'updater.workers.attainments', opts)
-      const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
-
-      const writeStudent = priority => async msg => {
-        let data = null
-        try {
-          const start = new Date()
-          data = JSON.parse(msg.getData())
-          if (data.task === 'meta') {
-            await updateMeta(data.data)
-          } else {
-            await updateStudent(data.data)
-          }
-          stan.publish(
-            'status',
-            JSON.stringify({ task: data.task, status: 'DONE', timems: new Date() - start }, priority),
-            err => {
-              if (err) console.log(err)
-            }
-          )
-        } catch (err) {
-          console.log('update failed', data.task, err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      }
-
-      sub.on('message', writeStudent(false))
-      prioSub.on('message', writeStudent(true))
-
-      attSub.on('message', async msg => {
-        try {
-          await updateAttainmentMeta()
-          stan.publish('status', JSON.stringify({ task: 'attainment', status: 'DONE' }), err => {
-            if (err) {
-              console.log('publish failed')
-            }
-          })
-        } catch (err) {
-          console.log('attainment meta update failed', err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      })
-    })
-  })
-  .catch(e => {
-    process.exitCode = 1
-    console.log(e)
-  })

old_oodikone_stuff/updater_writer/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_writer/updater/database_updater.js

@@ -1,217 +0,0 @@
-const { notIn } = require('sequelize').Op
-const { sequelize } = require('../database/connection')
-const { sortBy, flatMap, uniqBy, sortedUniqBy } = require('lodash')
-
-const {
-  Student,
-  Credit,
-  Course,
-  CreditTeacher,
-  Teacher,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { updateAttainmentDates } = require('./update_attainment_dates')
-
-const getColumnsToUpdate = arr => (arr[0] ? Object.keys(arr[0]) : [])
-
-const updateAttainments = async (studyAttainments, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const courses = sortedUniqBy(sortBy(studyAttainments.map(e => e.course), 'code'), 'code')
-  const disciplines = sortBy(flatMap(studyAttainments, e => e.course.disciplines || []), 'discipline_id', 'course_id')
-  const providers = sortedUniqBy(
-    sortBy(flatMap(studyAttainments, e => e.course.providers), 'providercode'),
-    'providercode'
-  )
-  const courseproviders = sortBy(flatMap(studyAttainments, e => e.course.courseproviders), 'coursecode', 'providercode')
-  const credits = sortedUniqBy(sortBy(studyAttainments.map(e => e.credit), 'id'), 'id')
-  const teachers = sortedUniqBy(sortBy(flatMap(studyAttainments, e => e.teachers || []), 'id'), 'id')
-  const creditTeachers = sortBy(flatMap(studyAttainments, e => e.creditTeachers), 'teacher_id', 'credit_id')
-
-  await Promise.all([
-    Course.bulkCreate(courses, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(courses)
-    }),
-    Provider.bulkCreate(providers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(providers)
-    }),
-    Credit.bulkCreate(credits, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(credits)
-    }),
-    Teacher.bulkCreate(teachers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(teachers)
-    })
-  ])
-
-  await Promise.all([
-    // must be after courses
-    CourseDisciplines.bulkCreate(disciplines, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after providers
-    CourseProvider.bulkCreate(courseproviders, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after teachers
-    CreditTeacher.bulkCreate(creditTeachers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudyRights = async (studyRights, studentnumber, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const studyRightExtents = sortedUniqBy(sortBy(studyRights.map(e => e.studyRightExtent), 'extentcode'), 'extentcode')
-  const studyrights = sortedUniqBy(sortBy(studyRights.map(e => e.studyright), 'studyrightid'), 'studyrightid')
-  const elementDetails = sortedUniqBy(sortBy(flatMap(studyRights, e => e.elementDetails), 'code'), 'code')
-  const studyRightElements = uniqBy(
-    sortBy(flatMap(studyRights, e => e.studyRightElements), 'startdate', 'enddate', 'studyrightid', 'code'),
-    s => `${s.startdate}${s.enddate}${s.studyrightid}${s.code}`
-  )
-  const transfers = sortBy(
-    flatMap(studyRights, e => e.transfers),
-    'transferdate',
-    'studentnumber',
-    'studyrightid',
-    'sourcecode',
-    'targetcode'
-  )
-
-  const studyrightids = studyrights.map(s => s.studyrightid)
-  await Studyright.destroy(
-    { where: { student_studentnumber: studentnumber, studyrightid: { [notIn]: studyrightids } } },
-    { transaction }
-  )
-  await StudyrightElement.destroy({ where: { studentnumber } }, { transaction })
-
-  await Promise.all([
-    StudyrightExtent.bulkCreate(studyRightExtents, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyRightExtents)
-    }),
-
-    Studyright.bulkCreate(studyrights, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyrights)
-    }),
-
-    ElementDetails.bulkCreate(elementDetails, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(elementDetails)
-    }),
-
-    StudyrightElement.bulkCreate(studyRightElements, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-
-    Transfers.bulkCreate(transfers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudent = async student => {
-  let { studentInfo, studyAttainments, semesterEnrollments, studyRights } = student
-
-  // sort data to avoid deadlocks
-  semesterEnrollments = uniqBy(
-    sortBy(semesterEnrollments, 'studentnumber', 'semestercode'),
-    sE => `${sE.semestercode}${sE.studentnumber}`
-  )
-
-  const transaction = await sequelize.transaction()
-  try {
-    await Student.upsert(studentInfo, { transaction })
-
-    // Change this to bulkCreate after https://github.com/sequelize/sequelize/issues/11569
-    // is merged and released
-    for (let i = 0; i < semesterEnrollments.length; i++) {
-      await SemesterEnrollment.upsert(semesterEnrollments[i], { transaction })
-    }
-
-    if (studyAttainments) await updateAttainments(studyAttainments, transaction)
-
-    if (studyRights) await updateStudyRights(studyRights, studentInfo.studentnumber, transaction)
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-const updateAttainmentMeta = async () => {
-  await updateAttainmentDates()
-}
-
-const updateMeta = async ({
-  faculties,
-  courseRealisationsTypes,
-  semesters,
-  creditTypeCodes,
-  courseTypeCodes,
-  disciplines
-}) => {
-  const transaction = await sequelize.transaction()
-
-  try {
-    await Promise.all([
-      CourseType.bulkCreate(uniqBy(courseTypeCodes, 'coursetypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseTypeCodes)
-      }),
-      Organisation.bulkCreate(uniqBy(faculties, 'code'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(faculties)
-      }),
-      CourseRealisationType.bulkCreate(uniqBy(courseRealisationsTypes, 'realisationtypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseRealisationsTypes)
-      }),
-      Semester.bulkCreate(uniqBy(semesters, 'semestercode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(semesters)
-      }),
-      CreditType.bulkCreate(uniqBy(creditTypeCodes, 'credittypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(creditTypeCodes)
-      }),
-      Discipline.bulkCreate(uniqBy(disciplines, 'discipline_id'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(disciplines)
-      })
-    ])
-
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-module.exports = {
-  updateStudent,
-  updateMeta,
-  updateAttainmentMeta
-}

old_oodikone_stuff/updater_writer/updater/database_updater.test.js

@@ -1,518 +0,0 @@
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./database_updater')
-const {
-  Student,
-  Credit,
-  Course,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  Teacher,
-  CreditTeacher,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { students } = require('./test_assets/test_students')
-const meta = require('./test_assets/meta')
-const { sequelize, forceSyncDatabase } = require('../database/connection')
-const { seedMigrations } = require('../database/seed_migrations')
-jest.setTimeout(10000)
-describe('Updater works', () => {
-  beforeAll(async () => {
-    await forceSyncDatabase()
-    await seedMigrations()
-
-    await updateMeta(meta)
-    await updateStudent(students[0])
-    await updateStudent(students[1])
-    await updateStudent(students[2])
-    await updateAttainmentMeta()
-  })
-
-  afterAll(async () => {
-    await sequelize.close()
-  })
-
-  describe('All metadata is written', () => {
-    test('Organisations are written to database', async () => {
-      const organizations = await Organisation.findAll()
-      expect(organizations.length).toBe(17)
-      expect(organizations.find(_ => _.code === 'H50').name.find(n => n.langcode === 'fi').text).toBe(
-        'Matemaattis-luonnontieteellinen tiedekunta'
-      )
-    })
-
-    test('Course realisation types are written to database', async () => {
-      const courseRealisationTypes = await CourseRealisationType.findAll()
-      expect(courseRealisationTypes.length).toBe(18)
-      expect(courseRealisationTypes.map(_ => _.realisationtypecode)).not.toContain(null)
-      expect(courseRealisationTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Semesters are written to database', async () => {
-      const semesters = await Semester.findAll()
-      expect(semesters.length).toBe(160)
-      expect(semesters.map(_ => _.semestercode)).toEqual(expect.arrayContaining([...Array(160).keys()].map(_ => ++_)))
-      expect(semesters.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-      expect(semesters.map(_ => _.startdate)).not.toContain(null)
-      expect(semesters.map(_ => _.enddate)).not.toContain(null)
-      expect(semesters.map(_ => _.createdAt)).not.toContain(null)
-      expect(semesters.map(_ => _.updatedAt)).not.toContain(null)
-      expect(new Set(semesters.map(_ => _.yearcode)).size).toBe(80)
-      expect(semesters.find(_ => _.yearcode === 1).yearname).toBe('1950-51')
-    })
-
-    test('Credit types are written to database', async () => {
-      const creditTypes = await CreditType.findAll()
-      expect(creditTypes.length).toBe(11)
-      expect(creditTypes.map(_ => _.credittypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(creditTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Course types are written to database', async () => {
-      const courseTypes = await CourseType.findAll()
-      expect(courseTypes.length).toBe(35)
-      expect(courseTypes.map(_ => _.coursetypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(courseTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Disciplines are written to database', async () => {
-      const disciplines = await Discipline.findAll()
-      expect(disciplines.length).toBe(430)
-      expect(disciplines.map(_ => _.discipline_id)).toEqual(expect.arrayContaining([expect.any(String)]))
-      expect(disciplines.map(_ => _.name)[0]).toMatchObject({
-        fi: expect.any(String)
-      })
-    })
-  })
-  describe('Updater writes students the right way', () => {
-    test('Student info is written to database', async () => {
-      const students = await Student.findAll({
-        order: [['studentnumber', 'DESC']]
-      })
-      expect(students.length).toBe(3)
-      const {
-        studentnumber,
-        lastname,
-        firstnames,
-        abbreviatedname,
-        birthdate,
-        communicationlanguage,
-        creditcount,
-        dateofuniversityenrollment,
-        city_fi,
-        city_sv,
-        gender_code,
-        gender_fi,
-        createdAt,
-        updatedAt
-      } = students[0]
-
-      expect(studentnumber).toBe('014441008')
-      expect(lastname).toBe('Orttenvuori')
-      expect(firstnames).toBe('Oona')
-      expect(abbreviatedname).toBe('Orttenvuori Oona')
-      expect(birthdate.toISOString()).toBe(new Date('1966-04-24 00:00:00+00').toISOString())
-      expect(communicationlanguage).toBe('Finnish')
-      expect(creditcount).toBe(34)
-      expect(dateofuniversityenrollment).toBe(null)
-      expect(city_fi).toBe('HELSINKI')
-      expect(city_sv).toBe('HELSINGFORS')
-      expect(gender_code).toBe(2)
-      expect(gender_fi).toBe('Nainen')
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-
-      const { creditcount: creditcount1, dateofuniversityenrollment: dofue } = students[1]
-
-      expect(creditcount1).toBe(380)
-      expect(dofue.toISOString()).toBe(new Date('2003-07-31 21:00:00+00').toISOString())
-    })
-
-    test('Courses are written to database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const {
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        startdate,
-        enddate,
-        max_attainment_date,
-        min_attainment_date,
-        coursetypecode
-      } = courses[0]
-
-      expect(courses.length).toBe(147)
-      expect(code).toBe('00031')
-      expect(name.en).toBe('Master of Science (Agriculture and Forestry)')
-      expect(latest_instance_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(is_study_module).toBeTruthy()
-      expect(startdate.toISOString()).toBe(new Date('1899-12-31 00:00:00+00').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2112-12-20 00:00:00+00').toISOString())
-      expect(max_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(min_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-
-      expect(coursetypecode).toBe(8)
-
-      const { code: code1, is_study_module: ism, coursetypecode: ctc } = courses[6]
-
-      expect(code1).toBe('402512')
-      expect(ism).toBe(false)
-      expect(ctc).toBe(2)
-    })
-
-    test('Credits are written to database', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const {
-        id,
-        grade,
-        student_studentnumber,
-        credits,
-        ordering,
-        attainment_date,
-        isStudyModule,
-        lastmodifieddate,
-        course_code,
-        credittypecode /*, semestercode*/
-      } = allCredits[0]
-
-      expect(allCredits.length).toBe(150)
-      expect(id).toBe('2340421')
-      expect(grade).toBe('4')
-      expect(allCredits.map(_ => _.grade)).not.toContain(null)
-      expect(student_studentnumber).toBe('014441008')
-      expect(credits).toBe(2)
-      expect(allCredits.map(_ => _.credits)).not.toContain(null)
-      expect(ordering).toBe('1997-10-21')
-      expect(attainment_date.toISOString()).toBe(new Date('1997-10-21 00:00:00+00').toISOString())
-      expect(isStudyModule).toBe(false)
-      expect(lastmodifieddate).not.toBe(null)
-      expect(course_code).toBe('65002')
-      expect(allCredits.map(_ => _.course_code)).not.toContain(null)
-      expect(credittypecode).toBe(4)
-      expect(allCredits.map(_ => _.semestercode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-    })
-
-    test('Teachers are written to database', async () => {
-      const teachers = await Teacher.findAll({ order: [['name', 'ASC']] })
-      const { id, code, name } = teachers[0]
-
-      expect(teachers.length).toBe(51)
-      expect(id).toBe('9016947')
-      expect(code).toBe('hpyrhone')
-      expect(name).toBe('Ahvenniemi Emilia')
-    })
-
-    test('Credit Teachers are written to database', async () => {
-      const creditteachers = await CreditTeacher.findAll()
-
-      expect(creditteachers.length).toBe(119)
-      expect(creditteachers.map(_ => _.credit_id)).not.toContain(null)
-      expect(creditteachers.map(_ => _.teacher_id)).not.toContain(null)
-    })
-
-    test('Semester enrollments are written to database', async () => {
-      const semesterEnrollments = await SemesterEnrollment.findAll({
-        order: [['semestercode', 'ASC']]
-      })
-      const { id, enrollmenttype, enrollment_date, studentnumber, semestercode } = semesterEnrollments[0]
-
-      expect(semesterEnrollments.length).toBe(28)
-      expect(id).toEqual(expect.any(String))
-      expect(enrollmenttype).toBe(1)
-      expect(enrollment_date.toISOString()).toBe(new Date('2003-08-17 00:00:00+00').toISOString())
-      expect(studentnumber).toBe('014272112')
-      expect(semestercode).toBe(107)
-    })
-
-    test('Providers are written to database', async () => {
-      const providers = await Provider.findAll({
-        order: [['providercode', 'DESC']]
-      })
-      const { providercode, name } = providers[0]
-
-      expect(providers.length).toBe(20)
-      expect(providercode).toBe('H906')
-      expect(name).toMatchObject({
-        en: 'Language Centre',
-        fi: 'Kielikeskus',
-        sv: 'Språkcentrum'
-      })
-      expect(providers.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course providers are written to database', async () => {
-      const courseproviders = await CourseProvider.findAll()
-
-      expect(courseproviders.length).toBe(123)
-      expect(courseproviders.map(_ => _.coursecode)).not.toContain(null)
-      expect(courseproviders.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course disciplines are written to database', async () => {
-      const courseDisciplines = await CourseDisciplines.findAll()
-      expect(courseDisciplines.length).toBe(40)
-      expect(courseDisciplines.map(_ => _.discipline_id)).not.toContain(null)
-      expect(courseDisciplines.map(_ => _.course_id)).not.toContain(null)
-    })
-
-    test('Study right extent are written to database', async () => {
-      const studyrightextent = await StudyrightExtent.findAll()
-      const { extentcode, name, createdAt, updatedAt } = studyrightextent[0]
-
-      expect(studyrightextent.length).toBe(2)
-      expect(extentcode).toBe(1)
-      expect(name.en).toBe("Bachelor's Degree")
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('Study rights are written to database', async () => {
-      const studyrights = await Studyright.findAll({
-        order: [['studyrightid', 'DESC']]
-      })
-      const {
-        studyrightid /*, canceldate*/,
-        cancelorganisation,
-        enddate,
-        givendate,
-        graduated,
-        prioritycode,
-        startdate,
-        studystartdate,
-        organization_code,
-        student_studentnumber,
-        extentcode
-      } = studyrights[0]
-
-      expect(studyrights.length).toBe(4)
-      expect(studyrightid).toBe('67628057')
-      expect(cancelorganisation).toBe('4325')
-      expect(enddate.toISOString()).toBe(new Date('2010-04-12 00:00:00+00').toISOString())
-      expect(givendate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(graduated).toBe(1)
-      expect(prioritycode).toBe(30)
-      expect(startdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(studystartdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(organization_code).toBe('4325')
-      expect(student_studentnumber).toBe('011120775')
-      expect(extentcode).toBe(1)
-    })
-
-    test('element details are written to database', async () => {
-      const elementDetails = await ElementDetails.findAll({
-        order: [['code', 'DESC']]
-      })
-      const { code, name, type, createdAt, updatedAt } = elementDetails[0]
-
-      expect(elementDetails.length).toBe(12)
-      expect(code).toBe('A2004')
-      expect(name.en).toBe('Government Decree (794/2004) on University Degrees')
-      expect(type).toBe(15)
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('study right elements are written to database', async () => {
-      const studyright_elements = await StudyrightElement.findAll({
-        order: ['startdate', 'enddate', 'studyrightid', 'code']
-      })
-      const { id, startdate, enddate, createdAt, updatedAt, studyrightid, code, studentnumber } = studyright_elements[0]
-
-      expect(studyright_elements.length).toBe(21)
-      expect(id).toBe(11)
-      expect(startdate.toISOString()).toBe(new Date('2003-07-31T21:00:00.000Z').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2007-08-24T21:00:00.000Z').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studyrightid).toBe('45853818')
-      expect(code).toBe('A1995MAAT')
-      expect(studentnumber).toBe('014272112')
-    })
-    test('transfers are written to database', async () => {
-      const transfers = await Transfers.findAll()
-      const {
-        id,
-        transferdate,
-        createdAt,
-        updatedAt,
-        studentnumber,
-        studyrightid,
-        sourcecode,
-        targetcode
-      } = transfers[0]
-
-      expect(transfers.length).toBe(1)
-      expect(id).toBe('1')
-      expect(transferdate.toISOString()).toBe(new Date('2009-07-30 21:00:00+00').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studentnumber).toBe('014272112')
-      expect(studyrightid).toBe('62346253')
-      expect(sourcecode).toBe('820016')
-      expect(targetcode).toBe('820042')
-    })
-  })
-  describe('Updater updates existing students correctly', () => {
-    beforeAll(async () => {
-      const { studentInfo, studyRights, studyAttainments } = students[0]
-      students[0].studentInfo = {
-        ...studentInfo,
-        email: 'Calgary.Flames@NHLonJätettä.fi',
-        mobile: '0400521981',
-        address: 'Yliopistonkatu 4',
-        creditcount: 322,
-        age: 41,
-        lastname: 'Tauriainen',
-        gender_code: 1,
-        abbreviatedname: 'Tauriainen Maria Helena'
-      }
-      const newCredit = {
-        id: 'ayyyyylmao',
-        grade: '5',
-        credits: 5,
-        ordering: '2019.05.04',
-        credittypecode: 1,
-        student_studentnumber: '011120775',
-        attainment_date: new Date(),
-        course_code: '420_666',
-        semestercode: 138,
-        isStudyModule: false
-      }
-      const newCourse = {
-        code: '420_666',
-        name: {
-          fi: 'Kaljan juonti',
-          en: 'Beer drinking',
-          sv: 'dryckade öl'
-        },
-        latest_instance_date: new Date(),
-        is_study_module: false,
-        coursetypecode: 1,
-        disciplines: [],
-        startdate: new Date(),
-        enddate: new Date(),
-        providers: [],
-        courseproviders: []
-      }
-
-      const newCreditTeachers = [
-        {
-          credit_id: 'ayyyyylmao',
-          teacher_id: '9016417'
-        }
-      ]
-      const newAttainment = {
-        credit: newCredit,
-        creditTeachers: newCreditTeachers,
-        course: newCourse
-      }
-      students[0].studyAttainments = [
-        ...studyAttainments,
-        newAttainment,
-        {
-          ...students[1].studyAttainments[2],
-          creditTeachers: [],
-          credit: {
-            ...students[1].studyAttainments[2].credit,
-            id: 'Kvas',
-            student_studentnumber: '011120775'
-          }
-        }
-      ]
-      students[0].studyAttainments[2] = {
-        ...students[0].studyAttainments[2],
-        credit: {
-          ...students[0].studyAttainments[2].credit,
-          grade: '0'
-        }
-      }
-      const newStudyRight = {
-        ...studyRights[0],
-        studyright: {
-          ...studyRights[0].studyright
-        }
-      }
-      students[0].studyRights = [newStudyRight]
-
-      await updateStudent(students[0])
-    })
-
-    test('StudentInfo for existing student is updated correclty', async () => {
-      const updatedStudent = await Student.findOne({
-        where: {
-          studentnumber: '011120775'
-        }
-      })
-      const {
-        studentnumber,
-        lastname,
-        abbreviatedname,
-        gender_code,
-        email,
-        age,
-        gender_fi,
-        address,
-        mobile
-      } = updatedStudent
-
-      expect(studentnumber).toBe('011120775')
-      expect(lastname).toBe('Tauriainen')
-      expect(abbreviatedname).toBe('Tauriainen Maria Helena')
-      expect(email).toBe('Calgary.Flames@NHLonJätettä.fi')
-      expect(gender_code).toBe(1)
-      expect(gender_fi).toBe('Nainen')
-      expect(age).toBe(41)
-      expect(address).toBe('Yliopistonkatu 4')
-      expect(mobile).toBe('0400521981')
-    })
-    test('Existing courses are updated (not duplicated) in database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const newCourse = courses.find(_ => _.code === '420_666')
-      expect(courses.length).toBe(148)
-
-      expect(newCourse).not.toBe(undefined)
-    })
-
-    test('Updating credits works', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const updatedCredit = allCredits.find(_ => _.id === students[0].studyAttainments[2].credit.id)
-      expect(allCredits.length).toBe(152)
-      expect(updatedCredit.grade).toBe('0')
-    })
-
-    test('Studyrights are rewritten', async () => {
-      const updatedStudyRights = await Studyright.findAll({
-        where: {
-          student_studentnumber: '011120775'
-        }
-      })
-      expect(updatedStudyRights.length).toBe(1)
-      // theres really not a lot to test here since everything related to studyrights are just rewritten
-    })
-  })
-})

old_oodikone_stuff/updater_writer/updater/update_attainment_dates.js

@@ -1,28 +0,0 @@
-const { sequelize } = require('../database/connection')
-
-const updateAttainmentDates = async () => {
-  const { schema } = sequelize.options
-  const transaction = await sequelize.transaction()
-  try {
-    await sequelize.query(
-      `
-      -- Block writing to table to avoid deadlocks:
-      LOCK TABLE ONLY ${schema}.course IN EXCLUSIVE MODE;
-
-      UPDATE  ${schema}.course
-      SET max_attainment_date = cr.max, min_attainment_date = cr.min
-      FROM (select course_code, max(attainment_date), min(attainment_date) from ${schema}.credit group by course_code) cr
-      WHERE course.code=cr.course_code;
-     `,
-      { transaction }
-    )
-    await transaction.commit()
-  } catch (e) {
-    await transaction.rollback()
-    throw e
-  }
-}
-
-module.exports = {
-  updateAttainmentDates
-}

package-lock.json

@@ -5,24 +5,25 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "oodikone",
       "license": "ISC",
       "devDependencies": {
         "@babel/eslint-parser": "^7.15.0",
         "@babel/plugin-syntax-jsx": "^7.14.5",
         "concurrently": "^6.2.1",
-        "cypress": "^8.2.0",
+        "cypress": "^8.3.1",
         "eslint": "^7.32.0",
         "eslint-config-airbnb": "^18.2.1",
         "eslint-config-prettier": "^8.3.0",
         "eslint-plugin-babel": "^5.3.1",
         "eslint-plugin-cypress": "^2.11.3",
-        "eslint-plugin-import": "^2.24.0",
+        "eslint-plugin-import": "^2.24.2",
         "eslint-plugin-jsx-a11y": "^6.4.1",
         "eslint-plugin-prefer-object-spread": "^1.2.1",
-        "eslint-plugin-prettier": "^3.4.0",
+        "eslint-plugin-prettier": "^3.4.1",
         "eslint-plugin-react": "^7.24.0",
         "eslint-plugin-react-hooks": "^4.2.0",
-        "husky": "^7.0.1",
+        "husky": "^7.0.2",
         "lint-staged": "^11.1.2",
         "prettier": "^2.3.2",
         "stylelint": "^13.13.1",
@@ -535,9 +536,9 @@
       }
     },
     "node_modules/@cypress/request": {
-      "version": "2.88.5",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz",
-      "integrity": "sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA==",
+      "version": "2.88.6",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.6.tgz",
+      "integrity": "sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ==",
       "dev": true,
       "dependencies": {
         "aws-sign2": "~0.7.0",
@@ -553,13 +554,12 @@
         "isstream": "~0.1.2",
         "json-stringify-safe": "~5.0.1",
         "mime-types": "~2.1.19",
-        "oauth-sign": "~0.9.0",
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
         "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
-        "uuid": "^3.3.2"
+        "uuid": "^8.3.2"
       },
       "engines": {
         "node": ">= 6"
@@ -686,6 +686,12 @@
         "postcss-syntax": ">=0.36.2"
       }
     },
+    "node_modules/@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
+      "dev": true
+    },
     "node_modules/@types/mdast": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
@@ -1468,19 +1474,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/concurrently/node_modules/date-fns": {
-      "version": "2.22.1",
-      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz",
-      "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.11"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/date-fns"
-      }
-    },
     "node_modules/concurrently/node_modules/supports-color": {
       "version": "8.1.1",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
@@ -1580,13 +1573,13 @@
       }
     },
     "node_modules/cypress": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
-      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.3.1.tgz",
+      "integrity": "sha512-1v6pfx+/5cXhaT5T6QKOvnkawmEHWHLiVzm3MYMoQN1fkX2Ma1C32STd3jBStE9qT5qPSTILjGzypVRxCBi40g==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@cypress/request": "^2.88.5",
+        "@cypress/request": "^2.88.6",
         "@cypress/xvfb": "^1.2.4",
         "@types/node": "^14.14.31",
         "@types/sinonjs__fake-timers": "^6.0.2",
@@ -1665,6 +1658,19 @@
         "node": ">=0.10"
       }
     },
+    "node_modules/date-fns": {
+      "version": "2.23.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.23.0.tgz",
+      "integrity": "sha512-5ycpauovVyAk0kXNZz6ZoB9AYMZB4DObse7P3BPWmyEjXNORTI8EJ6X0uaSAq4sCHzM1uajzrkr6HnsLQpxGXA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.11"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/date-fns"
+      }
+    },
     "node_modules/dayjs": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.5.tgz",
@@ -2067,9 +2073,9 @@
       }
     },
     "node_modules/eslint-import-resolver-node": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
-      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz",
+      "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==",
       "dev": true,
       "dependencies": {
         "debug": "^3.2.7",
@@ -2144,26 +2150,26 @@
       }
     },
     "node_modules/eslint-plugin-import": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
-      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
+      "version": "2.24.2",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz",
+      "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==",
       "dev": true,
       "dependencies": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-import-resolver-node": "^0.3.6",
         "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
-        "is-core-module": "^2.4.0",
+        "is-core-module": "^2.6.0",
         "minimatch": "^3.0.4",
-        "object.values": "^1.1.3",
+        "object.values": "^1.1.4",
         "pkg-up": "^2.0.0",
         "read-pkg-up": "^3.0.0",
         "resolve": "^1.20.0",
-        "tsconfig-paths": "^3.9.0"
+        "tsconfig-paths": "^3.11.0"
       },
       "engines": {
         "node": ">=4"
@@ -2237,9 +2243,9 @@
       }
     },
     "node_modules/eslint-plugin-prettier": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz",
-      "integrity": "sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.1.tgz",
+      "integrity": "sha512-htg25EUYUeIhKHXjOinK4BgCcDwtLHjqaxCDsMy5nbnUMkKFvIhMVCp+5GFUXQ4Nr8lBsPqtGAqBenbpFqAA2g==",
       "dev": true,
       "dependencies": {
         "prettier-linter-helpers": "^1.0.0"
@@ -3034,6 +3040,7 @@
       "version": "5.1.5",
       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
       "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "deprecated": "this library is no longer supported",
       "dev": true,
       "dependencies": {
         "ajv": "^6.12.3",
@@ -3148,9 +3155,9 @@
       }
     },
     "node_modules/husky": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.1.tgz",
-      "integrity": "sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.2.tgz",
+      "integrity": "sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg==",
       "dev": true,
       "bin": {
         "husky": "lib/bin.js"
@@ -3355,9 +3362,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz",
+      "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==",
       "dev": true,
       "dependencies": {
         "has": "^1.0.3"
@@ -4345,21 +4352,21 @@
       }
     },
     "node_modules/mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
       "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
     },
     "node_modules/mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
       "dev": true,
       "dependencies": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.49.0"
       },
       "engines": {
         "node": ">= 0.6"
@@ -4487,15 +4494,6 @@
       "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
       "dev": true
     },
-    "node_modules/oauth-sign": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
-      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
-      "dev": true,
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -4761,9 +4759,9 @@
       }
     },
     "node_modules/path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "node_modules/path-type": {
@@ -5625,6 +5623,11 @@
         "safer-buffer": "^2.0.2",
         "tweetnacl": "~0.14.0"
       },
+      "bin": {
+        "sshpk-conv": "bin/sshpk-conv",
+        "sshpk-sign": "bin/sshpk-sign",
+        "sshpk-verify": "bin/sshpk-verify"
+      },
       "engines": {
         "node": ">=0.10.0"
       }
@@ -6135,16 +6138,29 @@
       }
     },
     "node_modules/tsconfig-paths": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz",
-      "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==",
+      "version": "3.11.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz",
+      "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==",
       "dev": true,
       "dependencies": {
-        "json5": "^2.2.0",
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.1",
         "minimist": "^1.2.0",
         "strip-bom": "^3.0.0"
       }
     },
+    "node_modules/tsconfig-paths/node_modules/json5": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+      "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.0"
+      },
+      "bin": {
+        "json5": "lib/cli.js"
+      }
+    },
     "node_modules/tslib": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
@@ -6327,12 +6343,12 @@
       "dev": true
     },
     "node_modules/uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true,
       "bin": {
-        "uuid": "bin/uuid"
+        "uuid": "dist/bin/uuid"
       }
     },
     "node_modules/v8-compile-cache": {
@@ -6928,9 +6944,9 @@
       }
     },
     "@cypress/request": {
-      "version": "2.88.5",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.5.tgz",
-      "integrity": "sha512-TzEC1XMi1hJkywWpRfD2clreTa/Z+lOrXDCxxBTBPEcY5azdPi56A6Xw+O4tWJnaJH3iIE7G5aDXZC6JgRZLcA==",
+      "version": "2.88.6",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.6.tgz",
+      "integrity": "sha512-z0UxBE/+qaESAHY9p9sM2h8Y4XqtsbDCt0/DPOrqA/RZgKi4PkxdpXyK4wCCnSk1xHqWHZZAE+gV6aDAR6+caQ==",
       "dev": true,
       "requires": {
         "aws-sign2": "~0.7.0",
@@ -6946,13 +6962,12 @@
         "isstream": "~0.1.2",
         "json-stringify-safe": "~5.0.1",
         "mime-types": "~2.1.19",
-        "oauth-sign": "~0.9.0",
         "performance-now": "^2.1.0",
         "qs": "~6.5.2",
         "safe-buffer": "^5.1.2",
         "tough-cookie": "~2.5.0",
         "tunnel-agent": "^0.6.0",
-        "uuid": "^3.3.2"
+        "uuid": "^8.3.2"
       }
     },
     "@cypress/xvfb": {
@@ -7055,6 +7070,12 @@
         "unist-util-find-all-after": "^3.0.2"
       }
     },
+    "@types/json5": {
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
+      "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
+      "dev": true
+    },
     "@types/mdast": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.4.tgz",
@@ -7653,12 +7674,6 @@
         "yargs": "^16.2.0"
       },
       "dependencies": {
-        "date-fns": {
-          "version": "2.22.1",
-          "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.22.1.tgz",
-          "integrity": "sha512-yUFPQjrxEmIsMqlHhAhmxkuH769baF21Kk+nZwZGyrMoyLA+LugaQtC0+Tqf9CBUUULWwUJt6Q5ySI3LJDDCGg==",
-          "dev": true
-        },
         "supports-color": {
           "version": "8.1.1",
           "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
@@ -7736,12 +7751,12 @@
       "dev": true
     },
     "cypress": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
-      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.3.1.tgz",
+      "integrity": "sha512-1v6pfx+/5cXhaT5T6QKOvnkawmEHWHLiVzm3MYMoQN1fkX2Ma1C32STd3jBStE9qT5qPSTILjGzypVRxCBi40g==",
       "dev": true,
       "requires": {
-        "@cypress/request": "^2.88.5",
+        "@cypress/request": "^2.88.6",
         "@cypress/xvfb": "^1.2.4",
         "@types/node": "^14.14.31",
         "@types/sinonjs__fake-timers": "^6.0.2",
@@ -7810,6 +7825,12 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "date-fns": {
+      "version": "2.23.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.23.0.tgz",
+      "integrity": "sha512-5ycpauovVyAk0kXNZz6ZoB9AYMZB4DObse7P3BPWmyEjXNORTI8EJ6X0uaSAq4sCHzM1uajzrkr6HnsLQpxGXA==",
+      "dev": true
+    },
     "dayjs": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.5.tgz",
@@ -8142,9 +8163,9 @@
       "requires": {}
     },
     "eslint-import-resolver-node": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
-      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz",
+      "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==",
       "dev": true,
       "requires": {
         "debug": "^3.2.7",
@@ -8210,26 +8231,26 @@
       }
     },
     "eslint-plugin-import": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
-      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
+      "version": "2.24.2",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz",
+      "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==",
       "dev": true,
       "requires": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-import-resolver-node": "^0.3.6",
         "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
-        "is-core-module": "^2.4.0",
+        "is-core-module": "^2.6.0",
         "minimatch": "^3.0.4",
-        "object.values": "^1.1.3",
+        "object.values": "^1.1.4",
         "pkg-up": "^2.0.0",
         "read-pkg-up": "^3.0.0",
         "resolve": "^1.20.0",
-        "tsconfig-paths": "^3.9.0"
+        "tsconfig-paths": "^3.11.0"
       },
       "dependencies": {
         "debug": {
@@ -8285,9 +8306,9 @@
       "requires": {}
     },
     "eslint-plugin-prettier": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.0.tgz",
-      "integrity": "sha512-UDK6rJT6INSfcOo545jiaOwB701uAIt2/dR7WnFQoGCVl1/EMqdANBmwUaqqQ45aXprsTGzSa39LI1PyuRBxxw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.4.1.tgz",
+      "integrity": "sha512-htg25EUYUeIhKHXjOinK4BgCcDwtLHjqaxCDsMy5nbnUMkKFvIhMVCp+5GFUXQ4Nr8lBsPqtGAqBenbpFqAA2g==",
       "dev": true,
       "requires": {
         "prettier-linter-helpers": "^1.0.0"
@@ -8960,9 +8981,9 @@
       "dev": true
     },
     "husky": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.1.tgz",
-      "integrity": "sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-7.0.2.tgz",
+      "integrity": "sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg==",
       "dev": true
     },
     "ignore": {
@@ -9091,9 +9112,9 @@
       }
     },
     "is-core-module": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz",
-      "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==",
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz",
+      "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==",
       "dev": true,
       "requires": {
         "has": "^1.0.3"
@@ -9817,18 +9838,18 @@
       }
     },
     "mime-db": {
-      "version": "1.47.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.30",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
       "dev": true,
       "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.49.0"
       }
     },
     "mimic-fn": {
@@ -9932,12 +9953,6 @@
       "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
       "dev": true
     },
-    "oauth-sign": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
-      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
-      "dev": true
-    },
     "object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -10127,9 +10142,9 @@
       "dev": true
     },
     "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "path-type": {
@@ -11174,14 +11189,26 @@
       "dev": true
     },
     "tsconfig-paths": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz",
-      "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==",
+      "version": "3.11.0",
+      "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz",
+      "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==",
       "dev": true,
       "requires": {
-        "json5": "^2.2.0",
+        "@types/json5": "^0.0.29",
+        "json5": "^1.0.1",
         "minimist": "^1.2.0",
         "strip-bom": "^3.0.0"
+      },
+      "dependencies": {
+        "json5": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+          "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.0"
+          }
+        }
       }
     },
     "tslib": {
@@ -11333,9 +11360,9 @@
       "dev": true
     },
     "uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true
     },
     "v8-compile-cache": {

package.json

@@ -2,10 +2,11 @@
   "name": "oodikone",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "concurrently": "concurrently",
     "cypress": "cypress",
+    "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
@@ -13,29 +14,8 @@
     "updater:real": "./run.sh updater real up --detach",
     "both": "./run.sh both anon up --detach",
     "both:real": "./run.sh both real up --detach",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install",
-    "lint:docker-compose": "docker-compose config -q && :",
-    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
-    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
-    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
-  },
-  "lint-staged": {
-    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
-    "*.{js,json,md}": "prettier --write",
-    "*.css": "stylelint --fix",
-    ".github/workflows/*": "actionlint",
-    "Dockerfile": "hadolint",
-    "*.sh": "shellcheck -x",
-    "docker-compose.yml": "npm run lint:docker-compose",
-    "docker-compose.real.yml": "npm run lint:docker-compose:real",
-    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
-    "docker-compose.test.yml": "npm run lint:docker-compose:test"
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "prepare": "husky install"
   },
   "repository": {
     "type": "git",
@@ -50,19 +30,19 @@
     "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
     "concurrently": "^6.2.1",
-    "cypress": "^8.2.0",
+    "cypress": "^8.3.1",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.24.0",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
-    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-prettier": "^3.4.1",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "husky": "^7.0.1",
+    "husky": "^7.0.2",
     "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",

run.sh

@@ -85,7 +85,7 @@ parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
 elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${compose_command}"
+  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

scripts/functions_for_setup.sh

@@ -18,15 +18,13 @@ SIS_DB_NAME="sis-db"
 SIS_IMPORTER_DB_NAME="sis-importer-db"
 USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
-OODI_DB_NAME="oodi-db" # TODO: Remove when oodi is removed
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+KONE_DB_REAL_DUMP_URL="svm-101:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
 SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+USER_DB_REAL_DUMP_URL="svm-101:/home/oodikone_user/backups/latest-user-db.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
-OODI_DB_REAL_DUMP_URL="svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/latest-pg.sqz" # TODO: Remove when oodi is removed
 
 # Source utility functions
 source "$PROJECT_ROOT"/scripts/utils.sh
@@ -73,7 +71,7 @@ reset_databases() {
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
-  docker-compose down
+  npm run docker:down
   docker-compose up -d "${databases[@]}"
 
   for database in "${databases[@]}"; do
@@ -120,14 +118,6 @@ reset_sis_importer_data() {
   reset_databases $database
 }
 
-reset_old_oodi_data() {
-  infomsg "Downloading old oodi-db dump"
-  local database=$OODI_DB_NAME
-  local url=$OODI_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
-}
-
 set_up_oodikone_from_scratch() {
   draw_mopo
   folders_to_set_up=(

services/backend/index.js

@@ -1,5 +1,6 @@
 require('./src/app')
+const logger = require('./src/util/logger')
 
 process.on('unhandledRejection', reason => {
-  console.log(reason)
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
 })

services/backend/package.json

@@ -12,65 +12,30 @@
   },
   "dependencies": {
     "@sentry/node": "^5.30.0",
-    "async": "^3.2.0",
-    "async-lock": "^1.2.8",
+    "@sentry/tracing": "^6.11.0",
+    "async": "^3.2.1",
+    "async-lock": "^1.3.0",
     "axios": "^0.21.1",
     "bluebird": "^3.7.2",
-    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.17.1",
+    "express-async-errors": "^3.1.1",
     "jsonwebtoken": "^8.5.1",
-    "kafka-node": "^4.1.3",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
-    "mkdirp": "^0.5.5",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
-    "nodemailer": "^4.7.0",
-    "pg": "^8.6.0",
-    "redis": "^3.1.1",
-    "sequelize": "^6.6.2",
+    "pg": "^8.7.1",
+    "redis": "^3.1.2",
+    "sequelize": "^6.6.5",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0",
-    "winston-transport": "^4.4.0"
+    "winston": "^3.3.3"
   },
   "devDependencies": {
-    "ava": "^2.4.0",
-    "faker": "^4.1.0",
-    "jest": "^24.9.0",
-    "nock": "^10.0.6",
-    "node-status": "^1.0.0",
-    "nodemon": "^1.19.4",
-    "nyc": "^13.3.0",
-    "supertest": "^3.4.2"
-  },
-  "jest": {
-    "setupFilesAfterEnv": [
-      "./test/jest/globals.js"
-    ],
-    "testEnvironment": "node"
-  },
-  "ava": {
-    "files": [
-      "test/**/*_test.js"
-    ],
-    "source": [
-      "**/*.{js,jsx}",
-      "!test/**/*"
-    ],
-    "concurrency": 5,
-    "failFast": true,
-    "failWithoutAssertions": true,
-    "tap": true,
-    "powerAssert": false,
-    "require": [
-      "babel-register"
-    ],
-    "babel": "inherit"
+    "nodemon": "^1.19.4"
   }
 }

services/backend/requests/courses_yearly.rest

@@ -1,7 +0,0 @@
-GET /api/courseyearlystats/?start=2013&end=2018&code=TKT10002 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token generated from login.rest to use

services/backend/requests/enableUser.rest

@@ -1,6 +0,0 @@
-POST http://localhost:8080/api/users/enable
-Content-Type: application/json
-
-{
-  "id": 2
-}

services/backend/requests/get_all_duplicate_codes.rest

@@ -1,6 +0,0 @@
-GET /api/courses/duplicatecodes/all HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_duplicate_codes.rest

@@ -1,7 +0,0 @@
-GET /api/courses/duplicatecodes/?code=TKT10005 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/get_mandatory_courses.rest

@@ -1,7 +0,0 @@
-GET /api/v2/studyprogrammes/{insertstudyprogrammecodehere}/mandatory_courses
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/get_student.rest

@@ -1,7 +0,0 @@
-GET /api/students/{insert student id}
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/requests/login.rest

@@ -1,7 +0,0 @@
-POST /api/login HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-Cache-Control: no-cache
-Postman-Token: d8a05d0f-fd91-e4e0-54a0-172d18ff3fad
-Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW

services/backend/requests/newUser.rest

@@ -1,4 +0,0 @@
-GET http://localhost:8080/api/login
-Content-Type: application/json
-eduPersonPrincipalName: totutotu@
-shib-session-id: IDEEE

services/backend/requests/post_duplicate_codes.rest

@@ -1,7 +0,0 @@
-POST /api/courses/duplicatecodes/TKT10005/205044 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/remove_duplicate_code.rest

@@ -1,7 +0,0 @@
-DELETE /api/courses/duplicatecodes/TKT10005/58123 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/requests/update.rest

@@ -1,11 +0,0 @@
-POST /api/updatedatabase HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-content-type: application/json
-
-[
-  "01234562",
-  "12302459"
-]

services/backend/scripts/create_fmsei_report.js

@@ -1,132 +0,0 @@
-/* eslint-disable no-console */
-
-const { Student, Credit, Studyright } = require('../src/models')
-const Sequelize = require('sequelize')
-const Op = Sequelize.Op
-
-const startDate = new Date(2019, 1, 1, 0, 0, 0, 0)
-
-const wasFMSEIStudent = (student, date) => {
-  const degreeright = student.studyrights.find(sr => sr.extentcode !== 9 && sr.startdate <= date && sr.enddate >= date)
-  return !degreeright
-}
-
-const courseCodes = [
-  'AYTKT200041',
-  'AYTKT21007',
-  'TKT21007',
-  'AYTKT21025en',
-  'AYTKT21009',
-  'TKT21009',
-  'AYTKT21010',
-  'TKT21010',
-  'AYTKT21018',
-  'AYTKT21018fi',
-  'AYTKT21018sv',
-  'AY5823951',
-  'AY5823952',
-  'AY5823953',
-  'AY5823954',
-  'AY5823955',
-  'AY5823956',
-]
-
-const createReport = async () => {
-  console.log('SPLIT')
-  const { fmseiStudentnumbers, otherStudentnumbers } = await courseCodes.reduce(
-    async (accPromise, courseCode) => {
-      const credits = await Credit.findAll({
-        where: {
-          course_code: courseCode,
-          attainment_date: {
-            [Op.gte]: startDate,
-          },
-        },
-      })
-
-      const { fmseiStudentnumbers, otherStudentnumbers } = await credits.reduce(
-        async (accPromise, credit) => {
-          const student = await Student.findOne({
-            where: { studentnumber: credit.student_studentnumber },
-            include: [{ model: Studyright }],
-          })
-          const acc = await accPromise
-
-          if (wasFMSEIStudent(student, credit.attainment_date)) {
-            // FMSEI student fmsei credits
-            console.log(
-              `${credit.student_studentnumber};${credit.course_code};${credit.grade};${credit.credits};${new Date(
-                credit.attainment_date
-              ).toISOString()}`
-            )
-            return {
-              fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(student.studentnumber),
-              otherStudentnumbers: acc.otherStudentnumbers,
-            }
-          }
-          return {
-            fmseiStudentnumbers: acc.fmseiStudentnumbers,
-            otherStudentnumbers: acc.otherStudentnumbers.concat(student.studentnumber),
-          }
-        },
-        { fmseiStudentnumbers: [], otherStudentnumbers: [] }
-      )
-
-      const acc = await accPromise
-
-      return {
-        courseStats: acc.courseStats.concat({
-          courseCode: courseCode,
-          fmsei: fmseiStudentnumbers.length,
-          other: otherStudentnumbers.length,
-        }),
-        fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(fmseiStudentnumbers),
-        otherStudentnumbers: acc.otherStudentnumbers.concat(otherStudentnumbers),
-      }
-    },
-    { courseStats: [], fmseiStudentnumbers: [], otherStudentnumbers: [] }
-  )
-  const fmseiStudents = await Student.findAll({
-    where: { studentnumber: fmseiStudentnumbers },
-    include: [
-      {
-        model: Credit,
-        separate: true,
-      },
-    ],
-  })
-
-  const otherStudents = await Student.findAll({
-    where: { studentnumber: otherStudentnumbers },
-  })
-  console.log('SPLIT')
-  otherStudents.forEach(s => console.log(`${new Date(s.birthdate).getFullYear()};${s.gender_fi}`))
-  console.log('SPLIT')
-
-  const getFirstName = namestring => {
-    const names = namestring.split(' ')
-    const i = names.findIndex(n => n.substring(0, 1) === '*')
-    return i != -1 ? names[i].substring(1) : names[0]
-  }
-
-  fmseiStudents.forEach(s =>
-    console.log(
-      `${s.studentnumber};${s.email || ''};${s.zipcode || ''};${s.city_fi || ''};${new Date(
-        s.birthdate
-      ).getFullYear()};${s.country_fi || ''};${s.home_country_fi || ''};${s.gender_fi};${getFirstName(s.firstnames)}`
-    )
-  )
-  console.log('SPLIT')
-  // FMSEI all credits
-  fmseiStudents.forEach(s => {
-    s.credits.forEach(c =>
-      console.log(
-        `${c.student_studentnumber};${c.course_code};${c.grade};${c.credits};${new Date(
-          c.attainment_date
-        ).toISOString()}`
-      )
-    )
-  })
-}
-
-createReport()

services/backend/scripts/force_sync_database.js

@@ -1,16 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-const { forceSyncDatabase } = require('../src/database/connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-sync()

services/backend/scripts/refresh_studyright_associations.js

@@ -1,8 +0,0 @@
-const { refreshAssociationsInRedis } = require('../src/services/studyrights')
-
-const run = async () => {
-  await refreshAssociationsInRedis()
-  process.exit(0)
-}
-
-run()

services/backend/scripts/seed_migrations.js

@@ -1,13 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-
-const run = async () => {
-  try {
-    await seedAllMigrations()
-    process.exit(0)
-  } catch (e) {
-    console.log(`Seeding migrations failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/scripts/update_top_teachers.js

@@ -1,38 +0,0 @@
-const { findAndSaveTopTeachers } = require('../src/services/topteachers')
-const logger = require('../src/util/logger')
-
-const findAndSaveTeachers = async (startcode, endcode) => {
-  for (let code = startcode; code <= endcode; code++) {
-    logger.info(`Saving top teachers from year ${code}`)
-    await findAndSaveTopTeachers(code)
-  }
-}
-
-const parseargs = (args, SEPARATOR = '=') =>
-  args
-    .filter(arg => arg.includes(SEPARATOR))
-    .reduce((acc, arg) => {
-      const [key, val] = arg.split(SEPARATOR)
-      return {
-        ...acc,
-        [key.trim()]: val.trim(),
-      }
-    }, {})
-
-const run = async () => {
-  try {
-    const { from, to } = parseargs(process.argv)
-    const start = from
-    const end = to || from
-    if (end < start) {
-      throw Error('"to" -argument has to be less than "from" -argument. ')
-    }
-    await findAndSaveTeachers(start, end)
-    process.exit(0)
-  } catch (error) {
-    logger.error(`Failed to update top teachers: ${error.message}`)
-    process.exit(1)
-  }
-}
-
-run()