Merge pull request #3150 from UniversityOfHelsinkiCS/trunk

Remove oodi from oodikone + tiny fixes

.eslintrc.json

@@ -0,0 +1,26 @@
+{
+  "env": {
+    "es6": true,
+    "node": true,
+    "jest": true
+  },
+  "parser": "@babel/eslint-parser",
+  "parserOptions": {
+    "requireConfigFile": false
+  },
+  "extends": [
+    "eslint:recommended",
+    "plugin:prettier/recommended",
+    "plugin:cypress/recommended"
+  ],
+  "plugins": [
+    "prettier",
+    "cypress"
+  ],
+  "rules": {
+    "no-console": "off",
+    "react/no-unescaped-entities": "off",
+    "no-async-promise-executor": "off",
+    "prettier/prettier": "error"
+  }
+}

.github/dependabot.yml

@@ -6,8 +6,10 @@ updates:
     interval: weekly
     time: "03:00"
   open-pull-requests-limit: 10
+  target-branch: "trunk"
 - package-ecosystem: "github-actions"
   directory: "/"
   schedule:
     interval: weekly
     time: "03:00"
+  target-branch: "trunk"

.github/workflows/release-oodikone-staging.yaml

@@ -1,4 +1,4 @@
-name: Release oodikone to staging
+name: "Release oodikone to staging"
 
 on:
   push:
@@ -7,39 +7,119 @@ on:
     paths:
       - '.github/workflows/release-oodikone-staging.yaml'
       - 'services/**'
-      - 'docker-compose.ci.yml'
 
 jobs:
-  build_and_release:
-    name: "Build images and release"
+
+  build_and_release_backend:
+    name: "Build and release backend"
     runs-on: ubuntu-latest
-    env:
-      TAG: staging
-      CACHE_TAG: staging
     steps:
-      - name: "Checkout repo"
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
       - name: "Login to DockerHub"
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build  images"
-        run: ./run.sh oodikone ci build analytics backend frontend userservice
-      - name: "Start frontend"
-        run: ./run.sh oodikone ci up -d frontend
-      - name: "Add frontend to sentry"
-        run: |
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh +e -c 'npm install sentry-cli && npm run sentry-cli -- releases info "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases delete "$(cat /SENTRY_RELEASE_VERSION)" || true'
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh -c 'npm run sentry-cli -- releases new --finalize "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases files "$(cat /SENTRY_RELEASE_VERSION)" upload-sourcemaps --rewrite dist'
-      - name: "Push to docker hub"
-        run: ./run.sh oodikone ci push analytics backend frontend userservice
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:trunk
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: true
+          tags: toska/oodikone2-backend:staging
+          target: production
+
+  build_and_release_frontend:
+    name: "Build and release frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:trunk
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: true
+          tags: toska/oodikone2-frontend:staging
+          target: production
+      - name: "Create a container from built image "
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: "Copy assets from the container"
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: "Create Sentry release"
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
+
+  build_and_release_userservice:
+    name: "Build and release userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:trunk
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: true
+          tags: toska/oodikone2-userservice:staging
+          target: production
+
+  report_success:
+    runs-on: ubuntu-latest
+    name: "Report deployment success to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: success()
+    steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -48,14 +128,16 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
-    - build_and_release
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: staging
+          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -1,40 +1,132 @@
-name: Release oodikone to production
+name: "Release oodikone to production"
 
 on:
   release:
     types: [published]
 
 jobs:
-  build_and_release:
-    name: "Build images and release"
+  build_and_release_backend:
+    name: "Build and release backend"
     runs-on: ubuntu-latest
-    env:
-      TAG: latest
-      CACHE_TAG: latest
     steps:
-      - name: "Checkout repo"
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
       - name: "Login to DockerHub"
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build  images"
-        run: ./run.sh oodikone ci build analytics backend frontend userservice
-      - name: "Start frontend"
-        run: ./run.sh oodikone ci up -d frontend
-      - name: "Add frontend to sentry"
-        run: |
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh +e -c 'npm install sentry-cli && npm run sentry-cli -- releases info "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases delete "$(cat /SENTRY_RELEASE_VERSION)" || true'
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh -c 'npm run sentry-cli -- releases new --finalize "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases files "$(cat /SENTRY_RELEASE_VERSION)" upload-sourcemaps --rewrite dist'
-      - name: "Push to docker hub"
-        run: ./run.sh oodikone ci push analytics backend frontend userservice
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:trunk
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: true
+          tags: toska/oodikone2-backend:latest
+          target: production
+
+  build_and_release_frontend:
+    name: "Build and release frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:trunk
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: true
+          tags: toska/oodikone2-frontend:latest
+          target: production
+      - name: "Create a container from built image "
+        run: docker create --name frontend toska/oodikone2-frontend:latest
+      - name: "Copy assets from the container"
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: "Create Sentry release"
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
+
+  build_and_release_userservice:
+    name: "Build and release userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:trunk
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: true
+          tags: toska/oodikone2-userservice:latest
+          target: production
+
+  report_success:
+    runs-on: ubuntu-latest
+    name: "Report deployment success to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: success()
+    steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -43,14 +135,16 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
-    - build_and_release
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: production
+          deployment-target: production

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,4 +1,4 @@
-name: Release sis-updater-scheduler
+name: "Release sis-updater-scheduler"
 
 on:
   push:
@@ -10,41 +10,45 @@ on:
 
 jobs:
   build_and_release:
+    name: "Build and release sis-updater-scheduler"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-
-    - name: Login to DockerHub
+    - uses: docker/setup-buildx-action@v1
+    - name: "Login to DockerHub"
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build and push
+    - name: "Build and push"
       uses: docker/build-push-action@v2
       with:
+        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+        cache-to: type=inline
         context: updater/sis-updater-scheduler/
         push: true
         tags: toska/sis-updater-scheduler:latest
-
+        target: production
     - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
       with:
         webhook-url: ${{ secrets.WEBHOOK_URL }}
         message-type: deployment
         softa-url: https://oodikone.cs.helsinki.fi
         deployment-target: production
+        softa-to-deploy: sis-updater-scheduler
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
     - build_and_release
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: production
+          deployment-target: production
+          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,4 +1,4 @@
-name: Release sis-updater-worker
+name: "Release sis-updater-worker"
 
 on:
   push:
@@ -10,41 +10,45 @@ on:
 
 jobs:
   build_and_release:
+    name: "Build and release sis-updater-worker"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-
-    - name: Login to DockerHub
+    - uses: docker/setup-buildx-action@v1
+    - name: "Login to DockerHub"
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build and push
+    - name: "Build and push"
       uses: docker/build-push-action@v2
       with:
+        cache-from: type=registry,ref=toska/sis-updater-worker:latest
+        cache-to: type=inline
         context: updater/sis-updater-worker/
         push: true
         tags: toska/sis-updater-worker:latest
-
+        target: production
     - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
       with:
         webhook-url: ${{ secrets.WEBHOOK_URL }}
         message-type: deployment
         softa-url: https://oodikone.cs.helsinki.fi
         deployment-target: production
+        softa-to-deploy: sis-updater-worker
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
     - build_and_release
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: production
+          deployment-target: production
+          softa-to-deploy: sis-updater-worker

.github/workflows/test-oodikone.yaml

@@ -12,370 +12,421 @@ on:
       - docker-compose.ci.yml
       - nginx.conf
       - package*.json
-      - run.sh
 
 jobs:
   # === Static tests / linting ===
 
-  lint_analytics:
-    name: "Lint analytics"
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/oodikone2-analytics
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
   lint_backend:
     name: "Lint backend"
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/backend/oodikone2-backend
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
+      - name: "Install dependencies"
+        run: npm ci && cd services/backend && npm ci
+      - name: "Run backend linter"
+        run: npm run eslint services/backend
+
   lint_frontend:
     name: "Lint frontend"
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/oodikone2-frontend
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
+      - name: "Install dependencies"
+        run: npm ci && cd services/frontend && npm ci
+      - name: "Run frontend linter"
+        run: npm run concurrently \
+              'eslint --ext .jsx,.js services/frontend' \
+              'stylelint services/frontend/src/**/*.css'
+
   lint_userservice:
     name: "Lint userservice"
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/oodikone2-userservice
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
+      - name: "Install dependencies"
+        run: npm ci && cd services/userservice && npm ci
+      - name: "Run frontend linter"
+        run: npm run eslint services/userservice
 
   # === Build images ===
 
-  build_analytics:
-    name: "Build analytics"
-    runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull analytics
-      - name: Build  image
-        run: ./run.sh oodikone ci build analytics
-      - name: Save built image
-        run: docker save toska/oodikone2-analytics:trunk -o /tmp/analytics.tar
-      - name: Upload image for other jobs
-        uses: actions/upload-artifact@v2
-        with:
-          path: /tmp/analytics.tar
   build_backend:
     name: "Build backend"
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull backend
-      - name: Build image
-        run: ./run.sh oodikone ci build backend
-      - name: Save built image
-        run: docker save toska/oodikone2-backend:trunk -o /tmp/backend.tar
-      - name: Upload image for other jobs
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Build image and export it"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:trunk
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: false
+          tags: toska/oodikone2-backend:trunk
+          target: production
+          outputs: type=docker,dest=/tmp/backend.tar
+      - name: "Upload image for other jobs"
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
+
   build_frontend:
     name: "Build frontend"
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull frontend
-      - name: Build image
-        run: ./run.sh oodikone ci build frontend
-      - name: Save built image
-        run: docker save toska/oodikone2-frontend:trunk -o /tmp/frontend.tar
-      - name: Upload image for other jobs
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Build image and export it"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:trunk
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: false
+          tags: toska/oodikone2-frontend:trunk
+          target: production
+          outputs: type=docker,dest=/tmp/frontend.tar
+      - name: "Upload image for other jobs"
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
+
   build_userservice:
     name: "Build userservice"
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull userservice
-      - name: Build image
-        run: ./run.sh oodikone ci build userservice
-      - name: Save built image
-        run: docker save toska/oodikone2-userservice:trunk -o /tmp/userservice.tar
-      - name: Upload image for other jobs
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Build image and export it"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:trunk
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: false
+          tags: toska/oodikone2-userservice:trunk
+          target: production
+          outputs: type=docker,dest=/tmp/userservice.tar
+      - name: "Upload image for other jobs"
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/userservice.tar
 
-  # === End-to-end tests ===
-
-  test_course_population:
-    name: "Test course population"
-    needs:
-      - build_analytics
-      - build_backend
-      - build_frontend
-      - build_userservice
+  # === Integration tests ===
+  run_userservice_integration_tests:
+    name: "Run userservice integration tests"
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
-        uses: actions/download-artifact@v2
-        with:
-          path: /tmp/
-      - name: Load built images
-        run: |
-          docker load --input /tmp/artifact/analytics.tar
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Course_population.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
+      - uses: actions/checkout@v2
+      - name: "Run tests included in docker build"
+        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+
+  # === End-to-end tests ===
 
   test_course_statistics:
     name: "Test course statistics"
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: "Download images"
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: "Load built images"
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: "Login to toska docker registry"
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: "Upload video if tests fail"
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
     name: "Test custom population"
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: "Download images"
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: "Load built images"
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: "Login to toska docker registry"
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: "Upload video if tests fail"
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
-  test_filters:
-    name: "Test filters"
+  test_population_statistics:
+    name: "Test population statistics"
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: "Download images"
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: "Load built images"
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: "Login to toska docker registry"
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Population_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_filters:
+    name: "Test filters"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: "Upload video if tests fail"
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
-  test_teachers_and_users:
-    name: "Test teachers and users"
+  test_student_statistics:
+    name: "Test student statistics"
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: "Download images"
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: "Load built images"
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: "Login to toska docker registry"
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Student_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_studyprogramme_overview:
+    name: "Test studyprogramme overview"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Studyprogramme_overview.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_teachers_and_users:
+    name: "Test teachers and users"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -384,113 +435,112 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: "Upload video if tests fail"
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
     name: "Check services run correctly"
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: "Download images"
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: "Load built images"
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: "Login to toska docker registry"
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Check that exit statuses are not errors
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Check that exit statuses are not errors"
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
-            ./run.sh oodikone ci logs;
             exit 1;
           fi
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   push_to_dockerhub:
     name: "Push images to docker hub for caching"
     needs:
-    - lint_analytics
     - lint_backend
     - lint_frontend
     - lint_userservice
-    - test_course_population
+    - run_userservice_integration_tests
     - test_course_statistics
     - test_custom_population
     - test_filters
+    - test_population_statistics
+    - test_student_statistics
+    - test_studyprogramme_overview
     - test_teachers_and_users
     - test_service_exit_statuses
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: "Download images"
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: "Load built images"
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to DockerHub
+      - name: "Login to DockerHub"
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: "Push to docker hub"
-        run: ./run.sh oodikone ci push
+        run: docker-compose --file docker-compose.ci.yml push
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report test failure to oodikone notifications
+    name: "Report test failure to oodikone notifications"
     needs:
-    - lint_analytics
     - lint_backend
     - lint_frontend
     - lint_userservice
-    - test_course_population
+    - run_userservice_integration_tests
     - test_course_statistics
     - test_custom_population
     - test_filters
+    - test_population_statistics
+    - test_student_statistics
+    - test_studyprogramme_overview
     - test_teachers_and_users
     - test_service_exit_statuses
     - push_to_dockerhub
     if: failure()
     steps:
+      - uses: actions/checkout@v2
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.gitignore

@@ -1,20 +1,48 @@
-**/node_modules
-.databasedumps
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
 
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 
-# These really needed?
-private.key
-private.key.pub
-kafka
-services/updater_writer/updater/test_assets/testi.bak
-services/updater_writer/updater/test_assets/011120775.json
-services/updater_writer/updater/test_assets/014272112.json
-services/updater_writer/updater/test_assets/014441008.json
-services/updater_writer/updater/test_assets/meta.json
-services/updater_scheduler/all_student_numbers.txt
-services/updater_scheduler/active_student_numbers.txt
-datastore
-services/updater_api/debug.log
-services/backend/updater_writer/dumps
-debug.log
-programme_module_dump.sql
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# dotenv environment variables file
+.env
+.env.test
+.env.production
+
+# Oodikone related stuff
+.databasedumps

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

.prettierrc

@@ -5,5 +5,5 @@
   "printWidth": 120,
   "tabWidth": 2,
   "useTabs": false,
-  "jsxSingleQuote": false
-}
+  "jsxSingleQuote": false,
+}

README.md

@@ -1,20 +1,19 @@
-# oodikone
+# Oodikone
 
-[![CircleCI](https://circleci.com/gh/UniversityOfHelsinkiCS/oodikone/tree/master.svg?style=svg)](https://app.circleci.com/pipelines/github/UniversityOfHelsinkiCS/oodikone)
+An application for analyzing university data, running at [https://oodikone.cs.helsinki.fi/](https://oodikone.cs.helsinki.fi/).
 
-An application for analyzing university data. The entire development environment runs in a Docker network that's defined in the `docker-compose.yml` file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.
-
-## Requirements
+## ✔️ Requirements
 
 To run Oodikone locally, you will need the following:
 
-1. [Docker](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose/install/) running on your system.
-2. Membership of [Toska's GitHub organisation](https://github.com/UniversityOfHelsinkiCS).
-3. SSH key set up for GitHub (see [docs](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh)).
-4. Local clone of this repository.
-5. Run the CLI (see [Installation](#Installation))
+1. Applications:
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
+3. (Optional) For real data setup, access to Toskas production servers.
 
-## Installation
+## 🚀 Installation
 
 Launch the CLI and follow the instructions:
 
@@ -22,248 +21,110 @@ Launch the CLI and follow the instructions:
 npm run cli
 ```
 
-*Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary* 😊
+_Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary_ 😊
 
-The acual oodikone can be accessed at [http://localhost:8081/](http://localhost:8081/) and Adminer at [http://localhost:5050/](http://localhost:5050/?pgsql=db&username=postgres). Adminer requires you to login with username `postgres` and password `postgres`.
+What different CLI options do:
 
-### Populate Redis on Fresh Install
+1. Set up oodikone from scratch:
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
+2. Reset all real data:
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
+3. Reset sis-importer data
+   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
 
-On a fresh install some features do not work because the redis store is empty. To populate redis, navigate to [http://localhost:8081/updater](http://localhost:8081/updater), and click *Refresh Statistics*.
+## ⌨️ Development
 
-### What the CLI does
+### Basics
 
-- Builds and runs the Dockerized development environment.
-- Downloads a database dump from the production servers.
-- Creates the database schema and populates it with the downloaded dump.
-- Adds git-hooks to run linting on push. (Use `git push --no-verify` to circumvent this behavior.)
+The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.
 
-### Daily Refresh
+Running oodikone with real data requires separated databases and redis, which are defined in `docker-compose.real.yml` file. Otherwise real data development environment uses the configuration as anonymized data development environment.
 
-It is recommended to run the "morning script" daily when starting work. It prunes Docker, refreshes the repository and rebuilds the containers before starting Oodikone again.
+Some useful commands are defined in `package.json` and can be run with `npm run <command>` as follows:
 
-```bash
-./oodikone.sh -m
-```
+- `npm run oodikone`: starts oodikone with anonymized data
+- `npm run oodikone:real`: starts oodikone with real data
+- `npm run updater`: starts updater with anonymized data
+- `npm run updater:real`: starts oodikone with real data
+- `npm run both`: starts oodikone and updater with anonymized data
+- `npm run both:real`: starts oodikone and updater with real data
+- `npm run docker:down`: stops the whole environment
 
-## Development Environment
+Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
-The development environment is entirely configured in the docker-compose.yml file located in this repository. The file defines the containers/environments for:
+### Run.sh script
 
-- Frontend
-- Backend
-- Userservice
-- The databases used by the services
-- Adminer & other dev tools
+As said, the development environment runs entirely inside docker containers. To keep `package.json` clean and not filled with predefined scripts, we have created a simple helper script called `./run.sh`. The script allows you to use docker-compose commands without the need to write long list of parameters. Try to run `./run.sh` in the root of the project and see what happens!
 
-The mapping of ports and environment variables are also defined in the docker-compose file. You can start, stop and manage the development environment by running the following commands from a terminal shell in this directory.
+`./run.sh` is simply a wrapper script to run oodikone, updater or both services in either anon or real mode. If you take a look at `package.json`, you can see that most of the predefined scripts above use `./run.sh` under the hood.
 
-### Start the development environment
+It is recommended to spend some time to become familiar with `docker` and `docker-compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:
 
-With anonymized data:
+- `./run.sh oodikone anon pull`: Pull all images related to oodikone development
+- `./run.sh updater real up --build --force-recreate --detach`: Start updater detached (=in the background) in real data mode, but build new images before starting
+- `docker-compose ps`: view the containers in the running environment
+- `docker-compose logs frontend`: print logs for just frontend
+- `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
+- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
-```bash
-npm start
-```
+### User types for development
 
-With real data:
+By default, you're login as `mluukkai` dev user when running oodikone in development mode. If you want to debug with certain type of user, you can use mocking: go to "Users" -page, click on "edit" on user you want to use and then click icon on the right corner of user card. Another way is to mock user headers - see frontend's api configuration for how this is done.
 
-```bash
-npm run start:real
-```
+Anon data contains some preset user types, corresponding to most usual user types in real data. These are used in testing too (see "Testing" below for more info).
 
-### Stop the development environment
+### Linting
 
-```bash
-npm run docker:down
-```
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-### Restart container(s)
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-All:
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
-```bash
-npm run docker:restart
-```
+However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-Only backend:
+## 🔨 Testing & CI
 
-```bash
-npm run docker:restart:backend
-```
+There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
-Only frontend:
+- Cypress
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+- Jest
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
-```bash
-npm run docker:restart:frontend
-```
+Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
-### View the containers in the running environment
+- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
+- after successful merge to `master`, oodikone is deployed to staging
+- after creating a release, oodikone is deployed to production
+- updater is deployed to production on every push to `trunk`.
 
-```bash
-docker-compose ps
-```
+## ❓FAQ
 
-### View logs
+### Modules are missing after updating package.json
 
-All:
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
 
-```bash
-npm run docker:logs
-```
+### Everything is broken, can't get oodikone running, data is not there etc.
 
-Only backend:
+Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
-```bash
-npm run docker:logs:backend
-```
+## ✌🏼 Maintainers and contribution
 
-Only frontend:
+Toska of course.
 
-```bash
-npm run docker:logs:frontend
-```
-
-### Attach a terminal shell to a container
-
-```bash
-docker exec -it <container> <command>
-```
-
-For example:
-
-```bash
-docker exec -it backend bash
-```
-
-### Use `psql` or other database CLI tools
-
-```bash
-docker exec -it -u <username> <container> psql -d <db name>
-```
-
-For example:
-
-```bash
-docker exec -it -u postgres oodi_db psql -d tkt_oodi
-```
-
-## Testing
-
-Testing is implemented as a combination of unit tests (jest) and end-to-end tests (Cypress). Oodikone must be using the anonymous dataset when running tests (see [Installation](#Installation)).
-
-The test suite is run in CI on every push to `trunk`. Take advantage of this as running the tests can take upwards of 15 minutes.
-
-**Note:** All containers must be up and ready before running tests. (Use `npm start`).
-
-### Run All Tests
-
-```bash
-npm test
-```
-
-### Run End-to-End Tests
-
-```bash
-npm run cypress:run
-```
-
-## Open Cypress GUI
-
-```bash
-npm run cypress:open
-```
-
-## How to Updater
-
-Before running updater you must run `importer-db` setup via the CLI (see [Installation](#Installation)).
-
-If you want to use the control panel UI, first start Oodikone in real data mode:
-
-```bash
-npm run docker:up:real
-```
-
-Followed by acual updater:
-
-```bash
-npm run updater:dev:up
-```
-
-When you are done, close hanging containers with:
-
-```bash
-npm run updater:dev:down
-```
-
-### updater scheduler
-
-Stop with HTTP GET
-
-```bash
-http://localhost:8082/v1/abort?token=dev
-```
-
-Schedule students with HTTP POST
-
-```bash
-http://localhost:8082/v1/students?token=dev
-```
-
-body
-
-```bash
-{ 
-    "studentnumbers": [
-        "014598456"
-    ] 
-}
-```
-
-## Other commands
-
-### How to `scp` backup files from oodikone via melkki-proxy
-
-This is how the setup script fetches the database dump from production servers. It will require you to have access to both melkki.cs.helsinki.fi as well as oodikone.cs.helsinki.fi. The command uses `scp` to transfer all backup files recursively from a known location on the production server by using the melkki server as a proxy, allowing you to get the dump even if you are not in the university network.
-
-#### Command
-
-```bash
-scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* ./backups/
-
--r : recursively copy files
--o : options for defining the proxy command
-```
-
-#### [Unix StackExchange answer](https://unix.stackexchange.com/questions/355640/how-to-scp-via-an-intermediate-machine)
-
-> It's possible and relatively easy, even when you need to use certificates > for authentication (typical in AWS environments).
->
-> The command below will copy files from a remotePath on server2 directly > into your machine at localPath. Internally the scp request is proxied via > server1.
->
-> ```bash
-> scp -i user2-cert.pem -o ProxyCommand="ssh -i user1-cert.pem -W %h:%p > user1@server1" user2@server2:/<remotePath> <localpath>
-> ```
->
-> If you use password authentication instead, try with:
->
-> ```bash
-> scp -o ProxyCommand="ssh -W %h:%p user1@server1" > user2@server2:/<remotePath> <localpath>
-> ```
->
-> If you use the same user credentials in both servers:
->
-> ```bash
-> scp -o ProxyCommand="ssh -W %h:%p commonuser@server1" > commonuser@server2:/<remotePath> <localpath>
-> ```
->
-
-## Known situations and ways to handle them
-
-- Problem: Oodikone doesn't start and database container (e.g. db_sis) logs include the line "can't open '/docker-entrypoint-initdb.d/'
-  - Possible reason: You probably have too restricting read rights in `scripts/docker-entrypoint-initdb.d` -folder. This could be for example caused by fairly strict `umask` setting, such as `077` in your OS.
-  - Possible solution: Either change the umask on whole OS (to `022` for example) or set the correct read rights for docker-entrypoint -folder by running `chmod 755 scripts/docker-entrypoint-initdb.d`
-- Problem: Packages aren't updated in development containers after update in package.json, even after rebuilding containers and images
-  - Possible reason: `node_modules` folder is binded to external docker volume (i.e not to disk) in our development environment. These volumes aren't updated unless explicitly told so.
-  - Possible solution: Either:
-    a) update packages inside the failing container by first logging to container `docker exec -it container_name sh` and then running `npm install`
-    b) remove volume that contains failing containers `node_modules` by stopping and removing the failing container followed by `docker volume prune`.
+University of Helsinki

adminer-plugins/login-servers.php

@@ -7,7 +7,6 @@ $driver = "pgsql";
 */
 return new AdminerLoginServers(
     $servers = array(
-        "analytics-db" => array("server" => "analytics-db", "driver" => $driver),
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
         "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),

babel.config.json

@@ -0,0 +1,5 @@
+{
+  "plugins": [
+    "@babel/plugin-syntax-jsx"
+  ]
+}

cli.sh

@@ -28,8 +28,10 @@ cleanup() {
 # === CLI ===
 
 show_welcome() {
-  local cashmoneyyellow=$(tput setaf 221)
-  local normal=$(tput sgr0)
+  local cashmoneyyellow
+  cashmoneyyellow=$(tput setaf 221)
+  local normal
+  normal=$(tput sgr0)
   if [ "$(tput cols)" -gt "76" ]; then
     while IFS="" read -r p || [ -n "$p" ]; do
       printf '%40s\n' "${cashmoneyyellow}$p${normal}"
@@ -37,9 +39,8 @@ show_welcome() {
   fi
   infomsg "Welcome to Oodikone CLI!"
   msg "This tool helps you in managing the project configuration. If you are new to
-Oodikone development, you should probably run \"Set up oodikone\" which will
-take care of setting up and starting Oodikone for you. See README for more
-details."
+Oodikone development, you should probably run \"Set up oodikone from scratch.\" which
+will take care of setting up Oodikone for you. See README for more details."
   msg ""
 }
 
@@ -47,10 +48,9 @@ details."
 PS3="Please enter your choice: "
 
 options=(
-  "Set up oodikone."
+  "Set up oodikone from scratch."
   "Reset all real data."
   "Reset sis-importer data."
-  "Reset old oodi data."
   "Quit."
 )
 
@@ -62,14 +62,12 @@ get_username
 while true; do
   select opt in "${options[@]}"; do
     case $opt in
-      "Set up oodikone.")
-        set_up_oodikone;;
+      "Set up oodikone from scratch.")
+        set_up_oodikone_from_scratch;;
       "Reset all real data.")
         reset_all_real_data;;
       "Reset sis-importer data.")
         reset_sis_importer_data;;
-      "Reset old oodi data.")
-        reset_old_oodi_data;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress.json

@@ -1,9 +1,7 @@
 {
   "projectId": "c3jsph",
-  "baseUrl": "http://localhost:8081",
-  "defaultCommandTimeout": 60000,
-  "requestTimeout": 60000,
-  "responseTimeout": 60000,
-  "viewportHeight": 1080,
-  "viewportWidth": 1920
-}
+  "baseUrl": "http://localhost:3000",
+  "defaultCommandTimeout": 30000,
+  "requestTimeout": 30000,
+  "videoCompression": false
+}

cypress/fixtures/example.json

@@ -1,5 +0,0 @@
-{
-  "name": "Using fixtures to represent data",
-  "email": "hello@cypress.io",
-  "body": "Fixtures are a great way to mock data for responses to routes"
-}

cypress/integration/Course_population.js

@@ -1,29 +0,0 @@
-/// <reference types="Cypress" />
-
-describe("Course population tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Course statistics").click();
-    cy.contains("Search for courses");
-  });
-
-  it("Can find course population", () => {
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("Search for courses");
-    cy.get("input[placeholder='Search by a course code']").type("TKT20003");
-    cy.contains("tr", "TKT20003").click();
-    cy.contains("Fetch statistics").should("be.enabled").click();
-
-    cy.contains("Käyttöjärjestelmät");
-    cy.contains("TKT20003");
-
-    cy.get(":nth-child(3) > :nth-child(1) > div > .item > .level").click();
-    cy.contains("Population of course Käyttöjärjestelmät 2019-2020");
-    cy.contains("TKT20003");
-    cy.contains("Students (all=24)");
-
-    cy.contains("Students (24)").click();
-    cy.contains("010135486");
-    cy.contains("010431753");
-  });
-});

cypress/integration/Custom_population.js

@@ -1,224 +1,195 @@
 /// <reference types="Cypress" />
 
 const openCustomPopupForm = () => {
-  cy.get("button").contains("Custom population").click();
-};
+  cy.get('button').contains('Custom population').click()
+}
 
 const fillName = () => {
-  const name = `TEST-${new Date().getTime()}`;
-  cy.contains("Insert name for this custom population if you wish to save it")
+  const name = `TEST-${new Date().getTime()}`
+  cy.contains('Insert name for this custom population if you wish to save it')
     .siblings()
-    .get("input[placeholder=name]")
-    .type(name);
-  return name;
-};
+    .get('input[placeholder=name]')
+    .type(name)
+  return name
+}
 
 const save = () => {
-  cy.get("button").contains("Save").click();
-};
+  cy.get('button').contains('Save').click()
+}
 
-const selectSavedPopulation = (name) => {
-  cy.get('[data-cy="history-search"]')
-    .children()
-    .eq(0)
-    .type(name)
-    .type("{enter}");
-};
+const selectSavedPopulation = name => {
+  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+}
 
 const deleteAllSearches = () => {
-  cy.get('[data-cy="history-search"]').children().eq(0).click();
-  cy.contains("Saved populations")
+  cy.get('[data-cy="history-search"]').children().eq(0).click()
+  cy.contains('Saved populations')
     .parent()
     .parent()
     .parent()
     .parent()
-    .get(".dropdown")
-    .then((d) => {
-      const searchItems = d.find("div[role=option] > span[class=text]");
+    .get('.dropdown')
+    .then(d => {
+      const searchItems = d.find('div[role=option] > span[class=text]')
       for (let i = 0; i < searchItems.length; i++) {
-        if (searchItems[i].textContent.includes("TEST-")) {
-          cy.get('[data-cy="history-search"]')
-            .children()
-            .eq(0)
-            .type(searchItems[i].textContent)
-            .type("{enter}");
-          cy.get("button").contains("Delete").click();
+        if (searchItems[i].textContent.includes('TEST-')) {
+          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('button').contains('Delete').click()
         }
       }
-    });
-};
+    })
+}
 
-const fillForm = (content) => {
-  cy.contains("Insert studentnumbers you wish to use for population here")
+const fillForm = content => {
+  cy.contains('Insert studentnumbers you wish to use for population here')
     .siblings()
-    .get("textarea")
-    .type(content.join("\n"));
-};
+    .get('textarea')
+    .type(content.join('\n'))
+}
 
 const search = () => {
-  cy.get("button").contains("Search population").click();
-};
+  cy.get('button').contains('Search population').click()
+}
 
-const searchFor = (studentnumbers) => {
-  openCustomPopupForm();
-  fillForm(studentnumbers);
-  search();
-};
+const searchFor = studentnumbers => {
+  openCustomPopupForm()
+  fillForm(studentnumbers)
+  search()
+}
 
 const hasLanded = () => {
-  cy.contains("Credit accumulation");
-  cy.contains("Programme distribution");
-  cy.contains("Courses of population");
-  cy.contains("Students");
-};
+  cy.contains('Credit accumulation')
+  cy.contains('Programme distribution')
+  cy.contains('Courses of population')
+  cy.contains('Students')
+}
 
 const containsAmountOfStudents = (amount = 0) => {
-  cy.contains(`Credit accumulation (for ${amount} students)`);
-};
+  cy.contains(`Credit accumulation (for ${amount} students)`)
+}
 
 const containsSpecificStudents = (studentnumbers = []) => {
-  cy.contains(`Students (${studentnumbers.length})`).click();
+  cy.contains(`Students (${studentnumbers.length})`).click()
 
-  studentnumbers.forEach((s) => cy.contains(s));
-};
+  studentnumbers.forEach(s => cy.contains(s))
+}
 
-const loginAs = (username) => {
-  cy.contains("Users").click();
-  cy.contains(username).siblings().contains("Edit").click();
-  cy.get(".spy").click();
-  cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-  cy.url().should("include", "/custompopulation");
-  cy.contains("Custom population");
-};
-
-describe("Custom population tests", () => {
-
-  const nonExistentStudentNumbers = ["123", "X", "-", " "];
+describe('Custom population tests', () => {
+  const nonExistentStudentNumbers = ['123', 'X', '-', ' ']
   beforeEach(() => {
-    cy.init("/custompopulation");
-    cy.url().should("include", "/custompopulation");
-    cy.contains("Custom population");
-  });
+    cy.init('/custompopulation')
+    cy.url().should('include', '/custompopulation')
+    cy.contains('Custom population')
+  })
 
   after(() => {
-    cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-    cy.url().should("include", "/custompopulation");
-    cy.contains("Custom population");
-    cy.get("button").contains("Custom population").click();
-    deleteAllSearches();
-  });
+    cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+    cy.url().should('include', '/custompopulation')
+    cy.contains('Custom population')
+    cy.get('button').contains('Custom population').click()
+    deleteAllSearches()
+  })
 
-  describe("Custom population searching", () => {
-    it("Finds a proper population", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1}) => {
+  describe('Custom population searching', () => {
+    it('Finds a proper population', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
         const students = studentNumbersForCSStudentsSet1
-        searchFor(students);
-        hasLanded();
-        containsAmountOfStudents(students.length);
-        containsSpecificStudents(students);
+        searchFor(students)
+        hasLanded()
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
       })
-    });
+    })
 
     it("Doesn't return non-existing students", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1}) => {
-        const students = [
-          ...studentNumbersForCSStudentsSet1,
-          ...nonExistentStudentNumbers,
-        ];
-        searchFor(students);
-        hasLanded();
-        containsAmountOfStudents(studentNumbersForCSStudentsSet1.length);
-        containsSpecificStudents(studentNumbersForCSStudentsSet1);
-      });
-    });
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
+        const students = [...studentNumbersForCSStudentsSet1, ...nonExistentStudentNumbers]
+        searchFor(students)
+        hasLanded()
+        containsAmountOfStudents(studentNumbersForCSStudentsSet1.length)
+        containsSpecificStudents(studentNumbersForCSStudentsSet1)
+      })
+    })
 
     it("Doesn't find empty custom population", () => {
-      searchFor(nonExistentStudentNumbers);
-      cy.contains("Credit accumulation").should("not.exist");
-      cy.contains("Programme distribution").should("not.exist");
-      cy.contains("Courses of population").should("not.exist");
-      cy.get("button").contains("Custom population");
-    });
+      searchFor(nonExistentStudentNumbers)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
 
-    // Check login as when anon user database rights have been corrected
-    // it("Doesn't return students to whom the user doesn't have rights to", () => {
-    //   const kasvatusStudents = ["010102273", "010112616", "010116531"];
-    //   const kapistelyStudents = ["010623419", "010942404", "010484975"];
-    //   loginAs("Normaalikäyttäjä");
+    it("Doesn't return students user has no right to", () => {
+      // TODO: check that this doesn't create false positives, since this test goes
+      // through even if following students are not in anon sis-db.
+      const studentsForEduBachStudents = ['014990067', '013069465', '014853890']
+      searchFor(studentsForEduBachStudents)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
+  })
 
-    //   searchFor(kasvatusStudents);
-    //   cy.contains("Credit accumulation").should("not.exist");
-    //   cy.contains("Programme distribution").should("not.exist");
-    //   cy.contains("Courses of population").should("not.exist");
-
-    //   searchFor(kapistelyStudents.concat(kasvatusStudents));
-    //   hasLanded();
-    //   containsAmountOfStudents(3);
-    //   containsSpecificStudents(kapistelyStudents);
-    // });
-  });
-
-  describe("Custom population search saving", () => {
-    it("Saves a custom population search", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1}) => {
+  describe('Custom population search saving', () => {
+    it('Saves a custom population search', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
         const students = studentNumbersForCSStudentsSet1
-        openCustomPopupForm();
-        const name = fillName();
-        fillForm(students);
-        save();
+        openCustomPopupForm()
+        const name = fillName()
+        fillForm(students)
+        save()
 
         // Round 1
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(students.length);
-        containsSpecificStudents(students);
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
 
         // Round 2
-        cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-        cy.url().should("include", "/custompopulation");
-        cy.contains("Custom population");
-        cy.get("button").contains("Custom population").click();
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(students.length);
-        containsSpecificStudents(students);
-      });
-    });
+        cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+        cy.url().should('include', '/custompopulation')
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
+      })
+    })
 
-    it("Updates a custom population search", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2}) => {
+    it('Updates a custom population search', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
         const students1 = studentNumbersForCSStudentsSet1
         const students2 = studentNumbersForCSStudentsSet2
-        openCustomPopupForm();
-        const name = fillName();
-        fillForm(students1);
-        save();
+        openCustomPopupForm()
+        const name = fillName()
+        fillForm(students1)
+        save()
 
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(students1.length);
-        containsSpecificStudents(students1);
-        cy.contains("Custom population");
-        cy.get("button").contains("Custom population").click();
-        selectSavedPopulation(name);
-        fillForm([...students1, ...students2]);
-        save();
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students1.length)
+        containsSpecificStudents(students1)
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        fillForm([...students1, ...students2])
+        save()
 
-        cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-        cy.url().should("include", "/custompopulation");
-        cy.contains("Custom population");
-        cy.get("button").contains("Custom population").click();
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(
-            students1.length + students2.length
-        );
-        containsSpecificStudents([...students1, ...students2]);
-      });
-    });
-  });
-});
+        cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+        cy.url().should('include', '/custompopulation')
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students1.length + students2.length)
+        containsSpecificStudents([...students1, ...students2])
+      })
+    })
+  })
+})

cypress/integration/Filters.js

@@ -1,13 +1,19 @@
 /// <reference types="Cypress" />
 
-const checkFilteringResult = (studentCount) => {
-  cy.contains(`Students (${studentCount})`);
-};
+// const baseUrl = Cypress.config().baseUrl
+
+const checkFilteringResult = studentCount => {
+  cy.contains(`Students (${studentCount})`)
+}
 
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find("i.delete").click();
+  cy.cs(dataCyAttribute).find('i.delete').click()
+}
+
+const clearSingleDropdownSelection = dataCyAttribute => {
+  cy.cs(dataCyAttribute).find('i.clear').click()
 }
 
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
@@ -15,234 +21,276 @@ const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
   return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click();
-    checkFilteringResult(amountWithoutFiltering);
-    testStepFunctionToRun();
-    checkFilteringResult(amountWithoutFiltering);
-    cy.cs(dataCyAttributeOfHeaderToClick).click();
+    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    checkFilteringResult(amountWithoutFiltering)
+    testStepFunctionToRun()
+    checkFilteringResult(amountWithoutFiltering)
+    cy.cs(dataCyAttributeOfHeaderToClick).click()
   }
 }
 
-describe("Population Statistics", () => {
+describe('Population Statistics', () => {
+  const pathToCSBach2018 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
+  const defaultAmountOfStudents = 148
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
-  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(40)
   before(() => {
-    cy.init();
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-  });
+    cy.init(pathToCSBach2018)
+  })
 
-  it("Graduation filter works", () => {
-    runTestStepWithPreAndPostParts("graduatedFromProgrammeFilter-header", () => {
-      cy.cs("graduatedFromProgrammeFilter-graduated-true").click()
-      checkFilteringResult(10);
-      cy.cs("graduatedFromProgrammeFilter-graduated-false").click()
-      checkFilteringResult(30);
-      cy.cs("graduatedFromProgrammeFilter-all").click()
+  it('Transfer filter is on not transferred by default', () => {
+    cy.contains('By default only students who have not transferred to this study programme are shown.')
+    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+  })
+
+  it('Graduation filter works', () => {
+    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+      const graduated = 11
+      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      checkFilteringResult(graduated)
+      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      checkFilteringResult(defaultAmountOfStudents - graduated)
+      cy.cs('graduatedFromProgrammeFilter-all').click()
     })
-  });
+  })
 
-  // Can't be tested yet, since anon data doesn't provide enough information for this, fix
-  it.skip("Transfer filter works", () => {
-    cy.cs("transferredToProgrammeFilter-header").click();
-    cy.cs("transferredToProgrammeFilter-have").click();
-    checkFilteringResult(25);
-    cy.cs("transferredToProgrammeFilter-havenot").click();
-    checkFilteringResult(194);
-    cy.cs("transferredToProgrammeFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it("Enrollment filter works", () => {
-    runTestStepWithPreAndPostParts("enrollmentStatusFilter-header", () => {
-      cy.selectFromDropdown("enrollmentStatusFilter-status", 0);
-      cy.selectFromDropdown("enrollmentStatusFilter-semesters", [0]);
-      checkFilteringResult(36);
-      clearSemanticUIMultipleDropDownSelection("enrollmentStatusFilter-semesters")
-    });
-  });
-
-  it("Credit filter works", () => {
-    runTestStepWithPreAndPostParts("credit-filter-header", () => {
-        cy.cs("credit-filter-min").type("50");
-        cy.cs("credit-filter-max").type("150");
-        checkFilteringResult(26);
-        cy.cs("credit-filter-min").find("input").clear();
-        cy.cs("credit-filter-max").find("input").clear();
-    });
-  });
-
-  it("Age filter works", () => {
-    runTestStepWithPreAndPostParts("ageFilter-header", () => {
-      cy.cs("ageFilter-min").type("20");
-      cy.cs("ageFilter-max").type("40");
-      checkFilteringResult(13);
-      cy.cs("ageFilter-min").find("input").clear();
-      cy.cs("ageFilter-max").find("input").clear();
+  it('Transfer filter works', () => {
+    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+      const transferred = 1
+      cy.cs('transferredToProgrammeFilter-have').click()
+      checkFilteringResult(transferred)
+      cy.cs('transferredToProgrammeFilter-all').click()
+      checkFilteringResult(defaultAmountOfStudents + transferred)
+      cy.cs('transferredToProgrammeFilter-havenot').click()
     })
-  });
+  })
 
-  // Can't be tested yet, since anon data doesn't provide enough information for this, fix
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(48);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(171);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it("Starting year filter works", () => {
-    runTestStepWithPreAndPostParts("startYearAtUni-header", () => {
-      cy.selectFromDropdown("startYearAtUni-dropdown", [0]);
-      checkFilteringResult(1);
-      clearSemanticUIMultipleDropDownSelection("startYearAtUni-dropdown")
+  it('Enrollment filter works', () => {
+    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
+      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
+      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+      checkFilteringResult(137)
+      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
-  });
+  })
 
-  it("Courses filter works", () => {
-    runTestStepWithPreAndPostParts("courseFilter-header", () => {
-      const courses = ["DATA15001", "MAT11003"]
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[0]).click();
-      checkFilteringResult(15);
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1);
-      checkFilteringResult(13);
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[1]).click();
-      checkFilteringResult(9);
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3);
-      checkFilteringResult(1);
+  it('Credit filter works', () => {
+    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+      cy.cs('credit-filter-min').type('50')
+      cy.cs('credit-filter-max').type('150')
+      checkFilteringResult(116)
+      cy.cs('credit-filter-min').find('input').clear()
+      cy.cs('credit-filter-max').find('input').clear()
+    })
+  })
+
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('40')
+      checkFilteringResult(43)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('genderFilter-header', () => {
+      cy.selectFromDropdown('genderFilter-dropdown', 0)
+      checkFilteringResult(36)
+      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      checkFilteringResult(112)
+      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      checkFilteringResult(0)
+      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      checkFilteringResult(121)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+      checkFilteringResult(5)
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      checkFilteringResult(10)
+      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
+    })
+  })
+
+  // plz fix
+  it.skip('Courses filter works', () => {
+    // courses takes some time to load, wait for it to complete
+    cy.wait(10000)
+    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      cy.cs('courseFilter-course-dropdown').click()
+      const courses = ['TKT20001', 'MAT11002']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(140)
+      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      checkFilteringResult(131)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(56)
+      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      checkFilteringResult(2)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
-  });
-});
+  })
 
-describe("Course Statistics", () => {
-
-  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(27)
-  before(() => {
-    cy.init();
-    cy.cs("navbar-courseStatistics").click();
-    cy.get("input[placeholder='Search by a course code']").type("TKT20001");
-    // Click uni course, not avoin
-    cy.contains("td", /^TKT20001/).click();
-    cy.contains("Fetch statistics").should("be.enabled").click();
-    cy.get(":nth-child(2) > :nth-child(1) > div > .item > .level").click();
-  });
-
-  it("Grade filter works", () => {
-    runTestStepWithPreAndPostParts("gradeFilter-header", () => {
-      cy.cs("gradeFilter-5").click();
-      checkFilteringResult(4);
-      cy.cs("gradeFilter-5").click();
-    });
-  });
-
-  it("Age filter works", () => {
-    runTestStepWithPreAndPostParts("ageFilter-header", () => {
-      cy.cs("ageFilter-min").type("20");
-      cy.cs("ageFilter-max").type("40");
-      checkFilteringResult(10);
-      cy.cs("ageFilter-min").find("input").clear();
-      cy.cs("ageFilter-max").find("input").clear();
+  it('Filter combinations work', () => {
+    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+        cy.cs('ageFilter-min').type('20')
+        cy.cs('ageFilter-max').type('30')
+        checkFilteringResult(11)
+        cy.cs('ageFilter-min').find('input').clear()
+        cy.cs('ageFilter-max').find('input').clear()
+        cy.cs('graduatedFromProgrammeFilter-all').click()
+      })
     })
-  });
+  })
+})
 
-  // Doesn't work since anon db genders are missing, fix
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(19);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(74);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-header").click();
-  });
-
-  it("Starting year filter works", () => {
-    runTestStepWithPreAndPostParts("startYearAtUni-header", () => {
-      cy.selectFromDropdown("startYearAtUni-dropdown", [0]);
-      checkFilteringResult(1);
-      clearSemanticUIMultipleDropDownSelection("startYearAtUni-dropdown")
-    })
-  });
-
-  it("Filter combinations work", () => {
-    runTestStepWithPreAndPostParts("gradeFilter-header", () => {
-      runTestStepWithPreAndPostParts("ageFilter-header", () => {
-        cy.cs("gradeFilter-3").click();
-        cy.cs("ageFilter-min").type("20");
-        cy.cs("ageFilter-max").type("30");
-        checkFilteringResult(1);
-        cy.cs("ageFilter-min").find("input").clear();
-        cy.cs("ageFilter-max").find("input").clear();
-        cy.cs("gradeFilter-3").click();
-      });
-    });
-  });
-});
-
-describe("Custom Population Statistics", () => {
-  let runTestStepWithPreAndPostParts;
+describe('Course Statistics', () => {
+  const pathToDSAndAlgoSpring2019 =
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
+  const defaultAmountOfStudents = 118
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
-    cy.init("/custompopulation");
-    cy.cs("custom-pop-search-button").click();
-    cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2}) => {
+    cy.init(pathToDSAndAlgoSpring2019)
+  })
+
+  it('Grade filter works', () => {
+    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+      cy.cs('gradeFilter-5').click()
+      checkFilteringResult(20)
+      cy.cs('gradeFilter-3').click()
+      checkFilteringResult(38)
+      cy.cs('gradeFilter-3').click()
+      cy.cs('gradeFilter-5').click()
+    })
+  })
+
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('40')
+      checkFilteringResult(33)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('genderFilter-header', () => {
+      cy.selectFromDropdown('genderFilter-dropdown', 0)
+      checkFilteringResult(27)
+      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      checkFilteringResult(91)
+      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      checkFilteringResult(0)
+      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      checkFilteringResult(29)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Filter combinations work', () => {
+    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+        cy.cs('gradeFilter-3').click()
+        cy.cs('ageFilter-min').type('20')
+        cy.cs('ageFilter-max').type('30')
+        checkFilteringResult(1)
+        cy.cs('ageFilter-min').find('input').clear()
+        cy.cs('ageFilter-max').find('input').clear()
+        cy.cs('gradeFilter-3').click()
+      })
+    })
+  })
+})
+
+describe('Custom Population Statistics', () => {
+  let runTestStepWithPreAndPostParts
+
+  before(() => {
+    cy.init('/custompopulation')
+    cy.cs('custom-pop-search-button').click()
+    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs("student-no-input").click().type(students.join("/n"));
+      cy.cs('student-no-input').click().type(students.join('/n'))
     })
-    cy.cs("search-button").click();
-  });
+    cy.cs('search-button').click()
+  })
 
-
-  it("Age filter works", () => {
-    runTestStepWithPreAndPostParts("ageFilter-header", () => {
-      cy.cs("ageFilter-min").type("20");
-      cy.cs("ageFilter-max").type("30");
-      checkFilteringResult(1);
-      cy.cs("ageFilter-min").find("input").clear();
-      cy.cs("ageFilter-max").find("input").clear();
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('30')
+      checkFilteringResult(1)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
     })
-  });
+  })
 
-  // Can't be tested yet, since anon data doesn't provide enough information for this, fix
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(48);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(171);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
+  it('Gender filter works', () => {
+    cy.cs('genderFilter-header').click()
+    cy.selectFromDropdown('genderFilter-dropdown', 0)
+    checkFilteringResult(3)
+    cy.selectFromDropdown('genderFilter-dropdown', 1)
+    checkFilteringResult(2)
+    cy.selectFromDropdown('genderFilter-dropdown', 2)
+    checkFilteringResult(0)
+    clearSingleDropdownSelection('genderFilter-dropdown')
+    checkFilteringResult(5, true)
+  })
 
-  it("Starting year filter works", () => {
-    runTestStepWithPreAndPostParts("startYearAtUni-header", () => {
-      cy.selectFromDropdown("startYearAtUni-dropdown", [0]);
-      checkFilteringResult(1);
-      clearSemanticUIMultipleDropDownSelection("startYearAtUni-dropdown")
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
-  });
+  })
 
-  // Can't be tested yet, since feature doesn't work correctly
-  it.skip("Courses filter works", () => {
-    runTestStepWithPreAndPostParts("courseFilter-header", () => {
-      const courses = ["MAT11001", "TKT20004"]
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[0]).click();
-      checkFilteringResult(5);
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1);
-      checkFilteringResult(5);
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[1]).click();
-      checkFilteringResult(3);
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3);
-      checkFilteringResult(1);
+  it('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(4)
+      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      checkFilteringResult(4)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(3)
+      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
-  });
-});
+  })
+})

cypress/integration/Population_statistics.js

@@ -0,0 +1,167 @@
+/// <reference types="Cypress" />
+
+const setPopStatsUntil = (until, includeSettings = []) => {
+  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+  includeSettings.forEach(setting => {
+    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+  })
+  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+  cy.contains('Fetch population with new settings').click()
+  cy.contains('Advanced settings')
+}
+
+describe('Population Statistics tests', () => {
+  const pathToCSBach2017 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  describe('when using basic user', () => {
+    beforeEach(() => {
+      cy.init('/populations')
+    })
+
+    it('Population statistics search form is usable', () => {
+      cy.contains('See population').should('be.disabled')
+      cy.contains('Search for population')
+      cy.contains('Class of')
+        .parent()
+        .within(() => {
+          cy.get('.form-control').as('enrollmentSelect')
+        })
+
+      cy.get('@enrollmentSelect')
+        .its(`${[0]}.value`)
+        .then(beforeVal => {
+          cy.get('@enrollmentSelect').click()
+          cy.get('.yearSelectInput .rdtPrev').click({ force: true })
+          cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
+          cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
+        })
+
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen maisteriohjelma')
+        .click()
+      cy.contains('See population').should('be.enabled')
+    })
+
+    it('Searching for population really shows population', () => {
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen kandiohjelma')
+        .click()
+      cy.contains('See population').click()
+      cy.contains('Population statistics')
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+    })
+
+    it('Population statistics is usable on general level', () => {
+      cy.visit(pathToCSBach2017)
+      setPopStatsUntil('toukokuu 2020')
+
+      cy.get('.card').within(() => {
+        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.contains('Sample size: 149 students')
+        cy.contains('Excludes exchange students')
+        cy.contains("Excludes students who haven't enrolled present nor absent")
+        cy.contains('Excludes students with non-degree study right')
+        cy.contains('Excludes students who have transferred out of this programme')
+      })
+      cy.contains('Courses of population').click({ force: true })
+
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=expand-TKT1]').click()
+      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+    })
+
+    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
+      cy.contains('No tags defined. You can define them here.').find('a').click()
+      cy.contains('Tags').click()
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Create new tag')
+    })
+
+    // TODO: this fails in CI, but not locally. Investigate and fix
+    it.skip('Advanced settings work', () => {
+      cy.visit(pathToCSBach2017)
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.contains('Statistics until')
+      // only spring
+      cy.cs('toggle-fall').click()
+      cy.contains('Fetch population').click()
+
+      // moar waiting hack
+      cy.wait(10000)
+
+      cy.contains('No statistics found for the given query.')
+
+      // only fall
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-fall').click()
+      cy.cs('toggle-spring').click()
+
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 149 students)')
+
+      // spring + fall and include cancelled
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-spring').click()
+      cy.cs('toggle-cancelled').click()
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 170 students)')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
+
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
+      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
+      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
+      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
+    })
+  })
+
+  describe('when using admin', () => {
+    beforeEach(() => {
+      cy.init(pathToCSBach2017, 'admin')
+    })
+    it('Student list checking works as intended', () => {
+      const existing = '010113437'
+      const nonExisting = '66666666'
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.contains(existing)
+      cy.contains(nonExisting).should('not.exist')
+      cy.contains('button', 'Check studentnumbers').click()
+      cy.contains('Check for studentnumbers')
+      cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
+      cy.contains('button', 'check students').click()
+      cy.contains('#checkstudentsresults', 'Results').within(() => {
+        cy.contains('Student numbers in list and in oodi').click()
+        cy.contains('#found', existing)
+        cy.contains('Student numbers in list but not in oodi').click()
+        cy.contains('#notfound', nonExisting)
+        cy.contains('Student numbers in oodi but not in list').click()
+        cy.contains('#notsearched', '010614509')
+      })
+    })
+  })
+})

cypress/integration/Student_statistics.js

@@ -0,0 +1,90 @@
+/// <reference types="Cypress" />
+
+describe('Student Statistics tests', () => {
+  beforeEach(() => {
+    cy.init()
+    cy.contains('Student statistics').click()
+    cy.contains('Show student names')
+  })
+
+  const student = {
+    firstnames: 'Matti Jaakko',
+    lastname: 'Nieminen',
+    studentnumber: '010654019',
+  }
+
+  it.only('Student statistics search form is usable', () => {
+    cy.contains('Show student names')
+    cy.url().should('include', '/students')
+    cy.get('.prompt').type(student.lastname)
+    cy.contains('Student number')
+    cy.contains('Started')
+    cy.contains('Credits')
+    cy.contains(student.firstnames).should('not.exist')
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.firstnames)
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.firstnames).should('not.exist')
+  })
+
+  it('can search with studentnumber too', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber)
+  })
+
+  it('Can get student specific page by clicking student', () => {
+    cy.url().should('include', '/students')
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.contains('Started: 01.08.2011')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains(student.lastname).should('not.exist')
+    cy.contains(student.firstnames).should('not.exist')
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.lastname)
+    cy.contains(student.firstnames)
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.lastname).should('not.exist')
+    cy.contains(student.firstnames).should('not.exist')
+  })
+
+  it('Can get back to search menu', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.get('.remove').click()
+    cy.contains('Student number').should('not.exist')
+    cy.contains('Credits').should('not.exist')
+  })
+
+  it('Can jump to course', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.contains('Introduction to Machine Learning (DATA11002)')
+      .siblings()
+      .within(() => {
+        cy.get('.level').click()
+      })
+    cy.url().should('include', '/coursestatistics')
+    cy.contains('Introduction to Machine Learning')
+  })
+
+  it('Searching with bad inputs doesnt yield results', () => {
+    cy.get('.prompt').type('SWAG LITTINEN')
+    cy.contains('Student number').should('not.exist')
+
+    cy.get('.prompt').clear().type('01114')
+    cy.contains('Student number').should('not.exist')
+  })
+
+  it.only('Can jump to population page', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.get('i.level.up.alternate.icon').eq(0).click()
+    cy.contains('Population statistics')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -0,0 +1,228 @@
+/// <reference types="Cypress" />
+
+const deleteTag = name => {
+  cy.contains(name).siblings().contains('Delete').click()
+  cy.contains('Are you sure you want to delete tag')
+  cy.contains('Confirm').click()
+  cy.contains(name).should('not.exist')
+}
+
+describe('Studyprogramme overview', () => {
+  // check is used by couple of tests
+  const checkProgressAndProductivity = (user = 'basic') => {
+    // Column amount depends on user. Note that with dev user, you'll see even more
+    // stuff than with admin or basic.
+    const CSBachPopulationProgress2018ForAdmin = [
+      '160',
+      '42 (26%)',
+      '118 (73%)',
+      '159 (99%)',
+      '159',
+      '11',
+      '11',
+      '1',
+      '0',
+      '150',
+      '123',
+      '76',
+      '41',
+      '16',
+    ]
+    const CSBachPopulationProgress2018ForBasic = [
+      '160',
+      '42 (26%)',
+      '118 (73%)',
+      '159 (99%)',
+      '159',
+      '11',
+      '1',
+      '150',
+      '123',
+      '76',
+      '41',
+      '16',
+    ] // cancelled and transferred from are missing
+
+    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
+    cy.contains('Population progress')
+    cy.contains('Yearly productivity')
+
+    cy.contains('2018-2019')
+      .siblings()
+      .each((elem, index) => {
+        cy.wrap(elem).contains(population[index])
+      })
+    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    cy.get('table')
+      .eq(1)
+      .contains('2019')
+      .siblings()
+      .each((elem, index) => {
+        cy.wrap(elem).contains(populationproductivity2019[index])
+      })
+  }
+
+  describe('when opening programme page with admin user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005', 'admin')
+    })
+
+    it('progress should not be recalculating when opened for the first time', () => {
+      cy.contains('Recalculating').should('not.exist')
+      cy.contains('Last updated').should('exist')
+    })
+
+    // Following vars are used to compare values before and after stats calculation
+    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
+    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
+    let originalProgressCalculatedText
+    let originalProductivityCalculatedText
+    const doRecalculation = () => {
+      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
+      cy.wait(2000)
+      cy.get('.attached').contains('Admin').click()
+      cy.contains('recalculate productivity').click()
+      cy.contains('recalculate throughput').click()
+      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
+      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
+      cy.wait(5000)
+      cy.reload()
+    }
+
+    it('renders progress and productivity tables with calculated status', () => {
+      doRecalculation()
+
+      // check contains also study programme options for admin
+      cy.contains('Study programme options')
+      cy.get('table').should('have.length', 3)
+
+      checkProgressAndProductivity('admin')
+
+      // Grab update dates to be compared later
+      cy.cs('throughputUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProgressCalculatedText = normalizeText(text)
+          expect(originalProgressCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+
+      cy.cs('productivityUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProductivityCalculatedText = normalizeText(text)
+          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+    })
+
+    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
+      doRecalculation()
+      checkProgressAndProductivity('admin')
+
+      // Check new calculation statuses are reported
+      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
+      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+
+      newProgressCalculatedTextElement.should(text => {
+        const newProgressCalculatedText = normalizeText(text)
+        expect(newProgressCalculatedText).not.to.contain('recalculating')
+        expect(newProgressCalculatedText).not.to.contain('refresh')
+        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+
+      newProductivityCalculatedTextElement.should(text => {
+        const newProductivityCalculatedText = normalizeText(text)
+        expect(newProductivityCalculatedText).not.to.contain('recalculating')
+        expect(newProductivityCalculatedText).not.to.contain('refresh')
+        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+    })
+  })
+
+  describe('when opening programme page with basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005')
+    })
+
+    it('renders progress and productivity tables with previosly calculated values', () => {
+      // check doesn't contain study programme options for admin
+      cy.contains('Study programme options').should('not.exist')
+      cy.get('table').should('have.length', 2)
+      checkProgressAndProductivity()
+    })
+
+    it('can open Thesis page', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('Thesis Courses').click()
+      cy.contains('Add thesis course').click()
+      cy.contains('No results')
+    })
+
+    it('can move to Population statistics page by clickin', () => {
+      cy.get('i.level.up.alternate.icon').eq(0).click()
+      cy.contains('Students (10)')
+    })
+
+    it('can create and delete tags for population', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      cy.contains('2018')
+      deleteTag(name)
+    })
+
+    it('can create personal tags', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+
+      cy.get('.ui > label').click()
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      deleteTag(name)
+    })
+
+    it('can add tags to students', () => {
+      const name = `tag-${new Date().getTime()}`
+
+      const student = '010113437'
+
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+
+      cy.contains('Add tags to students').click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+
+      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+
+      cy.get('textarea').type('010113437')
+      cy.get('.positive').click()
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name)
+
+      cy.go('back')
+      cy.go('back')
+
+      deleteTag(name)
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name).should('not.exist')
+    })
+  })
+})

cypress/integration/Teachers.js

@@ -1,53 +1,52 @@
 /// <reference types="Cypress" />
 
-describe("Teachers page tests", () => {
+describe('Teachers page tests', () => {
   beforeEach(() => {
-    cy.init();
-    cy.contains("Teachers").click();
-    cy.contains("Teacher statistics by course providers");
-  });
+    // login as admin = has teacher rights
+    cy.init('/teachers', 'admin')
+  })
 
-  const teacher1 = "Landgraf Leo"
-  const teacher2 = "Vinogradov Jussi Petteri"
+  const teacher1 = 'Landgraf Leo'
+  const teacher2 = 'Vinogradov Jussi Petteri'
 
-  it("Check Statistics", () => {
-    cy.get(":nth-child(1) > .ui > .search").click();
-    cy.contains("Autumn 2020").click();
-    cy.cs("course-providers").click();
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.cs("course-providers").children(".icon").click();
-    cy.get(".form > .fluid").click();
-    cy.contains("Name");
-    cy.contains(teacher1);
-    cy.contains(teacher2);
-  });
+  it('Check Statistics', () => {
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('Autumn 2020').click()
+    cy.cs('course-providers').click()
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+    cy.cs('course-providers').children('.icon').click()
+    cy.get('.form > .fluid').click()
+    cy.contains('Name')
+    cy.contains(teacher1)
+    cy.contains(teacher2)
+  })
 
-  it("Teacher search works", () => {
-    cy.url().should("include", "/teachers");
-    cy.get(".borderless > :nth-child(3)").click();
-    cy.get(".prompt").type(teacher1.split()[0]);
-    cy.contains(teacher1);
-  });
+  it('Teacher search works', () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type(teacher1.split()[0])
+    cy.contains(teacher1)
+  })
 
-  it("Can check teacher page", () => {
-    cy.url().should("include", "/teachers");
-    cy.get(".borderless > :nth-child(3)").click();
-    cy.get(".prompt").type(teacher1.split()[0]);
-    cy.contains(teacher1).click();
-    cy.contains("Aineopintojen harjoitustyö: Tietokantasovellus");
-  });
+  it('Can check teacher page', () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type(teacher1.split()[0])
+    cy.contains(teacher1).click()
+    cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
+  })
 
-  it("Check leaderboad works", () => {
+  it('Check leaderboad works', () => {
     cy.get('.borderless > :nth-child(2)').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
     cy.contains('2019-2020').click({ force: true })
-    cy.contains("Recalculate this year").click({ force: true })
+    cy.contains('Recalculate this year').click({ force: true })
     cy.wait(5000)
     cy.reload()
     cy.contains('2019-2020').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains("Passed")
-    cy.contains("Ahokas Jarmo Ilkka")
+    cy.contains('Passed')
+    cy.contains('Ahokas Jarmo Ilkka')
   })
-});
+})

cypress/integration/Users.js

@@ -1,81 +1,62 @@
 /// <reference types="Cypress" />
 
-describe("Users tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Users").click();
-    cy.contains("Oodikone users");
-  });
+describe('Users tests', () => {
+  describe('Using user with just grp-oodikone-user, no other rights', () => {
+    it('shows only frontpage, trends feedback', () => {
+      cy.init('', 'norights') // login with norights user
+      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('.navBar').contains('Give feedback').should('exist')
 
-  it("Admin mocking normal user shows only the mocked user's programmes", () => {
-    cy.contains("mocking").should("not.exist");
-    cy.contains("tr", "Normaalikäyttäjä").within(($row) => {
-      cy.contains(".button", "Edit").click();
-    });
-    cy.contains("Access rights")
-      .siblings()
-      .within(($row) => {
-        cy.get("div[role='listitem']")
-          .should("have.length", 1)
-          .contains("Tietojenkäsittelytieteen kandiohjelma");
-      });
+      cy.get('.navBar').contains('Study programme').should('not.exist')
+      cy.get('.navBar').contains('Student statistics').should('not.exist')
+      cy.get('.navBar').contains('Course statistics').should('not.exist')
+      cy.get('.navBar').contains('Teachers').should('not.exist')
+      cy.get('.navBar').contains('Users').should('not.exist')
+      cy.get('.navBar').contains('Faculty').should('not.exist')
+      cy.get('.navBar').contains('Updater').should('not.exist')
+    })
+  })
 
-    cy.route("POST", "/api/superlogin/normk").as("superlogin");
-    cy.get("i.spy").click();
-    cy.wait("@superlogin");
-    cy.contains("mocking as normk");
-    cy.wait(1000);
-    cy.contains("Study programme")
-      .click()
-      .siblings()
-      .contains("Search by class")
-      .click();
-    cy.contains("label", "Study programme");
-    cy.contains("label", "Study programme")
-      .siblings()
-      .within(($row) => {
-        cy.get("div[role='option']")
-          .should("have.length", 1)
-          .contains("Tietojenkäsittelytieteen kandiohjelma");
-      });
-  });
+  // TODO: Add user right checks for onlycoursestats user (should be like above + should
+  // show course stats
+  //
+  // TODO: Add user right checks for basic user (should show also study programme,
+  // student stats, course stats
 
-  it("Admin mocking courseStatistics-user can see only Trends and Course Statistics", () => {
-    cy.contains("mocking").should("not.exist");
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains(".button", "Edit").click();
-    });
+  describe('Using as admin', () => {
+    beforeEach(() => {
+      cy.init('/users', 'admin')
+    })
 
-    cy.get("[data-cy=access-groups-form]")
-      .click()
-      .contains("courseStatistics")
-      .click();
-    cy.get("[data-cy=access-groups-save]").click();
+    it('should see more stuff than others', () => {
+      cy.get('.navBar').contains('Study programme').should('exist')
+      cy.get('.navBar').contains('Student statistics').should('exist')
+      cy.get('.navBar').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
+      cy.get('.navBar').contains('Users').should('exist')
+      cy.get('.navBar').contains('Trends').should('exist')
+    })
 
-    cy.route("POST", "/api/superlogin/usermk").as("superlogin");
-    cy.get("i.spy").click();
-    cy.wait("@superlogin");
-    cy.contains("mocking as usermk");
-    cy.wait(1000);
-
-    cy.get("[data-cy=navbar-studyProgramme").should("not.exist");
-    cy.get("[data-cy=navbar-students]").should("not.exist");
-    cy.get("[data-cy=navbar-teachers]").should("not.exist");
-    cy.get("[data-cy=navbar-users").should("not.exist");
-
-    cy.contains("Stop mocking").click();
-    cy.wait(1000);
-
-    cy.contains("Users").click();
-    cy.contains("Oodikone users");
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains(".button", "Edit").click();
-    });
-    cy.contains("Access Groups")
-      .siblings()
-      .within(($row) => {
-        cy.get("i[class='dropdown icon clear']").click();
-      });
-    cy.get("[data-cy=access-groups-save]").click();
-  });
-});
+    it("mocking normal user shows only the mocked user's programmes", () => {
+      cy.contains('mocking').should('not.exist')
+      cy.contains('tr', 'basic').within($row => {
+        cy.contains('.button', 'Edit').click()
+      })
+      cy.intercept({
+        method: 'POST',
+        path: '/api/superlogin/basic',
+      }).as('superlogin')
+      cy.get('i.spy').click()
+      cy.wait('@superlogin')
+      cy.contains('mocking as basic')
+      cy.wait(1000)
+      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('label', 'Study programme')
+      cy.contains('label', 'Study programme')
+        .siblings()
+        .within($row => {
+          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
+        })
+    })
+  })
+})

cypress/not_checked_integration/Population_statistics.js

@@ -1,167 +0,0 @@
-/// <reference types="Cypress" />
-
-const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains("Advanced settings")
-    .siblings()
-    .get("[data-cy=advanced-toggle]")
-    .click();
-  includeSettings.forEach((setting) => {
-    cy.contains("Advanced settings")
-      .parent()
-      .siblings()
-      .contains(setting)
-      .click();
-  });
-  cy.get(".adv-stats-until > .form-control").click().clear().type(until);
-  cy.contains("Fetch population with new settings").click();
-  cy.contains("Advanced settings");
-};
-
-describe("Population Statistics tests", () => {
-  beforeEach(() => {
-    cy.init();
-  });
-
-  const checkAmountOfStudents = (assertion) => {
-    let students = 0;
-    cy.contains("Credit accumulation")
-      .click()
-      .invoke("text")
-      .then((text) => {
-        students = Number(text.match(/\d+/g)[0]);
-        expect(students).to.equal(assertion);
-      });
-  };
-
-  it("Population statistics search form is usable", () => {
-    cy.cs("navbar-studyProgramme").click();
-    cy.cs("navbar-class").click();
-    cy.contains("See population").should("be.disabled");
-    cy.url().should("include", "/populations");
-    cy.contains("Search for population");
-    cy.contains("Class of")
-      .parent()
-      .within(() => {
-        cy.get(".form-control").as("enrollmentSelect");
-      });
-
-    cy.get("@enrollmentSelect")
-      .its(`${[0]}.value`)
-      .then((beforeVal) => {
-        cy.get("@enrollmentSelect").click();
-        // go back to 2010-2019
-        cy.get(".yearSelectInput .rdtPrev").click({ force: true });
-        cy.get(".yearSelectInput table")
-          .contains("2014-2015")
-          .click({ force: true });
-        cy.get("@enrollmentSelect").should("not.have.value", beforeVal);
-      });
-
-    cy.contains("Select study programme")
-      .click()
-      .siblings()
-      .contains("Tietojenkäsittelytieteen koulutusohjelma")
-      .click();
-    cy.contains("Select degree")
-      .click()
-      .siblings()
-      .contains("Luonnontieteiden kandidaatti");
-  });
-
-  // FIXME: re-enable when CI has mandatory courses in the db
-  it.skip("Population statistics is usable on general level", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
-    setPopStatsUntil("September 2019");
-
-    cy.get(".card").within(() => {
-      cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
-      cy.contains("Sample size: 29 students");
-      cy.contains("Excludes exchange students");
-      cy.contains("Excludes students who haven't enrolled present nor absent");
-    });
-    cy.contains("Courses of population").click({ force: true });
-
-    cy.route("/api/v3/courseyearlystats**").as("coursePage");
-    cy.wait(150);
-    cy.cs("expand-CSM10000").click();
-    cy.cs("coursestats-link-CSM12101").click();
-    cy.wait("@coursePage");
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("CSM12101");
-  });
-
-  it.skip("Student list checking works as intended", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
-    cy.contains("Students (16)").click();
-    cy.contains("010429464");
-    cy.contains("666666666").should("not.exist");
-    cy.contains("button", "Check studentnumbers").click();
-    cy.contains("Check for studentnumbers");
-    cy.get("textarea").type("010429464").type("{enter}").type("666666666");
-    cy.contains("button", "check students").click();
-    cy.contains("#checkstudentsresults", "Results").within((e) => {
-      cy.contains("Student numbers in list and in oodi").click();
-      cy.contains("#found", "010429464");
-      cy.contains("Student numbers in list but not in oodi").click();
-      cy.contains("#notfound", "666666666");
-      cy.contains("Student numbers in oodi but not in list").click();
-      cy.contains("#notsearched", "010533091");
-    });
-  });
-
-  it.skip("Empty 'tags' tab has a link to the page where tags can be created", () => {
-    cy.selectStudyProgramme("Kielten kandiohjelma");
-    cy.contains("Students (5)").click();
-    cy.get("[data-cy=student-table-tabs]").contains("Tags").click();
-    cy.contains("No tags defined. You can define them here.").find("a").click();
-    cy.contains("Kielten kandiohjelma");
-    cy.contains("Create new tag");
-  });
-
-  it.skip("Advanced settings work", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.contains("Statistics until");
-    // only spring
-    cy.cs("toggle-fall").click();
-    cy.contains("Fetch population").click();
-
-    cy.contains("Credit accumulation (for 0 students)");
-
-    // only fall
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.cs("toggle-fall").click();
-    cy.cs("toggle-spring").click();
-
-    cy.contains("Fetch population").click();
-
-    // FIXME: This fails randomly both locally and on CI.
-    cy.contains("Credit accumulation (for 194 students)");
-
-    // spring + fall and include cancelled
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.cs("toggle-spring").click();
-    cy.cs("toggle-cancelled").click();
-    cy.contains("Fetch population").click();
-
-    cy.contains("Credit accumulation (for 202 students)");
-  });
-
-  it.skip("Credit Statistics, Statistics pane works", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-    cy.contains("Credit statistics").click();
-    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click();
-
-    cy.get("[data-cy='credit-stats-table-name-header']").should(
-      "contain",
-      "Statistic for n = 194 Students"
-    );
-    cy.get("[data-cy='credit-stats-mean']").should("contain", "45.10");
-    cy.get("[data-cy='credit-stats-stdev']").should("contain", "30.61");
-    cy.get("[data-cy='credit-stats-min']").should("contain", "0");
-    cy.get("[data-cy='credit-stats-q1']").should("contain", "19");
-    cy.get("[data-cy='credit-stats-q2']").should("contain", "47");
-    cy.get("[data-cy='credit-stats-q3']").should("contain", "66");
-    cy.get("[data-cy='credit-stats-max']").should("contain", "137");
-  });
-});

cypress/not_checked_integration/Student_statistics.js

@@ -1,75 +0,0 @@
-/// <reference types="Cypress" />
-
-describe("Student Statistics tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Student statistics").click();
-    cy.contains("Show student names");
-  });
-
-  it("Student statistics search form is usable", () => {
-    cy.contains("Show student names");
-    cy.url().should("include", "/students");
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("Student number");
-    cy.contains("Started");
-    cy.contains("Credits");
-    cy.contains("Heidi Eeva").should("not.exist");
-
-    cy.get("label").click();
-    cy.contains("Heidi Eeva");
-
-    cy.get("label").click();
-    cy.contains("Heidi Eeva").should("not.exist");
-  });
-  it("Can get student specific page by clicking student", () => {
-    cy.url().should("include", "/students");
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.contains("Updated at 02.10.2019");
-    cy.contains("Musiikin didaktiikka (EDUK4212)");
-    cy.contains("Oinonen").should("not.exist");
-
-    cy.get("label").click();
-    cy.contains("Oinonen");
-
-    cy.get("label").click();
-    cy.contains("Oinonen").should("not.exist");
-  });
-
-  it("Can get back to search menu", () => {
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.get(".remove").click();
-    cy.contains("Student number").should("not.exist");
-    cy.contains("Credits").should("not.exist");
-  });
-
-  it("Can jump to course", () => {
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.contains("Musiikin didaktiikka (EDUK4212)")
-      .siblings()
-      .within(() => {
-        cy.get(".level").click();
-      });
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("Musiikin didaktiikka");
-  });
-
-  it("Searching with bad inputs doesnt yield results", () => {
-    cy.get(".prompt").type("Oin");
-    cy.contains("Student number").should("not.exist");
-
-    cy.get(".prompt").clear().type("01114");
-    cy.contains("Student number").should("not.exist");
-  });
-
-  it("Can jump to population page", () => {
-    cy.get(".prompt").type("Veli-Matti");
-    cy.contains("014824094").click();
-    cy.get("i.level.up.alternate.icon").eq(0).click();
-    cy.contains("Population statistics");
-    cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
-  });
-});

cypress/not_checked_integration/Studyprogramme_overview.js

@@ -1,162 +0,0 @@
-/// <reference types="Cypress" />
-
-const deleteTag = (name) => {
-  cy.contains(name).siblings().contains("Delete").click();
-  cy.contains("Are you sure you want to delete tag");
-  cy.contains("Confirm").click();
-  cy.contains(name).should("not.exist");
-};
-
-describe("Studyprogramme overview", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Study programme")
-      .click()
-      .siblings()
-      .contains("Overview")
-      .click();
-    cy.contains("Study Programme", { timeout: 100000 });
-  });
-
-  // FIXME: re-enable when CI has mandatory courses in the db
-  it.skip("can search for course mappings", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Code Mapper").click();
-    cy.wait(150);
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.get("input").type("582219");
-    });
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.get(".results").contains("Käyttöjärjestelmät (582219)");
-    });
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.contains("button", "Add");
-    });
-  });
-
-  it("can view course groups", () => {
-    cy.contains("Kasvatustieteiden kandiohjelma").click();
-    cy.contains("Course Groups").click();
-
-    cy.contains("tr", "Test course group").get("i.edit").click();
-    cy.contains("Edit group");
-    cy.get(".prompt").type("Professori Pekka");
-    cy.contains("Add teacher").parent().contains("000960").click();
-    cy.contains("Teachers in group").parent().contains("000960");
-
-    cy.get("i.reply.link.icon").click();
-    cy.contains("tr a", "Test course group").click();
-    cy.contains("Total teachers");
-    cy.get("i.reply.icon").click();
-  });
-
-  it("renders progress and productivity tables", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Admin").click();
-    cy.contains("productivity").click();
-    cy.contains("throughput").click();
-
-    cy.wait(1000);
-    cy.get(".attached > :nth-child(1)").click();
-    cy.get("table").should("have.length", 3);
-    cy.contains("Population progress");
-    cy.contains("Yearly productivity");
-    cy.contains("2017-2018")
-      .siblings()
-      .contains("228")
-      .siblings()
-      .contains("178");
-
-    cy.get("table")
-      .eq(1)
-      .contains("2018")
-      .siblings()
-      .contains("3023.00")
-      .siblings()
-      .contains("2631.00");
-    cy.get("table")
-      .eq(1)
-      .contains("2017")
-      .siblings()
-      .contains("3159.00")
-      .siblings()
-      .contains("2914.00");
-  });
-
-  it("can open Thesis page", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Thesis Courses").click();
-    cy.contains("Add thesis course").click();
-    cy.contains("No results");
-  });
-
-  it("can move to Population statistics page by clickin", () => {
-    cy.contains("Tietojenkäsittelytieteen maisteriohjelma").click();
-    cy.get("i.level.up.alternate.icon").eq(0).click();
-    // cy.contains("Credit accumulation (for 16 students)");
-  });
-
-  it("can create and delete tags for population", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-    cy.contains("2018");
-    deleteTag(name);
-  });
-
-  it("can create personal tags", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-
-    cy.get(".ui > label").click();
-    cy.contains("Create new tag").click();
-    cy.get(".purple");
-    cy.contains(name);
-    deleteTag(name);
-  });
-
-  it("can add tags to students", () => {
-    const name = `tag-${new Date().getTime()}`;
-
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-
-    cy.contains("Add tags to students").click();
-    cy.get(".form > .field > .dropdown")
-      .click()
-      .get(".ui > .search")
-      .type(name)
-      .click();
-
-    cy.get(".form > .field > .dropdown > .visible").contains(name).click();
-
-    cy.get("textarea").type("010623419");
-    cy.get(".positive").click();
-
-    cy.contains("Student statistics").click();
-    cy.get(".prompt").type("010623419");
-    cy.contains("10").click();
-    cy.contains(name);
-
-    cy.go("back");
-    cy.go("back");
-
-    deleteTag(name);
-
-    cy.contains("Student statistics").click();
-    cy.get(".prompt").type("010623419");
-    cy.contains("10").click();
-    cy.contains(name).should("not.exist");
-  });
-});

cypress/not_checked_integration/Tags.js

@@ -1,48 +0,0 @@
-/// <reference types="Cypress" />
-
-const deleteTag = (name) => {
-  cy.contains("Create tags for study programme");
-  cy.contains(name).siblings().contains("Delete").click();
-  cy.contains("Are you sure you want to delete tag");
-  cy.contains("Confirm").click();
-  cy.contains(name).should("not.exist");
-};
-
-describe("Tag tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Study programme").click();
-    cy.contains("Overview").click();
-    cy.contains("Datatieteen maisteriohjelma").click();
-    cy.contains("Tags").click();
-  });
-
-  it("Tagged population works", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.get(":nth-child(1) > .field > .ui > input").type(name);
-    cy.get(".form-control").type("2016");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-    cy.contains("2016");
-
-    cy.contains("Add tags to students").click();
-    cy.get(".form > .field > .dropdown")
-      .click()
-      .get(".ui > .search")
-      .type(name)
-      .click();
-
-    cy.get(".form > .field > .dropdown > .visible").contains(name).click();
-
-    cy.get("textarea").type("014022579\n011122249");
-    cy.get(".positive").click();
-    cy.contains(name).find(".level").click();
-
-    cy.contains("Credit accumulation (for 1 students)");
-    cy.contains("Students (1)").click();
-    cy.contains("014022579");
-
-    cy.go("back");
-    deleteTag(name);
-  });
-});

cypress/plugins/index.js

@@ -19,4 +19,4 @@
 module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
-};
+}

cypress/support/commands.js

@@ -1,67 +1,91 @@
 /// <reference types="Cypress" />
+//
+// Note: here we need to set keys to be all lowercase, since
+// we're replacing headers after they've left browser / frontend.
+// All of these users are available in anon user-db
+const adminUserHeaders = {
+  uid: 'admin',
+  displayname: 'Admin User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member;employee;faculty',
+  mail: 'grp-toska+mockadminuser@helsinki.fi',
+}
+
+const basicUserHeaders = {
+  uid: 'basic',
+  displayname: 'Basic User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mockbasicuser@helsinki.fi',
+}
+
+const onlycoursestatisticsUserHeaders = {
+  uid: 'onlycoursestatistics',
+  displayName: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-session',
+  hyGroupCn: 'grp-oodikone-basic-users',
+  eduPersonAffiliation: 'member',
+  mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+}
+
+const norightsUserHeaders = {
+  uid: 'norights',
+  displayname: 'Norights User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mocknorightuser@helsinki.fi',
+}
+
+const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
 
 /**
- * Initialize headers and load the base URL or optional path.
+ Set up headers to login, set up correct user (admin / basic / etc.) and open given path.
  */
-Cypress.Commands.add("init", (path = "") => {
-  cy.server({
-    onAnyRequest: function (route, proxy) {
-      proxy.xhr.setRequestHeader("uid", "tktl");
-      proxy.xhr.setRequestHeader("shib-session-id", "mock-shibboleth");
-      proxy.xhr.setRequestHeader("hygroupcn", "grp-oodikone-users");
-      proxy.xhr.setRequestHeader("edupersonaffiliation", "asdasd");
-    },
-  });
-
-  // Babel throws an error probably because of markdown files. I'm sorry about this :lul:.
-  cy.on("uncaught:exception", (err, runnable) => {
-    expect(err.message).to.include(
-      "Cannot read property 'helpers' of undefined"
-    );
-    done();
-    return false;
-  });
-
+Cypress.Commands.add('init', (path = '', userId = 'basic') => {
+  cy.intercept('', req => {
+    const headersToUse = userHeaders.find(({ uid }) => uid === userId)
+    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
+    req.headers = headersToUse
+  })
+  // Hide SIS warning flag to avoid screen size problems when running headlessly
+  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
-  cy.visit(baseUrl.concat(path));
-});
+  cy.visit(baseUrl.concat(path))
+})
 
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add("cs", (name) => {
-  return cy.get(`[data-cy='${name}']`);
-});
+Cypress.Commands.add('cs', name => {
+  return cy.get(`[data-cy='${name}']`)
+})
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add("selectFromDropdown", (name, index) => {
-  const indexes = Array.isArray(index) ? index : [index];
+Cypress.Commands.add('selectFromDropdown', (name, index) => {
+  const indexes = Array.isArray(index) ? index : [index]
 
-  indexes.forEach((i) => {
-    cy.cs(name)
-      .click()
-      .children(".menu")
-      .children()
-      .eq(i)
-      .click({ force: true });
-  });
+  indexes.forEach(i => {
+    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+  })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children(".icon").click({ force: true });
+    cy.cs(name).children('.icon').click({ force: true })
   }
-});
-
+})
 
 /**
  * Move to page with studyProgramme population
  */
-Cypress.Commands.add("selectStudyProgramme", (name) => {
-  cy.cs("navbar-studyProgramme").click();
-  cy.cs("navbar-class").click();
-  cy.cs("select-study-programme").click().children().contains(name).click();
-  cy.contains("See population").click();
-});
+Cypress.Commands.add('selectStudyProgramme', name => {
+  cy.cs('navbar-studyProgramme').click()
+  cy.cs('navbar-class').click()
+  cy.cs('select-study-programme').click().children().contains(name).click()
+  cy.contains('See population').click()
+})

cypress/support/index.js

@@ -13,4 +13,4 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-import "./commands";
+import './commands'

docker-compose.ci.yml

@@ -3,56 +3,16 @@ version: "3.9"
 x-defaults: &defaults
   restart: unless-stopped
 
-x-build_defaults: &build_defaults
-  build:
-    args:
-      TAG: ${TAG}
-
 services:
-  analytics:
-    <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-      - toska/oodikone2-userservice:trunk
-      - toska/oodikone2-userservice:staging
-      - toska/oodikone2-userservice:latest
-      context: services/oodikone2-analytics
-    container_name: analytics
-    depends_on:
-      - analytics-db
-    image: toska/oodikone2-analytics:${TAG}
-    environment:
-      DB_URL: postgres://postgres@analytics-db:5432/analytics-db
-    ports:
-      - "4568:4568"
-
-  analytics-db:
-    <<: *defaults
-    container_name: analytics-db
-    image: registry.toska.cs.helsinki.fi/analytics-db
-    ports:
-      - "5423:5432"
 
   backend:
     <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-        - toska/oodikone2-backend:trunk
-        - toska/oodikone2-backend:staging
-        - toska/oodikone2-backend:latest
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    container_name: backen
+    container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
     environment:
-      ANALYTICS_URL: http://analytics:4568
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
@@ -62,39 +22,17 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:${TAG}
-    ports:
-      - "9229:9229"
+    image: toska/oodikone2-backend:trunk
 
   frontend:
     <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-        - toska/oodikone2-frontend:trunk
-        - toska/oodikone2-frontend:staging
-        - toska/oodikone2-frontend:latest
-      context: services/oodikone2-frontend
     container_name: frontend
-    environment:
-      BACKEND_ADDR: backend
-    image: toska/oodikone2-frontend:${TAG}
-    ports:
-      - "5000:5000"
+    image: toska/oodikone2-frontend:trunk
 
   kone-db:
     <<: *defaults
     container_name: kone-db
     image: registry.toska.cs.helsinki.fi/kone-db
-    ports:
-      - "5425:5432"
-
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    ports:
-      - "5421:5432"
 
   redis:
     <<: *defaults
@@ -102,31 +40,20 @@ services:
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - 6379:6379
 
   sis-db:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
-    ports:
-      - "5426:5432"
 
   user-db:
     <<: *defaults
     container_name: user-db
     image: registry.toska.cs.helsinki.fi/user-db
-    ports:
-      - "5422:5432"
 
   userservice:
     <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-        - toska/oodikone2-userservice:trunk
-        - toska/oodikone2-userservice:staging
-        - toska/oodikone2-userservice:latest
-      context: services/oodikone2-userservice
     container_name: userservice
     depends_on:
       - user-db
@@ -134,9 +61,7 @@ services:
       DB_URL: postgres://postgres@user-db:5432/user-db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:${TAG}
-    ports:
-      - "4567:4567"
+    image: toska/oodikone2-userservice:trunk
 
   nginx:
     image: nginx:latest
@@ -145,7 +70,6 @@ services:
       - frontend
       - backend
       - userservice
-      - analytics
     ports:
       -  1337:80
     volumes:

docker-compose.real.yml

@@ -1,11 +1,7 @@
 services:
-  analytics:
-    environment:
-      DB_URL: postgres://postgres@analytics-db:5432/analytics-db-real
 
   backend:
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 

docker-compose.test.yml

@@ -0,0 +1,10 @@
+services:
+
+  userservice-test:
+    build:
+      context: services/userservice
+      target: test
+    container_name: userservice-test
+    environment:
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -1,15 +1,11 @@
 x-defaults: &defaults
   restart: unless-stopped
 
-x-command_default: &command_default
-  command: ["npm", "run", "dev"]
-
 x-build_defaults: &build_defaults
-  args:
-    NODE_ENV: development
-    TAG: development
+  target: development
 
 services:
+
   adminer:
     <<: *defaults
     container_name: adminer
@@ -17,58 +13,23 @@ services:
       ADMINER_DESIGN: pepa-linha
     image: adminer:4.8.0
     ports:
-      - "5050:8080"
+      - 5050:8080
     volumes:
       - ./adminer-plugins:/var/www/html/plugins-enabled
 
-  analytics:
-    <<: *defaults
-    <<: *command_default
-    build:
-      <<: *build_defaults
-      context: services/oodikone2-analytics
-    container_name: analytics
-    depends_on:
-      - analytics-db
-    environment:
-      DB_URL: postgres://postgres@analytics-db:5432/analytics-db
-    ports:
-      - "4568:4568"
-    profiles:
-      - oodikone
-    volumes:
-      - analytics-nodemod:/usr/src/app/node_modules
-      - ./services/oodikone2-analytics/:/usr/src/app
-
-  analytics-db:
-    <<: *defaults
-    container_name: analytics-db
-    image: registry.toska.cs.helsinki.fi/analytics-db
-    ports:
-      - "5423:5432"
-    profiles:
-      - oodikone
-    volumes:
-      - analytics-db-data:/data
-
   backend:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
       context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
     environment:
-      ANALYTICS_URL: http://analytics:4568
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      FRONT_URL: http://localhost:8081
+      FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
@@ -77,59 +38,36 @@ services:
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
     ports:
-      - "8080:8080"
+      - 8080:8080
     profiles:
       - oodikone
     volumes:
       - backend-nodemod:/usr/src/app/node_modules
-      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/:/usr/src/app
 
   frontend:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
-      context: services/oodikone2-frontend
+      context: services/frontend
     container_name: frontend
-    environment:
-      ADMINER_URL: http://localhost:5050/?pgsql=oodi-db&username=postgres
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics-db&username=postgres
-      BACKEND_ADDR: backend
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=kone-db&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=sis-db&username=postgres
-      SIS_IMPORTER_ADMINER_URL: http://localhost:5050/?pgsql=sis-importer-db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user-db&username=postgres
     ports:
-      - "8081:8081"
-      - "7777:7777"
+      - 3000:3000
     profiles:
       - oodikone
     volumes:
       - frontend-nodemod:/usr/src/app/node_modules
-      - ./services/oodikone2-frontend/:/usr/src/app
+      - ./services/frontend/:/usr/src/app
 
   kone-db:
     <<: *defaults
     container_name: kone-db
     image: registry.toska.cs.helsinki.fi/kone-db
-    ports:
-      - "5425:5432"
     profiles:
       - oodikone
     volumes:
       - kone-db-data:/data
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    ports:
-      - "5421:5432"
-    profiles:
-      - oodikone
-    volumes:
-      - oodi-db-data:/data
-
   redis:
     <<: *defaults
     command: "redis-server --appendonly yes"
@@ -144,8 +82,6 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
-    ports:
-      - "5426:5432"
     volumes:
       - sis-db-data:/data
 
@@ -168,7 +104,6 @@ services:
 
   sis-updater-scheduler:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
       context: updater/sis-updater-scheduler
@@ -187,7 +122,7 @@ services:
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
     ports:
-      - "8082:8082"
+      - 8082:8082
     profiles:
       - updater
     volumes:
@@ -196,7 +131,6 @@ services:
 
   sis-updater-worker:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
@@ -224,8 +158,6 @@ services:
     <<: *defaults
     container_name: user-db
     image: registry.toska.cs.helsinki.fi/user-db
-    ports:
-      - "5422:5432"
     profiles:
       - oodikone
     volumes:
@@ -233,10 +165,9 @@ services:
 
   userservice:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
-      context: services/oodikone2-userservice
+      context: services/userservice
     container_name: userservice
     depends_on:
       - user-db
@@ -245,20 +176,17 @@ services:
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
     ports:
-      - "4567:4567"
+      - 4567:4567
     profiles:
       - oodikone
     volumes:
       - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/oodikone2-userservice/:/usr/src/app
+      - ./services/userservice/:/usr/src/app
 
 volumes:
-  analytics-db-data:
-  analytics-nodemod:
   backend-nodemod:
   frontend-nodemod:
   kone-db-data:
-  oodi-db-data: # TODO: Remove when oodi stuff has been removed from oodikone
   redis-data:
   sis-db-data:
   sis-importer-db-data:

package.json

@@ -1,45 +1,73 @@
 {
   "name": "oodikone",
-  "version": "1.0.0",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "cli": "./cli.sh",
-    "oodikone": "./run.sh oodikone anon up --detach && ./run.sh oodikone logs --follow --tail 100",
-    "oodikone:real": "./run.sh oodikone real up --detach && ./run.sh oodikone logs --follow --tail 100",
-    "updater": "./run.sh updater anon up --detach && ./run.sh updater logs --follow --tail 100",
-    "updater:real": "./run.sh updater real up --detach && ./run.sh updater logs --follow --tail 100",
-    "both": "./run.sh both anon up --detach && ./run.sh both logs --follow --tail 100",
-    "both:real": "./run.sh both real up --detach && ./run.sh both logs --follow --tail 100",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "lint:analytics": "npm run lint --prefix services/oodikone2-analytics",
-    "lint:backend": "npm run lint --prefix services/backend/oodikone2-backend",
-    "lint:frontend": "npm run lint --prefix services/oodikone2-frontend",
-    "lint:userservice": "npm run lint --prefix services/oodikone2-userservice",
-    "lint": "concurrently 'npm run lint:*'",
-    "lintfix:analytics": "npm run lintfix --prefix services/oodikone2-analytics",
-    "lintfix:backend": "npm run lintfix --prefix services/backend/oodikone2-backend",
-    "lintfix:frontend": "npm run lintfix --prefix services/oodikone2-frontend",
-    "lintfix:userservice": "npm run lintfix --prefix services/oodikone2-userservice",
-    "lintfix": "concurrently 'npm run lintfix:*'",
+    "concurrently": "concurrently",
     "cypress": "cypress",
+    "eslint": "eslint",
+    "stylelint": "stylelint",
+    "cli": "./cli.sh",
+    "oodikone": "./run.sh oodikone anon up --detach",
+    "oodikone:real": "./run.sh oodikone real up --detach",
+    "updater": "./run.sh updater anon up --detach",
+    "updater:real": "./run.sh updater real up --detach",
+    "both": "./run.sh both anon up --detach",
+    "both:real": "./run.sh both real up --detach",
+    "docker:down": "./run.sh both down --remove-orphans",
     "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
     "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
     "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
     "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js"
+    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
+    "prepare": "husky install",
+    "lint:docker-compose": "docker-compose config -q && :",
+    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
+    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
+    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
+  },
+  "lint-staged": {
+    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
+    "*.{js,json,md}": "prettier --write",
+    "*.css": "stylelint --fix",
+    ".github/workflows/*": "actionlint",
+    "Dockerfile": "hadolint",
+    "*.sh": "shellcheck -x",
+    "docker-compose.yml": "npm run lint:docker-compose",
+    "docker-compose.real.yml": "npm run lint:docker-compose:real",
+    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
+    "docker-compose.test.yml": "npm run lint:docker-compose:test"
   },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/UniversityOfHelsinkiCS/oodikone.git"
   },
-  "author": "",
   "license": "ISC",
   "bugs": {
     "url": "https://github.com/UniversityOfHelsinkiCS/oodikone/issues"
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "concurrently": "^6.2.0",
-    "cypress": "^7.6.0"
+    "@babel/eslint-parser": "^7.15.0",
+    "@babel/plugin-syntax-jsx": "^7.14.5",
+    "concurrently": "^6.2.1",
+    "cypress": "^8.2.0",
+    "eslint": "^7.32.0",
+    "eslint-config-airbnb": "^18.2.1",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-babel": "^5.3.1",
+    "eslint-plugin-cypress": "^2.11.3",
+    "eslint-plugin-import": "^2.24.0",
+    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-prefer-object-spread": "^1.2.1",
+    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-react": "^7.24.0",
+    "eslint-plugin-react-hooks": "^4.2.0",
+    "husky": "^7.0.1",
+    "lint-staged": "^11.1.2",
+    "prettier": "^2.3.2",
+    "stylelint": "^13.13.1",
+    "stylelint-config-prettier": "^8.0.2",
+    "stylelint-config-standard": "^22.0.0",
+    "stylelint-prettier": "^1.2.0"
   }
 }

run.sh

@@ -13,7 +13,7 @@ set -euoE pipefail
 # Set up constants
 PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 
-# Set up logging
+# Set up messages and exiting
 source "$PROJECT_ROOT"/scripts/utils.sh
 
 usage() {
@@ -21,8 +21,8 @@ usage() {
 Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 
 Parameters:
-* Option: oodikone/updater/both/morning
-* Version: anon/real/ci. Not necessary in all cases, such as when running down or logs.
+* Option: oodikone/updater/both
+* Version: anon/real. Not necessary when running down/logs/pull.
 * Command: will be passed to docker-compose.
 EOF
 }
@@ -34,24 +34,21 @@ parse_params() {
   [[ ${#args[@]} -eq 0 ]] && usage && die "Wrong number of arguments"
   option=${args[0]}
 
-  # If option is morning, other parameters aren't needed
-  [[ "$option" == "morning" ]] && return 0
-
-  # Else, parse arguments
+  #  parse arguments
   [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
-  usage && die "Wrong option: $option"
+    usage && die "Wrong option: $option"
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
 
-  # Down or logs can be passed without version. Otherwise parse version and then pass
+  # Some arguments can be passed without version. Otherwise parse version and then pass
   # rest to compose
-  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" ]]; then
+  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" || "${args[1]}" == "pull" ]]; then
     version=""
     compose_command=${args[*]:1}
   else
     version=${args[1]}
-    [[ "$version" != "anon" && "$version" != "real" && "$version" != "ci" ]] && \
-usage && die "Wrong version $version"
+    [[ "$version" != "anon" && "$version" != "real" ]] && usage && \
+      die "Wrong version $version"
     compose_command=${args[*]:2}
   fi
   return 0
@@ -72,10 +69,7 @@ parse_profiles() {
 parse_env() {
   env=""
   if [[ "$version" == "real" ]]; then
-    env="-f docker-compose.yml -f docker-compose.real.yml"
-  elif [[ "$version" == "ci" ]]; then
-    env="-f docker-compose.ci.yml"
-    profiles=""
+    env="--file docker-compose.yml --file docker-compose.real.yml"
   fi
   return 0
 }
@@ -84,14 +78,6 @@ parse_env() {
 
 parse_params "$@"
 
-# Do only morning cleanup for morning option
-if [[ "$option" == "morning" ]];then
-  git checkout trunk
-  git pull
-  docker-compose down --rmi all --remove-orphans
-  return 0
-fi
-
 # Create command that will be run. Empty command and "down" command will be handled
 # differently.
 parse_profiles

scripts/functions_for_setup.sh

@@ -11,24 +11,20 @@ PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 USER_DATA_FILE="$DUMP_DIR/hyuserdata"
 
-## Following the naming convention in docker-compose, these are names for services
-## and for the anonymous database. Real databases have suffix "-real".
-ANALYTICS_DB_NAME="analytics-db"
+## Following the naming convention in docker-compose, these are names for database
+## services. The real data databases inside service will have names suffixed by "-real".
 KONE_DB_NAME="kone-db"
 SIS_DB_NAME="sis-db"
 SIS_IMPORTER_DB_NAME="sis-importer-db"
 USER_DB_NAME="user-db"
-DATABASES=("$ANALYTICS_DB_NAME" "$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
-OODI_DB_NAME="oodi-db" # TODO: Remove when oodi is removed
+DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
 
-## Urls should be in same order as databases
-ANALYTICS_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-analytics-pg.sqz"
+## Urls should be in same order as databases as both are iterated through by indexes.
 KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
 SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
 USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
-REAL_DUMP_URLS=("$ANALYTICS_DB_REAL_DUMP_URL" "$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
-OODI_DB_REAL_DUMP_URL="svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/latest-pg.sqz" # TODO: Remove when oodi is removed
+REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
 
 # Source utility functions
 source "$PROJECT_ROOT"/scripts/utils.sh
@@ -36,10 +32,12 @@ source "$PROJECT_ROOT"/scripts/utils.sh
 # === Function ===
 
 draw_mopo() {
-  # Some hacks to print mopo as green, since colours from common config don't work
-  # Please feel free to fix this. And tell otahontas how you did it!
-  local mopogreen=$(tput setaf 34)
-  local normal=$(tput sgr0)
+  # Some hacks to print mopo as green, since colours from common config don't work when
+  # catting images. Please feel free to fix this. And tell otahontas how you did it!
+  local mopogreen
+  mopogreen=$(tput setaf 34)
+  local normal
+  normal=$(tput sgr0)
   if [ "$(tput cols)" -gt "100" ]; then
     while IFS="" read -r p || [ -n "$p" ]; do
       printf '%40s\n' "${mopogreen}$p${normal}"
@@ -47,7 +45,7 @@ draw_mopo() {
   fi
 }
 
-retry () {
+retry() {
   sleep 5
   for i in {1..60}; do
     "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
@@ -69,18 +67,17 @@ check_if_postgres_is_ready() {
 
 reset_databases() {
   local databases=("$@")
-  local database_name_suffix="-real"
   local database_dump_dir=$DUMP_DIR
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
   docker-compose down
-  docker-compose up -d ${databases[*]}
+  docker-compose up -d "${databases[@]}"
 
-  for database in ${databases[@]}; do
+  for database in "${databases[@]}"; do
     local database_dump="$database_dump_dir/$database.sqz"
     local database_container="$database"
-    local database_name="$database$database_name_suffix"
+    local database_name="$database-real"
 
     infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
 
@@ -110,7 +107,7 @@ reset_all_real_data() {
     local url="${REAL_DUMP_URLS[$i]}"
     download_real_dump "$database" "$url"
   done
-  reset_databases ${DATABASES[*]}
+  reset_databases "${DATABASES[@]}"
 }
 
 reset_sis_importer_data() {
@@ -121,38 +118,41 @@ reset_sis_importer_data() {
   reset_databases $database
 }
 
-reset_old_oodi_data() {
-  infomsg "Downloading old oodi-db dump"
-  local database=$OODI_DB_NAME
-  local url=$OODI_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
-}
-
-set_up_oodikone() {
+set_up_oodikone_from_scratch() {
   draw_mopo
-
-  infomsg "Installing npm packages locally to enable linting"
-
   folders_to_set_up=(
     "$PROJECT_ROOT"
-    "$PROJECT_ROOT/services/oodikone2-analytics"
-    "$PROJECT_ROOT/services/oodikone2-frontend"
-    "$PROJECT_ROOT/services/oodikone2-userservice"
-    "$PROJECT_ROOT/services/backend/oodikone2-backend"
+    "$PROJECT_ROOT/services/frontend"
+    "$PROJECT_ROOT/services/userservice"
+    "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/updater/sis-updater-scheduler"
+    "$PROJECT_ROOT/updater/sis-updater-worker"
   )
 
+  infomsg "Cleaning up docker setup and node_modules from possible previous installations"
+  local compose_down_args="--remove-orphans --volumes --rmi all"
+  "$PROJECT_ROOT"/run.sh both down "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.test.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.ci.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
-    cd "$folder" || return 1
-    ([[ -d node_modules ]] && warningmsg "Packages already installed in $folder") || npm ci
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
   done
-  cd "$PROJECT_ROOT" || return 1
 
-  infomsg "Cleaning up previous docker containers, volumes and networks"
-  "$PROJECT_ROOT"/run.sh both down --remove-orphans --volumes
+  infomsg "Installing npm packages locally to root and each subfolder to enable \
+linting and formatting"
+
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    ([[ -d node_modules ]] && die "$folder already has node_modules, exiting!") || npm ci
+  done
+  cd "$PROJECT_ROOT" || die "Couldn't change directory to project root"
+
+  infomsg "Setting up husky for pre-commit linting"
+  npm run prepare
 
   infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh oodikone anon pull
 
   infomsg "Building images"
   "$PROJECT_ROOT"/run.sh oodikone anon build

services/backend/.dockerignore

@@ -0,0 +1,6 @@
+.dockerignore
+Dockerfile
+node_modules/
+requests/
+scripts/
+*.log

services/backend/.gitignore

@@ -0,0 +1,3 @@
+# Student diff output
+/student_diff_*
+/diff_partial_att_data.json

services/backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:14.17-alpine3.13 as base
+
+WORKDIR /usr/src/app
+EXPOSE 8080
+
+FROM base as development
+ENV NODE_ENV=development
+COPY ./package* /usr/src/app/
+RUN npm ci
+COPY . /usr/src/app/
+CMD ["npm", "run", "dev"]
+
+FROM base as production
+ENV NODE_ENV=production
+ARG SENTRY_ENVIRONMENT
+ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+ARG SENTRY_RELEASE
+ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+USER node
+CMD ["npm", "start"]

services/backend/index.js

@@ -1,4 +1,3 @@
-//require('./src/tester')
 require('./src/app')
 
 process.on('unhandledRejection', reason => {

services/backend/oodikone2-backend/.dockerignore

@@ -1 +0,0 @@
-node_modules

services/backend/oodikone2-backend/.eslintignore

@@ -1 +0,0 @@
-*.sql

services/backend/oodikone2-backend/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

services/backend/oodikone2-backend/.gitignore

@@ -1,18 +0,0 @@
-node_modules
-.vscode
-.nyc_output
-src/tester.js
-update.sh
-.env
-logs/*
-run_update_script.sh
-student_numbers
-debug.log
-npm-debug.log
-
-studentnumbers.txt
-studentnumbers*.txt
-src/anonymized_API
-
-/student_diff_*
-/diff_partial_att_data.json

services/backend/oodikone2-backend/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

services/backend/oodikone2-backend/Dockerfile

@@ -1,16 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-EXPOSE 8080
-CMD npm start
-
-COPY ./oodikone2-backend/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-ARG TAG=local
-ENV TAG=${TAG}
-
-COPY ./oodikone2-backend /usr/src/app

services/backend/oodikone2-backend/README.md

@@ -1,259 +0,0 @@
-# oodikone2-backend
-
-## Dev
-
-### Getting started
-
-To install docker & docker-compose to run postgres and redis follow these steps:
-
-Install docker CE: https://docs.docker.com/engine/installation/ 
-
-Install docker-compose: https://docs.docker.com/compose/install/
-
-Create docker-compose.yml file containing following:
-
-```
-version: '3'
-
-services:
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-    container_name: oodi_db
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-```
-
-To start a redis and a database, run:
-
-```docker-compose up -d```
-
-Copy a backup dump from oodikone
-
-`scp oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/2018-08-01.bak <local folder>`
-
-Create database tkt_oodi:
-
-`docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi"`
-
-Fill database with data from backup file: 
-
-`cat 2018-08-01.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi`
-
-Check that you have all the data you ever wanted:
-
-`docker exec -it -u postgres oodi_db psql -d tkt_oodi`
-
-`\dt`
-
-Create test database:
-
-`docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_test"`
-
-Fill database with data from test.bak dump file: 
-
-`cat test.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi_test`
-
-### Running 
-
-#### Dev
-
-Run `npm run dev`
-
-#### Testing
-
-Run `npm test`
-
-## Deployment
-
-Always deploy first to staging! 
-
-When merging from trunk to master Travis builds a docker image for staging and pushes it automatically to dockerh-hub. Currently **staging** is also automatically deployed after the image is pushed.
-
-When a release (or tag) is created in master branch Travis builds a docker image for production and pushes it automatically.
-
-### To update: 
-
-Log into svm-59 (aka oodikone)
-
-`ssh {username}@oodikone.cs.helsinki.fi`
-
-Switch user to tkt_oodi
-
-`sudo su - tkt_oodi`
-
-Navigate to correct folder
-`cd oodikone.cs.helsinki.fi` or `cd oodikone.cs.helsinki.fi/staging`
-
-Use the update script to restart the software
-`./update.sh`
-
-Be amazed!
-
-## In case you mess up your DB
-
-Drop your current db
-
-`docker exec -u postgres oodi_db dropdb tkt_oodi`
-
-Copy a backup dump from oodikone
-
-`scp oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/2018-08-01.bak <local folder>`
-
-Create a new database and fill it with the backup dump following the instructions above
-
-## Accessing production/staging DB
-
-To access the database in oodikone.cs.helsinki.fi you can use `postgres_connect.sh` script in production/staging directory. Alternatively you can use commands: 
-
-`docker exec -it -u postgres db psql -d tkt_oodi`
-
-or
-
-`docker exec -it -u postgres db_staging psql -d tkt_oodi_staging`
-
-Production db backups are taken daily and can be found in /var/toska/tkt_oodi/backups/postgres/. Symlink to latest backup can always be found as /home/tkt_oodi/backups/latest-pg.bak.bz2.
-
-To create a dump of the production db manually, run: 
-
-``docker exec -u postgres db pg_dump tkt_oodi > `date +%Y-%m-%d`.bak``
-
-## Updater
-
-Oodikone2-backend includes the update-script that fetches student data from WebOodi. Script is split on two parts. Both production and staging environments have separate updater set up, in directories /oodikone.cs.helsinki.fi/updater and /oodikone.cs.helsinki.fi/staging/updater respectively. Both of updater directories have README file with summary of commands. Container names are production_updater and staging_updater, replace in examples below CONTAINER_NAME with correct one. Both containers have updater_files-directory that is mounted within container to /updater_files.
-
-Updater uses respective backend-image (production or staging, respectively).
-
-### Student list updater
-
-Finds out the valid student nubers. 
-
-Run in updater directory with _update_student_list.sh_
-
-Updater is run in separate detached container, see graylog or _debug.log_ to find out when it is finished.
-
-Environment variable _INCREMENT_ (currently not set) controls how many new student numbers are searched (from the currently known greatest student number).
-
-At the moment script uses the local postgres (see .env) to save student numbers.
-
-### Student info updater (database updater)
-
-To update your database run `docker-compose run -d CONTAINER_NAME npm run update_database` with optional args `file="<filename>"` and `index=<number>` with former telling the studentnumber filename to update and latter telling which row to start from.
-
-
-### Anonymized data & E2E environment
-
-Use [oodikone-cli](https://github.com/UniversityOfHelsinkiCS/oodikone) to setup you're environments.
-
-Before running cli make yourself a deployment key for Anonyymioodi private repository and put it in your cli folder as `private.key`. This allows you to download the anonymized dumps from the repository.
-
-For local development use with anonymized data use `2) Anon setup` (you can run e2e tests with this).
-
-`1) e2e setup` is designed for use in travis. But you can still try it locally if you want to.
-
-Run E2E tests in ToskaWatch with `npm run cypress:run` or `npm run cypress:open` for local tests.
-
-
-## Teacher leader board statistics
-
-The statistics for the teacher leader board have to be calculated separately with a script.
-
-```
-# Update top teachers for academic years 50 (1999-20) through 70 (2018-19).
-docker-compose run -d CONTAINER_NAME npm run update_top_teachers from=50 to=70
-
-# Omit the to–argument to update teachers just for a single academic year. 
-docker-compose run -d CONTAINER_NAME npm run update_top_teachers from=50
-```
-
-## DEFA report
-
-The npm script `create_defa_report` can quickly aggregate data about DEFA students and display it in neat csv format. Here's how you can use it on the Oodikone production server. It does not work if run locally.
-
-### Input files
-
-Input files should be placed in the `/home/tkt_oodi/oodikone.cs.helsinki.fi/defa` directory unless a different directory has been included into the backend docker container as `/usr/src/app/defa`.
-
-* `params.json`
-  * Defines the input/output files. File paths are for the container, so the working directory is `/usr/src/app/defa/`.
-  * Defines the timeframe to look at: from `params.in.timeframe.start` to `params.in.timeframe.end`. Studyattainments outside this timeframe will not be considered to be part of DEFA. Changing the timeframe is the main reason you'd want to make changes to this file.
-  * Example:
-```
-{
-  "out": {
-    "report": "/usr/src/app/defa/defa-report.csv",
-    "stats": "/usr/src/app/defa/defa-stats.csv"
-  },
-  "in": {
-    "course_ids": "/usr/src/app/defa/ids.txt",
-    "included_courses": "/usr/src/app/defa/included.txt",
-    "required_courses": "/usr/src/app/defa/required.txt",
-    "timeframe": {
-      "start": "2018-09-01T01:00:00.000Z",
-      "end": "2019-06-30T01:00:00.000Z"
-    }
-  }
-}
-```
-
-* `ids.txt` (or another name specified in `params.in.course_ids`)
-  * Holds the courseunitrealisation ids of DEFA courses to be included in the report.
-  * Example:
-```
-126240934
-126237007
-126238082
-126239325
-12699580
-```
-
-* `included.txt` (or another name specified in `params.in.included_courses`)
-  * Holds the learningopportunity ids of courses to be included in the report. Basically put all DEFA course codes here.
-  * One id per line.
-  * AY courses like `AYTKT0000` can be entered without the `AY` prefix to match both with and without it. e.g. `TKT0000` matches both `TKT0000` and `AYTKT0000`. `AYTKT0000` will only match `AYTKT0000`.
-  * Example:
-```
-TKT10002
-TKT10003
-MAT11001
-TKT10004
-TKT20001
-TKT20004
-TKT20009
-TKT21007
-MAT12001
-MAT12002
-```
-
-* `required.txt` (or another name specified in `params.in.required_courses`)
-  * Holds the learningopportunity ids of courses that are required to complete for a DEFA student to qualify.
-  * Same format as `included.txt`.
-  * All entries in this file should also be in `included.txt`.
-  * Example: same as `included.txt`
-
-### Running
-
-1. Make sure the backend docker container is running.
-2. run `docker exec <backend container> npm run create_defa_report defa/<parameter json file>` or use the ready made script `/home/tkt_oodi/oodikone.cs.helsinki.fi/defa/create_report.sh`.
-3. Check the output files.
-
-### Output files
-
-Output files will be created in the same directory where the input files are. Old output files will be overwritten unless they have been renamed/moved. 
-
-* `defa-report.csv` (or another name defined in `params.out.report`)
-  * Contains individual information about each DEFA student and individual inforamtion about each DEFA course.
-  * Contains aggregate information about all DEFA students.
-
-* `defa-stats.csv` (or another name defined in `params.out.stats`)
-  * Contains the credit amount distribution of DEFA students.

services/backend/oodikone2-backend/scripts/create_course_groups_kayttis.js

@@ -1,34 +0,0 @@
-const { createCourseGroup } = require('../src/services/courseGroups')
-
-const run = async () => {
-  await Promise.all(groups.map(name => createCourseGroup(name)))
-  process.exit(0)
-}
-
-const groups = [
-  'Yleinen ja aikuiskasvatustiede',
-  'Kasvatustiede, yleinen didaktiikka',
-  'Kasvatuspsykologia',
-  'Varhaiskasvatus (VO+VAKAMO)',
-  'Erityispedagogiikka',
-  'Kotitaloustiede',
-  'Käsityötiede',
-  'Aikuisopetus',
-  'Biologia ja maantiede',
-  'Historia ja yhteiskuntaoppi',
-  'Uskonto',
-  'Elämänkatsomustieto',
-  'Suomen kieli ja kirjallisuus',
-  'Vieraat kielet',
-  'Matematiikka',
-  'Fysiikka ja kemia',
-  'Musiikki',
-  'Kuvataide',
-  'Liikunta',
-  'Draama',
-  'Mediakasvatus',
-  'Tiedekeskuspedagogiikka',
-  'Svenska studier'
-]
-
-run()

services/backend/oodikone2-backend/src/anonymizer/anonDisciplines.js

@@ -1,115 +0,0 @@
-const disciplines = {
-  data: {
-    data: [
-      {
-        discipline_id: '0',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Witchcraft'
-          }
-        ]
-      },
-      {
-        discipline_id: '1',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Magic'
-          }
-        ]
-      },
-      {
-        discipline_id: '2',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Alchemy'
-          }
-        ]
-      },
-      {
-        discipline_id: '3',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Arithmancy'
-          }
-        ]
-      },
-      {
-        discipline_id: '4',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Runes'
-          }
-        ]
-      },
-      {
-        discipline_id: '5',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Creatures'
-          }
-        ]
-      },
-      {
-        discipline_id: '6',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Astronomy'
-          }
-        ]
-      },
-      {
-        discipline_id: '7',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'History'
-          }
-        ]
-      },
-      {
-        discipline_id: '8',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Charms'
-          }
-        ]
-      },
-      {
-        discipline_id: '9',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Flight'
-          }
-        ]
-      },
-      {
-        discipline_id: '10',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Necromancy'
-          }
-        ]
-      },
-      {
-        discipline_id: '11',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Pyromancy'
-          }
-        ]
-      }
-    ]
-  }
-}
-module.exports = { disciplines }

services/backend/oodikone2-backend/src/anonymizer/anonStudyRights.js

@@ -1,63 +0,0 @@
-const studyRightCodes = {
-  a: {
-    name: 'Transfiguration',
-    code: '666ABA'
-  },
-  b: {
-    name: 'Defence Against the Dark Arts',
-    code: '666ABB'
-  },
-  c: {
-    name: 'Charms',
-    code: '666ABC'
-  },
-  d: {
-    name: 'Potions',
-    code: '666ABD'
-  },
-  e: {
-    name: 'Astronomy',
-    code: '666ABE'
-  },
-  f: {
-    name: 'History of Magic',
-    code: '666ABF'
-  },
-  g: {
-    name: 'Herbology',
-    code: '666ABG'
-  },
-  h: {
-    name: 'Arithmancy',
-    code: '666ABH'
-  },
-  i: {
-    name: 'Muggle Studies',
-    code: '666ABI'
-  },
-  j: {
-    name: 'Divination',
-    code: '666ABJ'
-  },
-  k: {
-    name: 'Study of Ancient Runes',
-    code: '666ABK'
-  },
-  l: {
-    name: 'Care of Magical Creatures',
-    code: '666ABL'
-  },
-  m: {
-    name: 'Alchemy',
-    code: '666ABM'
-  },
-  bachelor: {
-    name: 'Bachelors Degree on Witchcraft and Wizardry',
-    code: '666bach'
-  },
-  master: {
-    name: 'Master of Witchcraft and Wizardry',
-    code: '666mast'
-  }
-}
-module.exports = { studyRightCodes }

services/backend/oodikone2-backend/src/anonymizer/anon_oodi_interface.js

@@ -1,104 +0,0 @@
-const axios = require('axios')
-const https = require('https')
-require('dotenv').config()
-
-const instance = axios.create({
-  httpsAgent: new https.Agent({
-    rejectUnauthorized: false
-  })
-})
-
-axios.defaults.auth = {
-  username: 'tktl'
-}
-
-instance.interceptors.request.use(config => {
-  config.params = { token: process.env.TOKEN }
-  return config
-})
-
-const getStudent = studentNumber => {
-  return instance
-    .get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)
-    .then(response => {
-      return response.data.data
-    })
-    .catch(error => {
-      return error
-    })
-}
-const getStudyAttainments = async studentNumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentNumber}/studyattainments`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getTeacher = async teacherId => {
-  const url = `${process.env.OODI_ADDR}/teachers/${teacherId}/info`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getStudyRights = async studentNumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentNumber}/studyrights`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getCourseTypeCodes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/learningopportunities/types`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/studyattainments/statuses`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getSemesters = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/semesters`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentnumber}/semesterenrollments`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/courseunitrealisations/types`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${process.env.OODI_ADDR}/learningopportunities/${id}`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const courseUnitRealisations = async () => {
-  const url = `${process.env.OODI_ADDR}/courseunitrealisations/changes/ids/0000-12-24`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${process.env.OODI_ADDR}/courseunitrealisations/${id}`
-  const response = await instance.get(url)
-  return response.data.data
-}
-module.exports = {
-  getStudyRights,
-  getStudent,
-  getTeacher,
-  getStudyAttainments,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisations,
-  getCourseUnitRealisation
-}

services/backend/oodikone2-backend/src/anonymizer/anonymizer.js

@@ -1,489 +0,0 @@
-const faker = require('faker')
-const moment = require('moment')
-const mkdirp = require('mkdirp')
-const getDirName = require('path').dirname
-const logger = require('../util/logger')
-const { disciplines } = require('./anonDisciplines')
-const { studyRightCodes } = require('./anonStudyRights')
-const oodi = require('./anon_oodi_interface')
-const fs = require('fs')
-const _ = require('lodash')
-const status = require('node-status')
-
-require('dotenv').config()
-const startTime = moment()
-
-let student_numbers = []
-
-// few objects to save connections between anonymized and real data to keep relations.
-let teacherRelations = {}
-let studentRelations = {}
-let learningOpportunityRelations = {}
-
-const startCodesStatusBar = () => {
-  status.start({ pattern: 'Running: {uptime.time} {spinner.monkey.blue} | {codes.bar} | codes anonymized: {codes}' })
-}
-const startStudentsStatusBar = () => {
-  status.start({
-    pattern: 'Running: {uptime.time} {spinner.hearts.magenta} | {students.bar} | students anonymized: {students}'
-  })
-}
-const startRealisationsStatusBar = () => {
-  status.start({
-    pattern: 'Running: {uptime.time} {spinner.clock.red} | {realisations.bar} | realisations anonymized: {realisations}'
-  })
-}
-const stopStatusBar = () => {
-  status.stamp()
-  status.stop()
-}
-
-const getAnonymizedTeacher = async teacherId => {
-  if (teacherRelations[teacherId]) {
-    return teacherRelations[teacherId]
-  }
-  let teacher = await oodi.getTeacher(teacherId)
-  const last_name = faker.name.lastName()
-  const first_names = faker.name.firstName()
-  teacher = {
-    ...teacher,
-    teacher_id: generateRandomId(),
-    userid: faker.internet.userName(),
-    last_name,
-    first_names,
-    calling_name: faker.name.firstName(),
-    full_name: `${last_name} ${first_names}`,
-    email: faker.internet.email(),
-    mobile_phone: faker.phone.phoneNumberFormat()
-  }
-  teacherRelations[teacherId] = teacher
-  return teacher
-}
-
-const generateRandomId = () => String(Math.floor(Math.random() * (999999999 - 100000000 + 1) + 100000000))
-
-const anonymizeStudentInfo = async id => {
-  let student = await oodi.getStudent(id)
-  let anonStudent = undefined
-  if (student) {
-    anonStudent = studentRelations[student.student_number]
-  }
-  if (!anonStudent) {
-    anonStudent = {
-      ...student,
-      first_names: faker.name.firstName(),
-      last_name: faker.name.lastName(),
-      birthdate: new Date(
-        moment(
-          moment('1940', 'YYYY').valueOf() +
-            Math.random() *
-              (moment()
-                .subtract(17, 'years')
-                .valueOf() -
-                moment('1940', 'YYYY').valueOf())
-        )
-      ),
-      email: faker.internet.email(),
-      mobile_phone: faker.phone.phoneNumberFormat(),
-      national_student_number: null,
-      phone: faker.phone.phoneNumberFormat(),
-      address1: faker.address.streetName(),
-      address2: faker.address.streetName(),
-      zipcode: faker.address.zipCode(),
-      student_number: `0${Math.floor(Math.random() * (1999999 - 1000000 + 1) + 1000000)}`,
-      person_id: generateRandomId(),
-      city: [
-        {
-          langcode: 'en',
-          text: 'Harlem'
-        },
-        {
-          langcode: 'fi',
-          text: 'Kajaani'
-        }
-      ],
-      country: [
-        {
-          langcode: 'en',
-          text: 'Canada'
-        },
-        {
-          langcode: 'fi',
-          text: 'Ruotsi'
-        }
-      ]
-    }
-  }
-  if (!student) {
-    return anonStudent
-  }
-  studentRelations[student.student_number] = anonStudent
-
-  return anonStudent
-}
-const getRandomAmount = attainments => {
-  const rate = Math.random() * (0.375 - 0.05) + 0.05
-  const amount = Math.floor(rate * attainments.length)
-  return amount
-}
-const dropRandomAttainments = attainments => {
-  if (student_numbers.length < 1) {
-    return attainments
-  }
-  const dropAmount = getRandomAmount(attainments)
-  let i = 0
-  while (i < dropAmount) {
-    let randomIndex = Math.floor(Math.random() * attainments.length)
-    attainments.splice(randomIndex, 1)
-    i++
-  }
-  return attainments
-}
-const addRandomAttainments = (attainments, studentnumber_attainments) => {
-  if (studentnumber_attainments.length < 1) {
-    return attainments
-  }
-  let studentnumbers = studentnumber_attainments.map(s => s.student_number)
-  const addAmount = getRandomAmount(attainments)
-  let i = 0
-  while (i < addAmount) {
-    let randomStudentNumber = _.sample(studentnumbers)
-    let randomAttainment = _.sample(
-      _.flattenDeep(
-        studentnumber_attainments.filter(s => s.student_number === randomStudentNumber).map(s => s.attainments)
-      )
-    )
-    if (!randomAttainment) {
-      continue
-    }
-    randomAttainment.semester_code = _.sample(attainments).semester_code // Make attainment happen at the same time as the students some other attainment.
-    attainments = attainments.concat(randomAttainment)
-    i++
-  }
-  return attainments
-}
-const populateAnonymizedTeachers = async teachers => {
-  let anonTeachers = []
-  for (let teacher of teachers) {
-    let anonTeacher = await getAnonymizedTeacher(teacher.teacher_id)
-    APIWriter(`./src/anonymized_API/teachers/${anonTeacher.teacher_id}/info`, { data: { data: anonTeacher } })
-    anonTeachers = anonTeachers.concat(anonTeacher)
-  }
-  return anonTeachers
-}
-
-const anonymizeStudentAttainments = async (id, studentInfo, studentnumber_attainments) => {
-  let attainments = await oodi.getStudyAttainments(id)
-  attainments = dropRandomAttainments(attainments)
-  attainments = addRandomAttainments(attainments, studentnumber_attainments)
-  let anonAttainments = []
-  for (let attainment of attainments) {
-    let anonLearningOpportunity = learningOpportunityRelations[attainment.learningopportunity_id]
-    if (!anonLearningOpportunity) {
-      const learningOpportunity = await oodi.getLearningOpportunity(attainment.learningopportunity_id)
-      if (learningOpportunity === null) {
-        continue
-      }
-      anonLearningOpportunity = {
-        ...learningOpportunity,
-        disciplines: [disciplines.data.data[Math.floor(Math.random() * 12)]],
-        learningopportunity_id: generateRandomId(),
-        names: [
-          {
-            langcode: 'en',
-            text: faker.company.bs()
-          },
-          {
-            langcode: 'fi',
-            text: faker.company.bs()
-          }
-        ],
-        descriptions: [],
-        materials: [],
-        organisations: [],
-        learningopportunity_internal: generateRandomId(),
-        start_date: moment(learningOpportunity.start_date)
-          .add(_.sample([-3, 0, 3]), 'months')
-          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days')),
-        end_date: moment(learningOpportunity.start_date)
-          .add(_.sample([-3, 0, 3]), 'months')
-          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days'))
-          .add(100, 'years')
-      }
-      learningOpportunityRelations[attainment.learningopportunity_id] = anonLearningOpportunity
-      APIWriter(`./src/anonymized_API/learningopportunities/${anonLearningOpportunity.learningopportunity_id}`, {
-        data: { data: anonLearningOpportunity }
-      })
-    }
-    let teachers = []
-    teachers = await populateAnonymizedTeachers(attainment.teachers)
-
-    // semester code is calculated as "code * 6 months since 1950-1-1", Im bucketing all study attainments in 3 month periods by semester code abstracting real study attainment dates
-    const attainment_date = new Date(
-      moment(moment('1950', 'YYYY').add(attainment.semester_code * 6 + _.sample([-3, 0, 3]), 'months'))
-    )
-    attainment = {
-      ...attainment,
-      studyattainment_id: generateRandomId(),
-      person_id: studentInfo.person_id,
-      attainment_date,
-      teachers,
-      learningopportunity_id: anonLearningOpportunity.learningopportunity_id,
-      learningopportunity_name: anonLearningOpportunity.names
-    }
-    anonAttainments = anonAttainments.concat(attainment)
-  }
-  return anonAttainments
-}
-
-const anonymizeStudentStudyRights = async (id, studentInfo) => {
-  let studyRights = await oodi.getStudyRights(id)
-  let anonStudyRights = []
-  for (let studyright of studyRights) {
-    let program = _.sample(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
-    let degree = 'bachelor'
-    if (studyright.elements.some(element => element.name.some(name => name.text.includes('Master')))) {
-      program = _.sample(['i', 'j', 'k', 'l', 'm', 'h'])
-      degree = 'master'
-    }
-    const elements = studyright.elements
-      .filter(element => element.element_id === 10 || element.element_id === 20)
-      .map(element => {
-        if (element.element_id === 10) {
-          return {
-            ...element,
-            code: studyRightCodes[degree].code,
-            name: [
-              {
-                langcode: 'en',
-                text: studyRightCodes[degree].name
-              },
-              {
-                langcode: 'fi',
-                text: studyRightCodes[degree].name
-              }
-            ]
-          }
-        } else if (element.element_id === 20) {
-          return {
-            ...element,
-            code: studyRightCodes[program].code,
-            name: [
-              {
-                langcode: 'en',
-                text: studyRightCodes[program].name
-              },
-              {
-                langcode: 'fi',
-                text: studyRightCodes[program].name
-              }
-            ]
-          }
-        }
-        return
-      })
-
-    let anonStudyright = {
-      ...studyright,
-      studyright_id: generateRandomId(),
-      person_id: studentInfo.person_id,
-      elements,
-      organisation_name: [
-        {
-          langcode: 'en',
-          text: 'Hogwarts School of Witchcraft and Wizardry'
-        },
-        {
-          langcode: 'fi',
-          text: 'Tylypahkan noitien ja velhojen koulu'
-        }
-      ]
-    }
-    anonStudyRights = anonStudyRights.concat(anonStudyright)
-  }
-  return anonStudyRights
-}
-const getElapsedTime = () =>
-  `${moment().diff(startTime, 'minutes')} m, ${moment().diff(startTime, 'seconds') /
-    (moment().diff(startTime, 'minutes') * 60)} s.`
-const anonymizeSemesterEnrollments = async id => {
-  const data = await oodi.getSemesterEnrollments(id)
-  let anonymizedData = []
-  for (const enrollment of data) {
-    anonymizedData = anonymizedData.concat({
-      ...enrollment,
-      yths_code: null,
-      absence_reason_code: null,
-      payment_amount: 100,
-      yths_description: null,
-      semester_enrollment_date: moment(moment(enrollment.semester_enrollment_date).format('YYYY-MM'), 'YYYY-MM'),
-      payment_date: moment(moment(enrollment.payment_date).format('YYYY-MM'), 'YYYY-MM')
-    })
-  }
-  return anonymizedData
-}
-const anonymizeStudent = async (id, student_numbers) => {
-  const info = await anonymizeStudentInfo(id)
-
-  const attainments = await anonymizeStudentAttainments(id, info, studentnumber_attainments)
-  studentnumber_attainments = studentnumber_attainments.concat({
-    student_number: info.student_number,
-    attainments: attainments
-  })
-  info.studyattainments = attainments
-    .filter(
-      attainment => ['Hyv.', '1', '2', '3', '4', '5'].includes(attainment.grade[0].text) && attainment.credits < 25
-    )
-    .map(attainment => attainment.credits)
-    .reduce((acc, curr) => acc + curr, 0)
-
-  const studyrights = await anonymizeStudentStudyRights(id, info)
-
-  const semesterEnrollments = await anonymizeSemesterEnrollments(id)
-  const infoPath = `./src/anonymized_API/students/${info.student_number}/info`
-  const semesterEnrollmentPath = `./src/anonymized_API/students/${info.student_number}/semesterenrollments`
-  const courseEnrollmentPath = `./src/anonymized_API/students/${info.student_number}/enrollments`
-  const attainmentPath = `./src/anonymized_API/students/${info.student_number}/studyattainments`
-  const studyRightPath = `./src/anonymized_API/students/${info.student_number}/studyrights`
-  student_numbers = student_numbers.concat(info.student_number)
-
-  APIWriter(infoPath, { data: { data: info } })
-  APIWriter(studyRightPath, { data: { data: studyrights } })
-  APIWriter(courseEnrollmentPath, { data: { data: [] } })
-  APIWriter(attainmentPath, { data: { data: attainments } })
-  APIWriter(semesterEnrollmentPath, { data: { data: semesterEnrollments } })
-  return student_numbers
-}
-const anonymizeCourseUnitRealisations = async () => {
-  const courseUnitRealisations = await oodi.courseUnitRealisations()
-  const realisationsCounter = status.addItem('realisations', { max: courseUnitRealisations.length })
-  startRealisationsStatusBar()
-  let anonRealisations = []
-  for (const realisation of courseUnitRealisations) {
-    continue
-    realisationsCounter.inc() // eslint-disable-line
-    if (!learningOpportunityRelations[realisation.learningopportunity_id]) {
-      continue
-    }
-    const anonLearningId = learningOpportunityRelations[realisation.learningopportunity_id].learningopportunity_id
-    const anonRealisation = {
-      ...realisation,
-      course_id: generateRandomId(),
-      learningopportunity_id: anonLearningId
-    }
-    anonRealisations = anonRealisations.concat(anonRealisation)
-    const courseUnitRealisation = await oodi.getCourseUnitRealisation(realisation.course_id)
-    if (!courseUnitRealisation) {
-      continue
-    }
-    let anonStudents = []
-    for (const student of courseUnitRealisation.students) {
-      if (student) {
-        anonStudents = anonStudents.concat(await anonymizeStudentInfo(student.student_number))
-      }
-    }
-
-    let anonTeachers = []
-    for (const teacher of courseUnitRealisation.teachers) {
-      anonTeachers = anonTeachers.concat(await getAnonymizedTeacher(teacher.teacher_id))
-    }
-
-    const anonCourseUnitRealisation = {
-      ...courseUnitRealisation,
-      realisation_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      students: anonStudents,
-      providing_organisation: [],
-      instruction_locality: [],
-      events: [],
-      officials: [],
-      child_ids: [],
-      courses_extra_organisations: [],
-      teachers: anonTeachers, // note that new teachers created here are not populated in the API! (for future possible issues) regards: Saus 31.7.2018
-      organisations: [],
-      course_id: anonRealisation.course_id,
-      providing_locality: [],
-      learningopportunity_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      realisation_root_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      learningopportunity_id: learningOpportunityRelations[realisation.learningopportunity_id].learningopportunity_id,
-      descriptions: [],
-      languages: []
-    }
-    await APIWriter(`src/anonymized_API/courseunitrealisations/${anonCourseUnitRealisation.course_id}`, {
-      data: { data: anonCourseUnitRealisation }
-    })
-  }
-  await APIWriter('src/anonymized_API/courseunitrealisations/changes/ids/0000-12-24', {
-    data: { data: anonRealisations }
-  })
-  stopStatusBar()
-  return
-}
-
-const APIWriter = async (path, data) => {
-  mkdirp(getDirName(path), async () => {
-    fs.writeFile(path, JSON.stringify(data, null, 4), error => {
-      if (error) {
-        console.log(error)
-      }
-    })
-    return 'succeeee'
-  })
-}
-
-let studentnumber_attainments = []
-
-const anonymize = async () => {
-  logger.verbose(`starting time: ${startTime.format('HH:mm:ss')}`)
-  const filename = './studentnumbers.txt'
-  const readStudentNumbersFromFile = async filename => {
-    let studentnumbers = fs
-      .readFileSync(filename, 'utf-8')
-      .split('\n')
-      .map(s => s.replace(' ', ''))
-    const randomCopies = _.sampleSize(
-      studentnumbers,
-      Math.floor((Math.random() * (1 - 0.33) + 0.33) * studentnumbers.length)
-    )
-    studentnumbers = studentnumbers.concat(randomCopies)
-    studentnumbers = _.shuffle(studentnumbers)
-    return studentnumbers.filter(studentnumber => !!studentnumber)
-  }
-  const codeCounter = status.addItem('codes', { max: 5 })
-  startCodesStatusBar()
-  const semesters = { data: { data: await oodi.getSemesters() } }
-  codeCounter.inc()
-  const courseUnitRealisationsTypes = { data: { data: await oodi.getCourseRealisationTypes() } }
-  codeCounter.inc()
-  const creditTypeCodes = { data: { data: await oodi.getStudyattainmentStatusCodes() } }
-  codeCounter.inc()
-  const courseTypeCodes = { data: { data: await oodi.getCourseTypeCodes() } }
-  codeCounter.inc()
-  const courseDisciplines = disciplines
-  codeCounter.inc()
-  await APIWriter('./src/anonymized_API/codes/semesters', semesters)
-  await APIWriter('./src/anonymized_API/codes/courseunitrealisations/types', courseUnitRealisationsTypes)
-  await APIWriter('./src/anonymized_API/codes/studyattainments/statuses', creditTypeCodes)
-  await APIWriter('./src/anonymized_API/codes/learningopportunities/types', courseTypeCodes)
-  await APIWriter('./src/anonymized_API/codes/learningopportunities/disciplines', courseDisciplines)
-  const numberList = await readStudentNumbersFromFile(filename)
-  const studentCounter = status.addItem('students', { max: numberList.length })
-  status.stamp()
-  startStudentsStatusBar()
-  let student_numbers = []
-  for (let id of numberList) {
-    studentCounter.inc()
-    try {
-      student_numbers = await anonymizeStudent(id, student_numbers)
-    } catch (error) {
-      logger.verbose(`FAILED ${id}, ${error}  elapsed time:  ${getElapsedTime()}`)
-      process.exit(1)
-    }
-  }
-  status.stamp()
-  await anonymizeCourseUnitRealisations()
-  console.log(student_numbers)
-  await APIWriter('./studentnumbersN.txt', student_numbers)
-  logger.verbose(`end time: ${moment()}, total run-time:  ${getElapsedTime()}`)
-}
-
-anonymize()

services/backend/oodikone2-backend/src/database/migrations/20190710_03_drop_ldapuser.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.dropTable('ldapuser', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20190806_00_transfers_delete_on_studyright_delete.js

@@ -1,39 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_studyrightid_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_sourcecode_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_studentnumber_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_targetcode_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_studyrightid_fkey" FOREIGN KEY (studyrightid) REFERENCES studyright(studyrightid) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_sourcecode_fkey" FOREIGN KEY (sourcecode) REFERENCES element_details(code) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_studentnumber_fkey" FOREIGN KEY (studentnumber) REFERENCES student(studentnumber) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_targetcode_fkey" FOREIGN KEY (targetcode) REFERENCES element_details(code) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'DELETE FROM transfers WHERE studyrightid IS NULL OR sourcecode IS NULL OR studentnumber IS NULL OR targetcode IS NULL',
-        { transaction }
-      )
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20190816_00_drop_mandatory_course_labels.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.dropTable('mandatory_course_labels', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191021_00_drop_highlevelname.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.removeColumn('studyright', 'highlevelname', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191028_00_add_more_indexes.js

@@ -1,16 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.addIndex('course_disciplines', ['course_id'], { transaction })
-      await queryInterface.addIndex('element_details', ['type'], { transaction })
-      await queryInterface.addIndex('studyright', ['canceldate'], { transaction })
-      await queryInterface.addIndex('studyright', ['extentcode'], { transaction })
-      await queryInterface.addIndex('studyright_elements', ['studentnumber'], { transaction })
-      await queryInterface.addIndex('studyright_elements', ['studyrightid'], { transaction })
-      await queryInterface.addIndex('studyright_elements', ['code'], { transaction })
-      await queryInterface.addIndex('transfers', ['sourcecode'], { transaction })
-      await queryInterface.addIndex('transfers', ['targetcode'], { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191205_00_add_column_to_studyright.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('studyright', 'faculty_code', {
-      type: Sequelize.STRING
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191207_00_add_column_to_elementdetails.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('element_details', 'faculty_code', {
-      type: Sequelize.STRING
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191217_00_remove_matriculationexamination.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.removeColumn('student', 'matriculationexamination', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200101_00_fix_transfers_duplicates.js

@@ -1,27 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    await queryInterface.sequelize.query(
-      `DELETE
-            FROM transfers a
-                USING transfers b
-            WHERE
-                a.id > b.id
-                AND a.transferdate = b.transferdate
-                AND a.studentnumber = b.studentnumber
-                AND a.studyrightid = b.studyrightid
-                AND a.sourcecode = b.sourcecode
-                AND a.targetcode = b.targetcode`,
-      {
-        raw: true
-      }
-    )
-    await queryInterface.addIndex(
-      'transfers',
-      ['studyrightid', 'sourcecode', 'targetcode', 'transferdate', 'studentnumber'],
-      {
-        unique: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200101_01_remove_fac_from_elmdetails.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.removeColumn('element_details', 'faculty_code', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200218_00_add_admin_view_funcs.js

@@ -1,38 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    await queryInterface.sequelize.query(
-      `CREATE OR REPLACE FUNCTION public.map_range(
-            input DOUBLE PRECISION,
-            inMin DOUBLE PRECISION,
-            inMax DOUBLE PRECISION,
-            outMin DOUBLE PRECISION,
-            outMax DOUBLE PRECISION)
-        RETURNS DOUBLE PRECISION AS $$
-            SELECT (input - inMin) * (outMax - outMin) / (inMax - inMin) + outMin;
-        $$ LANGUAGE SQL;
-        
-        CREATE OR REPLACE FUNCTION public.is_in_target(
-            currentDate TIMESTAMP WITH TIME ZONE,
-            studyStartDate TIMESTAMP WITH TIME ZONE,
-            targetDate TIMESTAMP WITH TIME ZONE,
-            studentCredits DOUBLE PRECISION,
-            targetCredits DOUBLE PRECISION)
-        RETURNS INTEGER AS $$
-            SELECT CASE WHEN studentCredits >= public.map_range(
-                EXTRACT(EPOCH FROM currentDate),
-                EXTRACT(EPOCH FROM studyStartDate),
-                EXTRACT(EPOCH FROM targetDate),
-                0,
-                targetCredits
-            )
-                THEN 1
-                ELSE 0
-            END;
-        $$ LANGUAGE SQL;`,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200324_00_fix_sr_elements_duplicates.js

@@ -1,24 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    await queryInterface.sequelize.query(
-      `DELETE
-            FROM studyright_elements a
-                USING studyright_elements b
-            WHERE
-                a.id < b.id
-                AND a."createdAt" < b."createdAt"
-                AND a.studentnumber = b.studentnumber
-                AND a.studyrightid = b.studyrightid
-                AND a.code = b.code`,
-      {
-        raw: true,
-        underscored: false
-      }
-    )
-    await queryInterface.removeIndex('studyright_elements', ['startdate', 'enddate', 'studyrightid', 'code'])
-    await queryInterface.addIndex('studyright_elements', ['startdate', 'studyrightid', 'code', 'studentnumber'], {
-      unique: true
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200409_00_more_admin_view_funcs.js

@@ -1,81 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    // add IMMUTABLE & STABLE attribute to functions created in 20200218_00_add_admin_view_funcs
-    // to let postgres optimize their use
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION public.map_range(
-          input DOUBLE PRECISION,
-          inMin DOUBLE PRECISION,
-          inMax DOUBLE PRECISION,
-          outMin DOUBLE PRECISION,
-          outMax DOUBLE PRECISION)
-      RETURNS DOUBLE PRECISION AS $$
-          SELECT (input - inMin) * (outMax - outMin) / (inMax - inMin) + outMin;
-      $$ LANGUAGE SQL IMMUTABLE;
-
-      CREATE OR REPLACE FUNCTION public.is_in_target(
-          currentDate TIMESTAMP WITH TIME ZONE,
-          studyStartDate TIMESTAMP WITH TIME ZONE,
-          targetDate TIMESTAMP WITH TIME ZONE,
-          studentCredits DOUBLE PRECISION,
-          targetCredits DOUBLE PRECISION)
-      RETURNS INTEGER AS $$
-          SELECT CASE WHEN studentCredits >= public.map_range(
-              EXTRACT(EPOCH FROM currentDate),
-              EXTRACT(EPOCH FROM studyStartDate),
-              EXTRACT(EPOCH FROM targetDate),
-              0,
-              targetCredits
-          )
-              THEN 1
-              ELSE 0
-          END;
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-
-    // new funcs
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION change_year(t timestamp with time zone, yr int) RETURNS timestamp with time zone AS $$
-          SELECT make_timestamptz(
-              yr,
-              date_part('month', t)::int,
-              date_part('day', t)::int,
-              date_part('hour', t)::int,
-              date_part('minute', t)::int,
-              date_part('second', t)
-          )
-      $$ LANGUAGE SQL STABLE;
-
-      CREATE OR REPLACE FUNCTION is_leap_day(t timestamp with time zone) RETURNS boolean AS $$
-          SELECT date_part('day', t) = 29 AND date_part('month', t) = 2
-      $$ LANGUAGE SQL STABLE;
-
-      CREATE OR REPLACE FUNCTION same_date_this_year(t timestamp with time zone) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN public.is_leap_day(t) THEN public.change_year(t + INTERVAL '1 day', date_part('year', CURRENT_TIMESTAMP)::int)
-                  ELSE public.change_year(t, date_part('year', CURRENT_TIMESTAMP)::int)
-              END
-      $$ LANGUAGE SQL STABLE;
-
-      CREATE OR REPLACE FUNCTION next_date_occurrence(t timestamp with time zone) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN CURRENT_TIMESTAMP > public.same_date_this_year(t) THEN public.same_date_this_year(t) + INTERVAL '1 year'
-                  ELSE public.same_date_this_year(t)
-              END
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200409_01_fix_next_date_occurrence.js

@@ -1,24 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    // next_date_occurrence would previously return the same
-    // date if now() was the same timestamp
-    //
-    // fix by checking with >= instead of >
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION next_date_occurrence(t timestamp with time zone) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN CURRENT_TIMESTAMP >= public.same_date_this_year(t)
-                      THEN public.same_date_this_year(t) + INTERVAL '1 year'
-                  ELSE public.same_date_this_year(t)
-              END
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200409_02_timeline_admin_view_funcs.js

@@ -1,37 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    // adds overloads for a couple of functions from 20200409_00_more_admin_view_funcs.js
-    // where the CURRENT_TIMESTAMP can be passed via argument
-    // for use in the cool-data-science timeline calculations
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION same_date_this_year(
-          t timestamp with time zone,
-          reference_date timestamp with time zone
-      ) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN public.is_leap_day(t) THEN public.change_year(t + INTERVAL '1 day', date_part('year', reference_date)::int)
-                  ELSE public.change_year(t, date_part('year', reference_date)::int)
-              END
-      $$ LANGUAGE SQL STABLE;
-      
-      CREATE OR REPLACE FUNCTION next_date_occurrence(
-          t timestamp with time zone,
-          reference_date timestamp with time zone
-      ) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN reference_date >= public.same_date_this_year(t, reference_date)
-                      THEN public.same_date_this_year(t, reference_date) + INTERVAL '1 year'
-                  ELSE public.same_date_this_year(t, reference_date)
-              END
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -1,28 +0,0 @@
-const fs = require('fs')
-const { Migration } = require('../models/index')
-const { MigrationKone } = require('../models/models_kone')
-
-const DEFAULT_PATH = `${process.cwd()}/src/database/migrations`
-const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`
-
-const saveMigrationsToDatabase = Migration => async filenames =>
-  Promise.all(filenames.map(name => Migration.upsert({ name })))
-
-const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(Migration)(filenames)
-}
-
-const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(MigrationKone)(filenames)
-}
-
-const seedAllMigrations = async () => {
-  await seedMigrations()
-  await seedMigrationsKone()
-}
-
-module.exports = {
-  seedAllMigrations
-}

services/backend/oodikone2-backend/src/databaseV2/connection.js

@@ -1,44 +0,0 @@
-const Sequelize = require('sequelize')
-const EventEmitter = require('events')
-const { SIS_DB_URL } = process.env
-
-class DbConnection extends EventEmitter {
-  constructor() {
-    super()
-    this.RETRY_ATTEMPTS = 15
-    this.established = false
-
-    this.sequelize = new Sequelize(SIS_DB_URL, {
-      dialect: 'postgres',
-      pool: {
-        max: 25,
-        min: 0,
-        acquire: 10000,
-        idle: 300000000
-      },
-      logging: false
-    })
-  }
-
-  async connect(attempt = 1) {
-    try {
-      await this.sequelize.authenticate()
-      this.emit('connect')
-      this.established = true
-    } catch (e) {
-      if (attempt > this.RETRY_ATTEMPTS) {
-        this.emit('error', e)
-        return
-      }
-      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
-      console.log(e)
-      setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
-    }
-  }
-}
-
-const dbConnections = new DbConnection()
-dbConnections
-module.exports = {
-  dbConnections
-}

services/backend/oodikone2-backend/src/models/index.js

@@ -1,569 +0,0 @@
-const Sequelize = require('sequelize')
-const { sequelize } = require('../database/connection')
-
-const Student = sequelize.define(
-  'student',
-  {
-    studentnumber: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    },
-    lastname: { type: Sequelize.STRING },
-    firstnames: { type: Sequelize.STRING },
-    abbreviatedname: { type: Sequelize.STRING },
-    birthdate: { type: Sequelize.DATE },
-    communicationlanguage: { type: Sequelize.STRING },
-    creditcount: { type: Sequelize.INTEGER },
-    dateofuniversityenrollment: { type: Sequelize.DATE },
-    email: { type: Sequelize.STRING },
-    phone: { type: Sequelize.STRING },
-    city_fi: { type: Sequelize.STRING },
-    city_sv: { type: Sequelize.STRING },
-    national_student_number: { type: Sequelize.STRING },
-    zipcode: { type: Sequelize.STRING },
-    address: { type: Sequelize.STRING },
-    address2: { type: Sequelize.STRING },
-    language_fi: { type: Sequelize.STRING },
-    language_sv: { type: Sequelize.STRING },
-    language_en: { type: Sequelize.STRING },
-    age: { type: Sequelize.INTEGER },
-    mobile: { type: Sequelize.STRING },
-    home_county_id: { type: Sequelize.INTEGER },
-    country_fi: { type: Sequelize.STRING },
-    country_sv: { type: Sequelize.STRING },
-    country_en: { type: Sequelize.STRING },
-    home_country_fi: { type: Sequelize.STRING },
-    home_country_sv: { type: Sequelize.STRING },
-    home_country_en: { type: Sequelize.STRING },
-    gender_code: { type: Sequelize.INTEGER },
-    gender_fi: { type: Sequelize.STRING },
-    gender_sv: { type: Sequelize.STRING },
-    gender_en: { type: Sequelize.STRING }
-  },
-  {
-    tableName: 'student',
-    timestamps: true
-  }
-)
-
-Student.hasStarted = student => {
-  return student.credits.length > 0
-}
-
-const Organisation = sequelize.define(
-  'organization',
-  {
-    code: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    },
-    name: Sequelize.JSONB
-  },
-  {
-    tableName: 'organization',
-    timestamps: false
-  }
-)
-
-const Credit = sequelize.define(
-  'credit',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    },
-    grade: { type: Sequelize.STRING },
-    student_studentnumber: { type: Sequelize.STRING },
-    credits: { type: Sequelize.DOUBLE },
-    ordering: { type: Sequelize.STRING },
-    attainment_date: { type: Sequelize.DATE },
-    isStudyModule: { type: Sequelize.BOOLEAN }
-  },
-  {
-    tableName: 'credit',
-    timestamps: true,
-    createdAt: 'createddate',
-    updatedAt: 'lastmodifieddate',
-    indexes: [
-      {
-        fields: ['student_studentnumber']
-      },
-      {
-        fields: ['course_code'],
-        name: 'credit_course_code'
-      },
-      {
-        fields: ['attainment_date'],
-        name: 'credit_attainment_date'
-      }
-    ]
-  }
-)
-
-Credit.notUnnecessary = credit => {
-  return credit.credits > 0 && credit.credits <= 12
-}
-
-const CREDIT_TYPE_CODES = {
-  PASSED: 4,
-  FAILED: 10,
-  IMPROVED: 7,
-  APPROVED: 9
-}
-
-Credit.passed = ({ credittypecode }) =>
-  credittypecode === CREDIT_TYPE_CODES.PASSED || credittypecode === CREDIT_TYPE_CODES.APPROVED
-Credit.failed = credit => credit.credittypecode === CREDIT_TYPE_CODES.FAILED
-Credit.improved = credit => credit.credittypecode === CREDIT_TYPE_CODES.IMPROVED
-
-const Studyright = sequelize.define(
-  'studyright',
-  {
-    studyrightid: {
-      primaryKey: true,
-      type: Sequelize.BIGINT
-    },
-    canceldate: { type: Sequelize.DATE },
-    cancelorganisation: { type: Sequelize.STRING },
-    enddate: { type: Sequelize.DATE },
-    givendate: { type: Sequelize.DATE },
-    graduated: { type: Sequelize.INTEGER },
-    prioritycode: { type: Sequelize.INTEGER },
-    startdate: { type: Sequelize.DATE },
-    studystartdate: { type: Sequelize.DATE },
-    organization_code: { type: Sequelize.STRING },
-    student_studentnumber: { type: Sequelize.STRING },
-    faculty_code: { type: Sequelize.STRING }
-  },
-  {
-    tableName: 'studyright',
-    timestamps: false,
-    indexes: [
-      {
-        fields: ['student_studentnumber'],
-        name: 'studyright_student_studentnumber'
-      }
-    ]
-  }
-)
-
-const StudyrightElement = sequelize.define(
-  'studyright_elements',
-  {
-    startdate: { type: Sequelize.DATE },
-    enddate: { type: Sequelize.DATE }
-  },
-  {
-    indexes: [
-      {
-        unique: true,
-        fields: ['startdate', 'enddate', 'studyrightid', 'code']
-      },
-      {
-        fields: ['startdate']
-      }
-    ],
-    tableName: 'studyright_elements'
-  }
-)
-
-const ElementDetails = sequelize.define(
-  'element_details',
-  {
-    code: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    },
-    name: { type: Sequelize.JSONB },
-    type: { type: Sequelize.INTEGER }
-  },
-  {
-    tableName: 'element_details'
-  }
-)
-
-const Course = sequelize.define(
-  'course',
-  {
-    code: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    },
-    name: { type: Sequelize.JSONB },
-    latest_instance_date: { type: Sequelize.DATE },
-    is_study_module: { type: Sequelize.BOOLEAN },
-    startdate: { type: Sequelize.DATE },
-    enddate: { type: Sequelize.DATE },
-    max_attainment_date: { type: Sequelize.DATE },
-    min_attainment_date: { type: Sequelize.DATE }
-  },
-  {
-    tableName: 'course',
-    timestamps: false
-  }
-)
-
-const Teacher = sequelize.define(
-  'teacher',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    },
-    code: { type: Sequelize.STRING },
-    name: { type: Sequelize.STRING }
-  },
-  {
-    tableName: 'teacher',
-    timestamps: false
-  }
-)
-
-const StudentList = sequelize.define(
-  'student_list',
-  {
-    id: {
-      type: Sequelize.INTEGER,
-      autoIncrement: true,
-      primaryKey: true
-    },
-    key: {
-      type: Sequelize.STRING
-    },
-    max: {
-      type: Sequelize.BIGINT
-    },
-    description: {
-      type: Sequelize.STRING
-    },
-    student_numbers: {
-      type: Sequelize.JSONB
-    }
-  },
-  {
-    tableName: 'student_list',
-    timestamps: false
-  }
-)
-
-const StudyrightExtent = sequelize.define('studyright_extent', {
-  extentcode: {
-    type: Sequelize.INTEGER,
-    primaryKey: true
-  },
-  name: {
-    type: Sequelize.JSONB
-  }
-})
-
-const Discipline = sequelize.define('discipline', {
-  discipline_id: {
-    type: Sequelize.STRING,
-    primaryKey: true
-  },
-  name: {
-    type: Sequelize.JSONB
-  }
-})
-
-const CourseDisciplines = sequelize.define('course_disciplines', {})
-
-const CourseType = sequelize.define('course_type', {
-  coursetypecode: {
-    type: Sequelize.INTEGER,
-    primaryKey: true
-  },
-  name: {
-    type: Sequelize.JSONB
-  }
-})
-
-const CreditType = sequelize.define('credit_type', {
-  credittypecode: {
-    type: Sequelize.INTEGER,
-    primaryKey: true
-  },
-  name: {
-    type: Sequelize.JSONB
-  }
-})
-
-const Semester = sequelize.define('semester', {
-  semestercode: {
-    type: Sequelize.INTEGER,
-    primaryKey: true
-  },
-  name: {
-    type: Sequelize.JSONB
-  },
-  startdate: {
-    type: Sequelize.DATE
-  },
-  enddate: {
-    type: Sequelize.DATE
-  },
-  yearcode: {
-    type: Sequelize.INTEGER
-  },
-  yearname: {
-    type: Sequelize.STRING
-  }
-})
-
-const SemesterEnrollment = sequelize.define(
-  'semester_enrollment',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.BIGINT,
-      autoIncrement: true
-    },
-    enrollmenttype: {
-      type: Sequelize.INTEGER
-    },
-    enrollment_date: {
-      type: Sequelize.DATE
-    },
-    semestercode: {
-      type: Sequelize.INTEGER
-    },
-    studentnumber: {
-      type: Sequelize.STRING
-    }
-  },
-  {
-    indexes: [
-      {
-        fields: ['semestercode', 'studentnumber'],
-        unique: true
-      },
-      {
-        fields: ['studentnumber'],
-        name: 'semester_enrollment_studentnumber'
-      }
-    ]
-  }
-)
-
-const Provider = sequelize.define('provider', {
-  providercode: {
-    primaryKey: true,
-    type: Sequelize.STRING
-  },
-  name: {
-    type: Sequelize.JSONB
-  }
-})
-
-const Transfers = sequelize.define(
-  'transfers',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.BIGINT,
-      autoIncrement: true
-    },
-    studyrightid: {
-      type: Sequelize.BIGINT
-    },
-    sourcecode: {
-      type: Sequelize.STRING
-    },
-    targetcode: {
-      type: Sequelize.STRING
-    },
-    transferdate: {
-      type: Sequelize.DATE
-    },
-    studentnumber: {
-      type: Sequelize.STRING
-    }
-  },
-  {
-    indexes: [
-      {
-        fields: ['studyrightid', 'sourcecode', 'targetcode', 'transferdate', 'studentnumber'],
-        unique: true
-      },
-      {
-        fields: ['studentnumber'],
-        name: 'transfers_studentnumber'
-      }
-    ]
-  }
-)
-
-const CourseProvider = sequelize.define(
-  'course_providers',
-  {},
-  {
-    indexes: [
-      {
-        fields: ['providercode', 'coursecode'],
-        unique: true
-      }
-    ]
-  }
-)
-
-const CourseRealisationType = sequelize.define('courserealisation_type', {
-  realisationtypecode: {
-    primaryKey: true,
-    type: Sequelize.STRING
-  },
-  name: {
-    type: Sequelize.JSONB
-  }
-})
-
-const CourseRealisation = sequelize.define('courserealisation', {
-  courserealisation_id: {
-    primaryKey: true,
-    type: Sequelize.STRING
-  },
-  name: {
-    type: Sequelize.JSONB
-  },
-  startdate: {
-    type: Sequelize.DATE
-  },
-  enddate: {
-    type: Sequelize.DATE
-  },
-  parent: {
-    type: Sequelize.STRING
-  },
-  root: {
-    type: Sequelize.STRING
-  },
-  coursecode: {
-    type: Sequelize.STRING
-  }
-})
-
-const CourseEnrollment = sequelize.define('course_enrollment', {})
-
-const Migration = sequelize.define(
-  'migrations',
-  {
-    name: {
-      type: Sequelize.STRING,
-      primaryKey: true
-    }
-  },
-  {
-    tableName: 'migrations',
-    timestamps: false
-  }
-)
-
-const CreditTeacher = sequelize.define('credit_teacher', {
-  credit_id: {
-    type: Sequelize.STRING,
-    primaryKey: true
-  },
-  teacher_id: {
-    type: Sequelize.STRING,
-    primaryKey: true
-  }
-})
-
-const ErrorData = sequelize.define('error_data', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.STRING
-  },
-  data: {
-    type: Sequelize.JSONB
-  }
-})
-
-Organisation.hasMany(Studyright, { foreignKey: 'faculty_code', sourceKey: 'code' })
-Studyright.belongsTo(Organisation, { foreignKey: 'faculty_code', sourceKey: 'code' })
-
-Credit.belongsTo(Student, { foreignKey: 'student_studentnumber', targetKey: 'studentnumber' })
-Student.hasMany(Credit, { foreignKey: 'student_studentnumber', sourceKey: 'studentnumber' })
-
-Credit.belongsTo(Course, { foreignKey: 'course_code' })
-Course.hasMany(Credit, { foreignKey: 'course_code' })
-
-Studyright.belongsTo(Student, { foreignKey: 'student_studentnumber', targetKey: 'studentnumber' })
-Student.hasMany(Studyright, { foreignKey: 'student_studentnumber', sourceKey: 'studentnumber' })
-
-StudyrightElement.belongsTo(Studyright, { foreignKey: 'studyrightid', targetKey: 'studyrightid' })
-Studyright.hasMany(StudyrightElement, { foreignKey: 'studyrightid', sourceKey: 'studyrightid' })
-
-StudyrightElement.belongsTo(ElementDetails, { foreignKey: 'code', targetKey: 'code' })
-ElementDetails.hasMany(StudyrightElement, { foreignKey: 'code', sourceKey: 'code' })
-
-StudyrightElement.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
-Student.hasMany(StudyrightElement, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
-
-StudyrightExtent.hasMany(Studyright, { foreignKey: 'extentcode', sourceKey: 'extentcode' })
-Studyright.belongsTo(StudyrightExtent, { foreignKey: 'extentcode', targetKey: 'extentcode' })
-
-CourseType.hasMany(Course, { foreignKey: 'coursetypecode', sourceKey: 'coursetypecode' })
-Course.belongsTo(CourseType, { foreignKey: 'coursetypecode', targetKey: 'coursetypecode' })
-
-Discipline.belongsToMany(Course, { through: CourseDisciplines, foreignKey: 'discipline_id' })
-Course.belongsToMany(Discipline, { through: CourseDisciplines, foreignKey: 'course_id' })
-
-CreditType.hasMany(Credit, { foreignKey: 'credittypecode', sourceKey: 'credittypecode' })
-Credit.belongsTo(CreditType, { foreignKey: 'credittypecode', targetKey: 'credittypecode' })
-
-SemesterEnrollment.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
-Student.hasMany(SemesterEnrollment, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
-
-SemesterEnrollment.belongsTo(Semester, { foreignKey: 'semestercode', targetKey: 'semestercode' })
-Semester.hasMany(SemesterEnrollment, { foreignKey: 'semestercode', sourceKey: 'semestercode' })
-
-Course.belongsToMany(Provider, { through: CourseProvider, foreignKey: 'coursecode' })
-Provider.belongsToMany(Course, { through: CourseProvider, foreignKey: 'providercode' })
-
-Transfers.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
-Student.hasMany(Transfers, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
-
-Transfers.belongsTo(Studyright, { foreignKey: 'studyrightid', targetKey: 'studyrightid' })
-Studyright.hasMany(Transfers, { foreignKey: 'studyrightid', sourceKey: 'studyrightid' })
-
-Transfers.belongsTo(ElementDetails, { as: 'source', foreignKey: 'sourcecode' })
-Transfers.belongsTo(ElementDetails, { as: 'target', foreignKey: 'targetcode' })
-
-CourseRealisation.belongsTo(CourseRealisationType, { foreignKey: 'realisationtypecode' })
-CourseRealisation.belongsTo(Course, { foreignKey: 'coursecode' })
-CourseRealisation.belongsToMany(Student, { through: CourseEnrollment, foreignKey: 'courserealisation_id' })
-Student.belongsToMany(CourseRealisation, { through: CourseEnrollment, foreignKey: 'studentnumber' })
-
-Credit.belongsToMany(Teacher, { through: CreditTeacher, foreignKey: 'credit_id' })
-Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id' })
-
-Credit.belongsTo(Semester, { foreignKey: { name: 'semestercode', allowNull: false } })
-
-module.exports = {
-  Student,
-  Credit,
-  Studyright,
-  Course,
-  Teacher,
-  Organisation,
-  StudentList,
-  StudyrightElement,
-  ElementDetails,
-  StudyrightExtent,
-  CourseType,
-  CreditType,
-  Discipline,
-  CourseDisciplines,
-  Semester,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Transfers,
-  CourseRealisationType,
-  CourseRealisation,
-  CourseEnrollment,
-  CreditTeacher,
-  ErrorData,
-  Migration,
-  sequelize
-}

services/backend/oodikone2-backend/src/models/queries.js

@@ -1,172 +0,0 @@
-const { sequelize, sequelizeKone } = require('../database/connection')
-
-const getStudentNumbers = () =>
-  sequelize.query(
-    `select
-      studentnumber
-      from student
-      order by studentnumber desc`,
-    { type: sequelize.QueryTypes.SELECT }
-  )
-
-const getCurrentAcademicYear = () =>
-  sequelize.query(
-    `select
-      semestercode,
-      yearname
-    from semesters
-      where startdate <= now()
-      and date_part('month', startdate) = 7
-    order by startdate desc
-    fetch first row only`,
-    { type: sequelize.QueryTypes.SELECT }
-  )
-
-const getAcademicYearsFrom = startSemesterCode =>
-  sequelize.query(
-    `select
-      distinct on (yearname) yearname,
-      semestercode
-    from semesters
-    where semestercode >= :startSemesterCode
-    and startdate <= now()
-      order by yearname, semestercode`,
-    {
-      replacements: { startSemesterCode },
-      type: sequelize.QueryTypes.SELECT
-    }
-  )
-
-const getAcademicYearStatistics = (teacherIds, startSemester) => {
-  const endSemester = startSemester + 1
-  return sequelize.query(
-    `select
-        count(distinct c.course_code) as courses,
-        sum(credits) as credits,
-        count(distinct student_studentnumber) as students
-      from credit_teachers ct
-        left join credit c on ct.credit_id = c.id
-      where
-        ct.teacher_id in (:teacherIds)
-      and
-        c.semestercode >= :startSemester
-      and
-        c.semestercode <= :endSemester`,
-    {
-      replacements: { teacherIds, startSemester, endSemester },
-      type: sequelize.QueryTypes.SELECT
-    }
-  )
-}
-
-const getTeacherAcademicYearStatisticsByIds = (teacherIds, startSemester) => {
-  const endSemester = startSemester + 1
-  return sequelize.query(
-    `select
-        ct.teacher_id as id,
-        count(distinct c.course_code) as courses,
-        sum(credits) as credits,
-        count(distinct student_studentnumber) as students
-      from credit_teachers ct
-        left join credit c on ct.credit_id = c.id
-      where
-        c.semestercode >= :startSemester
-      and
-        c.semestercode <= :endSemester
-      and
-        ct.teacher_id in (:teacherIds)
-      group by ct.teacher_id`,
-    {
-      replacements: { teacherIds, startSemester, endSemester },
-      type: sequelize.QueryTypes.SELECT
-    }
-  )
-}
-
-const getAcademicYearCoursesByTeacherIds = (teacherIds, startSemester) => {
-  const endSemester = startSemester + 1
-  return sequelize.query(
-    `select
-          course_code as coursecode,
-          co.name as coursenames,
-          t.code as teachercode,
-          t.name as teachername,
-          sum(credits) as credits,
-          count(distinct student_studentnumber) as students
-        from credit_teachers ct
-          left join credit c on ct.credit_id = c.id
-          left join teacher t on ct.teacher_id = t.id
-          left join course co on c.course_code = co.code
-        where
-          c.semestercode >= :startSemester
-        and
-          c.semestercode <= :endSemester
-        and
-          ct.teacher_id in (:teacherIds)
-        group by course_code, t.name, t.code, co.name`,
-    {
-      replacements: { teacherIds, startSemester, endSemester },
-      type: sequelize.QueryTypes.SELECT
-    }
-  )
-}
-
-const getAcademicYearStatisticsForStudyProgramme = async (programmeid, startSemester) => {
-  const endSemester = startSemester + 1
-  const courseGroupData = await sequelizeKone.query(
-    `
-SELECT cg.id AS id, cg.name AS name, tcg.teacher_id as teacher_id, cg.programmeid as programmeid
-FROM   course_groups cg
-    left join teacher_course_groups tcg
-           ON tcg.course_group_id = cg.id
-WHERE cg.programmeid = :programmeid
-`,
-    {
-      replacements: { programmeid },
-      type: sequelize.QueryTypes.SELECT
-    }
-  )
-
-  const courseGroupIdToCGData = courseGroupData.reduce((acc, cgd) => {
-    acc[cgd.id] = acc[cgd.id] || { name: cgd.name, id: cgd.id, programmeid: cgd.programmeid, teacherids: [] }
-    if (cgd.teacher_id) acc[cgd.id].teacherids.push(cgd.teacher_id)
-    return acc
-  }, {})
-
-  const resultPromises = Object.keys(courseGroupIdToCGData).map(async cgid => {
-    const { name, id, programmeid, teacherids } = courseGroupIdToCGData[cgid]
-    if (teacherids.length === 0) return { name, id, programmeid, courses: 0, credits: 0, students: 0 }
-    const stats = await sequelize.query(
-      `
-SELECT
-    COUNT(DISTINCT c.course_code) AS courses,
-    COALESCE(SUM(credits), 0) AS credits,
-    COUNT(DISTINCT student_studentnumber) AS students
-FROM   credit_teachers ct
-    left join credit c
-            ON c.id = ct.credit_id
-              AND c.semestercode >= :startSemester
-              AND c.semestercode <= :endSemester
-WHERE ct.teacher_id IN (:teacherids)
-  `,
-      {
-        replacements: { startSemester, endSemester, teacherids },
-        type: sequelize.QueryTypes.SELECT
-      }
-    )
-    if (stats.length !== 1) throw 'expecting single row'
-    const { courses, credits, students } = stats[0]
-    return { name, id, programmeid, courses, credits, students }
-  })
-  return Promise.all(resultPromises)
-}
-
-module.exports = {
-  getStudentNumbers,
-  getCurrentAcademicYear,
-  getAcademicYearsFrom,
-  getAcademicYearStatistics,
-  getAcademicYearStatisticsForStudyProgramme,
-  getTeacherAcademicYearStatisticsByIds,
-  getAcademicYearCoursesByTeacherIds
-}

services/backend/oodikone2-backend/src/routes/coolDataScience.js

@@ -1,81 +0,0 @@
-const router = require('express').Router()
-const {
-  withErr,
-  getProtoC,
-  getProtoCProgramme,
-  getStatus,
-  getUber,
-  getStartYears,
-  getGraduatedStatus
-} = require('../services/coolDataScience')
-const useSisRouter = require('../util/useSisRouter')
-const trendsV2 = require('../routesV2/trends')
-
-router.get('/start-years', async (req, res) => {
-  const years = await getStartYears()
-  res.json(years.map(({ studystartdate }) => studystartdate))
-})
-
-router.get(
-  '/uber-data',
-  withErr(async (req, res) => {
-    const data = await getUber(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data-programme',
-  withErr(async (req, res) => {
-    const data = await getProtoCProgramme(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data',
-  withErr(async (req, res) => {
-    const data = await getProtoC(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status-graduated',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const data = await getGraduatedStatus(date.getTime(), showByYear)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const status = await getStatus(date.getTime(), showByYear)
-    res.json(status)
-  })
-)
-
-module.exports = useSisRouter(trendsV2, router)

services/backend/oodikone2-backend/src/routes/courseGroups.js

@@ -1,61 +0,0 @@
-const router = require('express').Router()
-const CourseGroupService = require('../services/courseGroups')
-
-router.get('/programme/:programmeId/:force?', async (req, res) => {
-  const { programmeId, force } = req.params
-  const { rights, roles } = req
-  const semesterCode = req.query.semester
-  if (rights.includes(programmeId) || (roles && roles.includes('admin'))) {
-    const courseGroups = await CourseGroupService.getCourseGroupsWithTotals(
-      programmeId,
-      semesterCode,
-      force === 'force'
-    )
-    return res.json(courseGroups)
-  }
-  return res.status(403).json({ error: 'No right to view programme data' })
-})
-
-router.get('/academic-years', async (req, res) => {
-  const academicYears = await CourseGroupService.getAcademicYears()
-  return res.json(academicYears)
-})
-
-router.get('/courses', async (req, res) => {
-  const { teacherIds: ids, semester } = req.query
-  const teacherIds = ids && JSON.parse(ids)
-
-  if (!teacherIds || !Array.isArray(teacherIds)) {
-    return res.sendStatus(400)
-  }
-
-  if (teacherIds.length === 0) return res.send([])
-
-  const courses = await CourseGroupService.getCoursesByTeachers(teacherIds, semester)
-
-  return courses ? res.send(courses) : res.sendStatus(404)
-})
-
-router.get('/:id', async (req, res) => {
-  const { id } = req.params
-  const { semester } = req.query
-  const groupData = await CourseGroupService.getCourseGroup(Number(id), Number(semester))
-
-  return groupData ? res.send(groupData) : res.sendStatus(404)
-})
-
-router.post('/:id/add/:teacherid', async (req, res) => {
-  const { id, teacherid } = req.params
-  const success = await CourseGroupService.addTeacher(Number(id), teacherid)
-
-  return success ? res.status(200).json('added') : res.sendStatus(404)
-})
-
-router.post('/:id/remove/:teacherid', async (req, res) => {
-  const { id, teacherid } = req.params
-  const success = await CourseGroupService.removeTeacher(Number(id), teacherid)
-
-  return success ? res.status(200).json('removed') : res.sendStatus(404)
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/courses.js

@@ -1,106 +0,0 @@
-const router = require('express').Router()
-const crypto = require('crypto')
-const Course = require('../services/courses')
-const { validateParamLength } = require('../util')
-const logger = require('../util/logger')
-const coursesV2 = require('../routesV2/courses')
-const useSisRouter = require('../util/useSisRouter')
-
-router.get('/courses', async (req, res) => {
-  let results = []
-  if (req.query.name) {
-    results = await Course.bySearchTerm(req.query.name, req.query.language)
-  }
-
-  res.json(results)
-})
-
-router.get('/coursesmulti', async (req, res) => {
-  let results = []
-  if (req.query.name || req.query.discipline || req.query.type) {
-    results = await Course.bySearchTermTypeAndDiscipline(
-      req.query.name,
-      req.query.type,
-      req.query.discipline,
-      req.query.language
-    ) // eslint-disable-line
-  }
-  res.json(results)
-})
-
-router.get('/v2/coursesmulti', async (req, res) => {
-  let results = { courses: [], groups: {}, groupMeta: {} }
-  const { name, code } = req.query
-
-  if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
-    return res.status(400).json({ error: 'name or code invalid' })
-  }
-
-  results = await Course.byNameAndOrCodeLike(name, code)
-  res.json(results)
-})
-
-router.get('/coursetypes', async (req, res) => {
-  const coursetypes = await Course.getAllCourseTypes()
-  res.json(coursetypes)
-})
-
-router.get('/coursedisciplines', async (req, res) => {
-  const courseDisciplines = await Course.getAllDisciplines()
-  res.json(courseDisciplines)
-})
-
-router.get('/v3/courseyearlystats', async (req, res) => {
-  try {
-    const { rights, roles } = req
-
-    const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
-
-    // If user has rights to see at least one programme, then they are allowed
-    // to see all of them
-    if (!allowedRoles && rights.length < 1) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
-
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
-
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      // Studentnumbers should be obfuscated to all other users except admins
-      // and users with rights to any specific study programmes
-      const anonymize = allowedRoles !== 'admin' && rights.length < 1
-      const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
-      res.json(results)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
-  }
-})
-
-router.get('/courses/duplicatecodes/:programme', async (req, res) => {
-  // const { programme } = req.params
-  const results = await Course.getMainCourseToCourseMap()
-  return res.json(results)
-})
-
-router.post('/courses/duplicatecodes/:code1/:code2', async (req, res) => {
-  const { code1, code2 } = req.params
-  await Course.setDuplicateCode(code1, code2)
-  const results = await Course.getMainCourseToCourseMap()
-  res.status(200).json(results)
-})
-
-router.delete('/courses/duplicatecodes/:code', async (req, res) => {
-  const { code } = req.params
-  await Course.deleteDuplicateCode(code)
-  const results = await Course.getMainCourseToCourseMap()
-  res.status(200).json(results)
-})
-
-module.exports = useSisRouter(coursesV2, router)

services/backend/oodikone2-backend/src/routes/diff.js

@@ -1,33 +0,0 @@
-const router = require('express').Router()
-const SisCourse = require('../servicesV2/courses')
-const OodiCourse = require('../services/courses')
-const diff = require('../services/diff')
-const logger = require('../util/logger')
-
-router.get('/diff/courseyearlystats', async (req, res) => {
-  try {
-    const { roles } = req
-    const admin = roles && roles.includes('admin')
-    if (!admin) {
-      return res.status(403).json({ error: 'No access to course stats diff' })
-    }
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
-
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      const oodiResults = await OodiCourse.courseYearlyStats(codes, separate, unifyOpenUniCourses)
-      const sisResults = await SisCourse.courseYearlyStats(codes, separate, unifyOpenUniCourses)
-      const result = diff.getCourseYearlyStatsDiff(sisResults, oodiResults)
-      res.json(result)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -1,269 +0,0 @@
-const router = require('express').Router()
-const { getStudentsUserCanAccess } = require('../services/userService')
-const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const MandatoryCourses = require('../services/mandatoryCourses')
-const { productivityStatsForCode, throughputStatsForCode, optionData } = require('../services/studyprogramme')
-const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-  getNonGraduatedStudents,
-  ping
-} = require('../services/analyticsService')
-const elementDetailsV2 = require('../routesV2/elementdetails')
-const useSisRouter = require('../util/useSisRouter')
-
-const programmeStatsSince = new Date('2017-07-31')
-
-router.get('/elementdetails/all', async (req, res) => {
-  try {
-    const elementdetails = await getAllElementDetails()
-    res.json(elementdetails)
-  } catch (e) {
-    res.status(500).json(e)
-  }
-})
-
-router.get('/studyprogrammes', async (req, res) => {
-  try {
-    const studyrights = await getAllDegreesAndProgrammes()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
-  try {
-    const {
-      roles,
-      decodedToken,
-      params: { id }
-    } = req
-    if (!id) return res.status(400).json({ error: 'programme id missing' })
-
-    const nonGraduatedStudents = await getNonGraduatedStudents(id)
-    if (!nonGraduatedStudents) return res.status(200).json({})
-    const {
-      data: { formattedData, studentnumbers }
-    } = nonGraduatedStudents
-
-    const filteredData = { ...formattedData }
-    const studentsUserCanAccess = await getStudentsUserCanAccess(studentnumbers, roles, decodedToken.userId)
-    Object.keys(filteredData).forEach(year => {
-      filteredData[year] = filteredData[year].filter(({ studentNumber }) => studentsUserCanAccess.has(studentNumber))
-    })
-
-    res.json(filteredData)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
-  if (req.params.id) {
-    const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
-    res.json(codes)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/ping', async (req, res) => {
-  try {
-    const result = await ping()
-    res.json(result)
-  } catch (e) {
-    res.status(500).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    if (code.includes('MH')) {
-      const data = await optionData(new Date('2017-07-31'), new Date(), code, 'MSC')
-      return res.json(data)
-    } else if (code.includes('KH')) {
-      const data = await optionData(new Date('2017-07-31'), new Date(), code, 'BSC')
-      return res.json(data)
-    } else {
-      return res.json([])
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      console.error(e)
-    }
-
-    if (!data) {
-      try {
-        const stats = await productivityStatsForCode(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        console.error(e)
-      }
-    }
-
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  console.log('Productivity stats recalculation starting')
-  const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
-  try {
-    await patchProductivity(
-      codes.reduce((acc, c) => {
-        acc[c] = { status: 'RECALCULATING' }
-        return acc
-      }, {})
-    )
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
-
-  let ready = 0
-  for (const code of codes) {
-    try {
-      if (code.includes('MH') || code.includes('KH')) {
-        const data = await productivityStatsForCode(code, new Date('2017-07-31'))
-        await setProductivity(data)
-      } else {
-        const data = await productivityStatsForCode(code, new Date('2000-07-31'))
-        await setProductivity(data)
-      }
-    } catch (e) {
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' }
-        })
-      } catch (e) {
-        console.error(e)
-        return
-      }
-      console.error(e)
-      console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-    }
-    ready += 1
-    console.log(`Productivity stats recalculation ${ready}/${codes.length} done`)
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      console.error(e)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForCode(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        console.error(e)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  console.log('Throughput stats recalculation starting')
-  const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
-  try {
-    const data = codes.reduce((acc, id) => ({ ...acc, [id]: { status: 'RECALCULATING' } }), {})
-    await patchThroughput(data)
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
-
-  let ready = 0
-  for (const code of codes) {
-    try {
-      if (code.includes('MH') || code.includes('KH')) {
-        const data = await throughputStatsForCode(code, 2017)
-        await setThroughput(data)
-      } else {
-        const data = await throughputStatsForCode(code, 2000)
-        await setThroughput(data)
-      }
-    } catch (e) {
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        console.error(e)
-        return
-      }
-      console.error(e)
-      console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-    }
-    ready += 1
-    console.log(`Throughput stats recalculation ${ready}/${codes.length} done`)
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  if (id) {
-    const thesis = await findProgrammeTheses(id)
-    res.json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  const { course, thesisType } = req.body
-  if (id && course && thesisType) {
-    const thesis = await createThesisCourse(id, course, thesisType)
-    res.status(201).json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
-  const { id, course } = req.params
-  if (id && course) {
-    const deleted = await deleteThesisCourse(id, course)
-    res.status(204).json(deleted)
-  } else {
-    res.status(422).end()
-  }
-})
-
-module.exports = useSisRouter(elementDetailsV2, router)

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -1,9 +0,0 @@
-const router = require('express').Router()
-const { all } = require('../services/organisations')
-
-router.get('/', async (req, res) => {
-  const faculties = await all()
-  res.json(faculties)
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/facultyStats.js

@@ -1,34 +0,0 @@
-const router = require('express').Router()
-const { all } = require('../services/organisations')
-const { byType: getElementDetailsByType } = require('../services/elementdetails')
-const { getFacultyYearlyStats } = require('../services/analyticsService')
-
-router.get('/faculties', async (req, res) => {
-  const { roles, faculties } = req
-  const allFaculties = await all()
-  res.json(roles.includes('admin') ? allFaculties : allFaculties.filter(({ code }) => faculties.has(code)))
-})
-
-router.get('/yearlystats', async (req, res) => {
-  try {
-    const { roles, faculties } = req
-    const facultyYearlyStats = await getFacultyYearlyStats()
-    res.json(roles.includes('admin') ? facultyYearlyStats : facultyYearlyStats.filter(({ id }) => faculties.has(id)))
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-router.get('/programmes', async (req, res) => {
-  try {
-    const { roles } = req
-    const isAdmin = roles.includes('admin')
-    const facultyProgrammes = await getElementDetailsByType(20)
-    const userRightsSet = new Set(req.rights)
-    res.json(facultyProgrammes.filter(({ code }) => userRightsSet.has(code) || isAdmin))
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/oodi.js

@@ -1,50 +0,0 @@
-const axios = require('axios')
-const https = require('https')
-const fs = require('fs')
-const { wrapper, router } = require('./router').routerWithWrapper()
-const { KEY_PATH, CERT_PATH, OODI_ADDR, OODI_SECRET, OODI_SECRET_HEADER_KEY } = require('../conf-backend')
-const logger = require('../util/logger')
-
-const agent =
-  KEY_PATH && CERT_PATH
-    ? new https.Agent({
-        cert: fs.readFileSync(CERT_PATH, 'utf8'),
-        key: fs.readFileSync(KEY_PATH, 'utf8')
-      })
-    : new https.Agent({
-        rejectUnauthorized: false
-      })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = instance.get
-
-const getOodiApi = async relative => {
-  const route = OODI_ADDR + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const validOodApiRequest = request => {
-  const secret = request.headers[OODI_SECRET_HEADER_KEY]
-  return !!OODI_SECRET && OODI_SECRET === secret
-}
-
-wrapper.post('', async (req, res) => {
-  const valid = validOodApiRequest(req)
-  if (!valid) {
-    return res.status(401).send()
-  }
-  const { route } = req.body
-  try {
-    const oodiApiResponse = await getOodiApi(route)
-    res.json(oodiApiResponse.data.data)
-  } catch (error) {
-    logger.error(error)
-    res.status(500).send()
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/semesters.js

@@ -1,11 +0,0 @@
-const router = require('express').Router()
-const { getSemestersAndYears } = require('../services/semesters')
-const semestersV2 = require('../routesV2/semesters')
-const useSisRouter = require('../util/useSisRouter')
-
-router.get('/semesters/codes', async (req, res) => {
-  const providers = await getSemestersAndYears()
-  res.json(providers)
-})
-
-module.exports = useSisRouter(semestersV2, router)

services/backend/oodikone2-backend/src/routes/tags.js

@@ -1,201 +0,0 @@
-const router = require('express').Router()
-const Tags = require('../services/tags')
-const TagStudent = require('../services/tagstudent')
-const Students = require('../services/students')
-const { difference } = require('lodash')
-const tagsV2 = require('../routesV2/tags')
-const useSisRouter = require('../util/useSisRouter')
-
-const filterRelevantTags = (tags, userId) => {
-  return tags.filter(tag => !tag.personal_user_id || tag.personal_user_id === userId)
-}
-
-const filterRelevantStudentTags = (studentTags, userId) => {
-  return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
-}
-
-router.get('/tags', async (req, res) => {
-  try {
-    const tags = await Tags.findTags()
-    res.status(200).json(tags)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/tags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.post('/tags', async (req, res) => {
-  try {
-    const {
-      tag: { studytrack, tagname, year, personal_user_id }
-    } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err.message)
-  }
-})
-
-router.delete('/tags', async (req, res) => {
-  try {
-    const { tag } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (
-      !rights.includes(tag.studytrack) &&
-      !(roles && roles.includes('admin')) &&
-      !(tag.personal_user_id === decodedToken.id)
-    )
-      return res.status(403).end()
-
-    await Tags.deleteTag(tag)
-    const t = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(filterRelevantTags(t, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/studenttags', async (req, res) => {
-  try {
-    const result = await TagStudent.getStudentTags()
-    res.status(200).json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/studenttags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { studentnumber } = req.params
-    const { decodedToken } = req
-    const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.post('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { tag, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    await TagStudent.createStudentTag(tag)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.post('/studenttags', async (req, res) => {
-  try {
-    const { tags, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const existingtags = await Tags.findTagsByStudytrack(studytrack)
-    const existingTagids = existingtags.map(t => t.tag_id)
-    const tagids = [...new Set(tags.map(t => t.tag_id))]
-    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
-
-    const studentnumbers = tags.map(t => t.studentnumber)
-    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
-    const missingStudents = difference(studentnumbers, students)
-    if (missingStudents.length !== 0)
-      return res
-        .status(400)
-        .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
-
-    await TagStudent.createMultipleStudentTags(tags)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.delete('/studenttags/delete_one', async (req, res) => {
-  try {
-    const { tag_id, studentnumber, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
-
-    await TagStudent.deleteStudentTag(studentnumber, tag_id)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.delete('/studenttags/delete_many', async (req, res) => {
-  try {
-    const { tagId, studentnumbers, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
-
-    await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).json(err)
-  }
-})
-
-module.exports = useSisRouter(tagsV2, router)

services/backend/oodikone2-backend/src/routes/tasks.js

@@ -1,20 +0,0 @@
-const router = require('express').Router()
-const { getTeacherUpdateStatus, startTopTeacherUpdate } = require('../services/tasks')
-
-router.get('/tasks/topteachers', (_, res) => {
-  const status = getTeacherUpdateStatus()
-  res.json(status)
-})
-
-router.post('/tasks/topteachers', async (req, res) => {
-  const { from = 50, to = 70 } = req.body
-  const { computing } = getTeacherUpdateStatus()
-  if (computing === true) {
-    res.status(409).send('Task already exists')
-  } else {
-    startTopTeacherUpdate(from, to)
-    res.status(201).send()
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -1,72 +0,0 @@
-const teachers = require('../services/teachers')
-const topteachers = require('../services/topteachers')
-const r = require('./router').routerWithWrapper()
-const { FEATURES } = require('../conf-backend')
-const { mapToProviders } = require('../util/utils')
-const teacherV2 = require('../routesV2/teachers')
-const useSisRouter = require('../util/useSisRouter')
-
-const router = FEATURES.ERROR_HANDLER ? r.wrapper : r.router
-
-router.get('/', async (req, res) => {
-  const { searchTerm } = req.query
-  if (!searchTerm) return res.status(400).json({ error: 'searchTerm missing' })
-
-  const trimmedSearchTerm = searchTerm.trim()
-  if (
-    !teachers.splitByEmptySpace(trimmedSearchTerm).find(t => t.length >= 4) ||
-    (Number(trimmedSearchTerm) && trimmedSearchTerm.length < 6)
-  ) {
-    return res.status(400).json({ error: 'invalid searchTerm' })
-  }
-
-  const result = await teachers.bySearchTerm(trimmedSearchTerm)
-  res.json(result)
-})
-
-router.get('/top', async (req, res) => {
-  const { yearcode, category = topteachers.ID.ALL } = req.query
-  if (!yearcode) {
-    return res.status(422).send('Missing required yearcode query param')
-  }
-  const result = await topteachers.getTeacherStats(category, yearcode)
-  res.json(result)
-})
-
-router.post('/top', async (req, res) => {
-  const { startyearcode, endyearcode } = req.body
-  res.status(200).end()
-  await topteachers.findAndSaveTeachers(startyearcode, endyearcode)
-})
-
-router.get('/top/categories', async (req, res) => {
-  const result = await topteachers.getCategoriesAndYears()
-  res.json(result)
-})
-
-router.get('/stats', async (req, res) => {
-  const { rights, roles } = req
-
-  const { providers, semesterStart, semesterEnd } = req.query
-  if (!providers || !semesterStart) {
-    return res.status(422).send('Missing required query parameters.')
-  }
-  const providerRights = mapToProviders(rights)
-
-  if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
-    return res.status(403).send('You do not have rights to see this data')
-  }
-  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd || semesterStart + 1)
-  res.json(result)
-})
-
-router.get('/:id', async (req, res) => {
-  const { id } = req.params
-  const result = await teachers.teacherStats(id)
-  if (!result) {
-    return res.status(404).send()
-  }
-  res.json(result)
-})
-
-module.exports = useSisRouter(teacherV2, r.router)

services/backend/oodikone2-backend/src/routes/updater.js

@@ -1,286 +0,0 @@
-const router = require('express').Router()
-const {
-  updateOldestStudents,
-  getStatuses,
-  updateActiveStudents,
-  updateAllStudents,
-  updateAttainments,
-  updateMetadata,
-  updateStudentlist,
-  rescheduleScheduled,
-  rescheduleFetched,
-  updateNoStudents,
-  updateDaily
-} = require('../services/updaterService')
-const {
-  updateSISMetadata,
-  updateSISStudents,
-  updateSISStudentsByProgramme,
-  updateSISProgrammes,
-  updateSISRedisCache,
-  updateStudentsByStudentNumber,
-  abort,
-  updateCoursesByCourseCode
-} = require('../services/sisUpdaterService')
-const { refreshStatistics, refreshStatisticsV2 } = require('../events')
-
-router.post('/update/oldest', async (req, res) => {
-  const { amount } = req.body
-  if (!Number.isInteger(Number(amount))) {
-    res.status(400).end()
-    return
-  }
-  try {
-    const response = await updateOldestStudents(amount)
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/all', async (req, res) => {
-  try {
-    const response = await updateAllStudents()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/active', async (req, res) => {
-  try {
-    const response = await updateActiveStudents()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/no_student', async (req, res) => {
-  try {
-    const response = await updateNoStudents()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/attainment', async (req, res) => {
-  try {
-    const response = await updateAttainments()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/meta', async (req, res) => {
-  try {
-    const response = await updateMetadata()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/studentlist', async (req, res) => {
-  try {
-    const response = await updateStudentlist()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.get('/status', async (req, res) => {
-  try {
-    const statuses = await getStatuses()
-    res.status(200).json(statuses)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json([{ label: 'request status', value: 'request failed' }])
-  }
-})
-
-router.post('/reschedule/scheduled', async (req, res) => {
-  try {
-    const response = await rescheduleScheduled()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/reschedule/fetched', async (req, res) => {
-  try {
-    const response = await rescheduleFetched()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/daily', async (req, res) => {
-  try {
-    const response = await updateDaily()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/refresh_statistics', async (req, res) => {
-  try {
-    refreshStatistics()
-    res.status(200).json('Refreshing statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/update/v2/meta', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISMetadata(req)
-    if (response) {
-      res.status(200).json('Update SIS meta scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISStudents()
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateStudentsByStudentNumber(req.body)
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/update/v2/courses', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateCoursesByCourseCode(req.body)
-    if (response) {
-      res.status(200).json('Update SIS courses scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/update/v2/programmes', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISProgrammes()
-    if (response) {
-      res.status(200).json('Update SIS programmes scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/update/v2/students_by_programme', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-  try {
-    const response = await updateSISStudentsByProgramme(req.body)
-    if (response) {
-      res.status(200).json(`Update for ${req.body.programme} scheduled`)
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/refresh_redis_cache', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISRedisCache()
-    if (response) {
-      res.status(200).json('Refreshing SIS redis cache scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/refresh_statistic_v2', async (req, res) => {
-  try {
-    refreshStatisticsV2()
-    res.status(200).json('Refreshing sis statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/abort', async (req, res) => {
-  try {
-    await abort()
-    res.status(200).json()
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routesV2/population.js

@@ -1,541 +0,0 @@
-const crypto = require('crypto')
-const Sentry = require('@sentry/node')
-const router = require('express').Router()
-/* const { isValidStudentId } = require('../util/index') */
-const Population = require('../servicesV2/populations')
-const Filters = require('../services/filters')
-/* const { updateStudents } = require('../services/updaterService') */
-const Student = require('../servicesV2/students')
-const StudyrightService = require('../servicesV2/studyrights')
-const UserService = require('../servicesV2/userService')
-const TagService = require('../services/tags')
-const CourseService = require('../servicesV2/courses')
-const StatMergeService = require('../services/statMerger')
-
-const filterPersonalTags = (population, userId) => {
-  return {
-    ...population,
-    students: population.students.map(student => {
-      student.tags = student.tags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
-      return student
-    })
-  }
-}
-
-// POST instead of GET because of too long params and "sensitive" data
-router.post('/v2/populationstatistics/courses', async (req, res) => {
-  try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
-      Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
-      res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
-      return
-    }
-    if (!Array.isArray(req.body.studyRights)) {
-      // studyRights should always be an array
-      req.body.studyRights = [req.body.studyRights]
-    }
-
-    if (req.body.months == null) {
-      req.body.months = 12
-    }
-
-    if (req.body.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multicoursestatPromises = Promise.all(
-        req.body.years.map(year => {
-          if (req.body.selectedStudentsByYear) {
-            req.body.selectedStudents = req.body.selectedStudentsByYear[year]
-          }
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const query = { ...req.body, year, months: newMonths }
-          const coursestatistics = Population.bottlenecksOf(query)
-          return coursestatistics
-        })
-      )
-      const multicoursestats = await multicoursestatPromises
-      const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
-      if (result.error) {
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    } else {
-      const result = await Population.bottlenecksOf(req.body)
-
-      if (result.error) {
-        Sentry.captureException(new Error(result.error))
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    }
-  } catch (e) {
-    console.log(e)
-    Sentry.captureException(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  try {
-    const { from, to, coursecodes } = req.body
-    if (!from || !to || !coursecodes) {
-      return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-    }
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-    if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-      return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-    const {
-      decodedToken: { userId },
-      roles
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
-  try {
-    const { tag } = req.body
-    if (!tag) {
-      res.status(400).json({ error: 'The body should have a tag defined' })
-      return
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByTag(tag)
-    const {
-      decodedToken: { userId },
-      roles
-    } = req
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-        tag
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  try {
-    if (!req.body.studentnumberlist) {
-      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-      return
-    }
-    let studentnumberlist
-    const {
-      decodedToken: { userId },
-      roles
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = req.body.studentnumberlist
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000
-      },
-      studentnumberlist
-    )
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: err })
-  }
-})
-
-router.get('/v3/populationstatistics', async (req, res) => {
-  const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { decodedToken } = req
-
-  try {
-    if (!year || !semesters || !studyRightsJSON) {
-      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
-      return
-    }
-    let studyRights = null
-    try {
-      studyRights = JSON.parse(studyRightsJSON)
-      const { rights, roles } = req
-
-      if (!roles || !roles.includes('admin')) {
-        if (!rights.includes(studyRights.programme)) {
-          res.status(403).json([])
-          return
-        }
-      }
-    } catch (e) {
-      console.error(e)
-      res.status(400).json({ error: 'The query had invalid studyRights' })
-      return
-    }
-
-    if (req.query.months == null) {
-      req.query.months = 12
-    }
-
-    if (req.query.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multipopulationstudentPromises = Promise.all(
-        req.query.years.map(year => {
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const populationStudents = Population.optimizedStatisticsOf({
-            ...req.query,
-            studyRights,
-            year,
-            months: newMonths
-          })
-          return populationStudents
-        })
-      )
-      const multipopulationstudents = await multipopulationstudentPromises
-
-      const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
-
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    } else {
-      const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    }
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.get('/v3/populationstatisticsbytag', async (req, res) => {
-  const { tag, studyRights: studyRightsJSON, months, year } = req.query
-  const { decodedToken } = req
-
-  if (!tag) return res.status(400).json({ error: 'The query should have a tag defined' })
-  const foundTag = await TagService.findTagById(tag)
-  if (!foundTag) return res.status(404).json({ error: 'Tag not found' })
-
-  const semesters = ['FALL', 'SPRING']
-  let studentnumberlist
-  const studentnumbers = await Student.findByTag(tag)
-  const {
-    decodedToken: { userId },
-    roles
-  } = req
-  if (roles && roles.includes('admin')) {
-    studentnumberlist = studentnumbers
-  } else {
-    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-  }
-  try {
-    const studyRights = JSON.parse(studyRightsJSON)
-    const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-    const yearDifference = Number(year) - Number(newStartYear)
-    const newMonths = Number(months) + 12 * yearDifference
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: newStartYear,
-        studyRights,
-        semesters,
-        months: newMonths,
-        tag: foundTag
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecodes, from, to, separate: sep } = req.query
-  const { decodedToken, roles } = req
-  const separate = sep ? JSON.parse(sep) : false
-
-  if (!coursecodes || !from || !to) {
-    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-  }
-
-  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(coursecodes))
-  const toFromDiff = Math.abs(to - from + 1)
-  // 2 semesters = 1 year
-  const requestedYearsToCreatePopulationFrom = Math.ceil(separate ? toFromDiff / 2 : toFromDiff)
-  if (requestedYearsToCreatePopulationFrom > maxYearsToCreatePopulationFrom) {
-    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-  }
-
-  const semesters = ['FALL', 'SPRING']
-  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        // Useless, because studentnumbers are already filtered above by from & to.
-        // We should probably refactor this to avoid more confusement.
-        year: 1900,
-        studyRights: [],
-        semesters,
-        months: 10000
-      },
-      studentnumbers
-    )
-
-    let studentsUserCanAccess
-    if (roles && roles.includes('admin')) {
-      studentsUserCanAccess = new Set(studentnumbers)
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-    }
-
-    const randomHash = crypto.randomBytes(12).toString('hex')
-    const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
-      courses,
-      studyrights,
-      gender_code,
-      studentNumber: crypto
-        .createHash('md5')
-        .update(`${studentNumber}${randomHash}`)
-        .digest('hex'),
-      firstnames: '',
-      lastname: '',
-      name: '',
-      email: '',
-      started: null,
-      credits: '',
-      tags: [],
-      obfuscated: true
-    })
-
-    result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
-  }
-})
-
-router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist } = req.body
-  const { roles, decodedToken } = req
-
-  const studentsUserCanAccess = await UserService.getStudentsUserCanAccess(
-    studentnumberlist,
-    roles,
-    decodedToken.userId
-  )
-  const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
-
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000
-      },
-      filteredStudentNumbers
-    )
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-    res.status(200).json(filterPersonalTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
-  }
-})
-
-router.get('/v2/populationstatistics/filters', async (req, res) => {
-  let results = []
-  let rights = req.query.studyRights
-  if (!Array.isArray(rights)) {
-    // studyRights should always be an array
-    rights = [rights]
-  }
-  try {
-    results = await Filters.findForPopulation(rights)
-    res.status(200).json(results)
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
-  }
-})
-
-router.post('/v2/populationstatistics/filters', async (req, res) => {
-  let results = []
-  const filter = req.body
-
-  try {
-    results = await Filters.createNewFilter(filter)
-    res.status(200).json(results)
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
-  }
-})
-
-router.delete('/v2/populationstatistics/filters', async (req, res) => {
-  let results = []
-  const filter = req.body
-  try {
-    results = await Filters.deleteFilter(filter)
-    res.status(200).json(results)
-  } catch (err) {
-    res.status(400).end()
-  }
-})
-
-/* router.post('/updatedatabase', async (req, res) => {
-  const studentnumbers = req.body
-  if (!(studentnumbers && studentnumbers.every(sn => isValidStudentId(sn)))) {
-    res.status(400).end()
-    return
-  }
-  try {
-    const response = await updateStudents(studentnumbers)
-    if (response) {
-      res.status(200).json('Scheduled')
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-}) */
-
-router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  try {
-    const { rights, roles } = req
-    if (roles && roles.includes('admin')) {
-      const studyrights = await StudyrightService.getAssociations()
-      res.json(studyrights)
-    } else {
-      const studyrights = await StudyrightService.getFilteredAssociations(rights)
-      res.json(studyrights)
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-/* router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, res) => {
-  try {
-    const studyrights = await StudyrightService.getAssociations()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
-}) */
-
-router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  try {
-    const { courseCodes } = req.query
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-    return res.json(maxYearsToCreatePopulationFrom)
-  } catch (err) {
-    console.log('err', err)
-    res.status(500).json(err)
-  }
-})
-
-router.use('*', (req, res, next) => next())
-
-module.exports = router

services/backend/oodikone2-backend/src/routesV2/students.js

@@ -1,105 +0,0 @@
-const router = require('express').Router()
-const Student = require('../servicesV2/students')
-const userService = require('../servicesV2/userService')
-const Unit = require('../servicesV2/units')
-const { mismatchedStudents } = require('../servicesV2/student_credit_total_mismatches')
-
-const filterStudentTags = (student, userId) => {
-  return {
-    ...student,
-    tags: student.tags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
-  }
-}
-
-const creditTotalMismatch = student => {
-  return {
-    ...student,
-    mismatch: mismatchedStudents.has(student.studentNumber)
-  }
-}
-
-router.get('/students', async (req, res) => {
-  const {
-    roles,
-    decodedToken: { userId },
-    query: { searchTerm }
-  } = req
-
-  const trimmedSearchTerm = searchTerm ? searchTerm.trim() : undefined
-
-  if (
-    trimmedSearchTerm &&
-    !Student.splitByEmptySpace(trimmedSearchTerm)
-      .slice(0, 2)
-      .find(t => t.length > 3)
-  ) {
-    return res.status(400).json({ error: 'at least one search term must be longer than 3 characters' })
-  }
-
-  if (roles && roles.includes('admin')) {
-    let results = []
-    if (trimmedSearchTerm) {
-      results = await Student.bySearchTerm(trimmedSearchTerm)
-    }
-    return res
-      .status(200)
-      .json(results)
-      .end()
-  } else {
-    const unitsUserCanAccess = await userService.getUnitsFromElementDetails(userId)
-    const codes = unitsUserCanAccess.map(unit => unit.id)
-    const matchingStudents = await Student.bySearchTermAndElements(trimmedSearchTerm, codes)
-    res.json(matchingStudents)
-  }
-})
-
-router.get('/students/:id', async (req, res) => {
-  const { id: studentId } = req.params
-  const { roles, decodedToken } = req
-
-  if (roles && roles.includes('admin')) {
-    const results = await Student.withId(studentId)
-    return results.error
-      ? res
-          .status(400)
-          .json({ error: 'error finding student' })
-          .end()
-      : res
-          .status(200)
-          .json(creditTotalMismatch(filterStudentTags(results, decodedToken.id)))
-          .end()
-  }
-
-  const uid = req.decodedToken.userId
-  const student = await Student.withId(studentId)
-  if (student.error) {
-    return res
-      .status(400)
-      .json({ error: 'error finding student' })
-      .end()
-  }
-  const units = await userService.getUnitsFromElementDetails(uid)
-
-  const rights = await Promise.all(
-    units.map(async unit => {
-      const jtn = await Unit.hasStudent(unit.id, student.studentNumber)
-      return jtn
-    })
-  )
-
-  if (rights.some(right => right !== null)) {
-    res
-      .status(200)
-      .json(filterStudentTags(student, decodedToken.id))
-      .end()
-  } else {
-    res
-      .status(400)
-      .json({ error: 'error finding student' })
-      .end()
-  }
-})
-
-router.use('*', (req, res, next) => next())
-
-module.exports = router

services/backend/oodikone2-backend/src/services/analyticsService.js

@@ -1,73 +0,0 @@
-const axios = require('axios')
-const { ANALYTICS_URL } = require('../conf-backend')
-
-const client = axios.create({ baseURL: ANALYTICS_URL })
-
-const ping = async () => {
-  const response = await client.get('/ping')
-  return response.data
-}
-
-const getProductivity = async id => {
-  const response = await client.get(`/productivity/${id}`)
-  return response.data
-}
-
-const setProductivity = async data => {
-  const response = await client.post('/productivity', { data })
-  return response.data
-}
-
-const patchProductivity = async data => {
-  const response = await client.patch('/productivity', { data })
-  return response.data
-}
-
-const getThroughput = async id => {
-  const response = await client.get(`/throughput/${id}`)
-  return response.data
-}
-
-const setThroughput = async data => {
-  const response = await client.post('/throughput', { data })
-  return response.data
-}
-
-const patchThroughput = async data => {
-  const response = await client.patch('/throughput', { data })
-  return response.data
-}
-
-const patchFacultyYearlyStats = async data => {
-  const response = await client.patch('/facultystats', { data })
-  return response.data
-}
-
-const getFacultyYearlyStats = async data => {
-  const response = await client.get('/facultystats', { data })
-  return response.data
-}
-
-const patchNonGraduatedStudents = async data => {
-  const response = await client.patch('/nongraduatedstudents', { data })
-  return response.data
-}
-
-const getNonGraduatedStudents = async id => {
-  const response = await client.get(`/nongraduatedstudents/${id}`)
-  return response.data
-}
-
-module.exports = {
-  ping,
-  getProductivity,
-  setProductivity,
-  patchProductivity,
-  getThroughput,
-  setThroughput,
-  patchThroughput,
-  patchFacultyYearlyStats,
-  getFacultyYearlyStats,
-  patchNonGraduatedStudents,
-  getNonGraduatedStudents
-}

services/backend/oodikone2-backend/src/services/cache.js

@@ -1,12 +0,0 @@
-const LRU = require('lru-cache')
-
-// Max 25 users can be stored in the cache
-const userDataCache = new LRU({
-  max: 25,
-  length: () => 1,
-  maxAge: 1000 * 60 * 60
-})
-
-module.exports = {
-  userDataCache
-}

services/backend/oodikone2-backend/src/services/courseGroups.js

@@ -1,169 +0,0 @@
-const { redisClient } = require('../services/redis')
-const { Teacher } = require('../models')
-const { CourseGroup, TeacherCourseGroup } = require('../models/models_kone')
-const { Op } = require('sequelize')
-const { flatMap } = require('lodash')
-
-const {
-  getCurrentAcademicYear,
-  getAcademicYearsFrom,
-  getAcademicYearStatistics,
-  getAcademicYearStatisticsForStudyProgramme,
-  getTeacherAcademicYearStatisticsByIds,
-  getAcademicYearCoursesByTeacherIds
-} = require('../models/queries')
-
-const ACADEMIC_YEAR_START_SEMESTER = 111 // academic year 2005-06
-
-const COURSE_GROUP_STATISTICS_KEY = (programmeId, semesterCode) =>
-  `course_group_statistics_${programmeId}_${semesterCode}`
-const TEACHER_COURSES_KEY = (teacherId, semesterCode) => `course_group_courses_${teacherId}_${semesterCode}`
-const REDIS_CACHE_TTL = 12 * 60 * 60
-
-const getAcademicYears = async () => getAcademicYearsFrom(ACADEMIC_YEAR_START_SEMESTER)
-
-const getCurrentAcademicYearSemesterCode = async () => {
-  const academicYear = await getCurrentAcademicYear()
-  return academicYear[0].semestercode
-}
-
-const getTeachersForCourseGroup = async courseGroupId => {
-  const courseGroups = await CourseGroup.findAll({
-    model: CourseGroup,
-    include: {
-      model: TeacherCourseGroup,
-      required: true
-    },
-    where: {
-      id: courseGroupId
-    }
-  })
-  const teacherIds = flatMap(courseGroups, e => e.teacher_course_groups.map(tcg => tcg.teacher_id))
-  const teachers = await Teacher.findAll({
-    attributes: ['name', 'code', 'id'],
-    where: {
-      id: {
-        [Op.in]: teacherIds
-      }
-    }
-  })
-  return teachers
-}
-
-const createCourseGroup = name => CourseGroup.create({ name })
-
-const getCourseGroupsWithTotals = async (programmeId, semesterCode, forceRefresh = false) => {
-  if (!forceRefresh) {
-    const cachedStats = await redisClient.getAsync(COURSE_GROUP_STATISTICS_KEY(programmeId, semesterCode))
-    if (cachedStats) {
-      return JSON.parse(cachedStats)
-    }
-  }
-  const startSemester = semesterCode || (await getCurrentAcademicYearSemesterCode())
-  const result = await getAcademicYearStatisticsForStudyProgramme(programmeId, startSemester)
-
-  await redisClient.setAsync(
-    COURSE_GROUP_STATISTICS_KEY(programmeId, semesterCode),
-    JSON.stringify(result),
-    'EX',
-    6 * 4 * 7 * 24 * 60 * 60 // 6 months in seconds
-  )
-  return result
-}
-
-const getCourseGroup = async (courseGroupId, semesterCode) => {
-  const startSemester = semesterCode || (await getCurrentAcademicYearSemesterCode())
-  const teacherBasicInfo = await getTeachersForCourseGroup(courseGroupId)
-  const courseGroup = await CourseGroup.findByPk(courseGroupId)
-
-  if (!courseGroup) return
-  if (!teacherBasicInfo) return
-
-  if (teacherBasicInfo.length === 0) {
-    return {
-      id: courseGroupId,
-      name: courseGroup.name,
-      teachers: [],
-      totalCredits: 0,
-      totalStudents: 0,
-      totalCourses: 0,
-      semester: startSemester
-    }
-  }
-
-  const teacherIds = teacherBasicInfo.map(t => t.id)
-  const statistics = await getAcademicYearStatistics(teacherIds, startSemester)
-  const teacherStats = await getTeacherAcademicYearStatisticsByIds(teacherIds, startSemester)
-
-  const { credits, students, courses } = statistics[0]
-
-  const mergedTeachers = [...teacherBasicInfo.map(e => e.get()), ...teacherStats].reduce((acc, cur) => {
-    if (acc[cur.id]) {
-      acc[cur.id] = { ...cur, ...acc[cur.id] }
-    } else {
-      acc[cur.id] = cur
-    }
-    return acc
-  }, {})
-
-  const teachers = Object.values(mergedTeachers).map(t => ({
-    ...t,
-    courses: t.courses ? Number(t.courses) : 0,
-    credits: t.credits ? Number(t.credits) : 0,
-    students: t.students ? Number(t.students) : 0
-  }))
-
-  const courseGroupInfo = {
-    id: courseGroupId,
-    name: courseGroup.name,
-    teachers,
-    totalCredits: credits || 0,
-    totalStudents: Number(students),
-    totalCourses: Number(courses),
-    semester: startSemester
-  }
-
-  return courseGroupInfo
-}
-
-const getCoursesByTeachers = async (teacherIds, semesterCode) => {
-  const startSemester = semesterCode || (await getCurrentAcademicYearSemesterCode())
-  const cacheKey = TEACHER_COURSES_KEY(teacherIds, startSemester)
-  const cachedData = await redisClient.getAsync(cacheKey)
-
-  if (cachedData) {
-    return JSON.parse(cachedData)
-  }
-
-  const courses = await getAcademicYearCoursesByTeacherIds(teacherIds, startSemester)
-
-  // Sequelize returns credis sums as string
-  courses.forEach(course => {
-    course.credits = Number(course.credits)
-    course.students = Number(course.students)
-  })
-
-  await redisClient.setAsync(cacheKey, JSON.stringify(courses), 'EX', REDIS_CACHE_TTL)
-
-  return courses
-}
-
-const addTeacher = async (courseGroupId, teacherid) => {
-  const teacher = await Teacher.findByPk(teacherid)
-  if (!teacher) return
-  return TeacherCourseGroup.create({ teacher_id: teacherid, course_group_id: courseGroupId })
-}
-
-const removeTeacher = (courseGroupId, teacherid) => {
-  return TeacherCourseGroup.destroy({ where: { teacher_id: teacherid, course_group_id: courseGroupId } })
-}
-
-module.exports = {
-  createCourseGroup,
-  getAcademicYears,
-  getCourseGroupsWithTotals,
-  getCourseGroup,
-  getCoursesByTeachers,
-  addTeacher,
-  removeTeacher
-}