Destructure props on function declaration

Bump axios from 0.21.1 to 0.21.2 in /services/frontend

.env.template

@@ -0,0 +1,3 @@
+# Populate and change this to .env
+IMPORTER_DB_API_TOKEN=
+PATE_TOKEN=

.eslintrc.json

@@ -0,0 +1,19 @@
+{
+  "env": {
+    "es6": true,
+    "node": true,
+    "jest": true
+  },
+  "parser": "@babel/eslint-parser",
+  "parserOptions": {
+    "requireConfigFile": false
+  },
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
+  "rules": {
+    "no-console": "error",
+    "react/no-unescaped-entities": "off",
+    "no-async-promise-executor": "off",
+    "prettier/prettier": "error"
+  }
+}

.github/dependabot.yml

@@ -6,8 +6,10 @@ updates:
     interval: weekly
     time: "03:00"
   open-pull-requests-limit: 10
+  target-branch: "trunk"
 - package-ecosystem: "github-actions"
   directory: "/"
   schedule:
     interval: weekly
     time: "03:00"
+  target-branch: "trunk"

.github/workflows/release-oodikone-staging.yaml

@@ -1,61 +0,0 @@
-name: Release oodikone to staging
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-oodikone-staging.yaml'
-      - 'services/**'
-      - 'docker-compose.ci.yml'
-
-jobs:
-  build_and_release:
-    name: "Build images and release"
-    runs-on: ubuntu-latest
-    env:
-      TAG: staging
-      CACHE_TAG: staging
-    steps:
-      - name: "Checkout repo"
-        uses: actions/checkout@v2
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build  images"
-        run: ./run.sh oodikone ci build analytics backend frontend userservice
-      - name: "Start frontend"
-        run: ./run.sh oodikone ci up -d frontend
-      - name: "Add frontend to sentry"
-        run: |
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh +e -c 'npm install sentry-cli && npm run sentry-cli -- releases info "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases delete "$(cat /SENTRY_RELEASE_VERSION)" || true'
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh -c 'npm run sentry-cli -- releases new --finalize "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases files "$(cat /SENTRY_RELEASE_VERSION)" upload-sourcemaps --rewrite dist'
-      - name: "Push to docker hub"
-        run: ./run.sh oodikone ci push analytics backend frontend userservice
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment
-          softa-url: https://oodikone-staging.cs.helsinki.fi/
-          deployment-target: staging
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -1,40 +1,138 @@
-name: Release oodikone to production
+name: "Release oodikone to production"
 
 on:
   release:
     types: [published]
 
 jobs:
-  build_and_release:
-    name: "Build images and release"
+  build_and_release_backend:
+    name: "Build and release backend"
     runs-on: ubuntu-latest
-    env:
-      TAG: latest
-      CACHE_TAG: latest
     steps:
-      - name: "Checkout repo"
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
       - name: "Login to DockerHub"
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Build  images"
-        run: ./run.sh oodikone ci build analytics backend frontend userservice
-      - name: "Start frontend"
-        run: ./run.sh oodikone ci up -d frontend
-      - name: "Add frontend to sentry"
-        run: |
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh +e -c 'npm install sentry-cli && npm run sentry-cli -- releases info "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases delete "$(cat /SENTRY_RELEASE_VERSION)" || true'
-          docker exec --env SENTRY_URL=${{ secrets.SENTRY_URL }} --env SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-            sh -c 'npm run sentry-cli -- releases new --finalize "$(cat /SENTRY_RELEASE_VERSION)" && \
-            npm run sentry-cli -- releases files "$(cat /SENTRY_RELEASE_VERSION)" upload-sourcemaps --rewrite dist'
-      - name: "Push to docker hub"
-        run: ./run.sh oodikone ci push analytics backend frontend userservice
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: true
+          tags: toska/oodikone2-backend:latest
+          target: production
+
+  build_and_release_frontend:
+    name: "Build and release frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: true
+          tags: toska/oodikone2-frontend:latest
+          target: production
+      - name: "Create a container from built image "
+        run: docker create --name frontend toska/oodikone2-frontend:latest
+      - name: "Copy assets from the container"
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: "Create Sentry release"
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
+
+  build_and_release_userservice:
+    name: "Build and release userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: true
+          tags: toska/oodikone2-userservice:latest
+          target: production
+
+  report_success:
+    runs-on: ubuntu-latest
+    name: "Report deployment success to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: success()
+    steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -43,14 +141,16 @@ jobs:
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
-    - build_and_release
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: production
+          deployment-target: production

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,50 +1,54 @@
-name: Release sis-updater-scheduler
+name: "Release sis-updater-scheduler"
 
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-scheduler.yaml'
       - 'updater/sis-updater-scheduler/**'
 
 jobs:
   build_and_release:
+    name: "Build and release sis-updater-scheduler"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-
-    - name: Login to DockerHub
+    - uses: docker/setup-buildx-action@v1
+    - name: "Login to DockerHub"
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build and push
+    - name: "Build and push"
       uses: docker/build-push-action@v2
       with:
+        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+        cache-to: type=inline
         context: updater/sis-updater-scheduler/
         push: true
         tags: toska/sis-updater-scheduler:latest
-
+        target: production
     - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
       with:
         webhook-url: ${{ secrets.WEBHOOK_URL }}
         message-type: deployment
         softa-url: https://oodikone.cs.helsinki.fi
         deployment-target: production
+        softa-to-deploy: sis-updater-scheduler
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
     - build_and_release
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: production
+          deployment-target: production
+          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,50 +1,54 @@
-name: Release sis-updater-worker
+name: "Release sis-updater-worker"
 
 on:
   push:
     branches:
-      - trunk
+      - master
     paths:
       - '.github/workflows/release-sis-updater-worker.yaml'
       - 'updater/sis-updater-worker/**'
 
 jobs:
   build_and_release:
+    name: "Build and release sis-updater-worker"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-
-    - name: Login to DockerHub
+    - uses: docker/setup-buildx-action@v1
+    - name: "Login to DockerHub"
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build and push
+    - name: "Build and push"
       uses: docker/build-push-action@v2
       with:
+        cache-from: type=registry,ref=toska/sis-updater-worker:latest
+        cache-to: type=inline
         context: updater/sis-updater-worker/
         push: true
         tags: toska/sis-updater-worker:latest
-
+        target: production
     - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
       with:
         webhook-url: ${{ secrets.WEBHOOK_URL }}
         message-type: deployment
         softa-url: https://oodikone.cs.helsinki.fi
         deployment-target: production
+        softa-to-deploy: sis-updater-worker
 
   report_failure:
     runs-on: ubuntu-latest
-    name: Report deployment failure to oodikone notifications
+    name: "Report deployment failure to oodikone notifications"
     needs:
     - build_and_release
     if: failure()
     steps:
       - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure
-          deployment-target: production
+          deployment-target: production
+          softa-to-deploy: sis-updater-worker

.github/workflows/test-oodikone.yaml

@@ -1,381 +1,444 @@
-name: "Test oodikone"
+name: 'Test oodikone and release to staging'
 
 on:
   push:
-    branches-ignore:
+    branches:
       - master
     paths:
-      - .github/workflows/test-oodikone.yaml
-      - cypress/**
-      - cypress.json
-      - services/**
-      - docker-compose.ci.yml
-      - nginx.conf
-      - package*.json
-      - run.sh
+      - '.github/workflows/test-oodikone.yaml'
+      - 'cypress/**'
+      - 'cypress.json'
+      - 'services/**'
+      - 'docker-compose.ci.yml'
+      - 'docker-compose.test.yml'
+      - 'nginx.conf'
+      - 'package*.json'
 
 jobs:
   # === Static tests / linting ===
 
-  lint_analytics:
-    name: "Lint analytics"
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/oodikone2-analytics
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
   lint_backend:
-    name: "Lint backend"
+    name: 'Lint backend'
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/backend/oodikone2-backend
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
+      - name: 'Install dependencies'
+        run: npm ci && cd services/backend && npm ci
+      - name: 'Run backend linter'
+        run: npm run eslint services/backend
+
   lint_frontend:
-    name: "Lint frontend"
+    name: 'Lint frontend'
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/oodikone2-frontend
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
+      - name: 'Install dependencies'
+        run: npm ci && cd services/frontend && npm ci
+      - name: 'Run frontend linter'
+        run: npm run concurrently \
+          'eslint --ext .jsx,.js services/frontend' \
+          'stylelint services/frontend/src/**/*.css'
+
   lint_userservice:
-    name: "Lint userservice"
+    name: 'Lint userservice'
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: services/oodikone2-userservice
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 14
-      - name: Install dependencies
-        run: npm ci
-      - name: Run linter
-        run: npm run lint
+      - name: 'Install dependencies'
+        run: npm ci && cd services/userservice && npm ci
+      - name: 'Run frontend linter'
+        run: npm run eslint services/userservice
 
   # === Build images ===
 
-  build_analytics:
-    name: "Build analytics"
-    runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull analytics
-      - name: Build  image
-        run: ./run.sh oodikone ci build analytics
-      - name: Save built image
-        run: docker save toska/oodikone2-analytics:trunk -o /tmp/analytics.tar
-      - name: Upload image for other jobs
-        uses: actions/upload-artifact@v2
-        with:
-          path: /tmp/analytics.tar
   build_backend:
-    name: "Build backend"
+    name: 'Build backend'
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull backend
-      - name: Build image
-        run: ./run.sh oodikone ci build backend
-      - name: Save built image
-        run: docker save toska/oodikone2-backend:trunk -o /tmp/backend.tar
-      - name: Upload image for other jobs
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build image and export it'
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: false
+          tags: toska/oodikone2-backend:staging
+          target: production
+          outputs: type=docker,dest=/tmp/backend.tar
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/backend.tar
+
   build_frontend:
-    name: "Build frontend"
+    name: 'Build frontend'
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull frontend
-      - name: Build image
-        run: ./run.sh oodikone ci build frontend
-      - name: Save built image
-        run: docker save toska/oodikone2-frontend:trunk -o /tmp/frontend.tar
-      - name: Upload image for other jobs
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Get current time'
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: 'Build image and export it'
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: false
+          tags: toska/oodikone2-frontend:staging
+          target: production
+          outputs: type=docker,dest=/tmp/frontend.tar
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/frontend.tar
+
   build_userservice:
-    name: "Build userservice"
+    name: 'Build userservice'
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Pull previous build from Docker hub
-        run: ./run.sh oodikone ci pull userservice
-      - name: Build image
-        run: ./run.sh oodikone ci build userservice
-      - name: Save built image
-        run: docker save toska/oodikone2-userservice:trunk -o /tmp/userservice.tar
-      - name: Upload image for other jobs
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build image and export it'
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: false
+          tags: toska/oodikone2-userservice:staging
+          target: production
+          outputs: type=docker,dest=/tmp/userservice.tar
+      - name: 'Upload image for other jobs'
         uses: actions/upload-artifact@v2
         with:
           path: /tmp/userservice.tar
 
+  # === Integration tests ===
+  #
+  run_userservice_integration_tests:
+    name: 'Run userservice integration tests'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Run tests included in docker build'
+        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+
   # === End-to-end tests ===
 
-  test_course_population:
-    name: "Test course population"
-    needs:
-      - build_analytics
-      - build_backend
-      - build_frontend
-      - build_userservice
-    runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
-        uses: actions/download-artifact@v2
-        with:
-          path: /tmp/
-      - name: Load built images
-        run: |
-          docker load --input /tmp/artifact/analytics.tar
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Course_population.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-
   test_course_statistics:
-    name: "Test course statistics"
+    name: 'Test course statistics'
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: 'Load built images'
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Course_statistics.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
-    name: "Test custom population"
+    name: 'Test custom population'
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: 'Load built images'
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Custom_population.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
-  test_filters:
-    name: "Test filters"
+  test_population_statistics:
+    name: 'Test population statistics'
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: 'Load built images'
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Population_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: 'Upload video if tests fail'
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_filters:
+    name: 'Test filters'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built images'
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
           spec: cypress/integration/Filters.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
-  test_teachers_and_users:
-    name: "Test teachers and users"
+  test_student_statistics:
+    name: 'Test student statistics'
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: 'Load built images'
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Run cypress
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Student_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: 'Upload video if tests fail'
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_studyprogramme_overview:
+    name: 'Test studyprogramme overview'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built images'
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Studyprogramme_overview.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: 'Upload video if tests fail'
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_teachers_and_users:
+    name: 'Test teachers and users'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Download images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load built images'
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Run cypress'
         uses: cypress-io/github-action@v2
         with:
           browser: chrome
@@ -384,113 +447,148 @@ jobs:
             cypress/integration/Users.js
         env:
           CYPRESS_baseUrl: http://localhost:1337
-      - name: Upload video
+      - name: 'Upload video if tests fail'
         uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
   test_service_exit_statuses:
-    name: "Check services run correctly"
+    name: 'Check services run correctly'
     needs:
-      - build_analytics
       - build_backend
       - build_frontend
       - build_userservice
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: 'Load built images'
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to toska docker registry
+      - name: 'Login to toska docker registry'
         uses: docker/login-action@v1
         with:
           registry: registry.toska.cs.helsinki.fi
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: Run services, pull rest of the needed images
-        run: ./run.sh oodikone ci up -d --no-recreate
-      - name: Check that exit statuses are not errors
+      - name: 'Run services, pull rest of the needed images'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: 'Check that exit statuses are not errors'
         run: |
           docker ps -a
           if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
             echo 'Some process had nonzero exit code';
-            ./run.sh oodikone ci logs;
             exit 1;
           fi
+      - name: 'Print logs on failure'
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
-  push_to_dockerhub:
-    name: "Push images to docker hub for caching"
-    needs:
-    - lint_analytics
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - test_course_population
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_teachers_and_users
-    - test_service_exit_statuses
+  # === Release ===
+
+  release_images:
+    name: 'Release built images'
     runs-on: ubuntu-latest
-    env:
-      TAG: trunk
-      CACHE_TAG: trunk
+    needs:
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-      - name: Download images
+      - uses: actions/checkout@v2
+      - name: 'Download images'
         uses: actions/download-artifact@v2
         with:
           path: /tmp/
-      - name: Load built images
+      - name: 'Load built images'
         run: |
-          docker load --input /tmp/artifact/analytics.tar
           docker load --input /tmp/artifact/backend.tar
           docker load --input /tmp/artifact/frontend.tar
           docker load --input /tmp/artifact/userservice.tar
-      - name: Login to DockerHub
+      - name: 'Login to DockerHub'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Push to docker hub"
-        run: ./run.sh oodikone ci push
+      - name: 'Create a container from frontend'
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: 'Copy assets from the container'
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: 'Create Sentry release'
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/static/js'
+          version: ${{ github.sha }}
+      - name: 'Push to docker hub'
+        run: |
+          docker push toska/oodikone2-backend:staging
+          docker push toska/oodikone2-frontend:staging
+          docker push toska/oodikone2-userservice:staging
+      - name: 'Send deployment success notification to slack'
+        if: success()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+      - name: 'Send deployment failure notification to slack'
+        if: failure()
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging
 
-  report_failure:
+  report_test_failure:
     runs-on: ubuntu-latest
-    name: Report test failure to oodikone notifications
+    name: 'Report test failure to oodikone notifications'
     needs:
-    - lint_analytics
-    - lint_backend
-    - lint_frontend
-    - lint_userservice
-    - test_course_population
-    - test_course_statistics
-    - test_custom_population
-    - test_filters
-    - test_teachers_and_users
-    - test_service_exit_statuses
-    - push_to_dockerhub
+      - lint_backend
+      - lint_frontend
+      - lint_userservice
+      - run_userservice_integration_tests
+      - test_course_statistics
+      - test_custom_population
+      - test_filters
+      - test_population_statistics
+      - test_student_statistics
+      - test_studyprogramme_overview
+      - test_teachers_and_users
+      - test_service_exit_statuses
     if: failure()
     steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4
+      - uses: actions/checkout@v2
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure

.gitignore

@@ -1,20 +1,48 @@
-**/node_modules
-.databasedumps
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
 
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 
-# These really needed?
-private.key
-private.key.pub
-kafka
-services/updater_writer/updater/test_assets/testi.bak
-services/updater_writer/updater/test_assets/011120775.json
-services/updater_writer/updater/test_assets/014272112.json
-services/updater_writer/updater/test_assets/014441008.json
-services/updater_writer/updater/test_assets/meta.json
-services/updater_scheduler/all_student_numbers.txt
-services/updater_scheduler/active_student_numbers.txt
-datastore
-services/updater_api/debug.log
-services/backend/updater_writer/dumps
-debug.log
-programme_module_dump.sql
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# dotenv environment variables file
+.env
+.env.test
+.env.production
+
+# Oodikone related stuff
+.databasedumps

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

.lintstagedrc.js

@@ -0,0 +1,26 @@
+const path = require('path')
+const cwd = process.cwd()
+const dockerCmdBase = `docker run --rm --volume ${cwd}:/oodikone --workdir /oodikone`
+const relativeFilePaths = files => [...files.map(file => path.relative(cwd, file))].join(' ')
+
+module.exports = {
+  '{services,updater}/**/*.{js,jsx}': files => `eslint --fix ${files.join(' ')}`,
+  '*.{js,json,md,yml,yaml,html}': files => `prettier --write ${files.join(' ')}`,
+  '*.css': files => `stylelint --fix ${files.join(' ')}`,
+  Dockerfile: files => `${dockerCmdBase} hadolint/hadolint:v2.7.0-alpine hadolint ${relativeFilePaths(files)}`,
+  '*.sh': files => `${dockerCmdBase} koalaman/shellcheck-alpine:v0.7.2 ${relativeFilePaths(files)} -x`,
+  '.github/workflows/*': files => `${dockerCmdBase} rhysd/actionlint:1.6.2 ${relativeFilePaths(files)}`,
+  'docker-compose*': files => {
+    const composeFiles = file => {
+      if (['docker-compose.ci.yml', 'docker-compose.test.yml'].some(f => file.includes(f))) {
+        return `--file ${file}`
+      }
+      if (file.includes('docker-compose.real.yml')) {
+        return `--file ${cwd}/docker-compose.yml --file ${file}`
+      }
+      return ''
+    }
+
+    return files.map(file => `docker-compose ${composeFiles(file)} config --quiet`)
+  },
+}

.prettierrc

@@ -5,5 +5,5 @@
   "printWidth": 120,
   "tabWidth": 2,
   "useTabs": false,
-  "jsxSingleQuote": false
-}
+  "jsxSingleQuote": false,
+}

README.md

@@ -1,269 +1,145 @@
-# oodikone
+# Oodikone
 
-[![CircleCI](https://circleci.com/gh/UniversityOfHelsinkiCS/oodikone/tree/master.svg?style=svg)](https://app.circleci.com/pipelines/github/UniversityOfHelsinkiCS/oodikone)
+An application for analyzing university data, running at [https://oodikone.cs.helsinki.fi/](https://oodikone.cs.helsinki.fi/).
 
-An application for analyzing university data. The entire development environment runs in a Docker network that's defined in the `docker-compose.yml` file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.
-
-## Requirements
+## ✔️ Requirements
 
 To run Oodikone locally, you will need the following:
 
-1. [Docker](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose/install/) running on your system.
-2. Membership of [Toska's GitHub organisation](https://github.com/UniversityOfHelsinkiCS).
-3. SSH key set up for GitHub (see [docs](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh)).
-4. Local clone of this repository.
-5. Run the CLI (see [Installation](#Installation))
+1. Applications:
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub,
+3. For real data setup, access to Toskas production servers.
+4. For some features to work, access to Toskas service tokens
 
-## Installation
+## 🚀 Installation
 
-Launch the CLI and follow the instructions:
+Pre-installation steps:
+
+- Clone this repo
+- Login to toskas docker hub (see Toskas Gitlab for more information)
+- Populate `.env.template` file with correct tokens and rename it to `.env`
+
+Then launch the CLI and follow the instructions:
 
 ```bash
 npm run cli
 ```
 
-*Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary* 😊
+_Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary_ 😊
 
-The acual oodikone can be accessed at [http://localhost:8081/](http://localhost:8081/) and Adminer at [http://localhost:5050/](http://localhost:5050/?pgsql=db&username=postgres). Adminer requires you to login with username `postgres` and password `postgres`.
+What different CLI options do:
 
-### Populate Redis on Fresh Install
+1. Set up oodikone from scratch:
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
+2. Reset all real data:
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
+3. Reset sis-importer data
+   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
 
-On a fresh install some features do not work because the redis store is empty. To populate redis, navigate to [http://localhost:8081/updater](http://localhost:8081/updater), and click *Refresh Statistics*.
+## ⌨️ Development
 
-### What the CLI does
+### Basics
 
-- Builds and runs the Dockerized development environment.
-- Downloads a database dump from the production servers.
-- Creates the database schema and populates it with the downloaded dump.
-- Adds git-hooks to run linting on push. (Use `git push --no-verify` to circumvent this behavior.)
+The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.
 
-### Daily Refresh
+Running oodikone with real data requires separated databases and redis, which are defined in `docker-compose.real.yml` file. Otherwise real data development environment uses the configuration as anonymized data development environment.
 
-It is recommended to run the "morning script" daily when starting work. It prunes Docker, refreshes the repository and rebuilds the containers before starting Oodikone again.
+Some useful commands are defined in `package.json` and can be run with `npm run <command>` as follows:
 
-```bash
-./oodikone.sh -m
-```
+- `npm run oodikone`: starts oodikone with anonymized data
+- `npm run oodikone:real`: starts oodikone with real data
+- `npm run updater`: starts updater with anonymized data
+- `npm run updater:real`: starts oodikone with real data
+- `npm run both`: starts oodikone and updater with anonymized data
+- `npm run both:real`: starts oodikone and updater with real data
+- `npm run docker:down`: stops the whole environment
 
-## Development Environment
+Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
-The development environment is entirely configured in the docker-compose.yml file located in this repository. The file defines the containers/environments for:
+### Run.sh script
 
-- Frontend
-- Backend
-- Userservice
-- The databases used by the services
-- Adminer & other dev tools
+As said, the development environment runs entirely inside docker containers. To keep `package.json` clean and not filled with predefined scripts, we have created a simple helper script called `./run.sh`. The script allows you to use docker-compose commands without the need to write long list of parameters. Try to run `./run.sh` in the root of the project and see what happens!
 
-The mapping of ports and environment variables are also defined in the docker-compose file. You can start, stop and manage the development environment by running the following commands from a terminal shell in this directory.
+`./run.sh` is simply a wrapper script to run oodikone, updater or both services in either anon or real mode. If you take a look at `package.json`, you can see that most of the predefined scripts above use `./run.sh` under the hood.
 
-### Start the development environment
+It is recommended to spend some time to become familiar with `docker` and `docker-compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:
 
-With anonymized data:
+- `./run.sh oodikone anon pull`: Pull all images related to oodikone development
+- `./run.sh updater real up --build --force-recreate --detach`: Start updater detached (=in the background) in real data mode, but build new images before starting
+- `docker-compose ps`: view the containers in the running environment
+- `docker-compose logs frontend`: print logs for just frontend
+- `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
+- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
-```bash
-npm start
-```
+### User types for development
 
-With real data:
+By default, you're login as `mluukkai` dev user when running oodikone in development mode. If you want to debug with certain type of user, you can use mocking: go to "Users" -page, click on "edit" on user you want to use and then click icon on the right corner of user card. Another way is to mock user headers - see frontend's api configuration for how this is done.
 
-```bash
-npm run start:real
-```
+Anon data contains some preset user types, corresponding to most usual user types in real data. These are used in testing too (see "Testing" below for more info).
 
-### Stop the development environment
+### Linting
 
-```bash
-npm run docker:down
-```
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-### Restart container(s)
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-All:
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
-```bash
-npm run docker:restart
-```
+However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-Only backend:
+## 🔨 Testing & CI
 
-```bash
-npm run docker:restart:backend
-```
+There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
-Only frontend:
+- Cypress
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+- Jest
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
-```bash
-npm run docker:restart:frontend
-```
+Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
-### View the containers in the running environment
+- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- tests are run on every push.
+- after successful test run, oodikone is deployed to staging
+- after creating a release, oodikone is deployed to production
+- updater is deployed to production on every push, no tests included
 
-```bash
-docker-compose ps
-```
+## ❓FAQ
 
-### View logs
+### Modules are missing after updating package.json
 
-All:
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
 
-```bash
-npm run docker:logs
-```
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
 
-Only backend:
+Make sure you have your VPN on
 
-```bash
-npm run docker:logs:backend
-```
+### Everything is broken, can't get oodikone running, data is not there etc.
 
-Only frontend:
+Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
-```bash
-npm run docker:logs:frontend
-```
+## ✌🏼 Maintainers and contribution
 
-### Attach a terminal shell to a container
+Toska of course.
 
-```bash
-docker exec -it <container> <command>
-```
+University of Helsinki
 
-For example:
+## Tietosuoja / data protection
 
-```bash
-docker exec -it backend bash
-```
-
-### Use `psql` or other database CLI tools
-
-```bash
-docker exec -it -u <username> <container> psql -d <db name>
-```
-
-For example:
-
-```bash
-docker exec -it -u postgres oodi_db psql -d tkt_oodi
-```
-
-## Testing
-
-Testing is implemented as a combination of unit tests (jest) and end-to-end tests (Cypress). Oodikone must be using the anonymous dataset when running tests (see [Installation](#Installation)).
-
-The test suite is run in CI on every push to `trunk`. Take advantage of this as running the tests can take upwards of 15 minutes.
-
-**Note:** All containers must be up and ready before running tests. (Use `npm start`).
-
-### Run All Tests
-
-```bash
-npm test
-```
-
-### Run End-to-End Tests
-
-```bash
-npm run cypress:run
-```
-
-## Open Cypress GUI
-
-```bash
-npm run cypress:open
-```
-
-## How to Updater
-
-Before running updater you must run `importer-db` setup via the CLI (see [Installation](#Installation)).
-
-If you want to use the control panel UI, first start Oodikone in real data mode:
-
-```bash
-npm run docker:up:real
-```
-
-Followed by acual updater:
-
-```bash
-npm run updater:dev:up
-```
-
-When you are done, close hanging containers with:
-
-```bash
-npm run updater:dev:down
-```
-
-### updater scheduler
-
-Stop with HTTP GET
-
-```bash
-http://localhost:8082/v1/abort?token=dev
-```
-
-Schedule students with HTTP POST
-
-```bash
-http://localhost:8082/v1/students?token=dev
-```
-
-body
-
-```bash
-{ 
-    "studentnumbers": [
-        "014598456"
-    ] 
-}
-```
-
-## Other commands
-
-### How to `scp` backup files from oodikone via melkki-proxy
-
-This is how the setup script fetches the database dump from production servers. It will require you to have access to both melkki.cs.helsinki.fi as well as oodikone.cs.helsinki.fi. The command uses `scp` to transfer all backup files recursively from a known location on the production server by using the melkki server as a proxy, allowing you to get the dump even if you are not in the university network.
-
-#### Command
-
-```bash
-scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* ./backups/
-
--r : recursively copy files
--o : options for defining the proxy command
-```
-
-#### [Unix StackExchange answer](https://unix.stackexchange.com/questions/355640/how-to-scp-via-an-intermediate-machine)
-
-> It's possible and relatively easy, even when you need to use certificates > for authentication (typical in AWS environments).
->
-> The command below will copy files from a remotePath on server2 directly > into your machine at localPath. Internally the scp request is proxied via > server1.
->
-> ```bash
-> scp -i user2-cert.pem -o ProxyCommand="ssh -i user1-cert.pem -W %h:%p > user1@server1" user2@server2:/<remotePath> <localpath>
-> ```
->
-> If you use password authentication instead, try with:
->
-> ```bash
-> scp -o ProxyCommand="ssh -W %h:%p user1@server1" > user2@server2:/<remotePath> <localpath>
-> ```
->
-> If you use the same user credentials in both servers:
->
-> ```bash
-> scp -o ProxyCommand="ssh -W %h:%p commonuser@server1" > commonuser@server2:/<remotePath> <localpath>
-> ```
->
-
-## Known situations and ways to handle them
-
-- Problem: Oodikone doesn't start and database container (e.g. db_sis) logs include the line "can't open '/docker-entrypoint-initdb.d/'
-  - Possible reason: You probably have too restricting read rights in `scripts/docker-entrypoint-initdb.d` -folder. This could be for example caused by fairly strict `umask` setting, such as `077` in your OS.
-  - Possible solution: Either change the umask on whole OS (to `022` for example) or set the correct read rights for docker-entrypoint -folder by running `chmod 755 scripts/docker-entrypoint-initdb.d`
-- Problem: Packages aren't updated in development containers after update in package.json, even after rebuilding containers and images
-  - Possible reason: `node_modules` folder is binded to external docker volume (i.e not to disk) in our development environment. These volumes aren't updated unless explicitly told so.
-  - Possible solution: Either:
-    a) update packages inside the failing container by first logging to container `docker exec -it container_name sh` and then running `npm install`
-    b) remove volume that contains failing containers `node_modules` by stopping and removing the failing container followed by `docker volume prune`.
+See [tietosuoja.md](tietosuoja.md).

adminer-plugins/login-servers.php

@@ -7,9 +7,7 @@ $driver = "pgsql";
 */
 return new AdminerLoginServers(
     $servers = array(
-        "analytics-db" => array("server" => "analytics-db", "driver" => $driver),
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
-        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),
         "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
         "user-db" => array("server" => "user-db", "driver" => $driver)

babel.config.json

@@ -0,0 +1,5 @@
+{
+  "plugins": [
+    "@babel/plugin-syntax-jsx"
+  ]
+}

cli.sh

@@ -28,8 +28,10 @@ cleanup() {
 # === CLI ===
 
 show_welcome() {
-  local cashmoneyyellow=$(tput setaf 221)
-  local normal=$(tput sgr0)
+  local cashmoneyyellow
+  cashmoneyyellow=$(tput setaf 221)
+  local normal
+  normal=$(tput sgr0)
   if [ "$(tput cols)" -gt "76" ]; then
     while IFS="" read -r p || [ -n "$p" ]; do
       printf '%40s\n' "${cashmoneyyellow}$p${normal}"
@@ -37,39 +39,39 @@ show_welcome() {
   fi
   infomsg "Welcome to Oodikone CLI!"
   msg "This tool helps you in managing the project configuration. If you are new to
-Oodikone development, you should probably run \"Set up oodikone\" which will
-take care of setting up and starting Oodikone for you. See README for more
-details."
+Oodikone development, you should probably run \"Set up oodikone from scratch.\" which
+will take care of setting up Oodikone for you. See README for more details."
   msg ""
 }
 
-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "
 
-options=(
-  "Set up oodikone."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Reset old oodi data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}
 
-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt
 
 while true; do
   select opt in "${options[@]}"; do
     case $opt in
-      "Set up oodikone.")
-        set_up_oodikone;;
+      "Set up oodikone from scratch.")
+        set_up_oodikone_from_scratch;;
       "Reset all real data.")
         reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
-      "Reset old oodi data.")
-        reset_old_oodi_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress.json

@@ -1,9 +1,7 @@
 {
   "projectId": "c3jsph",
-  "baseUrl": "http://localhost:8081",
-  "defaultCommandTimeout": 60000,
-  "requestTimeout": 60000,
-  "responseTimeout": 60000,
-  "viewportHeight": 1080,
-  "viewportWidth": 1920
-}
+  "baseUrl": "http://localhost:3000",
+  "defaultCommandTimeout": 30000,
+  "requestTimeout": 30000,
+  "videoCompression": false
+}

cypress/fixtures/example.json

@@ -1,5 +0,0 @@
-{
-  "name": "Using fixtures to represent data",
-  "email": "hello@cypress.io",
-  "body": "Fixtures are a great way to mock data for responses to routes"
-}

cypress/integration/Course_population.js

@@ -1,29 +0,0 @@
-/// <reference types="Cypress" />
-
-describe("Course population tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Course statistics").click();
-    cy.contains("Search for courses");
-  });
-
-  it("Can find course population", () => {
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("Search for courses");
-    cy.get("input[placeholder='Search by a course code']").type("TKT20003");
-    cy.contains("tr", "TKT20003").click();
-    cy.contains("Fetch statistics").should("be.enabled").click();
-
-    cy.contains("Käyttöjärjestelmät");
-    cy.contains("TKT20003");
-
-    cy.get(":nth-child(3) > :nth-child(1) > div > .item > .level").click();
-    cy.contains("Population of course Käyttöjärjestelmät 2019-2020");
-    cy.contains("TKT20003");
-    cy.contains("Students (all=24)");
-
-    cy.contains("Students (24)").click();
-    cy.contains("010135486");
-    cy.contains("010431753");
-  });
-});

cypress/integration/Custom_population.js

@@ -1,224 +1,195 @@
 /// <reference types="Cypress" />
 
 const openCustomPopupForm = () => {
-  cy.get("button").contains("Custom population").click();
-};
+  cy.get('button').contains('Custom population').click()
+}
 
 const fillName = () => {
-  const name = `TEST-${new Date().getTime()}`;
-  cy.contains("Insert name for this custom population if you wish to save it")
+  const name = `TEST-${new Date().getTime()}`
+  cy.contains('Insert name for this custom population if you wish to save it')
     .siblings()
-    .get("input[placeholder=name]")
-    .type(name);
-  return name;
-};
+    .get('input[placeholder=name]')
+    .type(name)
+  return name
+}
 
 const save = () => {
-  cy.get("button").contains("Save").click();
-};
+  cy.get('button').contains('Save').click()
+}
 
-const selectSavedPopulation = (name) => {
-  cy.get('[data-cy="history-search"]')
-    .children()
-    .eq(0)
-    .type(name)
-    .type("{enter}");
-};
+const selectSavedPopulation = name => {
+  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+}
 
 const deleteAllSearches = () => {
-  cy.get('[data-cy="history-search"]').children().eq(0).click();
-  cy.contains("Saved populations")
+  cy.get('[data-cy="history-search"]').children().eq(0).click()
+  cy.contains('Saved populations')
     .parent()
     .parent()
     .parent()
     .parent()
-    .get(".dropdown")
-    .then((d) => {
-      const searchItems = d.find("div[role=option] > span[class=text]");
+    .get('.dropdown')
+    .then(d => {
+      const searchItems = d.find('div[role=option] > span[class=text]')
       for (let i = 0; i < searchItems.length; i++) {
-        if (searchItems[i].textContent.includes("TEST-")) {
-          cy.get('[data-cy="history-search"]')
-            .children()
-            .eq(0)
-            .type(searchItems[i].textContent)
-            .type("{enter}");
-          cy.get("button").contains("Delete").click();
+        if (searchItems[i].textContent.includes('TEST-')) {
+          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('button').contains('Delete').click()
         }
       }
-    });
-};
+    })
+}
 
-const fillForm = (content) => {
-  cy.contains("Insert studentnumbers you wish to use for population here")
+const fillForm = content => {
+  cy.contains('Insert studentnumbers you wish to use for population here')
     .siblings()
-    .get("textarea")
-    .type(content.join("\n"));
-};
+    .get('textarea')
+    .type(content.join('\n'))
+}
 
 const search = () => {
-  cy.get("button").contains("Search population").click();
-};
+  cy.get('button').contains('Search population').click()
+}
 
-const searchFor = (studentnumbers) => {
-  openCustomPopupForm();
-  fillForm(studentnumbers);
-  search();
-};
+const searchFor = studentnumbers => {
+  openCustomPopupForm()
+  fillForm(studentnumbers)
+  search()
+}
 
 const hasLanded = () => {
-  cy.contains("Credit accumulation");
-  cy.contains("Programme distribution");
-  cy.contains("Courses of population");
-  cy.contains("Students");
-};
+  cy.contains('Credit accumulation')
+  cy.contains('Programme distribution')
+  cy.contains('Courses of population')
+  cy.contains('Students')
+}
 
 const containsAmountOfStudents = (amount = 0) => {
-  cy.contains(`Credit accumulation (for ${amount} students)`);
-};
+  cy.contains(`Credit accumulation (for ${amount} students)`)
+}
 
 const containsSpecificStudents = (studentnumbers = []) => {
-  cy.contains(`Students (${studentnumbers.length})`).click();
+  cy.contains(`Students (${studentnumbers.length})`).click()
 
-  studentnumbers.forEach((s) => cy.contains(s));
-};
+  studentnumbers.forEach(s => cy.contains(s))
+}
 
-const loginAs = (username) => {
-  cy.contains("Users").click();
-  cy.contains(username).siblings().contains("Edit").click();
-  cy.get(".spy").click();
-  cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-  cy.url().should("include", "/custompopulation");
-  cy.contains("Custom population");
-};
-
-describe("Custom population tests", () => {
-
-  const nonExistentStudentNumbers = ["123", "X", "-", " "];
+describe('Custom population tests', () => {
+  const nonExistentStudentNumbers = ['123', 'X', '-', ' ']
   beforeEach(() => {
-    cy.init("/custompopulation");
-    cy.url().should("include", "/custompopulation");
-    cy.contains("Custom population");
-  });
+    cy.init('/custompopulation')
+    cy.url().should('include', '/custompopulation')
+    cy.contains('Custom population')
+  })
 
   after(() => {
-    cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-    cy.url().should("include", "/custompopulation");
-    cy.contains("Custom population");
-    cy.get("button").contains("Custom population").click();
-    deleteAllSearches();
-  });
+    cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+    cy.url().should('include', '/custompopulation')
+    cy.contains('Custom population')
+    cy.get('button').contains('Custom population').click()
+    deleteAllSearches()
+  })
 
-  describe("Custom population searching", () => {
-    it("Finds a proper population", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1}) => {
+  describe('Custom population searching', () => {
+    it('Finds a proper population', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
         const students = studentNumbersForCSStudentsSet1
-        searchFor(students);
-        hasLanded();
-        containsAmountOfStudents(students.length);
-        containsSpecificStudents(students);
+        searchFor(students)
+        hasLanded()
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
       })
-    });
+    })
 
     it("Doesn't return non-existing students", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1}) => {
-        const students = [
-          ...studentNumbersForCSStudentsSet1,
-          ...nonExistentStudentNumbers,
-        ];
-        searchFor(students);
-        hasLanded();
-        containsAmountOfStudents(studentNumbersForCSStudentsSet1.length);
-        containsSpecificStudents(studentNumbersForCSStudentsSet1);
-      });
-    });
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
+        const students = [...studentNumbersForCSStudentsSet1, ...nonExistentStudentNumbers]
+        searchFor(students)
+        hasLanded()
+        containsAmountOfStudents(studentNumbersForCSStudentsSet1.length)
+        containsSpecificStudents(studentNumbersForCSStudentsSet1)
+      })
+    })
 
     it("Doesn't find empty custom population", () => {
-      searchFor(nonExistentStudentNumbers);
-      cy.contains("Credit accumulation").should("not.exist");
-      cy.contains("Programme distribution").should("not.exist");
-      cy.contains("Courses of population").should("not.exist");
-      cy.get("button").contains("Custom population");
-    });
+      searchFor(nonExistentStudentNumbers)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
 
-    // Check login as when anon user database rights have been corrected
-    // it("Doesn't return students to whom the user doesn't have rights to", () => {
-    //   const kasvatusStudents = ["010102273", "010112616", "010116531"];
-    //   const kapistelyStudents = ["010623419", "010942404", "010484975"];
-    //   loginAs("Normaalikäyttäjä");
+    it("Doesn't return students user has no right to", () => {
+      // TODO: check that this doesn't create false positives, since this test goes
+      // through even if following students are not in anon sis-db.
+      const studentsForEduBachStudents = ['014990067', '013069465', '014853890']
+      searchFor(studentsForEduBachStudents)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
+  })
 
-    //   searchFor(kasvatusStudents);
-    //   cy.contains("Credit accumulation").should("not.exist");
-    //   cy.contains("Programme distribution").should("not.exist");
-    //   cy.contains("Courses of population").should("not.exist");
-
-    //   searchFor(kapistelyStudents.concat(kasvatusStudents));
-    //   hasLanded();
-    //   containsAmountOfStudents(3);
-    //   containsSpecificStudents(kapistelyStudents);
-    // });
-  });
-
-  describe("Custom population search saving", () => {
-    it("Saves a custom population search", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1}) => {
+  describe('Custom population search saving', () => {
+    it('Saves a custom population search', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
         const students = studentNumbersForCSStudentsSet1
-        openCustomPopupForm();
-        const name = fillName();
-        fillForm(students);
-        save();
+        openCustomPopupForm()
+        const name = fillName()
+        fillForm(students)
+        save()
 
         // Round 1
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(students.length);
-        containsSpecificStudents(students);
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
 
         // Round 2
-        cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-        cy.url().should("include", "/custompopulation");
-        cy.contains("Custom population");
-        cy.get("button").contains("Custom population").click();
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(students.length);
-        containsSpecificStudents(students);
-      });
-    });
+        cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+        cy.url().should('include', '/custompopulation')
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
+      })
+    })
 
-    it("Updates a custom population search", () => {
-      cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2}) => {
+    it('Updates a custom population search', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
         const students1 = studentNumbersForCSStudentsSet1
         const students2 = studentNumbersForCSStudentsSet2
-        openCustomPopupForm();
-        const name = fillName();
-        fillForm(students1);
-        save();
+        openCustomPopupForm()
+        const name = fillName()
+        fillForm(students1)
+        save()
 
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(students1.length);
-        containsSpecificStudents(students1);
-        cy.contains("Custom population");
-        cy.get("button").contains("Custom population").click();
-        selectSavedPopulation(name);
-        fillForm([...students1, ...students2]);
-        save();
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students1.length)
+        containsSpecificStudents(students1)
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        fillForm([...students1, ...students2])
+        save()
 
-        cy.visit(`${Cypress.config().baseUrl}/custompopulation`);
-        cy.url().should("include", "/custompopulation");
-        cy.contains("Custom population");
-        cy.get("button").contains("Custom population").click();
-        selectSavedPopulation(name);
-        search();
-        cy.contains(`Custom population "${name}"`);
-        containsAmountOfStudents(
-            students1.length + students2.length
-        );
-        containsSpecificStudents([...students1, ...students2]);
-      });
-    });
-  });
-});
+        cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+        cy.url().should('include', '/custompopulation')
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students1.length + students2.length)
+        containsSpecificStudents([...students1, ...students2])
+      })
+    })
+  })
+})

cypress/integration/Filters.js

@@ -1,13 +1,19 @@
 /// <reference types="Cypress" />
 
-const checkFilteringResult = (studentCount) => {
-  cy.contains(`Students (${studentCount})`);
-};
+// const baseUrl = Cypress.config().baseUrl
+
+const checkFilteringResult = studentCount => {
+  cy.contains(`Students (${studentCount})`)
+}
 
 // Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
 // This function tries to click "x" on all selections inside dropdown matching given attribute
 const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
-  cy.cs(dataCyAttribute).find("i.delete").click();
+  cy.cs(dataCyAttribute).find('i.delete').click()
+}
+
+const clearSingleDropdownSelection = dataCyAttribute => {
+  cy.cs(dataCyAttribute).find('i.clear').click()
 }
 
 // Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
@@ -15,234 +21,277 @@ const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
 // functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
 const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
   return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
-    cy.cs(dataCyAttributeOfHeaderToClick).click();
-    checkFilteringResult(amountWithoutFiltering);
-    testStepFunctionToRun();
-    checkFilteringResult(amountWithoutFiltering);
-    cy.cs(dataCyAttributeOfHeaderToClick).click();
+    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    checkFilteringResult(amountWithoutFiltering)
+    testStepFunctionToRun()
+    checkFilteringResult(amountWithoutFiltering)
+    cy.cs(dataCyAttributeOfHeaderToClick).click()
   }
 }
 
-describe("Population Statistics", () => {
+describe('Population Statistics', () => {
+  const pathToCSBach2018 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
+  const defaultAmountOfStudents = 148
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
-  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(40)
   before(() => {
-    cy.init();
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-  });
+    cy.init(pathToCSBach2018)
+  })
 
-  it("Graduation filter works", () => {
-    runTestStepWithPreAndPostParts("graduatedFromProgrammeFilter-header", () => {
-      cy.cs("graduatedFromProgrammeFilter-graduated-true").click()
-      checkFilteringResult(10);
-      cy.cs("graduatedFromProgrammeFilter-graduated-false").click()
-      checkFilteringResult(30);
-      cy.cs("graduatedFromProgrammeFilter-all").click()
+  it('Transfer filter is on not transferred by default', () => {
+    cy.contains('By default only students who have not transferred to this study programme are shown.')
+    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
+  })
+
+  it('Graduation filter works', () => {
+    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+      const graduated = 11
+      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      checkFilteringResult(graduated)
+      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      checkFilteringResult(defaultAmountOfStudents - graduated)
+      cy.cs('graduatedFromProgrammeFilter-all').click()
     })
-  });
+  })
 
-  // Can't be tested yet, since anon data doesn't provide enough information for this, fix
-  it.skip("Transfer filter works", () => {
-    cy.cs("transferredToProgrammeFilter-header").click();
-    cy.cs("transferredToProgrammeFilter-have").click();
-    checkFilteringResult(25);
-    cy.cs("transferredToProgrammeFilter-havenot").click();
-    checkFilteringResult(194);
-    cy.cs("transferredToProgrammeFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it("Enrollment filter works", () => {
-    runTestStepWithPreAndPostParts("enrollmentStatusFilter-header", () => {
-      cy.selectFromDropdown("enrollmentStatusFilter-status", 0);
-      cy.selectFromDropdown("enrollmentStatusFilter-semesters", [0]);
-      checkFilteringResult(36);
-      clearSemanticUIMultipleDropDownSelection("enrollmentStatusFilter-semesters")
-    });
-  });
-
-  it("Credit filter works", () => {
-    runTestStepWithPreAndPostParts("credit-filter-header", () => {
-        cy.cs("credit-filter-min").type("50");
-        cy.cs("credit-filter-max").type("150");
-        checkFilteringResult(26);
-        cy.cs("credit-filter-min").find("input").clear();
-        cy.cs("credit-filter-max").find("input").clear();
-    });
-  });
-
-  it("Age filter works", () => {
-    runTestStepWithPreAndPostParts("ageFilter-header", () => {
-      cy.cs("ageFilter-min").type("20");
-      cy.cs("ageFilter-max").type("40");
-      checkFilteringResult(13);
-      cy.cs("ageFilter-min").find("input").clear();
-      cy.cs("ageFilter-max").find("input").clear();
+  it('Transfer filter works', () => {
+    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+      const transferred = 1
+      cy.cs('transferredToProgrammeFilter-have').click()
+      checkFilteringResult(transferred)
+      cy.cs('transferredToProgrammeFilter-all').click()
+      checkFilteringResult(defaultAmountOfStudents + transferred)
+      cy.cs('transferredToProgrammeFilter-havenot').click()
     })
-  });
+  })
 
-  // Can't be tested yet, since anon data doesn't provide enough information for this, fix
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(48);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(171);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it("Starting year filter works", () => {
-    runTestStepWithPreAndPostParts("startYearAtUni-header", () => {
-      cy.selectFromDropdown("startYearAtUni-dropdown", [0]);
-      checkFilteringResult(1);
-      clearSemanticUIMultipleDropDownSelection("startYearAtUni-dropdown")
+  it('Enrollment filter works', () => {
+    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
+      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
+      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+      checkFilteringResult(137)
+      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
-  });
+  })
 
-  it("Courses filter works", () => {
-    runTestStepWithPreAndPostParts("courseFilter-header", () => {
-      const courses = ["DATA15001", "MAT11003"]
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[0]).click();
-      checkFilteringResult(15);
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1);
-      checkFilteringResult(13);
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[1]).click();
-      checkFilteringResult(9);
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3);
-      checkFilteringResult(1);
+  it('Credit filter works', () => {
+    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+      cy.cs('credit-filter-min').type('50')
+      cy.cs('credit-filter-max').type('150')
+      checkFilteringResult(116)
+      cy.cs('credit-filter-min').find('input').clear()
+      cy.cs('credit-filter-max').find('input').clear()
+    })
+  })
+
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('40')
+      checkFilteringResult(42)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('genderFilter-header', () => {
+      cy.selectFromDropdown('genderFilter-dropdown', 0)
+      checkFilteringResult(36)
+      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      checkFilteringResult(112)
+      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      checkFilteringResult(0)
+      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      checkFilteringResult(121)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+      checkFilteringResult(5)
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      checkFilteringResult(10)
+      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
+    })
+  })
+
+  // plz fix
+  it.skip('Courses filter works', () => {
+    // courses takes some time to load, wait for it to complete
+    cy.wait(10000)
+    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      cy.cs('courseFilter-course-dropdown').click()
+      const courses = ['TKT20001', 'MAT11002']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(140)
+      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      checkFilteringResult(131)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(56)
+      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      checkFilteringResult(2)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
-  });
-});
+  })
 
-describe("Course Statistics", () => {
-
-  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(27)
-  before(() => {
-    cy.init();
-    cy.cs("navbar-courseStatistics").click();
-    cy.get("input[placeholder='Search by a course code']").type("TKT20001");
-    // Click uni course, not avoin
-    cy.contains("td", /^TKT20001/).click();
-    cy.contains("Fetch statistics").should("be.enabled").click();
-    cy.get(":nth-child(2) > :nth-child(1) > div > .item > .level").click();
-  });
-
-  it("Grade filter works", () => {
-    runTestStepWithPreAndPostParts("gradeFilter-header", () => {
-      cy.cs("gradeFilter-5").click();
-      checkFilteringResult(4);
-      cy.cs("gradeFilter-5").click();
-    });
-  });
-
-  it("Age filter works", () => {
-    runTestStepWithPreAndPostParts("ageFilter-header", () => {
-      cy.cs("ageFilter-min").type("20");
-      cy.cs("ageFilter-max").type("40");
-      checkFilteringResult(10);
-      cy.cs("ageFilter-min").find("input").clear();
-      cy.cs("ageFilter-max").find("input").clear();
+  it('Filter combinations work', () => {
+    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+        cy.cs('ageFilter-min').type('20')
+        cy.cs('ageFilter-max').type('30')
+        checkFilteringResult(11)
+        cy.cs('ageFilter-min').find('input').clear()
+        cy.cs('ageFilter-max').find('input').clear()
+        cy.cs('graduatedFromProgrammeFilter-all').click()
+      })
     })
-  });
+  })
+})
 
-  // Doesn't work since anon db genders are missing, fix
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(19);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(74);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-header").click();
-  });
-
-  it("Starting year filter works", () => {
-    runTestStepWithPreAndPostParts("startYearAtUni-header", () => {
-      cy.selectFromDropdown("startYearAtUni-dropdown", [0]);
-      checkFilteringResult(1);
-      clearSemanticUIMultipleDropDownSelection("startYearAtUni-dropdown")
-    })
-  });
-
-  it("Filter combinations work", () => {
-    runTestStepWithPreAndPostParts("gradeFilter-header", () => {
-      runTestStepWithPreAndPostParts("ageFilter-header", () => {
-        cy.cs("gradeFilter-3").click();
-        cy.cs("ageFilter-min").type("20");
-        cy.cs("ageFilter-max").type("30");
-        checkFilteringResult(1);
-        cy.cs("ageFilter-min").find("input").clear();
-        cy.cs("ageFilter-max").find("input").clear();
-        cy.cs("gradeFilter-3").click();
-      });
-    });
-  });
-});
-
-describe("Custom Population Statistics", () => {
-  let runTestStepWithPreAndPostParts;
+describe('Course Statistics', () => {
+  const pathToDSAndAlgoSpring2019 =
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
+  const defaultAmountOfStudents = 118
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
-    cy.init("/custompopulation");
-    cy.cs("custom-pop-search-button").click();
-    cy.fixture('customPopulations').then(({studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2}) => {
+    cy.init(pathToDSAndAlgoSpring2019)
+  })
+
+  it('Grade filter works', () => {
+    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+      cy.cs('gradeFilter-5').click()
+      checkFilteringResult(20)
+      cy.cs('gradeFilter-3').click()
+      checkFilteringResult(38)
+      cy.cs('gradeFilter-3').click()
+      cy.cs('gradeFilter-5').click()
+    })
+  })
+
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('40')
+      checkFilteringResult(32)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('genderFilter-header', () => {
+      cy.selectFromDropdown('genderFilter-dropdown', 0)
+      checkFilteringResult(27)
+      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      checkFilteringResult(91)
+      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      checkFilteringResult(0)
+      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      checkFilteringResult(29)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Filter combinations work', () => {
+    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+        cy.cs('gradeFilter-3').click()
+        cy.cs('ageFilter-min').type('20')
+        cy.cs('ageFilter-max').type('30')
+        checkFilteringResult(1)
+        cy.cs('ageFilter-min').find('input').clear()
+        cy.cs('ageFilter-max').find('input').clear()
+        cy.cs('gradeFilter-3').click()
+      })
+    })
+  })
+})
+
+describe('Custom Population Statistics', () => {
+  let runTestStepWithPreAndPostParts
+
+  before(() => {
+    cy.init('/custompopulation')
+    cy.cs('custom-pop-search-button').click()
+    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
       const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
       runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
-      cy.cs("student-no-input").click().type(students.join("/n"));
+      cy.cs('student-no-input').click().type(students.join('/n'))
     })
-    cy.cs("search-button").click();
-  });
+    cy.cs('search-button').click()
+  })
 
-
-  it("Age filter works", () => {
-    runTestStepWithPreAndPostParts("ageFilter-header", () => {
-      cy.cs("ageFilter-min").type("20");
-      cy.cs("ageFilter-max").type("30");
-      checkFilteringResult(1);
-      cy.cs("ageFilter-min").find("input").clear();
-      cy.cs("ageFilter-max").find("input").clear();
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('30')
+      checkFilteringResult(1)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
     })
-  });
+  })
 
-  // Can't be tested yet, since anon data doesn't provide enough information for this, fix
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(48);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(171);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
+  it('Gender filter works', () => {
+    cy.cs('genderFilter-header').click()
+    cy.selectFromDropdown('genderFilter-dropdown', 0)
+    checkFilteringResult(3)
+    cy.selectFromDropdown('genderFilter-dropdown', 1)
+    checkFilteringResult(2)
+    cy.selectFromDropdown('genderFilter-dropdown', 2)
+    checkFilteringResult(0)
+    clearSingleDropdownSelection('genderFilter-dropdown')
+    checkFilteringResult(5, true)
+  })
 
-  it("Starting year filter works", () => {
-    runTestStepWithPreAndPostParts("startYearAtUni-header", () => {
-      cy.selectFromDropdown("startYearAtUni-dropdown", [0]);
-      checkFilteringResult(1);
-      clearSemanticUIMultipleDropDownSelection("startYearAtUni-dropdown")
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
-  });
+  })
 
-  // Can't be tested yet, since feature doesn't work correctly
-  it.skip("Courses filter works", () => {
-    runTestStepWithPreAndPostParts("courseFilter-header", () => {
-      const courses = ["MAT11001", "TKT20004"]
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[0]).click();
-      checkFilteringResult(5);
-      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1);
-      checkFilteringResult(5);
-      cy.cs("courseFilter-course-dropdown").click().contains(courses[1]).click();
-      checkFilteringResult(3);
-      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3);
-      checkFilteringResult(1);
+  it('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(5)
+      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      checkFilteringResult(5)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(3)
+      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      checkFilteringResult(1)
       courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
     })
-  });
-});
+  })
+})

cypress/integration/Population_statistics.js

@@ -0,0 +1,218 @@
+/// <reference types="Cypress" />
+
+const moment = require('moment')
+const _ = require('lodash')
+
+const setPopStatsUntil = (until, includeSettings = []) => {
+  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+  includeSettings.forEach(setting => {
+    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+  })
+  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+  cy.contains('Fetch population with new settings').click()
+  cy.contains('Advanced settings')
+}
+
+describe('Population Statistics tests', () => {
+  const pathToCSBach2017 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
+  describe('when using basic user', () => {
+    beforeEach(() => {
+      cy.init('/populations')
+    })
+
+    it('Population statistics search form is usable', () => {
+      cy.contains('See population').should('be.disabled')
+      cy.contains('Search for population')
+      cy.contains('Class of')
+        .parent()
+        .within(() => {
+          cy.get('.form-control').as('enrollmentSelect')
+        })
+
+      cy.get('@enrollmentSelect')
+        .its(`${[0]}.value`)
+        .then(beforeVal => {
+          cy.get('@enrollmentSelect').click()
+          cy.get('.yearSelectInput .rdtPrev').click({ force: true })
+          cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
+          cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
+        })
+
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen maisteriohjelma')
+        .click()
+      cy.contains('See population').should('be.enabled')
+    })
+
+    it('Searching for population really shows population', () => {
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen kandiohjelma')
+        .click()
+      cy.contains('See population').click()
+      cy.contains('Population statistics')
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+    })
+
+    it('Population statistics is usable on general level', () => {
+      cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
+      setPopStatsUntil('toukokuu 2020')
+
+      cy.get('.card').within(() => {
+        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.contains('Sample size: 149 students')
+        cy.contains('Excludes exchange students')
+        cy.contains("Excludes students who haven't enrolled present nor absent")
+        cy.contains('Excludes students with non-degree study right')
+        cy.contains('Excludes students who have transferred out of this programme')
+      })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()
+
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=expand-TKT1]').click()
+      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+    })
+
+    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
+      cy.contains('No tags defined. You can define them here.').find('a').click()
+      cy.contains('Tags').click()
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Create new tag')
+    })
+
+    // TODO: this fails in CI, but not locally. Investigate and fix
+    it.skip('Advanced settings work', () => {
+      cy.visit(pathToCSBach2017)
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.contains('Statistics until')
+      // only spring
+      cy.cs('toggle-fall').click()
+      cy.contains('Fetch population').click()
+
+      // moar waiting hack
+      cy.wait(10000)
+
+      cy.contains('No statistics found for the given query.')
+
+      // only fall
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-fall').click()
+      cy.cs('toggle-spring').click()
+
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 149 students)')
+
+      // spring + fall and include cancelled
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-spring').click()
+      cy.cs('toggle-cancelled').click()
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 170 students)')
+    })
+
+    it('Credit Statistics, Credits Gained tab works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
+      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
+
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
+    })
+  })
+
+  describe('when using admin', () => {
+    beforeEach(() => {
+      cy.init(pathToCSBach2017, 'admin')
+    })
+    it('Student list checking works as intended', () => {
+      const existing = '010113437'
+      const nonExisting = '66666666'
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.contains(existing)
+      cy.contains(nonExisting).should('not.exist')
+      cy.contains('button', 'Check studentnumbers').click()
+      cy.contains('Check for studentnumbers')
+      cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
+      cy.contains('button', 'check students').click()
+      cy.contains('#checkstudentsresults', 'Results').within(() => {
+        cy.contains('Student numbers in list and in oodi').click()
+        cy.contains('#found', existing)
+        cy.contains('Student numbers in list but not in oodi').click()
+        cy.contains('#notfound', nonExisting)
+        cy.contains('Student numbers in oodi but not in list').click()
+        cy.contains('#notsearched', '010614509')
+      })
+    })
+  })
+})

cypress/integration/Student_statistics.js

@@ -0,0 +1,90 @@
+/// <reference types="Cypress" />
+
+describe('Student Statistics tests', () => {
+  beforeEach(() => {
+    cy.init()
+    cy.contains('Student statistics').click()
+    cy.contains('Show student names')
+  })
+
+  const student = {
+    firstnames: 'Matti Jaakko',
+    lastname: 'Nieminen',
+    studentnumber: '010654019',
+  }
+
+  it.only('Student statistics search form is usable', () => {
+    cy.contains('Show student names')
+    cy.url().should('include', '/students')
+    cy.get('.prompt').type(student.lastname)
+    cy.contains('Student number')
+    cy.contains('Started')
+    cy.contains('Credits')
+    cy.contains(student.firstnames).should('not.exist')
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.firstnames)
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.firstnames).should('not.exist')
+  })
+
+  it('can search with studentnumber too', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber)
+  })
+
+  it('Can get student specific page by clicking student', () => {
+    cy.url().should('include', '/students')
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.contains('Started: 01.08.2011')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains(student.lastname).should('not.exist')
+    cy.contains(student.firstnames).should('not.exist')
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.lastname)
+    cy.contains(student.firstnames)
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.lastname).should('not.exist')
+    cy.contains(student.firstnames).should('not.exist')
+  })
+
+  it('Can get back to search menu', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.get('.remove').click()
+    cy.contains('Student number').should('not.exist')
+    cy.contains('Credits').should('not.exist')
+  })
+
+  it('Can jump to course', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.contains('Introduction to Machine Learning (DATA11002)')
+      .siblings()
+      .within(() => {
+        cy.get('.level').click()
+      })
+    cy.url().should('include', '/coursestatistics')
+    cy.contains('Introduction to Machine Learning')
+  })
+
+  it('Searching with bad inputs doesnt yield results', () => {
+    cy.get('.prompt').type('SWAG LITTINEN')
+    cy.contains('Student number').should('not.exist')
+
+    cy.get('.prompt').clear().type('01114')
+    cy.contains('Student number').should('not.exist')
+  })
+
+  it.only('Can jump to population page', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.get('i.level.up.alternate.icon').eq(0).click()
+    cy.contains('Population statistics')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -0,0 +1,228 @@
+/// <reference types="Cypress" />
+
+const deleteTag = name => {
+  cy.contains(name).siblings().contains('Delete').click()
+  cy.contains('Are you sure you want to delete tag')
+  cy.contains('Confirm').click()
+  cy.contains(name).should('not.exist')
+}
+
+describe('Studyprogramme overview', () => {
+  // check is used by couple of tests
+  const checkProgressAndProductivity = (user = 'basic') => {
+    // Column amount depends on user. Note that with dev user, you'll see even more
+    // stuff than with admin or basic.
+    const CSBachPopulationProgress2018ForAdmin = [
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
+      '1',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
+      '16',
+    ]
+    const CSBachPopulationProgress2018ForBasic = [
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
+      '1',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
+      '16',
+    ] // cancelled and transferred from are missing
+
+    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
+    cy.contains('Population progress')
+    cy.contains('Yearly productivity')
+
+    cy.contains('2018-2019')
+      .siblings()
+      .each((elem, index) => {
+        cy.wrap(elem).contains(population[index])
+      })
+    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
+    cy.get('table')
+      .eq(1)
+      .contains('2019')
+      .siblings()
+      .each((elem, index) => {
+        cy.wrap(elem).contains(populationproductivity2019[index])
+      })
+  }
+
+  describe('when opening programme page with admin user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005', 'admin')
+    })
+
+    it('progress should not be recalculating when opened for the first time', () => {
+      cy.contains('Recalculating').should('not.exist')
+      cy.contains('Last updated').should('exist')
+    })
+
+    // Following vars are used to compare values before and after stats calculation
+    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
+    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
+    let originalProgressCalculatedText
+    let originalProductivityCalculatedText
+    const doRecalculation = () => {
+      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
+      cy.wait(5000)
+      cy.get('.attached').contains('Admin').click()
+      cy.contains('recalculate productivity').click()
+      cy.contains('recalculate throughput').click()
+      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
+      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
+      cy.wait(5000)
+      cy.reload()
+    }
+
+    it('renders progress and productivity tables with calculated status', () => {
+      doRecalculation()
+
+      // check contains also study programme options for admin
+      cy.contains('Study programme options')
+      cy.get('table').should('have.length', 3)
+
+      checkProgressAndProductivity('admin')
+
+      // Grab update dates to be compared later
+      cy.cs('throughputUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProgressCalculatedText = normalizeText(text)
+          expect(originalProgressCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+
+      cy.cs('productivityUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProductivityCalculatedText = normalizeText(text)
+          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+    })
+
+    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
+      doRecalculation()
+      checkProgressAndProductivity('admin')
+
+      // Check new calculation statuses are reported
+      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
+      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+
+      newProgressCalculatedTextElement.should(text => {
+        const newProgressCalculatedText = normalizeText(text)
+        expect(newProgressCalculatedText).not.to.contain('recalculating')
+        expect(newProgressCalculatedText).not.to.contain('refresh')
+        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+
+      newProductivityCalculatedTextElement.should(text => {
+        const newProductivityCalculatedText = normalizeText(text)
+        expect(newProductivityCalculatedText).not.to.contain('recalculating')
+        expect(newProductivityCalculatedText).not.to.contain('refresh')
+        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+    })
+  })
+
+  describe('when opening programme page with basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005')
+    })
+
+    it('renders progress and productivity tables with previosly calculated values', () => {
+      // check doesn't contain study programme options for admin
+      cy.contains('Study programme options').should('not.exist')
+      cy.get('table').should('have.length', 2)
+      checkProgressAndProductivity()
+    })
+
+    it('can open Thesis page', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('Thesis Courses').click()
+      cy.contains('Add thesis course').click()
+      cy.contains('No results')
+    })
+
+    it('can move to Population statistics page by clickin', () => {
+      cy.get('i.level.up.alternate.icon').eq(0).click()
+      cy.contains('Students (10)')
+    })
+
+    it('can create and delete tags for population', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      cy.contains('2018')
+      deleteTag(name)
+    })
+
+    it('can create personal tags', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+
+      cy.get('.ui > label').click()
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      deleteTag(name)
+    })
+
+    it('can add tags to students', () => {
+      const name = `tag-${new Date().getTime()}`
+
+      const student = '010113437'
+
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+
+      cy.contains('Add tags to students').click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+
+      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+
+      cy.get('textarea').type('010113437')
+      cy.get('.positive').click()
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name)
+
+      cy.go('back')
+      cy.go('back')
+
+      deleteTag(name)
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name).should('not.exist')
+    })
+  })
+})

cypress/integration/Teachers.js

@@ -1,53 +1,52 @@
 /// <reference types="Cypress" />
 
-describe("Teachers page tests", () => {
+describe('Teachers page tests', () => {
   beforeEach(() => {
-    cy.init();
-    cy.contains("Teachers").click();
-    cy.contains("Teacher statistics by course providers");
-  });
+    // login as admin = has teacher rights
+    cy.init('/teachers', 'admin')
+  })
 
-  const teacher1 = "Landgraf Leo"
-  const teacher2 = "Vinogradov Jussi Petteri"
+  const teacher1 = 'Landgraf Leo'
+  const teacher2 = 'Vinogradov Jussi Petteri'
 
-  it("Check Statistics", () => {
-    cy.get(":nth-child(1) > .ui > .search").click();
-    cy.contains("Autumn 2020").click();
-    cy.cs("course-providers").click();
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.cs("course-providers").children(".icon").click();
-    cy.get(".form > .fluid").click();
-    cy.contains("Name");
-    cy.contains(teacher1);
-    cy.contains(teacher2);
-  });
+  it('Check Statistics', () => {
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('Autumn 2020').click()
+    cy.cs('course-providers').click()
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+    cy.cs('course-providers').children('.icon').click()
+    cy.get('.form > .fluid').click()
+    cy.contains('Name')
+    cy.contains(teacher1)
+    cy.contains(teacher2)
+  })
 
-  it("Teacher search works", () => {
-    cy.url().should("include", "/teachers");
-    cy.get(".borderless > :nth-child(3)").click();
-    cy.get(".prompt").type(teacher1.split()[0]);
-    cy.contains(teacher1);
-  });
+  it('Teacher search works', () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type(teacher1.split()[0])
+    cy.contains(teacher1)
+  })
 
-  it("Can check teacher page", () => {
-    cy.url().should("include", "/teachers");
-    cy.get(".borderless > :nth-child(3)").click();
-    cy.get(".prompt").type(teacher1.split()[0]);
-    cy.contains(teacher1).click();
-    cy.contains("Aineopintojen harjoitustyö: Tietokantasovellus");
-  });
+  it('Can check teacher page', () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type(teacher1.split()[0])
+    cy.contains(teacher1).click()
+    cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
+  })
 
-  it("Check leaderboad works", () => {
+  it('Check leaderboad works', () => {
     cy.get('.borderless > :nth-child(2)').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
     cy.contains('2019-2020').click({ force: true })
-    cy.contains("Recalculate this year").click({ force: true })
+    cy.contains('Recalculate this year').click({ force: true })
     cy.wait(5000)
     cy.reload()
     cy.contains('2019-2020').click({ force: true })
     cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains("Passed")
-    cy.contains("Ahokas Jarmo Ilkka")
+    cy.contains('Passed')
+    cy.contains('Ahokas Jarmo Ilkka')
   })
-});
+})

cypress/integration/Users.js

@@ -1,81 +1,62 @@
 /// <reference types="Cypress" />
 
-describe("Users tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Users").click();
-    cy.contains("Oodikone users");
-  });
+describe('Users tests', () => {
+  describe('Using user with just grp-oodikone-user, no other rights', () => {
+    it('shows only frontpage, trends feedback', () => {
+      cy.init('', 'norights') // login with norights user
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
 
-  it("Admin mocking normal user shows only the mocked user's programmes", () => {
-    cy.contains("mocking").should("not.exist");
-    cy.contains("tr", "Normaalikäyttäjä").within(($row) => {
-      cy.contains(".button", "Edit").click();
-    });
-    cy.contains("Access rights")
-      .siblings()
-      .within(($row) => {
-        cy.get("div[role='listitem']")
-          .should("have.length", 1)
-          .contains("Tietojenkäsittelytieteen kandiohjelma");
-      });
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
+    })
+  })
 
-    cy.route("POST", "/api/superlogin/normk").as("superlogin");
-    cy.get("i.spy").click();
-    cy.wait("@superlogin");
-    cy.contains("mocking as normk");
-    cy.wait(1000);
-    cy.contains("Study programme")
-      .click()
-      .siblings()
-      .contains("Search by class")
-      .click();
-    cy.contains("label", "Study programme");
-    cy.contains("label", "Study programme")
-      .siblings()
-      .within(($row) => {
-        cy.get("div[role='option']")
-          .should("have.length", 1)
-          .contains("Tietojenkäsittelytieteen kandiohjelma");
-      });
-  });
+  // TODO: Add user right checks for onlycoursestats user (should be like above + should
+  // show course stats
+  //
+  // TODO: Add user right checks for basic user (should show also study programme,
+  // student stats, course stats
 
-  it("Admin mocking courseStatistics-user can see only Trends and Course Statistics", () => {
-    cy.contains("mocking").should("not.exist");
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains(".button", "Edit").click();
-    });
+  describe('Using as admin', () => {
+    beforeEach(() => {
+      cy.init('/users', 'admin')
+    })
 
-    cy.get("[data-cy=access-groups-form]")
-      .click()
-      .contains("courseStatistics")
-      .click();
-    cy.get("[data-cy=access-groups-save]").click();
+    it('should see more stuff than others', () => {
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+    })
 
-    cy.route("POST", "/api/superlogin/usermk").as("superlogin");
-    cy.get("i.spy").click();
-    cy.wait("@superlogin");
-    cy.contains("mocking as usermk");
-    cy.wait(1000);
-
-    cy.get("[data-cy=navbar-studyProgramme").should("not.exist");
-    cy.get("[data-cy=navbar-students]").should("not.exist");
-    cy.get("[data-cy=navbar-teachers]").should("not.exist");
-    cy.get("[data-cy=navbar-users").should("not.exist");
-
-    cy.contains("Stop mocking").click();
-    cy.wait(1000);
-
-    cy.contains("Users").click();
-    cy.contains("Oodikone users");
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains(".button", "Edit").click();
-    });
-    cy.contains("Access Groups")
-      .siblings()
-      .within(($row) => {
-        cy.get("i[class='dropdown icon clear']").click();
-      });
-    cy.get("[data-cy=access-groups-save]").click();
-  });
-});
+    it("mocking normal user shows only the mocked user's programmes", () => {
+      cy.contains('mocking').should('not.exist')
+      cy.contains('tr', 'basic').within($row => {
+        cy.contains('.button', 'Edit').click()
+      })
+      cy.intercept({
+        method: 'POST',
+        path: '/api/superlogin/basic',
+      }).as('superlogin')
+      cy.get('i.spy').click()
+      cy.wait('@superlogin')
+      cy.contains('mocking as basic')
+      cy.wait(1000)
+      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('label', 'Study programme')
+      cy.contains('label', 'Study programme')
+        .siblings()
+        .within($row => {
+          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
+        })
+    })
+  })
+})

cypress/not_checked_integration/Population_statistics.js

@@ -1,167 +0,0 @@
-/// <reference types="Cypress" />
-
-const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains("Advanced settings")
-    .siblings()
-    .get("[data-cy=advanced-toggle]")
-    .click();
-  includeSettings.forEach((setting) => {
-    cy.contains("Advanced settings")
-      .parent()
-      .siblings()
-      .contains(setting)
-      .click();
-  });
-  cy.get(".adv-stats-until > .form-control").click().clear().type(until);
-  cy.contains("Fetch population with new settings").click();
-  cy.contains("Advanced settings");
-};
-
-describe("Population Statistics tests", () => {
-  beforeEach(() => {
-    cy.init();
-  });
-
-  const checkAmountOfStudents = (assertion) => {
-    let students = 0;
-    cy.contains("Credit accumulation")
-      .click()
-      .invoke("text")
-      .then((text) => {
-        students = Number(text.match(/\d+/g)[0]);
-        expect(students).to.equal(assertion);
-      });
-  };
-
-  it("Population statistics search form is usable", () => {
-    cy.cs("navbar-studyProgramme").click();
-    cy.cs("navbar-class").click();
-    cy.contains("See population").should("be.disabled");
-    cy.url().should("include", "/populations");
-    cy.contains("Search for population");
-    cy.contains("Class of")
-      .parent()
-      .within(() => {
-        cy.get(".form-control").as("enrollmentSelect");
-      });
-
-    cy.get("@enrollmentSelect")
-      .its(`${[0]}.value`)
-      .then((beforeVal) => {
-        cy.get("@enrollmentSelect").click();
-        // go back to 2010-2019
-        cy.get(".yearSelectInput .rdtPrev").click({ force: true });
-        cy.get(".yearSelectInput table")
-          .contains("2014-2015")
-          .click({ force: true });
-        cy.get("@enrollmentSelect").should("not.have.value", beforeVal);
-      });
-
-    cy.contains("Select study programme")
-      .click()
-      .siblings()
-      .contains("Tietojenkäsittelytieteen koulutusohjelma")
-      .click();
-    cy.contains("Select degree")
-      .click()
-      .siblings()
-      .contains("Luonnontieteiden kandidaatti");
-  });
-
-  // FIXME: re-enable when CI has mandatory courses in the db
-  it.skip("Population statistics is usable on general level", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
-    setPopStatsUntil("September 2019");
-
-    cy.get(".card").within(() => {
-      cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
-      cy.contains("Sample size: 29 students");
-      cy.contains("Excludes exchange students");
-      cy.contains("Excludes students who haven't enrolled present nor absent");
-    });
-    cy.contains("Courses of population").click({ force: true });
-
-    cy.route("/api/v3/courseyearlystats**").as("coursePage");
-    cy.wait(150);
-    cy.cs("expand-CSM10000").click();
-    cy.cs("coursestats-link-CSM12101").click();
-    cy.wait("@coursePage");
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("CSM12101");
-  });
-
-  it.skip("Student list checking works as intended", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
-    cy.contains("Students (16)").click();
-    cy.contains("010429464");
-    cy.contains("666666666").should("not.exist");
-    cy.contains("button", "Check studentnumbers").click();
-    cy.contains("Check for studentnumbers");
-    cy.get("textarea").type("010429464").type("{enter}").type("666666666");
-    cy.contains("button", "check students").click();
-    cy.contains("#checkstudentsresults", "Results").within((e) => {
-      cy.contains("Student numbers in list and in oodi").click();
-      cy.contains("#found", "010429464");
-      cy.contains("Student numbers in list but not in oodi").click();
-      cy.contains("#notfound", "666666666");
-      cy.contains("Student numbers in oodi but not in list").click();
-      cy.contains("#notsearched", "010533091");
-    });
-  });
-
-  it.skip("Empty 'tags' tab has a link to the page where tags can be created", () => {
-    cy.selectStudyProgramme("Kielten kandiohjelma");
-    cy.contains("Students (5)").click();
-    cy.get("[data-cy=student-table-tabs]").contains("Tags").click();
-    cy.contains("No tags defined. You can define them here.").find("a").click();
-    cy.contains("Kielten kandiohjelma");
-    cy.contains("Create new tag");
-  });
-
-  it.skip("Advanced settings work", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.contains("Statistics until");
-    // only spring
-    cy.cs("toggle-fall").click();
-    cy.contains("Fetch population").click();
-
-    cy.contains("Credit accumulation (for 0 students)");
-
-    // only fall
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.cs("toggle-fall").click();
-    cy.cs("toggle-spring").click();
-
-    cy.contains("Fetch population").click();
-
-    // FIXME: This fails randomly both locally and on CI.
-    cy.contains("Credit accumulation (for 194 students)");
-
-    // spring + fall and include cancelled
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.cs("toggle-spring").click();
-    cy.cs("toggle-cancelled").click();
-    cy.contains("Fetch population").click();
-
-    cy.contains("Credit accumulation (for 202 students)");
-  });
-
-  it.skip("Credit Statistics, Statistics pane works", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-    cy.contains("Credit statistics").click();
-    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click();
-
-    cy.get("[data-cy='credit-stats-table-name-header']").should(
-      "contain",
-      "Statistic for n = 194 Students"
-    );
-    cy.get("[data-cy='credit-stats-mean']").should("contain", "45.10");
-    cy.get("[data-cy='credit-stats-stdev']").should("contain", "30.61");
-    cy.get("[data-cy='credit-stats-min']").should("contain", "0");
-    cy.get("[data-cy='credit-stats-q1']").should("contain", "19");
-    cy.get("[data-cy='credit-stats-q2']").should("contain", "47");
-    cy.get("[data-cy='credit-stats-q3']").should("contain", "66");
-    cy.get("[data-cy='credit-stats-max']").should("contain", "137");
-  });
-});

cypress/not_checked_integration/Student_statistics.js

@@ -1,75 +0,0 @@
-/// <reference types="Cypress" />
-
-describe("Student Statistics tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Student statistics").click();
-    cy.contains("Show student names");
-  });
-
-  it("Student statistics search form is usable", () => {
-    cy.contains("Show student names");
-    cy.url().should("include", "/students");
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("Student number");
-    cy.contains("Started");
-    cy.contains("Credits");
-    cy.contains("Heidi Eeva").should("not.exist");
-
-    cy.get("label").click();
-    cy.contains("Heidi Eeva");
-
-    cy.get("label").click();
-    cy.contains("Heidi Eeva").should("not.exist");
-  });
-  it("Can get student specific page by clicking student", () => {
-    cy.url().should("include", "/students");
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.contains("Updated at 02.10.2019");
-    cy.contains("Musiikin didaktiikka (EDUK4212)");
-    cy.contains("Oinonen").should("not.exist");
-
-    cy.get("label").click();
-    cy.contains("Oinonen");
-
-    cy.get("label").click();
-    cy.contains("Oinonen").should("not.exist");
-  });
-
-  it("Can get back to search menu", () => {
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.get(".remove").click();
-    cy.contains("Student number").should("not.exist");
-    cy.contains("Credits").should("not.exist");
-  });
-
-  it("Can jump to course", () => {
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.contains("Musiikin didaktiikka (EDUK4212)")
-      .siblings()
-      .within(() => {
-        cy.get(".level").click();
-      });
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("Musiikin didaktiikka");
-  });
-
-  it("Searching with bad inputs doesnt yield results", () => {
-    cy.get(".prompt").type("Oin");
-    cy.contains("Student number").should("not.exist");
-
-    cy.get(".prompt").clear().type("01114");
-    cy.contains("Student number").should("not.exist");
-  });
-
-  it("Can jump to population page", () => {
-    cy.get(".prompt").type("Veli-Matti");
-    cy.contains("014824094").click();
-    cy.get("i.level.up.alternate.icon").eq(0).click();
-    cy.contains("Population statistics");
-    cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
-  });
-});

cypress/not_checked_integration/Studyprogramme_overview.js

@@ -1,162 +0,0 @@
-/// <reference types="Cypress" />
-
-const deleteTag = (name) => {
-  cy.contains(name).siblings().contains("Delete").click();
-  cy.contains("Are you sure you want to delete tag");
-  cy.contains("Confirm").click();
-  cy.contains(name).should("not.exist");
-};
-
-describe("Studyprogramme overview", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Study programme")
-      .click()
-      .siblings()
-      .contains("Overview")
-      .click();
-    cy.contains("Study Programme", { timeout: 100000 });
-  });
-
-  // FIXME: re-enable when CI has mandatory courses in the db
-  it.skip("can search for course mappings", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Code Mapper").click();
-    cy.wait(150);
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.get("input").type("582219");
-    });
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.get(".results").contains("Käyttöjärjestelmät (582219)");
-    });
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.contains("button", "Add");
-    });
-  });
-
-  it("can view course groups", () => {
-    cy.contains("Kasvatustieteiden kandiohjelma").click();
-    cy.contains("Course Groups").click();
-
-    cy.contains("tr", "Test course group").get("i.edit").click();
-    cy.contains("Edit group");
-    cy.get(".prompt").type("Professori Pekka");
-    cy.contains("Add teacher").parent().contains("000960").click();
-    cy.contains("Teachers in group").parent().contains("000960");
-
-    cy.get("i.reply.link.icon").click();
-    cy.contains("tr a", "Test course group").click();
-    cy.contains("Total teachers");
-    cy.get("i.reply.icon").click();
-  });
-
-  it("renders progress and productivity tables", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Admin").click();
-    cy.contains("productivity").click();
-    cy.contains("throughput").click();
-
-    cy.wait(1000);
-    cy.get(".attached > :nth-child(1)").click();
-    cy.get("table").should("have.length", 3);
-    cy.contains("Population progress");
-    cy.contains("Yearly productivity");
-    cy.contains("2017-2018")
-      .siblings()
-      .contains("228")
-      .siblings()
-      .contains("178");
-
-    cy.get("table")
-      .eq(1)
-      .contains("2018")
-      .siblings()
-      .contains("3023.00")
-      .siblings()
-      .contains("2631.00");
-    cy.get("table")
-      .eq(1)
-      .contains("2017")
-      .siblings()
-      .contains("3159.00")
-      .siblings()
-      .contains("2914.00");
-  });
-
-  it("can open Thesis page", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Thesis Courses").click();
-    cy.contains("Add thesis course").click();
-    cy.contains("No results");
-  });
-
-  it("can move to Population statistics page by clickin", () => {
-    cy.contains("Tietojenkäsittelytieteen maisteriohjelma").click();
-    cy.get("i.level.up.alternate.icon").eq(0).click();
-    // cy.contains("Credit accumulation (for 16 students)");
-  });
-
-  it("can create and delete tags for population", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-    cy.contains("2018");
-    deleteTag(name);
-  });
-
-  it("can create personal tags", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-
-    cy.get(".ui > label").click();
-    cy.contains("Create new tag").click();
-    cy.get(".purple");
-    cy.contains(name);
-    deleteTag(name);
-  });
-
-  it("can add tags to students", () => {
-    const name = `tag-${new Date().getTime()}`;
-
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-
-    cy.contains("Add tags to students").click();
-    cy.get(".form > .field > .dropdown")
-      .click()
-      .get(".ui > .search")
-      .type(name)
-      .click();
-
-    cy.get(".form > .field > .dropdown > .visible").contains(name).click();
-
-    cy.get("textarea").type("010623419");
-    cy.get(".positive").click();
-
-    cy.contains("Student statistics").click();
-    cy.get(".prompt").type("010623419");
-    cy.contains("10").click();
-    cy.contains(name);
-
-    cy.go("back");
-    cy.go("back");
-
-    deleteTag(name);
-
-    cy.contains("Student statistics").click();
-    cy.get(".prompt").type("010623419");
-    cy.contains("10").click();
-    cy.contains(name).should("not.exist");
-  });
-});

cypress/not_checked_integration/Tags.js

@@ -1,48 +0,0 @@
-/// <reference types="Cypress" />
-
-const deleteTag = (name) => {
-  cy.contains("Create tags for study programme");
-  cy.contains(name).siblings().contains("Delete").click();
-  cy.contains("Are you sure you want to delete tag");
-  cy.contains("Confirm").click();
-  cy.contains(name).should("not.exist");
-};
-
-describe("Tag tests", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Study programme").click();
-    cy.contains("Overview").click();
-    cy.contains("Datatieteen maisteriohjelma").click();
-    cy.contains("Tags").click();
-  });
-
-  it("Tagged population works", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.get(":nth-child(1) > .field > .ui > input").type(name);
-    cy.get(".form-control").type("2016");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-    cy.contains("2016");
-
-    cy.contains("Add tags to students").click();
-    cy.get(".form > .field > .dropdown")
-      .click()
-      .get(".ui > .search")
-      .type(name)
-      .click();
-
-    cy.get(".form > .field > .dropdown > .visible").contains(name).click();
-
-    cy.get("textarea").type("014022579\n011122249");
-    cy.get(".positive").click();
-    cy.contains(name).find(".level").click();
-
-    cy.contains("Credit accumulation (for 1 students)");
-    cy.contains("Students (1)").click();
-    cy.contains("014022579");
-
-    cy.go("back");
-    deleteTag(name);
-  });
-});

cypress/plugins/index.js

@@ -19,4 +19,4 @@
 module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
-};
+}

cypress/support/commands.js

@@ -1,67 +1,95 @@
 /// <reference types="Cypress" />
+//
+// Note: here we need to set keys to be all lowercase, since
+// we're replacing headers after they've left browser / frontend.
+// All of these users are available in anon user-db
+const adminUserHeaders = {
+  uid: 'admin',
+  displayname: 'Admin User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member;employee;faculty',
+  mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
+}
+
+const basicUserHeaders = {
+  uid: 'basic',
+  displayname: 'Basic User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
+}
+
+const onlycoursestatisticsUserHeaders = {
+  uid: 'onlycoursestatistics',
+  displayName: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-session',
+  hyGroupCn: 'grp-oodikone-basic-users',
+  eduPersonAffiliation: 'member',
+  mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
+}
+
+const norightsUserHeaders = {
+  uid: 'norights',
+  displayname: 'Norights User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
+}
+
+const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
 
 /**
- * Initialize headers and load the base URL or optional path.
+ Set up headers to login, set up correct user (admin / basic / etc.) and open given path.
  */
-Cypress.Commands.add("init", (path = "") => {
-  cy.server({
-    onAnyRequest: function (route, proxy) {
-      proxy.xhr.setRequestHeader("uid", "tktl");
-      proxy.xhr.setRequestHeader("shib-session-id", "mock-shibboleth");
-      proxy.xhr.setRequestHeader("hygroupcn", "grp-oodikone-users");
-      proxy.xhr.setRequestHeader("edupersonaffiliation", "asdasd");
-    },
-  });
-
-  // Babel throws an error probably because of markdown files. I'm sorry about this :lul:.
-  cy.on("uncaught:exception", (err, runnable) => {
-    expect(err.message).to.include(
-      "Cannot read property 'helpers' of undefined"
-    );
-    done();
-    return false;
-  });
-
+Cypress.Commands.add('init', (path = '', userId = 'basic') => {
+  cy.intercept('', req => {
+    const headersToUse = userHeaders.find(({ uid }) => uid === userId)
+    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
+    req.headers = headersToUse
+  })
+  // Hide SIS warning flag to avoid screen size problems when running headlessly
+  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
-  cy.visit(baseUrl.concat(path));
-});
+  cy.visit(baseUrl.concat(path))
+})
 
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add("cs", (name) => {
-  return cy.get(`[data-cy='${name}']`);
-});
+Cypress.Commands.add('cs', name => {
+  return cy.get(`[data-cy='${name}']`)
+})
 
 /**
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add("selectFromDropdown", (name, index) => {
-  const indexes = Array.isArray(index) ? index : [index];
+Cypress.Commands.add('selectFromDropdown', (name, index) => {
+  const indexes = Array.isArray(index) ? index : [index]
 
-  indexes.forEach((i) => {
-    cy.cs(name)
-      .click()
-      .children(".menu")
-      .children()
-      .eq(i)
-      .click({ force: true });
-  });
+  indexes.forEach(i => {
+    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
+  })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children(".icon").click({ force: true });
+    cy.cs(name).children('.icon').click({ force: true })
   }
-});
-
+})
 
 /**
  * Move to page with studyProgramme population
  */
-Cypress.Commands.add("selectStudyProgramme", (name) => {
-  cy.cs("navbar-studyProgramme").click();
-  cy.cs("navbar-class").click();
-  cy.cs("select-study-programme").click().children().contains(name).click();
-  cy.contains("See population").click();
-});
+Cypress.Commands.add('selectStudyProgramme', name => {
+  cy.cs('navbar-studyProgramme').click()
+  cy.cs('navbar-class').click()
+  cy.cs('select-study-programme').click().children().contains(name).click()
+  cy.contains('See population').click()
+})

cypress/support/index.js

@@ -13,4 +13,4 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-import "./commands";
+import './commands'

docker-compose.ci.yml

@@ -1,58 +1,17 @@
-version: "3.9"
-
 x-defaults: &defaults
   restart: unless-stopped
 
-x-build_defaults: &build_defaults
-  build:
-    args:
-      TAG: ${TAG}
-
 services:
-  analytics:
-    <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-      - toska/oodikone2-userservice:trunk
-      - toska/oodikone2-userservice:staging
-      - toska/oodikone2-userservice:latest
-      context: services/oodikone2-analytics
-    container_name: analytics
-    depends_on:
-      - analytics-db
-    image: toska/oodikone2-analytics:${TAG}
-    environment:
-      DB_URL: postgres://postgres@analytics-db:5432/analytics-db
-    ports:
-      - "4568:4568"
-
-  analytics-db:
-    <<: *defaults
-    container_name: analytics-db
-    image: registry.toska.cs.helsinki.fi/analytics-db
-    ports:
-      - "5423:5432"
 
   backend:
     <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-        - toska/oodikone2-backend:trunk
-        - toska/oodikone2-backend:staging
-        - toska/oodikone2-backend:latest
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    container_name: backen
+    container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
     environment:
-      ANALYTICS_URL: http://analytics:4568
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
+      CI: "true"
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
@@ -62,39 +21,17 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
-    image: toska/oodikone2-backend:${TAG}
-    ports:
-      - "9229:9229"
+    image: toska/oodikone2-backend:staging
 
   frontend:
     <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-        - toska/oodikone2-frontend:trunk
-        - toska/oodikone2-frontend:staging
-        - toska/oodikone2-frontend:latest
-      context: services/oodikone2-frontend
     container_name: frontend
-    environment:
-      BACKEND_ADDR: backend
-    image: toska/oodikone2-frontend:${TAG}
-    ports:
-      - "5000:5000"
+    image: toska/oodikone2-frontend:staging
 
   kone-db:
     <<: *defaults
     container_name: kone-db
     image: registry.toska.cs.helsinki.fi/kone-db
-    ports:
-      - "5425:5432"
-
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    ports:
-      - "5421:5432"
 
   redis:
     <<: *defaults
@@ -102,41 +39,29 @@ services:
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - 6379:6379
 
   sis-db:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
-    ports:
-      - "5426:5432"
 
   user-db:
     <<: *defaults
     container_name: user-db
     image: registry.toska.cs.helsinki.fi/user-db
-    ports:
-      - "5422:5432"
 
   userservice:
     <<: *defaults
-    <<: *build_defaults
-    build:
-      cache_from:
-        - toska/oodikone2-userservice:trunk
-        - toska/oodikone2-userservice:staging
-        - toska/oodikone2-userservice:latest
-      context: services/oodikone2-userservice
     container_name: userservice
     depends_on:
       - user-db
     environment:
+      CI: "true"
       DB_URL: postgres://postgres@user-db:5432/user-db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
-    image: toska/oodikone2-userservice:${TAG}
-    ports:
-      - "4567:4567"
+    image: toska/oodikone2-userservice:staging
 
   nginx:
     image: nginx:latest
@@ -145,7 +70,6 @@ services:
       - frontend
       - backend
       - userservice
-      - analytics
     ports:
       -  1337:80
     volumes:

docker-compose.real.yml

@@ -1,11 +1,7 @@
 services:
-  analytics:
-    environment:
-      DB_URL: postgres://postgres@analytics-db:5432/analytics-db-real
 
   backend:
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 

docker-compose.test.yml

@@ -0,0 +1,10 @@
+services:
+
+  userservice-test:
+    build:
+      context: services/userservice
+      target: test
+    container_name: userservice-test
+    environment:
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -1,13 +1,8 @@
 x-defaults: &defaults
   restart: unless-stopped
 
-x-command_default: &command_default
-  command: ["npm", "run", "dev"]
-
 x-build_defaults: &build_defaults
-  args:
-    NODE_ENV: development
-    TAG: development
+  target: development
 
 services:
   adminer:
@@ -17,58 +12,25 @@ services:
       ADMINER_DESIGN: pepa-linha
     image: adminer:4.8.0
     ports:
-      - "5050:8080"
+      - 5050:8080
     volumes:
       - ./adminer-plugins:/var/www/html/plugins-enabled
 
-  analytics:
-    <<: *defaults
-    <<: *command_default
-    build:
-      <<: *build_defaults
-      context: services/oodikone2-analytics
-    container_name: analytics
-    depends_on:
-      - analytics-db
-    environment:
-      DB_URL: postgres://postgres@analytics-db:5432/analytics-db
-    ports:
-      - "4568:4568"
-    profiles:
-      - oodikone
-    volumes:
-      - analytics-nodemod:/usr/src/app/node_modules
-      - ./services/oodikone2-analytics/:/usr/src/app
-
-  analytics-db:
-    <<: *defaults
-    container_name: analytics-db
-    image: registry.toska.cs.helsinki.fi/analytics-db
-    ports:
-      - "5423:5432"
-    profiles:
-      - oodikone
-    volumes:
-      - analytics-db-data:/data
-
   backend:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
       context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
+    env_file:
+      - .env
     environment:
-      ANALYTICS_URL: http://analytics:4568
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
-      FRONT_URL: http://localhost:8081
+      FRONT_URL: http://localhost:3000
       REDIS: redis
       SECRET_TOKEN: dev
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
@@ -77,66 +39,43 @@ services:
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USERSERVICE_URL: http://userservice:4567
     ports:
-      - "8080:8080"
+      - 8080:8080
     profiles:
       - oodikone
     volumes:
       - backend-nodemod:/usr/src/app/node_modules
-      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/:/usr/src/app
 
   frontend:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
-      context: services/oodikone2-frontend
+      context: services/frontend
     container_name: frontend
-    environment:
-      ADMINER_URL: http://localhost:5050/?pgsql=oodi-db&username=postgres
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics-db&username=postgres
-      BACKEND_ADDR: backend
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=kone-db&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=sis-db&username=postgres
-      SIS_IMPORTER_ADMINER_URL: http://localhost:5050/?pgsql=sis-importer-db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user-db&username=postgres
     ports:
-      - "8081:8081"
-      - "7777:7777"
+      - 3000:3000
     profiles:
       - oodikone
     volumes:
       - frontend-nodemod:/usr/src/app/node_modules
-      - ./services/oodikone2-frontend/:/usr/src/app
+      - ./services/frontend/:/usr/src/app
 
   kone-db:
     <<: *defaults
     container_name: kone-db
     image: registry.toska.cs.helsinki.fi/kone-db
-    ports:
-      - "5425:5432"
     profiles:
       - oodikone
     volumes:
       - kone-db-data:/data
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    ports:
-      - "5421:5432"
-    profiles:
-      - oodikone
-    volumes:
-      - oodi-db-data:/data
-
   redis:
     <<: *defaults
-    command: "redis-server --appendonly yes"
+    command: 'redis-server --appendonly yes'
     container_name: redis
     image: redis
     ports:
-      - "6379:6379"
+      - '6379:6379'
     volumes:
       - redis-data:/data
 
@@ -144,10 +83,9 @@ services:
     <<: *defaults
     container_name: sis-db
     image: registry.toska.cs.helsinki.fi/sis-db
-    ports:
-      - "5426:5432"
     volumes:
       - sis-db-data:/data
+    shm_size: 2gb
 
   sis-importer-db:
     <<: *defaults
@@ -160,7 +98,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -168,7 +106,6 @@ services:
 
   sis-updater-scheduler:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
       context: updater/sis-updater-scheduler
@@ -187,7 +124,7 @@ services:
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
     ports:
-      - "8082:8082"
+      - 8082:8082
     profiles:
       - updater
     volumes:
@@ -196,7 +133,6 @@ services:
 
   sis-updater-worker:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
       context: updater/sis-updater-worker
@@ -224,8 +160,6 @@ services:
     <<: *defaults
     container_name: user-db
     image: registry.toska.cs.helsinki.fi/user-db
-    ports:
-      - "5422:5432"
     profiles:
       - oodikone
     volumes:
@@ -233,10 +167,9 @@ services:
 
   userservice:
     <<: *defaults
-    <<: *command_default
     build:
       <<: *build_defaults
-      context: services/oodikone2-userservice
+      context: services/userservice
     container_name: userservice
     depends_on:
       - user-db
@@ -245,20 +178,17 @@ services:
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       TOKEN_SECRET: IWannaBeTheVeryBest
     ports:
-      - "4567:4567"
+      - 4567:4567
     profiles:
       - oodikone
     volumes:
       - userservice-nodemod:/usr/src/app/node_modules
-      - ./services/oodikone2-userservice/:/usr/src/app
+      - ./services/userservice/:/usr/src/app
 
 volumes:
-  analytics-db-data:
-  analytics-nodemod:
   backend-nodemod:
   frontend-nodemod:
   kone-db-data:
-  oodi-db-data: # TODO: Remove when oodi stuff has been removed from oodikone
   redis-data:
   sis-db-data:
   sis-importer-db-data:

package.json

@@ -1,45 +1,57 @@
 {
   "name": "oodikone",
-  "version": "1.0.0",
   "description": "An application for analyzing university data.",
   "scripts": {
-    "cli": "./cli.sh",
-    "oodikone": "./run.sh oodikone anon up --detach && ./run.sh oodikone logs --follow --tail 100",
-    "oodikone:real": "./run.sh oodikone real up --detach && ./run.sh oodikone logs --follow --tail 100",
-    "updater": "./run.sh updater anon up --detach && ./run.sh updater logs --follow --tail 100",
-    "updater:real": "./run.sh updater real up --detach && ./run.sh updater logs --follow --tail 100",
-    "both": "./run.sh both anon up --detach && ./run.sh both logs --follow --tail 100",
-    "both:real": "./run.sh both real up --detach && ./run.sh both logs --follow --tail 100",
-    "docker:down": "./run.sh both down --remove-orphans",
-    "lint:analytics": "npm run lint --prefix services/oodikone2-analytics",
-    "lint:backend": "npm run lint --prefix services/backend/oodikone2-backend",
-    "lint:frontend": "npm run lint --prefix services/oodikone2-frontend",
-    "lint:userservice": "npm run lint --prefix services/oodikone2-userservice",
-    "lint": "concurrently 'npm run lint:*'",
-    "lintfix:analytics": "npm run lintfix --prefix services/oodikone2-analytics",
-    "lintfix:backend": "npm run lintfix --prefix services/backend/oodikone2-backend",
-    "lintfix:frontend": "npm run lintfix --prefix services/oodikone2-frontend",
-    "lintfix:userservice": "npm run lintfix --prefix services/oodikone2-userservice",
-    "lintfix": "concurrently 'npm run lintfix:*'",
     "cypress": "cypress",
-    "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js"
+    "concurrently": "concurrently",
+    "eslint": "eslint",
+    "stylelint": "stylelint",
+    "prettier": "prettier",
+    "cli": "./cli.sh",
+    "oodikone": "./run.sh oodikone anon up --detach",
+    "oodikone:real": "./run.sh oodikone real up --detach",
+    "updater": "./run.sh updater anon up --detach",
+    "updater:real": "./run.sh updater real up --detach",
+    "both": "./run.sh both anon up --detach",
+    "both:real": "./run.sh both real up --detach",
+    "docker:down": "./run.sh both anon down --remove-orphans; ./run.sh both real down --remove-orphans",
+    "prepare": "husky install"
   },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/UniversityOfHelsinkiCS/oodikone.git"
   },
-  "author": "",
   "license": "ISC",
   "bugs": {
     "url": "https://github.com/UniversityOfHelsinkiCS/oodikone/issues"
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "concurrently": "^6.2.0",
-    "cypress": "^7.6.0"
+    "@babel/eslint-parser": "^7.15.0",
+    "@babel/plugin-syntax-jsx": "^7.14.5",
+    "concurrently": "^6.2.1",
+    "cypress": "^8.3.1",
+    "eslint": "^7.32.0",
+    "eslint-config-airbnb": "^18.2.1",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-babel": "^5.3.1",
+    "eslint-plugin-cypress": "^2.11.3",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-prefer-object-spread": "^1.2.1",
+    "eslint-plugin-prettier": "^3.4.1",
+    "eslint-plugin-react": "^7.24.0",
+    "eslint-plugin-react-hooks": "^4.2.0",
+    "husky": "^7.0.2",
+    "lint-staged": "^11.1.2",
+    "prettier": "^2.3.2",
+    "stylelint": "^13.13.1",
+    "stylelint-config-prettier": "^8.0.2",
+    "stylelint-config-standard": "^22.0.0",
+    "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
   }
 }

run.sh

@@ -13,7 +13,7 @@ set -euoE pipefail
 # Set up constants
 PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 
-# Set up logging
+# Set up messages and exiting
 source "$PROJECT_ROOT"/scripts/utils.sh
 
 usage() {
@@ -21,8 +21,8 @@ usage() {
 Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 
 Parameters:
-* Option: oodikone/updater/both/morning
-* Version: anon/real/ci. Not necessary in all cases, such as when running down or logs.
+* Option: oodikone/updater/both
+* Version: anon/real. Not necessary when running down/logs/pull.
 * Command: will be passed to docker-compose.
 EOF
 }
@@ -34,24 +34,21 @@ parse_params() {
   [[ ${#args[@]} -eq 0 ]] && usage && die "Wrong number of arguments"
   option=${args[0]}
 
-  # If option is morning, other parameters aren't needed
-  [[ "$option" == "morning" ]] && return 0
-
-  # Else, parse arguments
+  #  parse arguments
   [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
-  usage && die "Wrong option: $option"
+    usage && die "Wrong option: $option"
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
 
-  # Down or logs can be passed without version. Otherwise parse version and then pass
+  # Some arguments can be passed without version. Otherwise parse version and then pass
   # rest to compose
-  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" ]]; then
+  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" || "${args[1]}" == "pull" ]]; then
     version=""
     compose_command=${args[*]:1}
   else
     version=${args[1]}
-    [[ "$version" != "anon" && "$version" != "real" && "$version" != "ci" ]] && \
-usage && die "Wrong version $version"
+    [[ "$version" != "anon" && "$version" != "real" ]] && usage && \
+      die "Wrong version $version"
     compose_command=${args[*]:2}
   fi
   return 0
@@ -67,15 +64,11 @@ parse_profiles() {
   return 0
 }
 
-# Set which docker-compose files to use based on version. In ci, profiles are also
-# emptied, since they're passed from github actions.
+# Set which docker-compose files to use based on version.
 parse_env() {
   env=""
   if [[ "$version" == "real" ]]; then
-    env="-f docker-compose.yml -f docker-compose.real.yml"
-  elif [[ "$version" == "ci" ]]; then
-    env="-f docker-compose.ci.yml"
-    profiles=""
+    env="--file docker-compose.yml --file docker-compose.real.yml"
   fi
   return 0
 }
@@ -84,14 +77,6 @@ parse_env() {
 
 parse_params "$@"
 
-# Do only morning cleanup for morning option
-if [[ "$option" == "morning" ]];then
-  git checkout trunk
-  git pull
-  docker-compose down --rmi all --remove-orphans
-  return 0
-fi
-
 # Create command that will be run. Empty command and "down" command will be handled
 # differently.
 parse_profiles
@@ -99,7 +84,7 @@ parse_env
 if [[ "$compose_command" == "" ]]; then
   final_command="docker-compose ${env}"
 elif [[ "$compose_command" == *"down"* ]]; then
-  final_command="docker-compose ${compose_command}"
+  final_command="docker-compose ${env} ${compose_command}"
 else
   final_command="docker-compose ${env} ${profiles} ${compose_command}"
 fi

scripts/functions_for_setup.sh

@@ -11,24 +11,20 @@ PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 USER_DATA_FILE="$DUMP_DIR/hyuserdata"
 
-## Following the naming convention in docker-compose, these are names for services
-## and for the anonymous database. Real databases have suffix "-real".
-ANALYTICS_DB_NAME="analytics-db"
+## Following the naming convention in docker-compose, these are names for database
+## services. The real data databases inside service will have names suffixed by "-real".
 KONE_DB_NAME="kone-db"
 SIS_DB_NAME="sis-db"
 SIS_IMPORTER_DB_NAME="sis-importer-db"
 USER_DB_NAME="user-db"
-DATABASES=("$ANALYTICS_DB_NAME" "$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
-OODI_DB_NAME="oodi-db" # TODO: Remove when oodi is removed
+DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
 
-## Urls should be in same order as databases
-ANALYTICS_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-analytics-pg.sqz"
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+## Urls should be in same order as databases as both are iterated through by indexes.
+KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
-REAL_DUMP_URLS=("$ANALYTICS_DB_REAL_DUMP_URL" "$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
-OODI_DB_REAL_DUMP_URL="svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/latest-pg.sqz" # TODO: Remove when oodi is removed
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
+REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
 
 # Source utility functions
 source "$PROJECT_ROOT"/scripts/utils.sh
@@ -36,10 +32,12 @@ source "$PROJECT_ROOT"/scripts/utils.sh
 # === Function ===
 
 draw_mopo() {
-  # Some hacks to print mopo as green, since colours from common config don't work
-  # Please feel free to fix this. And tell otahontas how you did it!
-  local mopogreen=$(tput setaf 34)
-  local normal=$(tput sgr0)
+  # Some hacks to print mopo as green, since colours from common config don't work when
+  # catting images. Please feel free to fix this. And tell otahontas how you did it!
+  local mopogreen
+  mopogreen=$(tput setaf 34)
+  local normal
+  normal=$(tput sgr0)
   if [ "$(tput cols)" -gt "100" ]; then
     while IFS="" read -r p || [ -n "$p" ]; do
       printf '%40s\n' "${mopogreen}$p${normal}"
@@ -47,7 +45,7 @@ draw_mopo() {
   fi
 }
 
-retry () {
+retry() {
   sleep 5
   for i in {1..60}; do
     "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
@@ -69,18 +67,17 @@ check_if_postgres_is_ready() {
 
 reset_databases() {
   local databases=("$@")
-  local database_name_suffix="-real"
   local database_dump_dir=$DUMP_DIR
 
   infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
 
-  docker-compose down
-  docker-compose up -d ${databases[*]}
+  npm run docker:down
+  docker-compose up -d "${databases[@]}"
 
-  for database in ${databases[@]}; do
+  for database in "${databases[@]}"; do
     local database_dump="$database_dump_dir/$database.sqz"
     local database_container="$database"
-    local database_name="$database$database_name_suffix"
+    local database_name="$database-real"
 
     infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
 
@@ -110,52 +107,97 @@ reset_all_real_data() {
     local url="${REAL_DUMP_URLS[$i]}"
     download_real_dump "$database" "$url"
   done
-  reset_databases ${DATABASES[*]}
+  reset_databases "${DATABASES[@]}"
 }
 
-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
-}
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
 
-reset_old_oodi_data() {
-  infomsg "Downloading old oodi-db dump"
-  local database=$OODI_DB_NAME
-  local url=$OODI_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
-}
-
-set_up_oodikone() {
-  draw_mopo
-
-  infomsg "Installing npm packages locally to enable linting"
-
-  folders_to_set_up=(
-    "$PROJECT_ROOT"
-    "$PROJECT_ROOT/services/oodikone2-analytics"
-    "$PROJECT_ROOT/services/oodikone2-frontend"
-    "$PROJECT_ROOT/services/oodikone2-userservice"
-    "$PROJECT_ROOT/services/backend/oodikone2-backend"
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
   )
 
-  for folder in "${folders_to_set_up[@]}"; do
-    cd "$folder" || return 1
-    ([[ -d node_modules ]] && warningmsg "Packages already installed in $folder") || npm ci
-  done
-  cd "$PROJECT_ROOT" || return 1
+  local database=""
+  local url=""
 
-  infomsg "Cleaning up previous docker containers, volumes and networks"
-  "$PROJECT_ROOT"/run.sh both down --remove-orphans --volumes
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
+}
+
+set_up_oodikone_from_scratch() {
+  draw_mopo
+  folders_to_set_up=(
+    "$PROJECT_ROOT"
+    "$PROJECT_ROOT/services/frontend"
+    "$PROJECT_ROOT/services/userservice"
+    "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/updater/sis-updater-scheduler"
+    "$PROJECT_ROOT/updater/sis-updater-worker"
+  )
+
+  infomsg "Cleaning up docker setup and node_modules from possible previous installations"
+  local compose_down_args="--remove-orphans --volumes --rmi all"
+  "$PROJECT_ROOT"/run.sh both down "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.test.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.ci.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
+  done
+
+  infomsg "Installing npm packages locally to root and each subfolder to enable \
+linting and formatting"
+
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    ([[ -d node_modules ]] && die "$folder already has node_modules, exiting!") || npm ci
+  done
+  cd "$PROJECT_ROOT" || die "Couldn't change directory to project root"
+
+  infomsg "Setting up husky for pre-commit linting"
+  npm run prepare
 
   infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon pull
 
   infomsg "Building images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon build
 
   successmsg "Setup ready, oodikone can be started! See README for more info."
 }

services/backend/.dockerignore

@@ -0,0 +1,6 @@
+.dockerignore
+Dockerfile
+node_modules/
+requests/
+scripts/
+*.log

services/backend/.gitignore

@@ -0,0 +1,3 @@
+# Student diff output
+/student_diff_*
+/diff_partial_att_data.json

services/backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:14.17-alpine3.13 as base
+
+WORKDIR /usr/src/app
+EXPOSE 8080
+
+FROM base as development
+ENV NODE_ENV=development
+COPY ./package* /usr/src/app/
+RUN npm ci
+COPY . /usr/src/app/
+CMD ["npm", "run", "dev"]
+
+FROM base as production
+ENV NODE_ENV=production
+ARG SENTRY_ENVIRONMENT
+ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+ARG SENTRY_RELEASE
+ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+USER node
+CMD ["npm", "start"]

services/backend/index.js

@@ -0,0 +1,6 @@
+require('./src/app')
+const logger = require('./src/util/logger')
+
+process.on('unhandledRejection', reason => {
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
+})

services/backend/oodikone2-backend/.dockerignore

@@ -1 +0,0 @@
-node_modules

services/backend/oodikone2-backend/.eslintignore

@@ -1 +0,0 @@
-*.sql

services/backend/oodikone2-backend/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

services/backend/oodikone2-backend/.gitignore

@@ -1,18 +0,0 @@
-node_modules
-.vscode
-.nyc_output
-src/tester.js
-update.sh
-.env
-logs/*
-run_update_script.sh
-student_numbers
-debug.log
-npm-debug.log
-
-studentnumbers.txt
-studentnumbers*.txt
-src/anonymized_API
-
-/student_diff_*
-/diff_partial_att_data.json

services/backend/oodikone2-backend/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

services/backend/oodikone2-backend/Dockerfile

@@ -1,16 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-EXPOSE 8080
-CMD npm start
-
-COPY ./oodikone2-backend/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-ARG TAG=local
-ENV TAG=${TAG}
-
-COPY ./oodikone2-backend /usr/src/app

services/backend/oodikone2-backend/README.md

@@ -1,259 +0,0 @@
-# oodikone2-backend
-
-## Dev
-
-### Getting started
-
-To install docker & docker-compose to run postgres and redis follow these steps:
-
-Install docker CE: https://docs.docker.com/engine/installation/ 
-
-Install docker-compose: https://docs.docker.com/compose/install/
-
-Create docker-compose.yml file containing following:
-
-```
-version: '3'
-
-services:
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-    container_name: oodi_db
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-```
-
-To start a redis and a database, run:
-
-```docker-compose up -d```
-
-Copy a backup dump from oodikone
-
-`scp oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/2018-08-01.bak <local folder>`
-
-Create database tkt_oodi:
-
-`docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi"`
-
-Fill database with data from backup file: 
-
-`cat 2018-08-01.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi`
-
-Check that you have all the data you ever wanted:
-
-`docker exec -it -u postgres oodi_db psql -d tkt_oodi`
-
-`\dt`
-
-Create test database:
-
-`docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_test"`
-
-Fill database with data from test.bak dump file: 
-
-`cat test.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi_test`
-
-### Running 
-
-#### Dev
-
-Run `npm run dev`
-
-#### Testing
-
-Run `npm test`
-
-## Deployment
-
-Always deploy first to staging! 
-
-When merging from trunk to master Travis builds a docker image for staging and pushes it automatically to dockerh-hub. Currently **staging** is also automatically deployed after the image is pushed.
-
-When a release (or tag) is created in master branch Travis builds a docker image for production and pushes it automatically.
-
-### To update: 
-
-Log into svm-59 (aka oodikone)
-
-`ssh {username}@oodikone.cs.helsinki.fi`
-
-Switch user to tkt_oodi
-
-`sudo su - tkt_oodi`
-
-Navigate to correct folder
-`cd oodikone.cs.helsinki.fi` or `cd oodikone.cs.helsinki.fi/staging`
-
-Use the update script to restart the software
-`./update.sh`
-
-Be amazed!
-
-## In case you mess up your DB
-
-Drop your current db
-
-`docker exec -u postgres oodi_db dropdb tkt_oodi`
-
-Copy a backup dump from oodikone
-
-`scp oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/2018-08-01.bak <local folder>`
-
-Create a new database and fill it with the backup dump following the instructions above
-
-## Accessing production/staging DB
-
-To access the database in oodikone.cs.helsinki.fi you can use `postgres_connect.sh` script in production/staging directory. Alternatively you can use commands: 
-
-`docker exec -it -u postgres db psql -d tkt_oodi`
-
-or
-
-`docker exec -it -u postgres db_staging psql -d tkt_oodi_staging`
-
-Production db backups are taken daily and can be found in /var/toska/tkt_oodi/backups/postgres/. Symlink to latest backup can always be found as /home/tkt_oodi/backups/latest-pg.bak.bz2.
-
-To create a dump of the production db manually, run: 
-
-``docker exec -u postgres db pg_dump tkt_oodi > `date +%Y-%m-%d`.bak``
-
-## Updater
-
-Oodikone2-backend includes the update-script that fetches student data from WebOodi. Script is split on two parts. Both production and staging environments have separate updater set up, in directories /oodikone.cs.helsinki.fi/updater and /oodikone.cs.helsinki.fi/staging/updater respectively. Both of updater directories have README file with summary of commands. Container names are production_updater and staging_updater, replace in examples below CONTAINER_NAME with correct one. Both containers have updater_files-directory that is mounted within container to /updater_files.
-
-Updater uses respective backend-image (production or staging, respectively).
-
-### Student list updater
-
-Finds out the valid student nubers. 
-
-Run in updater directory with _update_student_list.sh_
-
-Updater is run in separate detached container, see graylog or _debug.log_ to find out when it is finished.
-
-Environment variable _INCREMENT_ (currently not set) controls how many new student numbers are searched (from the currently known greatest student number).
-
-At the moment script uses the local postgres (see .env) to save student numbers.
-
-### Student info updater (database updater)
-
-To update your database run `docker-compose run -d CONTAINER_NAME npm run update_database` with optional args `file="<filename>"` and `index=<number>` with former telling the studentnumber filename to update and latter telling which row to start from.
-
-
-### Anonymized data & E2E environment
-
-Use [oodikone-cli](https://github.com/UniversityOfHelsinkiCS/oodikone) to setup you're environments.
-
-Before running cli make yourself a deployment key for Anonyymioodi private repository and put it in your cli folder as `private.key`. This allows you to download the anonymized dumps from the repository.
-
-For local development use with anonymized data use `2) Anon setup` (you can run e2e tests with this).
-
-`1) e2e setup` is designed for use in travis. But you can still try it locally if you want to.
-
-Run E2E tests in ToskaWatch with `npm run cypress:run` or `npm run cypress:open` for local tests.
-
-
-## Teacher leader board statistics
-
-The statistics for the teacher leader board have to be calculated separately with a script.
-
-```
-# Update top teachers for academic years 50 (1999-20) through 70 (2018-19).
-docker-compose run -d CONTAINER_NAME npm run update_top_teachers from=50 to=70
-
-# Omit the to–argument to update teachers just for a single academic year. 
-docker-compose run -d CONTAINER_NAME npm run update_top_teachers from=50
-```
-
-## DEFA report
-
-The npm script `create_defa_report` can quickly aggregate data about DEFA students and display it in neat csv format. Here's how you can use it on the Oodikone production server. It does not work if run locally.
-
-### Input files
-
-Input files should be placed in the `/home/tkt_oodi/oodikone.cs.helsinki.fi/defa` directory unless a different directory has been included into the backend docker container as `/usr/src/app/defa`.
-
-* `params.json`
-  * Defines the input/output files. File paths are for the container, so the working directory is `/usr/src/app/defa/`.
-  * Defines the timeframe to look at: from `params.in.timeframe.start` to `params.in.timeframe.end`. Studyattainments outside this timeframe will not be considered to be part of DEFA. Changing the timeframe is the main reason you'd want to make changes to this file.
-  * Example:
-```
-{
-  "out": {
-    "report": "/usr/src/app/defa/defa-report.csv",
-    "stats": "/usr/src/app/defa/defa-stats.csv"
-  },
-  "in": {
-    "course_ids": "/usr/src/app/defa/ids.txt",
-    "included_courses": "/usr/src/app/defa/included.txt",
-    "required_courses": "/usr/src/app/defa/required.txt",
-    "timeframe": {
-      "start": "2018-09-01T01:00:00.000Z",
-      "end": "2019-06-30T01:00:00.000Z"
-    }
-  }
-}
-```
-
-* `ids.txt` (or another name specified in `params.in.course_ids`)
-  * Holds the courseunitrealisation ids of DEFA courses to be included in the report.
-  * Example:
-```
-126240934
-126237007
-126238082
-126239325
-12699580
-```
-
-* `included.txt` (or another name specified in `params.in.included_courses`)
-  * Holds the learningopportunity ids of courses to be included in the report. Basically put all DEFA course codes here.
-  * One id per line.
-  * AY courses like `AYTKT0000` can be entered without the `AY` prefix to match both with and without it. e.g. `TKT0000` matches both `TKT0000` and `AYTKT0000`. `AYTKT0000` will only match `AYTKT0000`.
-  * Example:
-```
-TKT10002
-TKT10003
-MAT11001
-TKT10004
-TKT20001
-TKT20004
-TKT20009
-TKT21007
-MAT12001
-MAT12002
-```
-
-* `required.txt` (or another name specified in `params.in.required_courses`)
-  * Holds the learningopportunity ids of courses that are required to complete for a DEFA student to qualify.
-  * Same format as `included.txt`.
-  * All entries in this file should also be in `included.txt`.
-  * Example: same as `included.txt`
-
-### Running
-
-1. Make sure the backend docker container is running.
-2. run `docker exec <backend container> npm run create_defa_report defa/<parameter json file>` or use the ready made script `/home/tkt_oodi/oodikone.cs.helsinki.fi/defa/create_report.sh`.
-3. Check the output files.
-
-### Output files
-
-Output files will be created in the same directory where the input files are. Old output files will be overwritten unless they have been renamed/moved. 
-
-* `defa-report.csv` (or another name defined in `params.out.report`)
-  * Contains individual information about each DEFA student and individual inforamtion about each DEFA course.
-  * Contains aggregate information about all DEFA students.
-
-* `defa-stats.csv` (or another name defined in `params.out.stats`)
-  * Contains the credit amount distribution of DEFA students.

services/backend/oodikone2-backend/enableUser.rest

@@ -1,6 +0,0 @@
-POST http://localhost:8080/api/users/enable
-Content-Type: application/json
-
-{
-  "id": 2
-}

services/backend/oodikone2-backend/index.js

@@ -1,6 +0,0 @@
-//require('./src/tester')
-require('./src/app')
-
-process.on('unhandledRejection', reason => {
-  console.log(reason)
-})

services/backend/oodikone2-backend/package.json

@@ -1,89 +0,0 @@
-{
-  "name": "oodikone-backend",
-  "version": "0.0.1",
-  "description": "Backend for Oodikone",
-  "main": "index.js",
-  "jest": {
-    "setupFilesAfterEnv": [
-      "./test/jest/globals.js"
-    ],
-    "testEnvironment": "node"
-  },
-  "scripts": {
-    "start": "node --max_old_space_size=4096 index.js",
-    "dev": "NODE_ENV=dev nodemon --max_old_space_size=4096 index.js",
-    "lint": "eslint .",
-    "lintfix": "prettier --write '**/*.{js,json}' && eslint --fix .",
-    "diff:students": "node /usr/src/app/src/sisoodi_diff/students/index.js",
-    "diff:populations": "node /usr/src/app/src/sisoodi_diff/populations.js",
-    "diff:courses": "node /usr/src/app/src/sisoodi_diff/courses/index.js",
-    "diff:transfers": "node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "node /usr/src/app/src/sisoodi_diff/trends.js"
-  },
-  "author": "Matti Luukkainen",
-  "license": "MIT",
-  "dependencies": {
-    "@sentry/node": "^5.30.0",
-    "async": "^3.2.0",
-    "async-lock": "^1.2.8",
-    "axios": "^0.21.1",
-    "bluebird": "^3.7.2",
-    "body-parser": "^1.19.0",
-    "cors": "^2.8.5",
-    "cron": "^1.8.2",
-    "dotenv": "^6.2.0",
-    "express": "^4.17.1",
-    "jsonwebtoken": "^8.5.1",
-    "kafka-node": "^4.1.3",
-    "lodash": "^4.17.21",
-    "lru-cache": "^5.1.1",
-    "mkdirp": "^0.5.5",
-    "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
-    "morgan": "^1.10.0",
-    "nodemailer": "^4.7.0",
-    "pg": "^8.6.0",
-    "redis": "^3.1.1",
-    "sequelize": "^6.6.2",
-    "umzug": "^2.3.0",
-    "unfuck-utf8-headers-middleware": "^1.0.1",
-    "uuid": "^3.4.0",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0",
-    "winston-transport": "^4.4.0"
-  },
-  "devDependencies": {
-    "ava": "^2.4.0",
-    "babel-eslint": "^8.2.6",
-    "eslint": "^5.16.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "faker": "^4.1.0",
-    "jest": "^24.9.0",
-    "nock": "^10.0.6",
-    "node-status": "^1.0.0",
-    "nodemon": "^1.19.4",
-    "nyc": "^13.3.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2",
-    "supertest": "^3.4.2"
-  },
-  "ava": {
-    "files": [
-      "test/**/*_test.js"
-    ],
-    "source": [
-      "**/*.{js,jsx}",
-      "!test/**/*"
-    ],
-    "concurrency": 5,
-    "failFast": true,
-    "failWithoutAssertions": true,
-    "tap": true,
-    "powerAssert": false,
-    "require": [
-      "babel-register"
-    ],
-    "babel": "inherit"
-  }
-}

services/backend/oodikone2-backend/requests/courses_yearly.rest

@@ -1,7 +0,0 @@
-GET /api/courseyearlystats/?start=2013&end=2018&code=TKT10002 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token generated from login.rest to use

services/backend/oodikone2-backend/requests/get_all_duplicate_codes.rest

@@ -1,6 +0,0 @@
-GET /api/courses/duplicatecodes/all HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/oodikone2-backend/requests/get_duplicate_codes.rest

@@ -1,7 +0,0 @@
-GET /api/courses/duplicatecodes/?code=TKT10005 HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/oodikone2-backend/requests/get_mandatory_courses.rest

@@ -1,7 +0,0 @@
-GET /api/v2/studyprogrammes/{insertstudyprogrammecodehere}/mandatory_courses
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/oodikone2-backend/requests/get_student.rest

@@ -1,7 +0,0 @@
-GET /api/students/{insert student id}
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest and a student id to use

services/backend/oodikone2-backend/requests/login.rest

@@ -1,7 +0,0 @@
-POST /api/login HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-Cache-Control: no-cache
-Postman-Token: d8a05d0f-fd91-e4e0-54a0-172d18ff3fad
-Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW

services/backend/oodikone2-backend/requests/post_duplicate_codes.rest

@@ -1,7 +0,0 @@
-POST /api/courses/duplicatecodes/TKT10005/205044 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token:
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/oodikone2-backend/requests/remove_duplicate_code.rest

@@ -1,7 +0,0 @@
-DELETE /api/courses/duplicatecodes/TKT10005/58123 HTTP/1.1
-Host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-
-## Insert valid access token (without quotes) generated from login.rest to use

services/backend/oodikone2-backend/requests/update.rest

@@ -1,11 +0,0 @@
-POST /api/updatedatabase HTTP/1.1
-host: localhost:8080
-uid: tktl
-shib-session-id: mock-session
-x-access-token: 
-content-type: application/json
-
-[
-  "01234562",
-  "12302459"
-]

services/backend/oodikone2-backend/rest/newUser.rest

@@ -1,4 +0,0 @@
-GET http://localhost:8080/api/login
-Content-Type: application/json
-eduPersonPrincipalName: totutotu@
-shib-session-id: IDEEE

services/backend/oodikone2-backend/scripts/create_course_groups_kayttis.js

@@ -1,34 +0,0 @@
-const { createCourseGroup } = require('../src/services/courseGroups')
-
-const run = async () => {
-  await Promise.all(groups.map(name => createCourseGroup(name)))
-  process.exit(0)
-}
-
-const groups = [
-  'Yleinen ja aikuiskasvatustiede',
-  'Kasvatustiede, yleinen didaktiikka',
-  'Kasvatuspsykologia',
-  'Varhaiskasvatus (VO+VAKAMO)',
-  'Erityispedagogiikka',
-  'Kotitaloustiede',
-  'Käsityötiede',
-  'Aikuisopetus',
-  'Biologia ja maantiede',
-  'Historia ja yhteiskuntaoppi',
-  'Uskonto',
-  'Elämänkatsomustieto',
-  'Suomen kieli ja kirjallisuus',
-  'Vieraat kielet',
-  'Matematiikka',
-  'Fysiikka ja kemia',
-  'Musiikki',
-  'Kuvataide',
-  'Liikunta',
-  'Draama',
-  'Mediakasvatus',
-  'Tiedekeskuspedagogiikka',
-  'Svenska studier'
-]
-
-run()

services/backend/oodikone2-backend/scripts/create_fmsei_report.js

@@ -1,132 +0,0 @@
-/* eslint-disable no-console */
-
-const { Student, Credit, Studyright } = require('../src/models')
-const Sequelize = require('sequelize')
-const Op = Sequelize.Op
-
-const startDate = new Date(2019, 1, 1, 0, 0, 0, 0)
-
-const wasFMSEIStudent = (student, date) => {
-  const degreeright = student.studyrights.find(sr => sr.extentcode !== 9 && sr.startdate <= date && sr.enddate >= date)
-  return !degreeright
-}
-
-const courseCodes = [
-  'AYTKT200041',
-  'AYTKT21007',
-  'TKT21007',
-  'AYTKT21025en',
-  'AYTKT21009',
-  'TKT21009',
-  'AYTKT21010',
-  'TKT21010',
-  'AYTKT21018',
-  'AYTKT21018fi',
-  'AYTKT21018sv',
-  'AY5823951',
-  'AY5823952',
-  'AY5823953',
-  'AY5823954',
-  'AY5823955',
-  'AY5823956'
-]
-
-const createReport = async () => {
-  console.log('SPLIT')
-  const { fmseiStudentnumbers, otherStudentnumbers } = await courseCodes.reduce(
-    async (accPromise, courseCode) => {
-      const credits = await Credit.findAll({
-        where: {
-          course_code: courseCode,
-          attainment_date: {
-            [Op.gte]: startDate
-          }
-        }
-      })
-
-      const { fmseiStudentnumbers, otherStudentnumbers } = await credits.reduce(
-        async (accPromise, credit) => {
-          const student = await Student.findOne({
-            where: { studentnumber: credit.student_studentnumber },
-            include: [{ model: Studyright }]
-          })
-          const acc = await accPromise
-
-          if (wasFMSEIStudent(student, credit.attainment_date)) {
-            // FMSEI student fmsei credits
-            console.log(
-              `${credit.student_studentnumber};${credit.course_code};${credit.grade};${credit.credits};${new Date(
-                credit.attainment_date
-              ).toISOString()}`
-            )
-            return {
-              fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(student.studentnumber),
-              otherStudentnumbers: acc.otherStudentnumbers
-            }
-          }
-          return {
-            fmseiStudentnumbers: acc.fmseiStudentnumbers,
-            otherStudentnumbers: acc.otherStudentnumbers.concat(student.studentnumber)
-          }
-        },
-        { fmseiStudentnumbers: [], otherStudentnumbers: [] }
-      )
-
-      const acc = await accPromise
-
-      return {
-        courseStats: acc.courseStats.concat({
-          courseCode: courseCode,
-          fmsei: fmseiStudentnumbers.length,
-          other: otherStudentnumbers.length
-        }),
-        fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(fmseiStudentnumbers),
-        otherStudentnumbers: acc.otherStudentnumbers.concat(otherStudentnumbers)
-      }
-    },
-    { courseStats: [], fmseiStudentnumbers: [], otherStudentnumbers: [] }
-  )
-  const fmseiStudents = await Student.findAll({
-    where: { studentnumber: fmseiStudentnumbers },
-    include: [
-      {
-        model: Credit,
-        separate: true
-      }
-    ]
-  })
-
-  const otherStudents = await Student.findAll({
-    where: { studentnumber: otherStudentnumbers }
-  })
-  console.log('SPLIT')
-  otherStudents.forEach(s => console.log(`${new Date(s.birthdate).getFullYear()};${s.gender_fi}`))
-  console.log('SPLIT')
-
-  const getFirstName = namestring => {
-    const names = namestring.split(' ')
-    const i = names.findIndex(n => n.substring(0, 1) === '*')
-    return i != -1 ? names[i].substring(1) : names[0]
-  }
-
-  fmseiStudents.forEach(s =>
-    console.log(
-      `${s.studentnumber};${s.email || ''};${s.zipcode || ''};${s.city_fi || ''};${new Date(
-        s.birthdate
-      ).getFullYear()};${s.country_fi || ''};${s.home_country_fi || ''};${s.gender_fi};${getFirstName(s.firstnames)}`
-    )
-  )
-  console.log('SPLIT')
-  // FMSEI all credits
-  fmseiStudents.forEach(s => {
-    s.credits.forEach(c =>
-      console.log(
-        `${c.student_studentnumber};${c.course_code};${c.grade};${c.credits};${new Date(
-          c.attainment_date
-        ).toISOString()}`
-      )
-    )
-  })
-}
-
-createReport()

services/backend/oodikone2-backend/scripts/force_sync_database.js

@@ -1,16 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-const { forceSyncDatabase } = require('../src/database/connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-sync()

services/backend/oodikone2-backend/scripts/refresh_studyright_associations.js

@@ -1,8 +0,0 @@
-const { refreshAssociationsInRedis } = require('../src/services/studyrights')
-
-const run = async () => {
-  await refreshAssociationsInRedis()
-  process.exit(0)
-}
-
-run()

services/backend/oodikone2-backend/scripts/seed_migrations.js

@@ -1,13 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-
-const run = async () => {
-  try {
-    await seedAllMigrations()
-    process.exit(0)
-  } catch (e) {
-    console.log(`Seeding migrations failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/oodikone2-backend/scripts/update_top_teachers.js

@@ -1,38 +0,0 @@
-const { findAndSaveTopTeachers } = require('../src/services/topteachers')
-const logger = require('../src/util/logger')
-
-const findAndSaveTeachers = async (startcode, endcode) => {
-  for (let code = startcode; code <= endcode; code++) {
-    logger.info(`Saving top teachers from year ${code}`)
-    await findAndSaveTopTeachers(code)
-  }
-}
-
-const parseargs = (args, SEPARATOR = '=') =>
-  args
-    .filter(arg => arg.includes(SEPARATOR))
-    .reduce((acc, arg) => {
-      const [key, val] = arg.split(SEPARATOR)
-      return {
-        ...acc,
-        [key.trim()]: val.trim()
-      }
-    }, {})
-
-const run = async () => {
-  try {
-    const { from, to } = parseargs(process.argv)
-    const start = from
-    const end = to || from
-    if (end < start) {
-      throw Error('"to" -argument has to be less than "from" -argument. ')
-    }
-    await findAndSaveTeachers(start, end)
-    process.exit(0)
-  } catch (error) {
-    logger.error(`Failed to update top teachers: ${error.message}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/oodikone2-backend/src/anonymizer/anonDisciplines.js

@@ -1,115 +0,0 @@
-const disciplines = {
-  data: {
-    data: [
-      {
-        discipline_id: '0',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Witchcraft'
-          }
-        ]
-      },
-      {
-        discipline_id: '1',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Magic'
-          }
-        ]
-      },
-      {
-        discipline_id: '2',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Alchemy'
-          }
-        ]
-      },
-      {
-        discipline_id: '3',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Arithmancy'
-          }
-        ]
-      },
-      {
-        discipline_id: '4',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Runes'
-          }
-        ]
-      },
-      {
-        discipline_id: '5',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Creatures'
-          }
-        ]
-      },
-      {
-        discipline_id: '6',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Astronomy'
-          }
-        ]
-      },
-      {
-        discipline_id: '7',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'History'
-          }
-        ]
-      },
-      {
-        discipline_id: '8',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Charms'
-          }
-        ]
-      },
-      {
-        discipline_id: '9',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Flight'
-          }
-        ]
-      },
-      {
-        discipline_id: '10',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Necromancy'
-          }
-        ]
-      },
-      {
-        discipline_id: '11',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Pyromancy'
-          }
-        ]
-      }
-    ]
-  }
-}
-module.exports = { disciplines }

services/backend/oodikone2-backend/src/anonymizer/anonStudyRights.js

@@ -1,63 +0,0 @@
-const studyRightCodes = {
-  a: {
-    name: 'Transfiguration',
-    code: '666ABA'
-  },
-  b: {
-    name: 'Defence Against the Dark Arts',
-    code: '666ABB'
-  },
-  c: {
-    name: 'Charms',
-    code: '666ABC'
-  },
-  d: {
-    name: 'Potions',
-    code: '666ABD'
-  },
-  e: {
-    name: 'Astronomy',
-    code: '666ABE'
-  },
-  f: {
-    name: 'History of Magic',
-    code: '666ABF'
-  },
-  g: {
-    name: 'Herbology',
-    code: '666ABG'
-  },
-  h: {
-    name: 'Arithmancy',
-    code: '666ABH'
-  },
-  i: {
-    name: 'Muggle Studies',
-    code: '666ABI'
-  },
-  j: {
-    name: 'Divination',
-    code: '666ABJ'
-  },
-  k: {
-    name: 'Study of Ancient Runes',
-    code: '666ABK'
-  },
-  l: {
-    name: 'Care of Magical Creatures',
-    code: '666ABL'
-  },
-  m: {
-    name: 'Alchemy',
-    code: '666ABM'
-  },
-  bachelor: {
-    name: 'Bachelors Degree on Witchcraft and Wizardry',
-    code: '666bach'
-  },
-  master: {
-    name: 'Master of Witchcraft and Wizardry',
-    code: '666mast'
-  }
-}
-module.exports = { studyRightCodes }

services/backend/oodikone2-backend/src/anonymizer/anon_oodi_interface.js

@@ -1,104 +0,0 @@
-const axios = require('axios')
-const https = require('https')
-require('dotenv').config()
-
-const instance = axios.create({
-  httpsAgent: new https.Agent({
-    rejectUnauthorized: false
-  })
-})
-
-axios.defaults.auth = {
-  username: 'tktl'
-}
-
-instance.interceptors.request.use(config => {
-  config.params = { token: process.env.TOKEN }
-  return config
-})
-
-const getStudent = studentNumber => {
-  return instance
-    .get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)
-    .then(response => {
-      return response.data.data
-    })
-    .catch(error => {
-      return error
-    })
-}
-const getStudyAttainments = async studentNumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentNumber}/studyattainments`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getTeacher = async teacherId => {
-  const url = `${process.env.OODI_ADDR}/teachers/${teacherId}/info`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getStudyRights = async studentNumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentNumber}/studyrights`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getCourseTypeCodes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/learningopportunities/types`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/studyattainments/statuses`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getSemesters = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/semesters`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentnumber}/semesterenrollments`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/courseunitrealisations/types`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${process.env.OODI_ADDR}/learningopportunities/${id}`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const courseUnitRealisations = async () => {
-  const url = `${process.env.OODI_ADDR}/courseunitrealisations/changes/ids/0000-12-24`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${process.env.OODI_ADDR}/courseunitrealisations/${id}`
-  const response = await instance.get(url)
-  return response.data.data
-}
-module.exports = {
-  getStudyRights,
-  getStudent,
-  getTeacher,
-  getStudyAttainments,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisations,
-  getCourseUnitRealisation
-}

services/backend/oodikone2-backend/src/anonymizer/anonymizer.js

@@ -1,489 +0,0 @@
-const faker = require('faker')
-const moment = require('moment')
-const mkdirp = require('mkdirp')
-const getDirName = require('path').dirname
-const logger = require('../util/logger')
-const { disciplines } = require('./anonDisciplines')
-const { studyRightCodes } = require('./anonStudyRights')
-const oodi = require('./anon_oodi_interface')
-const fs = require('fs')
-const _ = require('lodash')
-const status = require('node-status')
-
-require('dotenv').config()
-const startTime = moment()
-
-let student_numbers = []
-
-// few objects to save connections between anonymized and real data to keep relations.
-let teacherRelations = {}
-let studentRelations = {}
-let learningOpportunityRelations = {}
-
-const startCodesStatusBar = () => {
-  status.start({ pattern: 'Running: {uptime.time} {spinner.monkey.blue} | {codes.bar} | codes anonymized: {codes}' })
-}
-const startStudentsStatusBar = () => {
-  status.start({
-    pattern: 'Running: {uptime.time} {spinner.hearts.magenta} | {students.bar} | students anonymized: {students}'
-  })
-}
-const startRealisationsStatusBar = () => {
-  status.start({
-    pattern: 'Running: {uptime.time} {spinner.clock.red} | {realisations.bar} | realisations anonymized: {realisations}'
-  })
-}
-const stopStatusBar = () => {
-  status.stamp()
-  status.stop()
-}
-
-const getAnonymizedTeacher = async teacherId => {
-  if (teacherRelations[teacherId]) {
-    return teacherRelations[teacherId]
-  }
-  let teacher = await oodi.getTeacher(teacherId)
-  const last_name = faker.name.lastName()
-  const first_names = faker.name.firstName()
-  teacher = {
-    ...teacher,
-    teacher_id: generateRandomId(),
-    userid: faker.internet.userName(),
-    last_name,
-    first_names,
-    calling_name: faker.name.firstName(),
-    full_name: `${last_name} ${first_names}`,
-    email: faker.internet.email(),
-    mobile_phone: faker.phone.phoneNumberFormat()
-  }
-  teacherRelations[teacherId] = teacher
-  return teacher
-}
-
-const generateRandomId = () => String(Math.floor(Math.random() * (999999999 - 100000000 + 1) + 100000000))
-
-const anonymizeStudentInfo = async id => {
-  let student = await oodi.getStudent(id)
-  let anonStudent = undefined
-  if (student) {
-    anonStudent = studentRelations[student.student_number]
-  }
-  if (!anonStudent) {
-    anonStudent = {
-      ...student,
-      first_names: faker.name.firstName(),
-      last_name: faker.name.lastName(),
-      birthdate: new Date(
-        moment(
-          moment('1940', 'YYYY').valueOf() +
-            Math.random() *
-              (moment()
-                .subtract(17, 'years')
-                .valueOf() -
-                moment('1940', 'YYYY').valueOf())
-        )
-      ),
-      email: faker.internet.email(),
-      mobile_phone: faker.phone.phoneNumberFormat(),
-      national_student_number: null,
-      phone: faker.phone.phoneNumberFormat(),
-      address1: faker.address.streetName(),
-      address2: faker.address.streetName(),
-      zipcode: faker.address.zipCode(),
-      student_number: `0${Math.floor(Math.random() * (1999999 - 1000000 + 1) + 1000000)}`,
-      person_id: generateRandomId(),
-      city: [
-        {
-          langcode: 'en',
-          text: 'Harlem'
-        },
-        {
-          langcode: 'fi',
-          text: 'Kajaani'
-        }
-      ],
-      country: [
-        {
-          langcode: 'en',
-          text: 'Canada'
-        },
-        {
-          langcode: 'fi',
-          text: 'Ruotsi'
-        }
-      ]
-    }
-  }
-  if (!student) {
-    return anonStudent
-  }
-  studentRelations[student.student_number] = anonStudent
-
-  return anonStudent
-}
-const getRandomAmount = attainments => {
-  const rate = Math.random() * (0.375 - 0.05) + 0.05
-  const amount = Math.floor(rate * attainments.length)
-  return amount
-}
-const dropRandomAttainments = attainments => {
-  if (student_numbers.length < 1) {
-    return attainments
-  }
-  const dropAmount = getRandomAmount(attainments)
-  let i = 0
-  while (i < dropAmount) {
-    let randomIndex = Math.floor(Math.random() * attainments.length)
-    attainments.splice(randomIndex, 1)
-    i++
-  }
-  return attainments
-}
-const addRandomAttainments = (attainments, studentnumber_attainments) => {
-  if (studentnumber_attainments.length < 1) {
-    return attainments
-  }
-  let studentnumbers = studentnumber_attainments.map(s => s.student_number)
-  const addAmount = getRandomAmount(attainments)
-  let i = 0
-  while (i < addAmount) {
-    let randomStudentNumber = _.sample(studentnumbers)
-    let randomAttainment = _.sample(
-      _.flattenDeep(
-        studentnumber_attainments.filter(s => s.student_number === randomStudentNumber).map(s => s.attainments)
-      )
-    )
-    if (!randomAttainment) {
-      continue
-    }
-    randomAttainment.semester_code = _.sample(attainments).semester_code // Make attainment happen at the same time as the students some other attainment.
-    attainments = attainments.concat(randomAttainment)
-    i++
-  }
-  return attainments
-}
-const populateAnonymizedTeachers = async teachers => {
-  let anonTeachers = []
-  for (let teacher of teachers) {
-    let anonTeacher = await getAnonymizedTeacher(teacher.teacher_id)
-    APIWriter(`./src/anonymized_API/teachers/${anonTeacher.teacher_id}/info`, { data: { data: anonTeacher } })
-    anonTeachers = anonTeachers.concat(anonTeacher)
-  }
-  return anonTeachers
-}
-
-const anonymizeStudentAttainments = async (id, studentInfo, studentnumber_attainments) => {
-  let attainments = await oodi.getStudyAttainments(id)
-  attainments = dropRandomAttainments(attainments)
-  attainments = addRandomAttainments(attainments, studentnumber_attainments)
-  let anonAttainments = []
-  for (let attainment of attainments) {
-    let anonLearningOpportunity = learningOpportunityRelations[attainment.learningopportunity_id]
-    if (!anonLearningOpportunity) {
-      const learningOpportunity = await oodi.getLearningOpportunity(attainment.learningopportunity_id)
-      if (learningOpportunity === null) {
-        continue
-      }
-      anonLearningOpportunity = {
-        ...learningOpportunity,
-        disciplines: [disciplines.data.data[Math.floor(Math.random() * 12)]],
-        learningopportunity_id: generateRandomId(),
-        names: [
-          {
-            langcode: 'en',
-            text: faker.company.bs()
-          },
-          {
-            langcode: 'fi',
-            text: faker.company.bs()
-          }
-        ],
-        descriptions: [],
-        materials: [],
-        organisations: [],
-        learningopportunity_internal: generateRandomId(),
-        start_date: moment(learningOpportunity.start_date)
-          .add(_.sample([-3, 0, 3]), 'months')
-          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days')),
-        end_date: moment(learningOpportunity.start_date)
-          .add(_.sample([-3, 0, 3]), 'months')
-          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days'))
-          .add(100, 'years')
-      }
-      learningOpportunityRelations[attainment.learningopportunity_id] = anonLearningOpportunity
-      APIWriter(`./src/anonymized_API/learningopportunities/${anonLearningOpportunity.learningopportunity_id}`, {
-        data: { data: anonLearningOpportunity }
-      })
-    }
-    let teachers = []
-    teachers = await populateAnonymizedTeachers(attainment.teachers)
-
-    // semester code is calculated as "code * 6 months since 1950-1-1", Im bucketing all study attainments in 3 month periods by semester code abstracting real study attainment dates
-    const attainment_date = new Date(
-      moment(moment('1950', 'YYYY').add(attainment.semester_code * 6 + _.sample([-3, 0, 3]), 'months'))
-    )
-    attainment = {
-      ...attainment,
-      studyattainment_id: generateRandomId(),
-      person_id: studentInfo.person_id,
-      attainment_date,
-      teachers,
-      learningopportunity_id: anonLearningOpportunity.learningopportunity_id,
-      learningopportunity_name: anonLearningOpportunity.names
-    }
-    anonAttainments = anonAttainments.concat(attainment)
-  }
-  return anonAttainments
-}
-
-const anonymizeStudentStudyRights = async (id, studentInfo) => {
-  let studyRights = await oodi.getStudyRights(id)
-  let anonStudyRights = []
-  for (let studyright of studyRights) {
-    let program = _.sample(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
-    let degree = 'bachelor'
-    if (studyright.elements.some(element => element.name.some(name => name.text.includes('Master')))) {
-      program = _.sample(['i', 'j', 'k', 'l', 'm', 'h'])
-      degree = 'master'
-    }
-    const elements = studyright.elements
-      .filter(element => element.element_id === 10 || element.element_id === 20)
-      .map(element => {
-        if (element.element_id === 10) {
-          return {
-            ...element,
-            code: studyRightCodes[degree].code,
-            name: [
-              {
-                langcode: 'en',
-                text: studyRightCodes[degree].name
-              },
-              {
-                langcode: 'fi',
-                text: studyRightCodes[degree].name
-              }
-            ]
-          }
-        } else if (element.element_id === 20) {
-          return {
-            ...element,
-            code: studyRightCodes[program].code,
-            name: [
-              {
-                langcode: 'en',
-                text: studyRightCodes[program].name
-              },
-              {
-                langcode: 'fi',
-                text: studyRightCodes[program].name
-              }
-            ]
-          }
-        }
-        return
-      })
-
-    let anonStudyright = {
-      ...studyright,
-      studyright_id: generateRandomId(),
-      person_id: studentInfo.person_id,
-      elements,
-      organisation_name: [
-        {
-          langcode: 'en',
-          text: 'Hogwarts School of Witchcraft and Wizardry'
-        },
-        {
-          langcode: 'fi',
-          text: 'Tylypahkan noitien ja velhojen koulu'
-        }
-      ]
-    }
-    anonStudyRights = anonStudyRights.concat(anonStudyright)
-  }
-  return anonStudyRights
-}
-const getElapsedTime = () =>
-  `${moment().diff(startTime, 'minutes')} m, ${moment().diff(startTime, 'seconds') /
-    (moment().diff(startTime, 'minutes') * 60)} s.`
-const anonymizeSemesterEnrollments = async id => {
-  const data = await oodi.getSemesterEnrollments(id)
-  let anonymizedData = []
-  for (const enrollment of data) {
-    anonymizedData = anonymizedData.concat({
-      ...enrollment,
-      yths_code: null,
-      absence_reason_code: null,
-      payment_amount: 100,
-      yths_description: null,
-      semester_enrollment_date: moment(moment(enrollment.semester_enrollment_date).format('YYYY-MM'), 'YYYY-MM'),
-      payment_date: moment(moment(enrollment.payment_date).format('YYYY-MM'), 'YYYY-MM')
-    })
-  }
-  return anonymizedData
-}
-const anonymizeStudent = async (id, student_numbers) => {
-  const info = await anonymizeStudentInfo(id)
-
-  const attainments = await anonymizeStudentAttainments(id, info, studentnumber_attainments)
-  studentnumber_attainments = studentnumber_attainments.concat({
-    student_number: info.student_number,
-    attainments: attainments
-  })
-  info.studyattainments = attainments
-    .filter(
-      attainment => ['Hyv.', '1', '2', '3', '4', '5'].includes(attainment.grade[0].text) && attainment.credits < 25
-    )
-    .map(attainment => attainment.credits)
-    .reduce((acc, curr) => acc + curr, 0)
-
-  const studyrights = await anonymizeStudentStudyRights(id, info)
-
-  const semesterEnrollments = await anonymizeSemesterEnrollments(id)
-  const infoPath = `./src/anonymized_API/students/${info.student_number}/info`
-  const semesterEnrollmentPath = `./src/anonymized_API/students/${info.student_number}/semesterenrollments`
-  const courseEnrollmentPath = `./src/anonymized_API/students/${info.student_number}/enrollments`
-  const attainmentPath = `./src/anonymized_API/students/${info.student_number}/studyattainments`
-  const studyRightPath = `./src/anonymized_API/students/${info.student_number}/studyrights`
-  student_numbers = student_numbers.concat(info.student_number)
-
-  APIWriter(infoPath, { data: { data: info } })
-  APIWriter(studyRightPath, { data: { data: studyrights } })
-  APIWriter(courseEnrollmentPath, { data: { data: [] } })
-  APIWriter(attainmentPath, { data: { data: attainments } })
-  APIWriter(semesterEnrollmentPath, { data: { data: semesterEnrollments } })
-  return student_numbers
-}
-const anonymizeCourseUnitRealisations = async () => {
-  const courseUnitRealisations = await oodi.courseUnitRealisations()
-  const realisationsCounter = status.addItem('realisations', { max: courseUnitRealisations.length })
-  startRealisationsStatusBar()
-  let anonRealisations = []
-  for (const realisation of courseUnitRealisations) {
-    continue
-    realisationsCounter.inc() // eslint-disable-line
-    if (!learningOpportunityRelations[realisation.learningopportunity_id]) {
-      continue
-    }
-    const anonLearningId = learningOpportunityRelations[realisation.learningopportunity_id].learningopportunity_id
-    const anonRealisation = {
-      ...realisation,
-      course_id: generateRandomId(),
-      learningopportunity_id: anonLearningId
-    }
-    anonRealisations = anonRealisations.concat(anonRealisation)
-    const courseUnitRealisation = await oodi.getCourseUnitRealisation(realisation.course_id)
-    if (!courseUnitRealisation) {
-      continue
-    }
-    let anonStudents = []
-    for (const student of courseUnitRealisation.students) {
-      if (student) {
-        anonStudents = anonStudents.concat(await anonymizeStudentInfo(student.student_number))
-      }
-    }
-
-    let anonTeachers = []
-    for (const teacher of courseUnitRealisation.teachers) {
-      anonTeachers = anonTeachers.concat(await getAnonymizedTeacher(teacher.teacher_id))
-    }
-
-    const anonCourseUnitRealisation = {
-      ...courseUnitRealisation,
-      realisation_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      students: anonStudents,
-      providing_organisation: [],
-      instruction_locality: [],
-      events: [],
-      officials: [],
-      child_ids: [],
-      courses_extra_organisations: [],
-      teachers: anonTeachers, // note that new teachers created here are not populated in the API! (for future possible issues) regards: Saus 31.7.2018
-      organisations: [],
-      course_id: anonRealisation.course_id,
-      providing_locality: [],
-      learningopportunity_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      realisation_root_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      learningopportunity_id: learningOpportunityRelations[realisation.learningopportunity_id].learningopportunity_id,
-      descriptions: [],
-      languages: []
-    }
-    await APIWriter(`src/anonymized_API/courseunitrealisations/${anonCourseUnitRealisation.course_id}`, {
-      data: { data: anonCourseUnitRealisation }
-    })
-  }
-  await APIWriter('src/anonymized_API/courseunitrealisations/changes/ids/0000-12-24', {
-    data: { data: anonRealisations }
-  })
-  stopStatusBar()
-  return
-}
-
-const APIWriter = async (path, data) => {
-  mkdirp(getDirName(path), async () => {
-    fs.writeFile(path, JSON.stringify(data, null, 4), error => {
-      if (error) {
-        console.log(error)
-      }
-    })
-    return 'succeeee'
-  })
-}
-
-let studentnumber_attainments = []
-
-const anonymize = async () => {
-  logger.verbose(`starting time: ${startTime.format('HH:mm:ss')}`)
-  const filename = './studentnumbers.txt'
-  const readStudentNumbersFromFile = async filename => {
-    let studentnumbers = fs
-      .readFileSync(filename, 'utf-8')
-      .split('\n')
-      .map(s => s.replace(' ', ''))
-    const randomCopies = _.sampleSize(
-      studentnumbers,
-      Math.floor((Math.random() * (1 - 0.33) + 0.33) * studentnumbers.length)
-    )
-    studentnumbers = studentnumbers.concat(randomCopies)
-    studentnumbers = _.shuffle(studentnumbers)
-    return studentnumbers.filter(studentnumber => !!studentnumber)
-  }
-  const codeCounter = status.addItem('codes', { max: 5 })
-  startCodesStatusBar()
-  const semesters = { data: { data: await oodi.getSemesters() } }
-  codeCounter.inc()
-  const courseUnitRealisationsTypes = { data: { data: await oodi.getCourseRealisationTypes() } }
-  codeCounter.inc()
-  const creditTypeCodes = { data: { data: await oodi.getStudyattainmentStatusCodes() } }
-  codeCounter.inc()
-  const courseTypeCodes = { data: { data: await oodi.getCourseTypeCodes() } }
-  codeCounter.inc()
-  const courseDisciplines = disciplines
-  codeCounter.inc()
-  await APIWriter('./src/anonymized_API/codes/semesters', semesters)
-  await APIWriter('./src/anonymized_API/codes/courseunitrealisations/types', courseUnitRealisationsTypes)
-  await APIWriter('./src/anonymized_API/codes/studyattainments/statuses', creditTypeCodes)
-  await APIWriter('./src/anonymized_API/codes/learningopportunities/types', courseTypeCodes)
-  await APIWriter('./src/anonymized_API/codes/learningopportunities/disciplines', courseDisciplines)
-  const numberList = await readStudentNumbersFromFile(filename)
-  const studentCounter = status.addItem('students', { max: numberList.length })
-  status.stamp()
-  startStudentsStatusBar()
-  let student_numbers = []
-  for (let id of numberList) {
-    studentCounter.inc()
-    try {
-      student_numbers = await anonymizeStudent(id, student_numbers)
-    } catch (error) {
-      logger.verbose(`FAILED ${id}, ${error}  elapsed time:  ${getElapsedTime()}`)
-      process.exit(1)
-    }
-  }
-  status.stamp()
-  await anonymizeCourseUnitRealisations()
-  console.log(student_numbers)
-  await APIWriter('./studentnumbersN.txt', student_numbers)
-  logger.verbose(`end time: ${moment()}, total run-time:  ${getElapsedTime()}`)
-}
-
-anonymize()

services/backend/oodikone2-backend/src/app.js

@@ -1,65 +0,0 @@
-const express = require('express')
-const cors = require('cors')
-const bodyParser = require('body-parser')
-const Sentry = require('@sentry/node')
-const conf = require('./conf-backend')
-const routes = require('./routes')
-const { startCron } = require('./events')
-const { PORT } = conf
-const { initializeDatabaseConnection } = require('./database/connection')
-const { dbConnections } = require('./databaseV2/connection')
-
-const IS_PRODUCTION = process.env.NODE_ENV === 'production'
-
-initializeDatabaseConnection()
-  .then(() => {
-    dbConnections.connect()
-
-    dbConnections.on('connect', () => {
-      console.log('Connected to sis db successfully')
-    })
-
-    dbConnections.on('error', () => {
-      console.log('Failed connecting to sis db')
-    })
-
-    const app = express()
-
-    if (IS_PRODUCTION && ['staging', 'latest'].includes(process.env.TAG)) {
-      Sentry.init({
-        dsn: 'https://020b79f0cbb14aad94cc9d69a1ea9d52@sentry.cs.helsinki.fi/2',
-        environment: process.env.TAG
-      })
-      app.use(Sentry.Handlers.requestHandler())
-    }
-
-    startCron()
-
-    app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-    app.use(bodyParser.json())
-
-    let BASE_URL = ''
-    if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
-      BASE_URL = '/api'
-    }
-
-    routes(app, BASE_URL)
-
-    app.get('*', async (req, res) => {
-      const results = { error: 'unknown endpoint' }
-      res.status(404).json(results)
-    })
-
-    app.use(Sentry.Handlers.errorHandler())
-
-    const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
-    process.on('SIGTERM', () => {
-      server.close(() => {
-        console.log('Process terminated')
-      })
-    })
-  })
-  .catch(e => {
-    process.exitCode = 1
-    console.log(e)
-  })

services/backend/oodikone2-backend/src/conf-backend.js

@@ -1,114 +0,0 @@
-require('dotenv').config()
-
-const { NODE_ENV } = process.env
-const isTest = process.env.NODE_ENV === 'test'
-let DB_URL = process.env.DB_URL
-let DB_URL_KONE = process.env.DB_URL_KONE
-if (isTest) {
-  DB_URL = process.env.TEST_DB
-  DB_URL_KONE = process.env.TEST_DB_KONE
-} else if (NODE_ENV === 'anon') {
-  DB_URL = process.env.ANON_DB
-  DB_URL_KONE = process.env.ANON_DB_KONE
-}
-
-let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
-if (isNaN(DB_MAX_CONNECTIONS)) {
-  DB_MAX_CONNECTIONS = 5 // sequelize's default
-}
-let DB_MAX_CRON_CONNECTIONS = DB_MAX_CONNECTIONS - 5
-if (DB_MAX_CRON_CONNECTIONS < 1) {
-  DB_MAX_CRON_CONNECTIONS = 1
-}
-
-const frontend_addr = process.env.FRONT_URL
-const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
-const SECRET_TOKEN = process.env.SECRET_TOKEN
-const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const CERT_PATH = process.env.CERT_PATH // production/staging only
-const KEY_PATH = process.env.KEY_PATH // production/staging only
-const USERSERVICE_URL = process.env.USERSERVICE_URL
-const UPDATER_URL = process.env.UPDATER_URL
-const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
-const ANALYTICS_URL = process.env.ANALYTICS_URL
-const PORT = isTest ? 8079 : 8080
-const OODI_SECRET = process.env.OODI_SECRET
-const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
-const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
-const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
-const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
-const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
-const MATOMO_URL = process.env.MATOMO_URL
-
-const FEATURES = {
-  ERROR_HANDLER: false
-}
-
-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
-
-if (process.env.NODE_ENV === 'dev' && process.env.FEATURES) {
-  const toggled = process.env.FEATURES.split(',')
-  toggled
-    .map(toggle => toggle.trim())
-    .forEach(feature => {
-      if (FEATURES[feature] !== undefined) {
-        FEATURES[feature] = true
-      }
-    })
-}
-
-const OODI = {
-  test: 'http://localhost',
-  anon: process.env.OODI_ADDR_ANON
-}
-
-const OODI_ADDR = OODI[process.env.NODE_ENV] || process.env.OODI_ADDR
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-const OODI_SECRET_HEADER_KEY = 'x-oodi-secret'
-
-let requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
-if (process.env.NODE_ENV === 'staging') {
-  requiredGroup = ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-}
-if (process.env.NODE_ENV === 'dev' || isTest) {
-  requiredGroup = null
-}
-
-module.exports = {
-  frontend_addr,
-  DB_URL,
-  DB_URL_KONE,
-  DB_MAX_CONNECTIONS,
-  DB_MAX_CRON_CONNECTIONS,
-  redis,
-  TOKEN_SECRET,
-  SECRET_TOKEN,
-  DB_SCHEMA,
-  DB_SCHEMA_KONE,
-  OODI_ADDR,
-  CERT_PATH,
-  KEY_PATH,
-  FEATURES,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
-  PORT,
-  ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  requiredGroup,
-  OODI_SECRET,
-  OODI_SECRET_HEADER_KEY,
-  isTest,
-  UPDATER_URL,
-  SIS_UPDATER_URL,
-  ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
-  ANALYTICS_INFLUXDB_USER,
-  ANALYTICS_INFLUXDB_PASSWORD,
-  ANALYTICS_INFLUXDB_DB,
-  MATOMO_SITE_ID,
-  MATOMO_URL: MATOMO_URL && addSlashToEnd(MATOMO_URL)
-}

services/backend/oodikone2-backend/src/database/connection.js

@@ -1,87 +0,0 @@
-const Sequelize = require('sequelize')
-const Umzug = require('umzug')
-const conf = require('../conf-backend')
-
-const sequelize = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA,
-  searchPath: conf.DB_SCHEMA,
-  logging: false,
-  pool: {
-    min: 0,
-    max: conf.DB_MAX_CONNECTIONS
-  }
-})
-sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
-
-const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
-  schema: conf.DB_SCHEMA_KONE,
-  searchPath: conf.DB_SCHEMA_KONE,
-  logging: false
-})
-sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
-
-const initializeDatabaseConnection = async () => {
-  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelize.authenticate()
-      await sequelizeKone.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
-      }
-      console.log('.')
-      await sleep(1000)
-    }
-  }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelizeKone,
-          tableName: 'migrations',
-          schema: conf.DB_SCHEMA_KONE
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelizeKone.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/src/database/migrations_kone`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA_KONE
-        }
-      })
-      const migrations = await migrator.up()
-
-      console.log('Kone Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
-  }
-}
-
-const forceSyncDatabase = async () => {
-  try {
-    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
-  } catch (e) {
-    // console.log(e)
-  }
-  try {
-    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
-  } catch (e) {
-    // console.log(e)
-  }
-  await sequelize.sync({ force: true })
-  await sequelizeKone.sync({ force: true })
-}
-
-module.exports = {
-  sequelize,
-  sequelizeKone,
-  initializeDatabaseConnection,
-  forceSyncDatabase
-}

services/backend/oodikone2-backend/src/database/migrations/20190710_03_drop_ldapuser.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.dropTable('ldapuser', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20190806_00_transfers_delete_on_studyright_delete.js

@@ -1,39 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_studyrightid_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_sourcecode_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_studentnumber_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query('ALTER TABLE transfers DROP CONSTRAINT "transfers_targetcode_fkey"', {
-        transaction
-      })
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_studyrightid_fkey" FOREIGN KEY (studyrightid) REFERENCES studyright(studyrightid) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_sourcecode_fkey" FOREIGN KEY (sourcecode) REFERENCES element_details(code) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_studentnumber_fkey" FOREIGN KEY (studentnumber) REFERENCES student(studentnumber) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'ALTER TABLE transfers ADD CONSTRAINT "transfers_targetcode_fkey" FOREIGN KEY (targetcode) REFERENCES element_details(code) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE',
-        { transaction }
-      )
-      await queryInterface.sequelize.query(
-        'DELETE FROM transfers WHERE studyrightid IS NULL OR sourcecode IS NULL OR studentnumber IS NULL OR targetcode IS NULL',
-        { transaction }
-      )
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20190816_00_drop_mandatory_course_labels.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.dropTable('mandatory_course_labels', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191021_00_drop_highlevelname.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.removeColumn('studyright', 'highlevelname', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191028_00_add_more_indexes.js

@@ -1,16 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.addIndex('course_disciplines', ['course_id'], { transaction })
-      await queryInterface.addIndex('element_details', ['type'], { transaction })
-      await queryInterface.addIndex('studyright', ['canceldate'], { transaction })
-      await queryInterface.addIndex('studyright', ['extentcode'], { transaction })
-      await queryInterface.addIndex('studyright_elements', ['studentnumber'], { transaction })
-      await queryInterface.addIndex('studyright_elements', ['studyrightid'], { transaction })
-      await queryInterface.addIndex('studyright_elements', ['code'], { transaction })
-      await queryInterface.addIndex('transfers', ['sourcecode'], { transaction })
-      await queryInterface.addIndex('transfers', ['targetcode'], { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191205_00_add_column_to_studyright.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('studyright', 'faculty_code', {
-      type: Sequelize.STRING
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191207_00_add_column_to_elementdetails.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('element_details', 'faculty_code', {
-      type: Sequelize.STRING
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20191217_00_remove_matriculationexamination.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.removeColumn('student', 'matriculationexamination', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200101_00_fix_transfers_duplicates.js

@@ -1,27 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    await queryInterface.sequelize.query(
-      `DELETE
-            FROM transfers a
-                USING transfers b
-            WHERE
-                a.id > b.id
-                AND a.transferdate = b.transferdate
-                AND a.studentnumber = b.studentnumber
-                AND a.studyrightid = b.studyrightid
-                AND a.sourcecode = b.sourcecode
-                AND a.targetcode = b.targetcode`,
-      {
-        raw: true
-      }
-    )
-    await queryInterface.addIndex(
-      'transfers',
-      ['studyrightid', 'sourcecode', 'targetcode', 'transferdate', 'studentnumber'],
-      {
-        unique: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200101_01_remove_fac_from_elmdetails.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.removeColumn('element_details', 'faculty_code', { transaction })
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200218_00_add_admin_view_funcs.js

@@ -1,38 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    await queryInterface.sequelize.query(
-      `CREATE OR REPLACE FUNCTION public.map_range(
-            input DOUBLE PRECISION,
-            inMin DOUBLE PRECISION,
-            inMax DOUBLE PRECISION,
-            outMin DOUBLE PRECISION,
-            outMax DOUBLE PRECISION)
-        RETURNS DOUBLE PRECISION AS $$
-            SELECT (input - inMin) * (outMax - outMin) / (inMax - inMin) + outMin;
-        $$ LANGUAGE SQL;
-        
-        CREATE OR REPLACE FUNCTION public.is_in_target(
-            currentDate TIMESTAMP WITH TIME ZONE,
-            studyStartDate TIMESTAMP WITH TIME ZONE,
-            targetDate TIMESTAMP WITH TIME ZONE,
-            studentCredits DOUBLE PRECISION,
-            targetCredits DOUBLE PRECISION)
-        RETURNS INTEGER AS $$
-            SELECT CASE WHEN studentCredits >= public.map_range(
-                EXTRACT(EPOCH FROM currentDate),
-                EXTRACT(EPOCH FROM studyStartDate),
-                EXTRACT(EPOCH FROM targetDate),
-                0,
-                targetCredits
-            )
-                THEN 1
-                ELSE 0
-            END;
-        $$ LANGUAGE SQL;`,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200324_00_fix_sr_elements_duplicates.js

@@ -1,24 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    await queryInterface.sequelize.query(
-      `DELETE
-            FROM studyright_elements a
-                USING studyright_elements b
-            WHERE
-                a.id < b.id
-                AND a."createdAt" < b."createdAt"
-                AND a.studentnumber = b.studentnumber
-                AND a.studyrightid = b.studyrightid
-                AND a.code = b.code`,
-      {
-        raw: true,
-        underscored: false
-      }
-    )
-    await queryInterface.removeIndex('studyright_elements', ['startdate', 'enddate', 'studyrightid', 'code'])
-    await queryInterface.addIndex('studyright_elements', ['startdate', 'studyrightid', 'code', 'studentnumber'], {
-      unique: true
-    })
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200409_00_more_admin_view_funcs.js

@@ -1,81 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    // add IMMUTABLE & STABLE attribute to functions created in 20200218_00_add_admin_view_funcs
-    // to let postgres optimize their use
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION public.map_range(
-          input DOUBLE PRECISION,
-          inMin DOUBLE PRECISION,
-          inMax DOUBLE PRECISION,
-          outMin DOUBLE PRECISION,
-          outMax DOUBLE PRECISION)
-      RETURNS DOUBLE PRECISION AS $$
-          SELECT (input - inMin) * (outMax - outMin) / (inMax - inMin) + outMin;
-      $$ LANGUAGE SQL IMMUTABLE;
-
-      CREATE OR REPLACE FUNCTION public.is_in_target(
-          currentDate TIMESTAMP WITH TIME ZONE,
-          studyStartDate TIMESTAMP WITH TIME ZONE,
-          targetDate TIMESTAMP WITH TIME ZONE,
-          studentCredits DOUBLE PRECISION,
-          targetCredits DOUBLE PRECISION)
-      RETURNS INTEGER AS $$
-          SELECT CASE WHEN studentCredits >= public.map_range(
-              EXTRACT(EPOCH FROM currentDate),
-              EXTRACT(EPOCH FROM studyStartDate),
-              EXTRACT(EPOCH FROM targetDate),
-              0,
-              targetCredits
-          )
-              THEN 1
-              ELSE 0
-          END;
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-
-    // new funcs
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION change_year(t timestamp with time zone, yr int) RETURNS timestamp with time zone AS $$
-          SELECT make_timestamptz(
-              yr,
-              date_part('month', t)::int,
-              date_part('day', t)::int,
-              date_part('hour', t)::int,
-              date_part('minute', t)::int,
-              date_part('second', t)
-          )
-      $$ LANGUAGE SQL STABLE;
-
-      CREATE OR REPLACE FUNCTION is_leap_day(t timestamp with time zone) RETURNS boolean AS $$
-          SELECT date_part('day', t) = 29 AND date_part('month', t) = 2
-      $$ LANGUAGE SQL STABLE;
-
-      CREATE OR REPLACE FUNCTION same_date_this_year(t timestamp with time zone) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN public.is_leap_day(t) THEN public.change_year(t + INTERVAL '1 day', date_part('year', CURRENT_TIMESTAMP)::int)
-                  ELSE public.change_year(t, date_part('year', CURRENT_TIMESTAMP)::int)
-              END
-      $$ LANGUAGE SQL STABLE;
-
-      CREATE OR REPLACE FUNCTION next_date_occurrence(t timestamp with time zone) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN CURRENT_TIMESTAMP > public.same_date_this_year(t) THEN public.same_date_this_year(t) + INTERVAL '1 year'
-                  ELSE public.same_date_this_year(t)
-              END
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}

services/backend/oodikone2-backend/src/database/migrations/20200409_01_fix_next_date_occurrence.js

@@ -1,24 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    // next_date_occurrence would previously return the same
-    // date if now() was the same timestamp
-    //
-    // fix by checking with >= instead of >
-    await queryInterface.sequelize.query(
-      `
-      CREATE OR REPLACE FUNCTION next_date_occurrence(t timestamp with time zone) RETURNS timestamp with time zone AS $$
-          SELECT
-              CASE
-                  WHEN CURRENT_TIMESTAMP >= public.same_date_this_year(t)
-                      THEN public.same_date_this_year(t) + INTERVAL '1 year'
-                  ELSE public.same_date_this_year(t)
-              END
-      $$ LANGUAGE SQL STABLE;
-      `,
-      {
-        multipleStatements: true
-      }
-    )
-  },
-  down: async () => {}
-}