Merge pull request #3150 from UniversityOfHelsinkiCS/trunk

Remove oodi from oodikone + tiny fixes

.circleci/config.yml

@@ -1,121 +0,0 @@
-version: 2.1
-
-orbs:
-  node: circleci/node@2.0.3
-
-executors:
-  my-executor:
-    working_directory: ~/app
-    machine:
-      image: ubuntu-1604:201903-01
-
-aliases:
-  - &check_exit_statuses
-    run:
-      name: check_exit_statuses
-      command: |
-        docker ps -a
-        if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
-          echo 'Some process had nonzero exit code';
-          npm run docker:ci -- logs
-          exit 1;
-        fi
-
-jobs:
-  build:
-    executor: my-executor
-    steps:
-      - run:
-          name: set_env_variables
-          command: |
-            echo $CIRCLE_BRANCH;
-            echo $CIRCLE_TAG;
-            if [[ -n "$CIRCLE_TAG" ]]; then
-              echo 'export TAG=latest' >> $BASH_ENV;
-              echo 'export DEPLOY=true' >> $BASH_ENV;
-              echo 'export CACHE_TAG=latest' >> $BASH_ENV;
-              echo 'USING RELEASE CONFIG';
-            elif [[ "$CIRCLE_BRANCH" = "master" && -z "$CIRCLE_PULL_REQUEST" ]]; then
-              echo 'export TAG=staging' >> $BASH_ENV;
-              echo 'export DEPLOY=true' >> $BASH_ENV;
-              echo 'export CACHE_TAG=staging' >> $BASH_ENV;
-              echo 'USING STAGING CONFIG';
-            else
-              echo 'export TAG=trunk' >> $BASH_ENV;
-              echo 'export DEPLOY=false' >> $BASH_ENV;
-              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
-              echo 'USING TRUNK CONFIG';
-            fi
-      - checkout
-      - node/install:
-          node-version: latest
-      - restore_cache:
-          keys:
-            - npmcache-v2-{{ checksum "package-lock.json" }}
-      - run: npm ci
-      # Temporarily disable cypress until we have sis test data available
-      # - run: npm run cypress -- verify
-      - save_cache:
-          key: npmcache-v2-{{ checksum "package-lock.json" }}
-          paths:
-            - ~/.cache
-      - run: npm run docker:ci -- pull --ignore-pull-failures
-      - run: npm run docker:ci -- build
-      - run: npm run docker:ci -- up -d user_db db_kone analytics_db
-      - *check_exit_statuses
-      # - run: git clone --depth 1 git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
-      # - run: source ./scripts/scripts.sh && init_dirs && db_anon_setup_full
-      # - *check_exit_statuses
-      - run: npm run docker:ci -- up -d --no-recreate
-      - run:
-          name: log_in_background
-          background: true
-          command: |
-            docker ps -a && npm run docker:ci -- logs -f
-      - *check_exit_statuses
-      - run:
-          name: test services
-          command: |
-            if [ "${DEPLOY}" != "true" ]; then
-              npm run test_services
-            fi
-      - *check_exit_statuses
-      # Temporarily disable cypress until we have sis test data available
-      # - run:
-      #     name: run cypress tests
-      #     command: |
-      #       if [ "${DEPLOY}" != "true" ]; then
-      #         timeout 5m bash -c 'until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf .; sleep 5; done'
-      #         npm run cypress:ci
-      #       fi
-      # - *check_exit_statuses
-      # - store_artifacts:
-      #     path: cypress/videos
-      # - store_artifacts:
-      #     path: cypress/screenshots
-      - deploy:
-          command: |
-            if [ "${DEPLOY}" == "true" ]; then
-              docker exec --env SENTRY_URL --env SENTRY_AUTH_TOKEN --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-                sh +e -c 'npm run sentry-cli -- releases info "$(cat /SENTRY_RELEASE_VERSION)" && \
-                npm run sentry-cli -- releases delete "$(cat /SENTRY_RELEASE_VERSION)" || true'
-
-              docker exec --env SENTRY_URL --env SENTRY_AUTH_TOKEN --env SENTRY_ORG=toska --env SENTRY_PROJECT=oodikone frontend \
-                sh -c 'npm run sentry-cli -- releases new --finalize "$(cat /SENTRY_RELEASE_VERSION)" && \
-                npm run sentry-cli -- releases files "$(cat /SENTRY_RELEASE_VERSION)" upload-sourcemaps --rewrite dist'
-            fi
-            echo "$DOCKER_PASS" > DOCKER_PASS
-            cat DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
-            npm run docker:ci -- push
-            if [ "${DEPLOY}" == "true" ]; then
-              curl --silent -X POST -H 'Content-type: application/json' --data "{\"text\":\"${CIRCLE_BRANCH}${CIRCLE_TAG} started auto deployment!\"}" ${SLACKBOT_URL}
-            fi
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - build:
-          filters:
-            tags:
-              only: /.*/

.eslintrc.json

@@ -0,0 +1,26 @@
+{
+  "env": {
+    "es6": true,
+    "node": true,
+    "jest": true
+  },
+  "parser": "@babel/eslint-parser",
+  "parserOptions": {
+    "requireConfigFile": false
+  },
+  "extends": [
+    "eslint:recommended",
+    "plugin:prettier/recommended",
+    "plugin:cypress/recommended"
+  ],
+  "plugins": [
+    "prettier",
+    "cypress"
+  ],
+  "rules": {
+    "no-console": "off",
+    "react/no-unescaped-entities": "off",
+    "no-async-promise-executor": "off",
+    "prettier/prettier": "error"
+  }
+}

.github/dependabot.yml

@@ -6,13 +6,10 @@ updates:
     interval: weekly
     time: "03:00"
   open-pull-requests-limit: 10
-  target-branch: trunk
-  ignore:
-  - dependency-name: cypress
-    versions:
-    - 7.0.1
-  - dependency-name: husky
-    versions:
-    - 5.0.9
-    - 5.1.0
-    - 5.1.2
+  target-branch: "trunk"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: weekly
+    time: "03:00"
+  target-branch: "trunk"

.github/workflows/release-oodikone-staging.yaml

@@ -0,0 +1,143 @@
+name: "Release oodikone to staging"
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/release-oodikone-staging.yaml'
+      - 'services/**'
+
+jobs:
+
+  build_and_release_backend:
+    name: "Build and release backend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:trunk
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: true
+          tags: toska/oodikone2-backend:staging
+          target: production
+
+  build_and_release_frontend:
+    name: "Build and release frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ github.sha }}
+            SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:trunk
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: true
+          tags: toska/oodikone2-frontend:staging
+          target: production
+      - name: "Create a container from built image "
+        run: docker create --name frontend toska/oodikone2-frontend:staging
+      - name: "Copy assets from the container"
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: "Create Sentry release"
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: staging
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
+
+  build_and_release_userservice:
+    name: "Build and release userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:trunk
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: true
+          tags: toska/oodikone2-userservice:staging
+          target: production
+
+  report_success:
+    runs-on: ubuntu-latest
+    name: "Report deployment success to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: success()
+    steps:
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone-staging.cs.helsinki.fi/
+          deployment-target: staging
+
+  report_failure:
+    runs-on: ubuntu-latest
+    name: "Report deployment failure to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: failure()
+    steps:
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: staging

.github/workflows/release-oodikone.yaml

@@ -0,0 +1,150 @@
+name: "Release oodikone to production"
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build_and_release_backend:
+    name: "Build and release backend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:trunk
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: true
+          tags: toska/oodikone2-backend:latest
+          target: production
+
+  build_and_release_frontend:
+    name: "Build and release frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+      - name: "Extract release version"
+        # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
+        # This steps extracts release version from GITHUB_REF env var
+        # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
+        id: extract_version
+        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
+            SENTRY_ENVIRONMENT=production
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:trunk
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: true
+          tags: toska/oodikone2-frontend:latest
+          target: production
+      - name: "Create a container from built image "
+        run: docker create --name frontend toska/oodikone2-frontend:latest
+      - name: "Copy assets from the container"
+        run: docker cp frontend:/usr/src/app/build ./build
+      - name: "Create Sentry release"
+        uses: getsentry/action-release@v1
+        env:
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: toska
+          SENTRY_PROJECT: oodikone
+        with:
+          environment: production
+          set_commits: 'skip'
+          sourcemaps: './build/static/js'
+          url_prefix: '~/oodikone/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
+
+  build_and_release_userservice:
+    name: "Build and release userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Build and push"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:trunk
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: true
+          tags: toska/oodikone2-userservice:latest
+          target: production
+
+  report_success:
+    runs-on: ubuntu-latest
+    name: "Report deployment success to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: success()
+    steps:
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi/
+          deployment-target: production
+
+  report_failure:
+    runs-on: ubuntu-latest
+    name: "Report deployment failure to oodikone notifications"
+    needs:
+    - build_and_release_backend
+    - build_and_release_frontend
+    - build_and_release_userservice
+    if: failure()
+    steps:
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: production

.github/workflows/release-sis-updater-scheduler.yaml

@@ -0,0 +1,54 @@
+name: "Release sis-updater-scheduler"
+
+on:
+  push:
+    branches:
+      - trunk
+    paths:
+      - '.github/workflows/release-sis-updater-scheduler.yaml'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_and_release:
+    name: "Build and release sis-updater-scheduler"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: docker/setup-buildx-action@v1
+    - name: "Login to DockerHub"
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: "Build and push"
+      uses: docker/build-push-action@v2
+      with:
+        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+        cache-to: type=inline
+        context: updater/sis-updater-scheduler/
+        push: true
+        tags: toska/sis-updater-scheduler:latest
+        target: production
+    - name: "Send notification to slack"
+      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      with:
+        webhook-url: ${{ secrets.WEBHOOK_URL }}
+        message-type: deployment
+        softa-url: https://oodikone.cs.helsinki.fi
+        deployment-target: production
+        softa-to-deploy: sis-updater-scheduler
+
+  report_failure:
+    runs-on: ubuntu-latest
+    name: "Report deployment failure to oodikone notifications"
+    needs:
+    - build_and_release
+    if: failure()
+    steps:
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: production
+          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -0,0 +1,54 @@
+name: "Release sis-updater-worker"
+
+on:
+  push:
+    branches:
+      - trunk
+    paths:
+      - '.github/workflows/release-sis-updater-worker.yaml'
+      - 'updater/sis-updater-worker/**'
+
+jobs:
+  build_and_release:
+    name: "Build and release sis-updater-worker"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: docker/setup-buildx-action@v1
+    - name: "Login to DockerHub"
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: "Build and push"
+      uses: docker/build-push-action@v2
+      with:
+        cache-from: type=registry,ref=toska/sis-updater-worker:latest
+        cache-to: type=inline
+        context: updater/sis-updater-worker/
+        push: true
+        tags: toska/sis-updater-worker:latest
+        target: production
+    - name: "Send notification to slack"
+      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+      with:
+        webhook-url: ${{ secrets.WEBHOOK_URL }}
+        message-type: deployment
+        softa-url: https://oodikone.cs.helsinki.fi
+        deployment-target: production
+        softa-to-deploy: sis-updater-worker
+
+  report_failure:
+    runs-on: ubuntu-latest
+    name: "Report deployment failure to oodikone notifications"
+    needs:
+    - build_and_release
+    if: failure()
+    steps:
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment-failure
+          deployment-target: production
+          softa-to-deploy: sis-updater-worker

.github/workflows/sis-updater-scheduler.yaml

@@ -1,28 +0,0 @@
-name: Release sis-updater-scheduler
-
-on:
-  push:
-    branches:
-      - trunk
-    paths:
-      - '.github/workflows/sis-updater-scheduler.yaml'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build: 
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2 
-
-    - name: Login to DockerHub
-      uses: docker/login-action@v1 
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build and push
-      uses: docker/build-push-action@v2
-      with:
-        context: updater/sis-updater-scheduler/
-        push: true
-        tags: toska/sis-updater-scheduler:latest

.github/workflows/sis-updater-worker.yaml

@@ -1,28 +0,0 @@
-name: Release sis-updater-worker
-
-on:
-  push:
-    branches:
-      - trunk
-    paths:
-      - '.github/workflows/sis-updater-worker.yaml'
-      - 'updater/sis-updater-worker/**'
-
-jobs:
-  build: 
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2 
-
-    - name: Login to DockerHub
-      uses: docker/login-action@v1 
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-    - name: Build and push
-      uses: docker/build-push-action@v2
-      with:
-        context: updater/sis-updater-worker/
-        push: true
-        tags: toska/sis-updater-worker:latest

.github/workflows/test-oodikone.yaml

@@ -0,0 +1,546 @@
+name: "Test oodikone"
+
+on:
+  push:
+    branches-ignore:
+      - master
+    paths:
+      - .github/workflows/test-oodikone.yaml
+      - cypress/**
+      - cypress.json
+      - services/**
+      - docker-compose.ci.yml
+      - nginx.conf
+      - package*.json
+
+jobs:
+  # === Static tests / linting ===
+
+  lint_backend:
+    name: "Lint backend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - name: "Install dependencies"
+        run: npm ci && cd services/backend && npm ci
+      - name: "Run backend linter"
+        run: npm run eslint services/backend
+
+  lint_frontend:
+    name: "Lint frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - name: "Install dependencies"
+        run: npm ci && cd services/frontend && npm ci
+      - name: "Run frontend linter"
+        run: npm run concurrently \
+              'eslint --ext .jsx,.js services/frontend' \
+              'stylelint services/frontend/src/**/*.css'
+
+  lint_userservice:
+    name: "Lint userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - name: "Install dependencies"
+        run: npm ci && cd services/userservice && npm ci
+      - name: "Run frontend linter"
+        run: npm run eslint services/userservice
+
+  # === Build images ===
+
+  build_backend:
+    name: "Build backend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Build image and export it"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-backend:trunk
+            type=registry,ref=toska/oodikone2-backend:staging
+            type=registry,ref=toska/oodikone2-backend:latest
+          cache-to: type=inline
+          context: services/backend
+          push: false
+          tags: toska/oodikone2-backend:trunk
+          target: production
+          outputs: type=docker,dest=/tmp/backend.tar
+      - name: "Upload image for other jobs"
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/backend.tar
+
+  build_frontend:
+    name: "Build frontend"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Build image and export it"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-frontend:trunk
+            type=registry,ref=toska/oodikone2-frontend:staging
+            type=registry,ref=toska/oodikone2-frontend:latest
+          cache-to: type=inline
+          context: services/frontend
+          push: false
+          tags: toska/oodikone2-frontend:trunk
+          target: production
+          outputs: type=docker,dest=/tmp/frontend.tar
+      - name: "Upload image for other jobs"
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/frontend.tar
+
+  build_userservice:
+    name: "Build userservice"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: "Build image and export it"
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: |
+            type=registry,ref=toska/oodikone2-userservice:trunk
+            type=registry,ref=toska/oodikone2-userservice:staging
+            type=registry,ref=toska/oodikone2-userservice:latest
+          cache-to: type=inline
+          context: services/userservice
+          push: false
+          tags: toska/oodikone2-userservice:trunk
+          target: production
+          outputs: type=docker,dest=/tmp/userservice.tar
+      - name: "Upload image for other jobs"
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/userservice.tar
+
+  # === Integration tests ===
+  run_userservice_integration_tests:
+    name: "Run userservice integration tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Run tests included in docker build"
+        run: docker-compose --file docker-compose.test.yml up --build userservice-test
+
+  # === End-to-end tests ===
+
+  test_course_statistics:
+    name: "Test course statistics"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Course_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_custom_population:
+    name: "Test custom population"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Custom_population.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_population_statistics:
+    name: "Test population statistics"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Population_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_filters:
+    name: "Test filters"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Filters.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_student_statistics:
+    name: "Test student statistics"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Student_statistics.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_studyprogramme_overview:
+    name: "Test studyprogramme overview"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: cypress/integration/Studyprogramme_overview.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  test_teachers_and_users:
+    name: "Test teachers and users"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Run cypress"
+        uses: cypress-io/github-action@v2
+        with:
+          browser: chrome
+          spec: |
+            cypress/integration/Teachers.js
+            cypress/integration/Users.js
+        env:
+          CYPRESS_baseUrl: http://localhost:1337
+      - name: "Upload video if tests fail"
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Cypress videos
+          path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  # === Other tests ===
+
+  test_service_exit_statuses:
+    name: "Check services run correctly"
+    needs:
+      - build_backend
+      - build_frontend
+      - build_userservice
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to toska docker registry"
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: "Run services, pull rest of the needed images"
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
+      - name: "Check that exit statuses are not errors"
+        run: |
+          docker ps -a
+          if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+            echo 'Some process had nonzero exit code';
+            exit 1;
+          fi
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
+
+  push_to_dockerhub:
+    name: "Push images to docker hub for caching"
+    needs:
+    - lint_backend
+    - lint_frontend
+    - lint_userservice
+    - run_userservice_integration_tests
+    - test_course_statistics
+    - test_custom_population
+    - test_filters
+    - test_population_statistics
+    - test_student_statistics
+    - test_studyprogramme_overview
+    - test_teachers_and_users
+    - test_service_exit_statuses
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Download images"
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: "Load built images"
+        run: |
+          docker load --input /tmp/artifact/backend.tar
+          docker load --input /tmp/artifact/frontend.tar
+          docker load --input /tmp/artifact/userservice.tar
+      - name: "Login to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Push to docker hub"
+        run: docker-compose --file docker-compose.ci.yml push
+
+  report_failure:
+    runs-on: ubuntu-latest
+    name: "Report test failure to oodikone notifications"
+    needs:
+    - lint_backend
+    - lint_frontend
+    - lint_userservice
+    - run_userservice_integration_tests
+    - test_course_statistics
+    - test_custom_population
+    - test_filters
+    - test_population_statistics
+    - test_student_statistics
+    - test_studyprogramme_overview
+    - test_teachers_and_users
+    - test_service_exit_statuses
+    - push_to_dockerhub
+    if: failure()
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Send notification to slack"
+        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: test-failure

.gitignore

@@ -1,30 +1,48 @@
-**/node_modules
-data
-pgdata
-pgdata_kone
-user_pgdata
-analytics_pgdata
-usage_pgdata
-analytics-redis-data
-redis-data
-mongodata
-backups
-private.key
-private.key.pub
-anonyymioodi
-nginx
-kafka
-node_modules
-services/updater_writer/updater/test_assets/testi.bak
-services/updater_writer/updater/test_assets/011120775.json
-services/updater_writer/updater/test_assets/014272112.json
-services/updater_writer/updater/test_assets/014441008.json
-services/updater_writer/updater/test_assets/meta.json
-services/updater_scheduler/all_student_numbers.txt
-services/updater_scheduler/active_student_numbers.txt
-datastore
-services/updater_api/debug.log
-services/backend/updater_writer/dumps
-debug.log
-programme_module_dump.sql
-hyuserdata
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# dotenv environment variables file
+.env
+.env.test
+.env.production
+
+# Oodikone related stuff
+.databasedumps

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

.prettierrc

@@ -5,5 +5,5 @@
   "printWidth": 120,
   "tabWidth": 2,
   "useTabs": false,
-  "jsxSingleQuote": false
-}
+  "jsxSingleQuote": false,
+}

README.md

@@ -1,20 +1,19 @@
-# oodikone
+# Oodikone
 
-[![CircleCI](https://circleci.com/gh/UniversityOfHelsinkiCS/oodikone/tree/master.svg?style=svg)](https://app.circleci.com/pipelines/github/UniversityOfHelsinkiCS/oodikone)
+An application for analyzing university data, running at [https://oodikone.cs.helsinki.fi/](https://oodikone.cs.helsinki.fi/).
 
-An application for analyzing university data. The entire development environment runs in a Docker network that's defined in the `docker-compose.yml` file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.
-
-## Requirements
+## ✔️ Requirements
 
 To run Oodikone locally, you will need the following:
 
-1. [Docker](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose/install/) running on your system.
-2. Membership of [Toska's GitHub organisation](https://github.com/UniversityOfHelsinkiCS).
-3. SSH key set up for GitHub (see [docs](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh)).
-4. Local clone of this repository.
-5. Run the CLI (see [Installation](#Installation))
+1. Applications:
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
+3. (Optional) For real data setup, access to Toskas production servers.
 
-## Installation
+## 🚀 Installation
 
 Launch the CLI and follow the instructions:
 
@@ -22,248 +21,110 @@ Launch the CLI and follow the instructions:
 npm run cli
 ```
 
-*Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary* 😊
+_Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary_ 😊
 
-The acual oodikone can be accessed at [http://localhost:8081/](http://localhost:8081/) and Adminer at [http://localhost:5050/](http://localhost:5050/?pgsql=db&username=postgres). Adminer requires you to login with username `postgres` and password `postgres`.
+What different CLI options do:
 
-### Populate Redis on Fresh Install
+1. Set up oodikone from scratch:
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
+2. Reset all real data:
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
+3. Reset sis-importer data
+   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
 
-On a fresh install some features do not work because the redis store is empty. To populate redis, navigate to [http://localhost:8081/updater](http://localhost:8081/updater), and click *Refresh Statistics*.
+## ⌨️ Development
 
-### What the CLI does
+### Basics
 
-- Builds and runs the Dockerized development environment.
-- Downloads a database dump from the production servers.
-- Creates the database schema and populates it with the downloaded dump.
-- Adds git-hooks to run linting on push. (Use `git push --no-verify` to circumvent this behavior.)
+The development environment is entirely configured in the `docker-compose.yml` file located in this repository. The file defines the services for oodikone's two main components: **oodikone** and **updater**.
 
-### Daily Refresh
+Running oodikone with real data requires separated databases and redis, which are defined in `docker-compose.real.yml` file. Otherwise real data development environment uses the configuration as anonymized data development environment.
 
-It is recommended to run the "morning script" daily when starting work. It prunes Docker, refreshes the repository and rebuilds the containers before starting Oodikone again.
+Some useful commands are defined in `package.json` and can be run with `npm run <command>` as follows:
 
-```bash
-./oodikone.sh -m
-```
+- `npm run oodikone`: starts oodikone with anonymized data
+- `npm run oodikone:real`: starts oodikone with real data
+- `npm run updater`: starts updater with anonymized data
+- `npm run updater:real`: starts oodikone with real data
+- `npm run both`: starts oodikone and updater with anonymized data
+- `npm run both:real`: starts oodikone and updater with real data
+- `npm run docker:down`: stops the whole environment
 
-## Development Environment
+Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
-The development environment is entirely configured in the docker-compose.yml file located in this repository. The file defines the containers/environments for:
+### Run.sh script
 
-- Frontend
-- Backend
-- Userservice
-- The databases used by the services
-- Adminer & other dev tools
+As said, the development environment runs entirely inside docker containers. To keep `package.json` clean and not filled with predefined scripts, we have created a simple helper script called `./run.sh`. The script allows you to use docker-compose commands without the need to write long list of parameters. Try to run `./run.sh` in the root of the project and see what happens!
 
-The mapping of ports and environment variables are also defined in the docker-compose file. You can start, stop and manage the development environment by running the following commands from a terminal shell in this directory.
+`./run.sh` is simply a wrapper script to run oodikone, updater or both services in either anon or real mode. If you take a look at `package.json`, you can see that most of the predefined scripts above use `./run.sh` under the hood.
 
-### Start the development environment
+It is recommended to spend some time to become familiar with `docker` and `docker-compose` cli commands. You can then use them directly or with `./run.sh` wrapper. Here is some examples for day-to-day development situations:
 
-With anonymized data:
+- `./run.sh oodikone anon pull`: Pull all images related to oodikone development
+- `./run.sh updater real up --build --force-recreate --detach`: Start updater detached (=in the background) in real data mode, but build new images before starting
+- `docker-compose ps`: view the containers in the running environment
+- `docker-compose logs frontend`: print logs for just frontend
+- `docker-compose logs --follow --tail 100 backend`: print last hundred rows of backend logs and begin to follow them in your terminal window
+- `docker exec -it userservice sh`: open bash terminal inside userservice container
+- `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
-```bash
-npm start
-```
+### User types for development
 
-With real data:
+By default, you're login as `mluukkai` dev user when running oodikone in development mode. If you want to debug with certain type of user, you can use mocking: go to "Users" -page, click on "edit" on user you want to use and then click icon on the right corner of user card. Another way is to mock user headers - see frontend's api configuration for how this is done.
 
-```bash
-npm run start:real
-```
+Anon data contains some preset user types, corresponding to most usual user types in real data. These are used in testing too (see "Testing" below for more info).
 
-### Stop the development environment
+### Linting
 
-```bash
-npm run docker:down
-```
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-### Restart container(s)
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-All:
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
-```bash
-npm run docker:restart
-```
+However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-Only backend:
+## 🔨 Testing & CI
 
-```bash
-npm run docker:restart:backend
-```
+There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
-Only frontend:
+- Cypress
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+- Jest
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
-```bash
-npm run docker:restart:frontend
-```
+Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
-### View the containers in the running environment
+- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
+- after successful merge to `master`, oodikone is deployed to staging
+- after creating a release, oodikone is deployed to production
+- updater is deployed to production on every push to `trunk`.
 
-```bash
-docker-compose ps
-```
+## ❓FAQ
 
-### View logs
+### Modules are missing after updating package.json
 
-All:
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
 
-```bash
-npm run docker:logs
-```
+### Everything is broken, can't get oodikone running, data is not there etc.
 
-Only backend:
+Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
-```bash
-npm run docker:logs:backend
-```
+## ✌🏼 Maintainers and contribution
 
-Only frontend:
+Toska of course.
 
-```bash
-npm run docker:logs:frontend
-```
-
-### Attach a terminal shell to a container
-
-```bash
-docker exec -it <container> <command>
-```
-
-For example:
-
-```bash
-docker exec -it backend bash
-```
-
-### Use `psql` or other database CLI tools
-
-```bash
-docker exec -it -u <username> <container> psql -d <db name>
-```
-
-For example:
-
-```bash
-docker exec -it -u postgres oodi_db psql -d tkt_oodi
-```
-
-## Testing
-
-Testing is implemented as a combination of unit tests (jest) and end-to-end tests (Cypress). Oodikone must be using the anonymous dataset when running tests (see [Installation](#Installation)).
-
-The test suite is run in CI on every push to `trunk`. Take advantage of this as running the tests can take upwards of 15 minutes.
-
-**Note:** All containers must be up and ready before running tests. (Use `npm start`).
-
-### Run All Tests
-
-```bash
-npm test
-```
-
-### Run End-to-End Tests
-
-```bash
-npm run cypress:run
-```
-
-## Open Cypress GUI
-
-```bash
-npm run cypress:open
-```
-
-## How to Updater
-
-Before running updater you must run `importer-db` setup via the CLI (see [Installation](#Installation)).
-
-If you want to use the control panel UI, first start Oodikone in real data mode:
-
-```bash
-npm run docker:up:real
-```
-
-Followed by acual updater:
-
-```bash
-npm run updater:dev:up
-```
-
-When you are done, close hanging containers with:
-
-```bash
-npm run updater:dev:down
-```
-
-### updater scheduler
-
-Stop with HTTP GET
-
-```bash
-http://localhost:8082/v1/abort?token=dev
-```
-
-Schedule students with HTTP POST
-
-```bash
-http://localhost:8082/v1/students?token=dev
-```
-
-body
-
-```bash
-{ 
-    "studentnumbers": [
-        "014598456"
-    ] 
-}
-```
-
-## Other commands
-
-### How to `scp` backup files from oodikone via melkki-proxy
-
-This is how the setup script fetches the database dump from production servers. It will require you to have access to both melkki.cs.helsinki.fi as well as oodikone.cs.helsinki.fi. The command uses `scp` to transfer all backup files recursively from a known location on the production server by using the melkki server as a proxy, allowing you to get the dump even if you are not in the university network.
-
-#### Command
-
-```bash
-scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* ./backups/
-
--r : recursively copy files
--o : options for defining the proxy command
-```
-
-#### [Unix StackExchange answer](https://unix.stackexchange.com/questions/355640/how-to-scp-via-an-intermediate-machine)
-
-> It's possible and relatively easy, even when you need to use certificates > for authentication (typical in AWS environments).
->
-> The command below will copy files from a remotePath on server2 directly > into your machine at localPath. Internally the scp request is proxied via > server1.
->
-> ```bash
-> scp -i user2-cert.pem -o ProxyCommand="ssh -i user1-cert.pem -W %h:%p > user1@server1" user2@server2:/<remotePath> <localpath>
-> ```
->
-> If you use password authentication instead, try with:
->
-> ```bash
-> scp -o ProxyCommand="ssh -W %h:%p user1@server1" > user2@server2:/<remotePath> <localpath>
-> ```
->
-> If you use the same user credentials in both servers:
->
-> ```bash
-> scp -o ProxyCommand="ssh -W %h:%p commonuser@server1" > commonuser@server2:/<remotePath> <localpath>
-> ```
->
-
-## Known situations and ways to handle them
-
-- Problem: Oodikone doesn't start and database container (e.g. db_sis) logs include the line "can't open '/docker-entrypoint-initdb.d/'
-  - Possible reason: You probably have too restricting read rights in `scripts/docker-entrypoint-initdb.d` -folder. This could be for example caused by fairly strict `umask` setting, such as `077` in your OS.
-  - Possible solution: Either change the umask on whole OS (to `022` for example) or set the correct read rights for docker-entrypoint -folder by running `chmod 755 scripts/docker-entrypoint-initdb.d`
-- Problem: Packages aren't updated in development containers after update in package.json, even after rebuilding containers and images
-  - Possible reason: `node_modules` folder is binded to external docker volume (i.e not to disk) in our development environment. These volumes aren't updated unless explicitly told so.
-  - Possible solution: Either:
-    a) update packages inside the failing container by first logging to container `docker exec -it container_name sh` and then running `npm install`
-    b) remove volume that contains failing containers `node_modules` by stopping and removing the failing container followed by `docker volume prune`.
+University of Helsinki

adminer-plugins/login-servers.php

@@ -7,11 +7,10 @@ $driver = "pgsql";
 */
 return new AdminerLoginServers(
     $servers = array(
-        "db" => array("server" => "db", "driver" => $driver),
-        "db_kone" => array("server" => "db_kone", "driver" => $driver),
-        "user_db" => array("server" => "user_db", "driver" => $driver),
-        "analytics_db" => array("server" => "analytics_db", "driver" => $driver),
-        "db_sis" => array("server" => "db_sis", "driver" => $driver),
-        "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver)
+        "kone-db" => array("server" => "kone-db", "driver" => $driver),
+        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
+        "sis-db" => array("server" => "sis-db", "driver" => $driver),
+        "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
+        "user-db" => array("server" => "user-db", "driver" => $driver)
     )
 );

babel.config.json

@@ -0,0 +1,5 @@
+{
+  "plugins": [
+    "@babel/plugin-syntax-jsx"
+  ]
+}

cli.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+# This script is used to setup oodikone with interactive cli. Base for script:
+# https://betterdev.blog/minimal-safe-bash-script-template/
+
+# === Config ===
+
+# Fail immediately if script fails, unbound variables are referenced
+# or command inside pipe fails. -E ensures cleanup trap fires in rare ERR cases.
+set -euoE pipefail
+
+# Set up constants
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+
+# Source functions for setup
+source "$PROJECT_ROOT"/scripts/functions_for_setup.sh
+
+# Try to run cleanup function if things fail.
+trap cleanup SIGINT SIGTERM ERR EXIT
+
+# Run docker-compose down on cleanup
+cleanup() {
+  trap - SIGINT SIGTERM ERR EXIT
+  warningmsg "Trying to run docker-compose down and remove orphans"
+  docker-compose down --remove-orphans
+}
+
+# === CLI ===
+
+show_welcome() {
+  local cashmoneyyellow
+  cashmoneyyellow=$(tput setaf 221)
+  local normal
+  normal=$(tput sgr0)
+  if [ "$(tput cols)" -gt "76" ]; then
+    while IFS="" read -r p || [ -n "$p" ]; do
+      printf '%40s\n' "${cashmoneyyellow}$p${normal}"
+    done < "$PROJECT_ROOT"/scripts/assets/logo.txt
+  fi
+  infomsg "Welcome to Oodikone CLI!"
+  msg "This tool helps you in managing the project configuration. If you are new to
+Oodikone development, you should probably run \"Set up oodikone from scratch.\" which
+will take care of setting up Oodikone for you. See README for more details."
+  msg ""
+}
+
+# Define custom shell prompt for the interactive select loop
+PS3="Please enter your choice: "
+
+options=(
+  "Set up oodikone from scratch."
+  "Reset all real data."
+  "Reset sis-importer data."
+  "Quit."
+)
+
+# Run scripts before showing interactive prompt
+show_welcome
+init_dirs
+get_username
+
+while true; do
+  select opt in "${options[@]}"; do
+    case $opt in
+      "Set up oodikone from scratch.")
+        set_up_oodikone_from_scratch;;
+      "Reset all real data.")
+        reset_all_real_data;;
+      "Reset sis-importer data.")
+        reset_sis_importer_data;;
+      "Quit.")
+        break 2;;
+      *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+";;
+    esac
+    break
+  done
+done

cypress.json

@@ -1,11 +1,7 @@
 {
   "projectId": "c3jsph",
-  "defaultCommandTimeout": 60000,
-  "execTimeout": 60000,
-  "taskTimeout": 60000,
-  "pageLoadTimeout": 60000,
-  "requestTimeout": 60000,
-  "responseTimeout": 60000,
-  "viewportHeight": 1080,
-  "viewportWidth": 1920
+  "baseUrl": "http://localhost:3000",
+  "defaultCommandTimeout": 30000,
+  "requestTimeout": 30000,
+  "videoCompression": false
 }

cypress/fixtures/customPopulations.json

@@ -0,0 +1,4 @@
+{
+  "studentNumbersForCSStudentsSet1": ["010450938", "010589388", "010614509"],
+  "studentNumbersForCSStudentsSet2": ["010262566", "010307146"]
+}

cypress/fixtures/example.json

@@ -1,5 +0,0 @@
-{
-  "name": "Using fixtures to represent data",
-  "email": "hello@cypress.io",
-  "body": "Fixtures are a great way to mock data for responses to routes"
-}

cypress/integration/Course_population.js

@@ -1,29 +0,0 @@
-/// <reference types="Cypress" />
-
-describe('Course population tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains("Course statistics").click()
-    cy.contains("Search for courses")
-  })
-
-  it('Can find course population', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains("tr", "TKT20003").click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-
-    cy.contains("Käyttöjärjestelmät")
-    cy.contains("TKT20003")
-
-    cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-    cy.contains('Population of course Käyttöjärjestelmät 2017-18')
-    cy.contains('TKT20003, 582219, 582640, 582497')
-    cy.contains('Students (all=39)')
-
-    cy.contains('Students (39)').click()
-    cy.contains('013614218')
-    cy.contains('014022579')
-  })
-})

cypress/integration/Course_statistics.js

@@ -1,629 +1,328 @@
 /// <reference types="Cypress" />
 
 describe('Course Statistics tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains("Course statistics").click()
-    cy.contains("Search for courses")
-  })
-
-  it('Searching single course having duplicate mappings shows course statistics', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains("tr", "TKT20003").click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-    cy.contains("Search for courses").should('not.exist')
-
-    cy.contains("Käyttöjärjestelmät")
-    cy.contains("TKT20003")
-    cy.contains("582640") // old mapped code
-
-    cy.contains(".tabular.menu a", "Table").click()
-    cy.contains("All")
-    cy.contains(".modeSelectorRow a", "Attempts").click()
-    cy.contains(".modeSelectorRow a", "Students").click()
-
-    cy.contains(".tabular.menu a", "Pass rate chart").click()
-    cy.get("div.modeSelectorContainer").click()
-    cy.contains("svg", "Pass rate chart")
-
-    cy.contains(".tabular.menu a", "Grade distribution chart").click()
-    cy.get("div.modeSelectorContainer").click()
-    cy.contains("svg", "Grades")
-
-    cy.contains("a", "New query").click()
-    cy.contains("Search for courses")
-  })
-
-  it('Searching multiple courses having duplicate mappings shows course statistics', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by a course code']").type('TKT')
-    cy.contains("td", "TKT20003").click()
-    cy.contains("td", /^TKT10002/).click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-    cy.contains("Search for courses").should('not.exist')
-
-    cy.contains('.courseNameCell', "Käyttöjärjestelmät").contains("TKT20003").click()
-    cy.contains('.courseNameCell', "Ohjelmoinnin perusteet").should('not.exist');
-    cy.contains("TKT20003")
-    cy.contains("582640") // old mapped code
-    cy.contains("Summary").click()
-
-    cy.contains('.courseNameCell', "Ohjelmoinnin perusteet").contains("TKT10002").click()
-    cy.contains('.courseNameCell', "Käyttöjärjestelmät").should('not.exist');
-    cy.contains("TKT10002")
-    cy.contains("581325") // old mapped code
-    cy.contains("Summary").click()
-
-    cy.contains("a", "New query").click()
-    cy.contains("Search for courses")
-  })
-
-  it('On consecutive searches should not crash and search should work', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains("td", "TKT20003").click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-    cy.contains("Show population")
-    cy.contains("TKT20003")
-
-    cy.contains("Course statistics").click()
-    cy.contains("Search for courses")
-    //cy.wait(500)
-    cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-    cy.contains("td", "TKT20001").click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-    cy.contains("Show population")
-    cy.contains("TKT20001")
-  })
-
-  it('Shows statistics when separating by semester', () => {
-    const courseCode = 'DATA11002'
-    cy.url().should('include', '/coursestatistics')
-    cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by a course code']").type(courseCode)
-    cy.contains("td", courseCode).click()
-    cy.contains("Separate statistics for Spring and Fall semesters").click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-    cy.contains("Kevät 2018")
-    cy.contains("Syksy 2017")
-    cy.get("tbody").contains("19")
-    cy.contains(courseCode)
-  })
-
-  describe('When searching unified course stats', () => {
+  describe('when using basic user', () => {
     beforeEach(() => {
+      cy.init('/coursestatistics')
+    })
+    it('Searching single course having substitution mappings shows course statistics', () => {
       cy.url().should('include', '/coursestatistics')
-      cy.contains("Search for courses")
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      // Click uni course, not avoin
+      cy.contains('td', /^TKT20001/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001')
+      cy.contains('58131') // old mapped code
+
+      cy.contains('.tabular.menu a', 'Table').click()
+      cy.contains('All')
+      cy.contains('.modeSelectorRow a', 'Attempts').click()
+      cy.contains('.modeSelectorRow a', 'Students').click()
+
+      cy.contains('.tabular.menu a', 'Pass rate chart').click()
+      cy.get('div.modeSelectorContainer').click()
+      cy.contains('svg', 'Pass rate chart')
+
+      cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+      cy.get('div.modeSelectorContainer').click()
+      cy.contains('svg', 'Grades')
+
+      cy.contains('a', 'New query').click()
+      cy.contains('Search for courses')
+    })
+
+    it('Searching multiple courses having substitution mappings shows course statistics', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.contains('td', /^TKT20001/).click()
+      cy.contains('td', /^TKT10002/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('.courseNameCell', 'Tietorakenteet ja algoritmit').contains('TKT20001').click()
+      cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').should('not.exist')
+      cy.contains('TKT20001')
+      cy.contains('58131') // old mapped code
+      cy.contains('Summary').click()
+
+      cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').contains('TKT10002').click()
+      cy.contains('.courseNameCell', 'Käyttöjärjestelmät').should('not.exist')
+      cy.contains('TKT10002')
+      cy.contains('581325') // old mapped code
+      cy.contains('Summary').click()
+
+      cy.contains('a', 'New query').click()
+      cy.contains('Search for courses')
+    })
+
+    it('On consecutive searches should not crash and search should work', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains('td', 'TKT20003').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Show population')
+      cy.contains('TKT20003')
+
+      cy.contains('Course statistics').click()
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      cy.contains('td', 'TKT20001').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Show population')
+      cy.contains('TKT20001')
+    })
+
+    it('Shows statistics when separating by semester', () => {
+      const courseCode = 'DATA11002'
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type(courseCode)
+      cy.contains('td', courseCode).click()
+      cy.contains('Separate statistics for Spring and Fall semesters').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Syksy 2019')
+      cy.contains('Syksy 2017')
+      cy.get('tbody').contains('19')
+      cy.contains(courseCode)
+    })
+
+    it('Searching course by name displays right courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
+
+      cy.contains('Tietokantojen perusteet')
+      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('A581328')
+      cy.contains('AYTKT10004')
+      cy.contains('TKT10004, 581328')
+      cy.contains('td', /^TKT10004/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.get('.right').click()
+      cy.contains('No results')
+
+      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
+      cy.contains('td', /^AYTKT10004/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+    })
+
+    it('Searching course by name displays right courses, 10 credit courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
+      cy.contains('AYTKT20001')
+      cy.contains('TKT20001, 58131')
+      cy.contains('td', /^TKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get('.right').click()
+      cy.contains('No results')
+
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', /^AYTKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
+
+      cy.get('.right').click()
+      cy.contains('No results')
       cy.get("input[name='unifyOpenUniCourses']").parent().click()
-      cy.get("input[placeholder='Search by a course code']").type('TKT10002')
-      cy.contains("td", "TKT10002, AYTKT10002").click()
-      cy.contains("Fetch statistics").should('be.enabled').click()
-      cy.contains("Search for courses").should('not.exist')
-      cy.contains("TKT10002, 581325, A581325, AYTKT10002 Ohjelmoinnin perusteet")
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
     })
 
-    const yearRange = { from: "2000-01", to: "2017-18" }
-    const attemptsTableContents = [
-      // [time, passed, failed, passrate]
-      ["Total", 354, 312, 42, "88.14 %"],
-      ["2017-18", 204, 175, 29, "85.78 %"],
-      ["2016-17", 68, 66, 2, "97.06 %"],
-      ["2015-16", 29, 27, 2, "93.10 %"],
-      ["2014-15", 16, 16, 0, "100.00 %"],
-      ["2013-14", 8, 6, 2, "75.00 %"],
-    ]
-    const studentTableContents = [
-      // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-      ["Total", 289, 254, 13,	"92.39 %", 22, "7.61 %"],
-      ["2017-18", 157, 132, 5, "87.26 %", 20, "12.74 %"],
-      ["2016-17", 62, 60, 1, "98.39 %", 1, "1.61 %"],
-      ["2015-16", 27, 25, 2, "100.00 %", 0, "0.00 %"],
-      ["2014-15", 16, 16, 0, "100.00 %", 0, "0.00 %"],
-      ["2013-14", 5, 3, 2, "100.00 %", 0, "0.00 %"],
-    ]
-    const gradesTableContents = [
-      // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-      ["Total", 354, 42, 6,	12,	24,	35,	218, 17],
-      ["2017-18", 204, 29, 5, 8, 10, 16, 127, 9],
-      ["2016-17", 68, 2, 0, 1, 6, 10, 49, 0],
-      ["2015-16", 29, 2, 1, 1, 0, 5, 16, 4],
-      ["2014-15", 16, 0, 0, 0, 1, 0, 11, 4],
-      ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
-    ]
+    it('Can find course population', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains('tr', 'TKT20003').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
 
-    it('shows stats', () => {
-      // Time range
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", yearRange.to)
-        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should("have.text", "2000-01")
-        cy.get("div[role='option']").should('have.length', 18)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", "2017-18")
-        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should("have.text", yearRange.from)
-        cy.get("div[role='option']").should('have.length', 18)
-      })
-      cy.contains("Show population").should('be.enabled')
+      cy.contains('Käyttöjärjestelmät')
+      cy.contains('TKT20003')
 
-      // Filters
-      const primaryContents = [
-        // [studentcount, label, code]
-        [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ]
-      const totalStudyProgrammes = 20
-      cy.get("div[name='primary']").within(() => {
-        primaryContents.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']").eq(i).within(() => {
-            cy.contains(label)
-            cy.contains("div>div:last-child>div.label", code)
-            cy.contains("div>div:first-child>div.label", count)
-          })
-        })
-        cy.get("div[role='option']").eq(10).within(() => {
-          cy.contains("Muu")
-          cy.contains("div>div:last-child>div.label", "OTHER")
-          cy.contains("div>div:first-child>div.label", "2")
-        })
-        cy.get("div[role='option']").should('have.length', totalStudyProgrammes)
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('TKT20003')
+
+      cy.contains('Students (127)').click()
+      cy.contains('010135486')
+      cy.contains('010431753')
+    })
+
+    it('Population of course shows grades for each student', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      cy.contains(/^TKT20001$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
+      cy.contains('Students (197)').click()
+      cy.contains('td', '010262566').siblings().contains('4')
+      cy.contains('td', '010674989').siblings().contains('1')
+    })
+
+    describe('When searching unified course stats', () => {
+      beforeEach(() => {
+        cy.url().should('include', '/coursestatistics')
+        cy.contains('Search for courses')
+        cy.get("input[name='unifyOpenUniCourses']").parent().click()
+        cy.get("input[placeholder='Search by a course code']").type('TKT10002')
+        cy.contains('td', 'TKT10002, AYTKT10002').click()
+        cy.contains('Fetch statistics').should('be.enabled').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
       })
-      const comparisonContents = [
-        [289, "All", "ALL"],
-        ...primaryContents
-      ]
-      cy.get("div[name='comparison']").within(() => {
-        comparisonContents.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']").eq(i).within(() => {
-            cy.contains(label)
-            cy.contains("div>div:last-child>div.label", code)
-            cy.contains("div>div:first-child>div.label", count)
-          })
-        })
-        cy.get("div[role='option']").eq(11).within(() => {
-          cy.contains("Muu")
-          cy.contains("div>div:last-child>div.label", "OTHER")
-          cy.contains("div>div:first-child>div.label", "2")
-        })
-        cy.get("div[role='option']").should('have.length', totalStudyProgrammes+1) // +1 comes from already selected option
-      })
-      cy.contains("Select excluded study programmes").should("be.disabled")
 
       // Statistics
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Attempts").click()
-      cy.get("#CourseStatPanes table>tbody").within(() => {
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-
-      cy.get("[data-cy=gradeToggle]", { force: true }).click({ force: true })
-      cy.get("#CourseStatPanes table>tbody").within(() => {
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Students").click()
-      cy.get("#CourseStatPanes table>tbody").within(() => {
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-    })
-
-    it('after changing time range shows same stats', () => {
-      const newYearRange = { from: "2014-15", to: "2016-17" }
-      cy.get("div[name='fromYear']").click().within(() => {
-        cy.contains(newYearRange.from).click()
-      })
-      cy.get("div[name='toYear']").click().within(() => {
-        cy.contains(newYearRange.to).click()
-      })
-
-      // Time range
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", newYearRange.to)
-        cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should("have.text", "2000-01")
-        cy.get("div[role='option']").should('have.length', 17)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", "2017-18")
-        cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should("have.text", newYearRange.from)
-        cy.get("div[role='option']").should('have.length', 4)
-      })
-      cy.contains("Show population").should('be.enabled')
-
-      // Filters
-      const primaryContentsAfterTimeChange = [
-        // [studentcount, label, code]
-        [79, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-        [15, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [10, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [7, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-      ]
-      const totalStudyProgrammesAfterTimeChange = 13
-      cy.get("div[name='primary']").within(() => {
-        primaryContentsAfterTimeChange.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']").eq(i).within(() => {
-            cy.contains(label)
-            cy.contains("div>div:last-child>div.label", code)
-            cy.contains("div>div:first-child>div.label", count)
-          })
-        })
-        cy.get("div[role='option']").eq(5).within(() => {
-          cy.contains("Muu")
-          cy.contains("div>div:last-child>div.label", "OTHER")
-          cy.contains("div>div:first-child>div.label", "2")
-        })
-        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterTimeChange)
-      })
-      const comparisonContentsAfterTimeChange = [
-        [107, "All", "ALL"],
-        ...primaryContentsAfterTimeChange
-      ]
-      cy.get("div[name='comparison']").within(() => {
-        comparisonContentsAfterTimeChange.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']").eq(i).within(() => {
-            cy.contains(label)
-            cy.contains("div>div:last-child>div.label", code)
-            cy.contains("div>div:first-child>div.label", count)
-          })
-        })
-        cy.get("div[role='option']").eq(6).within(() => {
-          cy.contains("Muu")
-          cy.contains("div>div:last-child>div.label", "OTHER")
-          cy.contains("div>div:first-child>div.label", "2")
-        })
-        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterTimeChange+1) // +1 comes from already selected option
-      })
-      cy.contains("Select excluded study programmes").should("be.disabled")
-      
-      const timeAttemptsTableContents = [
+      const yearRange = { from: '2000-2001', to: '2020-2021' }
+      const attemptsTableContents = [
         // [time, passed, failed, passrate]
-        ["Total", 113, 109, 4, "96.46 %"],
-        ["2017-18", 204, 175, 29, "85.78 %"],
-        ["2016-17", 68, 66, 2, "97.06 %"],
-        ["2015-16", 29, 27, 2, "93.10 %"],
-        ["2014-15", 16, 16, 0, "100.00 %"],
-        ["2013-14", 8, 6, 2, "75.00 %"],
+        ['Total', 511, 486, 25],
       ]
 
-      const timeStudentTableContents = [
-        // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-        ["Total", 105, 101, 3,	"99.05 %", 1, "0.95 %"],
-        ["2017-18", 195, 131, 43, "89.23 %", 1, "10.77 %"],
-        ["2016-17", 62, 60, 1, "98.39 %", 1, "1.61 %"],
-        ["2015-16", 27, 25, 2, "100.00 %", 0, "0.00 %"],
-        ["2014-15", 16, 16, 0, "100.00 %", 0, "0.00 %"],
-        ["2013-14", 5, 2, 3, "100.00 %", 0, "0.00 %"],
-      ]
-
-      const timeGradesTableContents = [
+      const gradesTableContents = [
         // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ["Total", 113, 4,	1, 2,	7, 15, 76, 8],
-        ["2017-18", 204, 29, 5, 8, 10, 16, 127, 9],
-        ["2016-17", 68, 2, 0, 1, 6, 10, 49, 0],
-        ["2015-16", 29, 2, 1, 1, 0, 5, 16, 4],
-        ["2014-15", 16, 0, 0, 0, 1, 0, 11, 4],
-        ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
+        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
       ]
 
-      // Statistics
-      const timeRangeFilter = (timeRange) => ([time]) => {
-        return (timeRange.from <= time && time <= timeRange.to) || time === 'Total'
-      }
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Attempts").click()
-      cy.get("#CourseStatPanes table>tbody").within(() => {
-        timeAttemptsTableContents.filter(timeRangeFilter(newYearRange)).forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
+      it('shows stats', () => {
+        // Time range
+        cy.get("div[name='fromYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', yearRange.to)
+          cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', '2000-2001')
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('Show population').should('not.be.enabled')
+
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContents.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
           })
+          cy.get('tr').should('have.length', 17)
+        })
+        cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          gradesTableContents.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+          cy.get('tr').should('have.length', 17)
         })
-        cy.get("tr").should('have.length', 4)
       })
 
-      cy.get("[data-cy=gradeToggle]", { force: true }).click({ force: true })
-      cy.get("#CourseStatPanes table>tbody").within(() => {
-        timeGradesTableContents.filter(timeRangeFilter(newYearRange)).forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
+      it('after changing time range shows same stats', () => {
+        const newYearRange = { from: '2016-2017', to: '2019-2020' }
+        cy.get("div[name='fromYear']")
+          .click()
+          .within(() => {
+            cy.contains(newYearRange.from).click()
+          })
+        cy.get("div[name='toYear']")
+          .click()
+          .within(() => {
+            cy.contains(newYearRange.to).click()
           })
-        })
-        cy.get("tr").should('have.length', 4)
-      })
 
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Students").click()
-      cy.get("#CourseStatPanes table>tbody").within(() => {
-        timeStudentTableContents.filter(timeRangeFilter(newYearRange)).forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
+        // Time range
+        cy.get("div[name='fromYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', newYearRange.to)
+          cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', '2000-2001')
+          cy.get("div[role='option']").should('have.length', 20)
         })
-        cy.get("tr").should('have.length', 4)
-      })
-
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-    })
-
-    it('after changing filters shows different stats', () => {
-      cy.get("div[name='primary']").click().within(() => {
-        cy.contains("div[role='option']", "Tietojenkäsittelytieteen kandiohjelma").click()
-      })
-      cy.get("body").type("{esc}") // close dropdown
-
-      // Time range (should not have changed)
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", yearRange.to)
-        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should("have.text", "2000-01")
-        cy.get("div[role='option']").should('have.length', 18)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", "2017-18")
-        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should("have.text", yearRange.from)
-        cy.get("div[role='option']").should('have.length', 18)
-      })
-      cy.contains("Show population").should('be.enabled')
-
-      // Filters
-      const primaryContentsAfterFilterChange = [
-        // [studentcount, label, code]
-        [289, "All", "ALL"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ]
-      const totalStudyProgrammesAfterFilterChange = 20
-      cy.get("div[name='primary']").within(() => {
-        primaryContentsAfterFilterChange.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']").eq(i).within(() => {
-            cy.contains(label)
-            cy.contains("div>div:last-child>div.label", code)
-            cy.contains("div>div:first-child>div.label", count)
-          })
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', newYearRange.from)
+          cy.get("div[role='option']").should('have.length', 5)
         })
-        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterFilterChange)
-      })
-      const comparisonContentsAfterFilterChange = [
-        // [studentcount, label, code]
-        [289, "All", "ALL"],
-        [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-        [122, "Excluded", "EXCLUDED"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ]
-      cy.get("div[name='comparison']").within(() => {
-        comparisonContentsAfterFilterChange.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']").eq(i).within(() => {
-            cy.contains(label)
-            cy.contains("div>div:last-child>div.label", code)
-            cy.contains("div>div:first-child>div.label", count)
-          })
-        })
-        cy.get("div[role='option']").should('have.length', totalStudyProgrammesAfterFilterChange+2) // +2 comes from EXCLUDED and the filtered programme
-      })
-      cy.contains("Select excluded study programmes").should("be.enabled").click()
-
-      // Statistics
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Attempts").click()
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const attemptsTableContents = [
-          // [time, passed, failed, passrate]
-          ["Total", 227, 215, 12, "94.71 %"],
-          ["2017-18", 136, 126, 10, "92.65 %"],
-          ["2016-17", 57, 56, 1, "98.25 %"],
-          ["2015-16", 21, 20, 1, "95.24 %"],
-          ["2014-15", 6, 6, 0, "100.00 %"],
-        ]
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const attemptsTableContents = [
-          // [time, passed, failed, passrate]
-          ["Total", 140, 110, 30, "78.57 %"],
-          ["2017-18", 71, 52, 19, "73.24 %"],
-          ["2016-17", 16, 15, 1, "93.75 %"],
-          ["2015-16", 10, 9, 1, "90.00 %"],
-          ["2014-15", 11, 11, 0, "100.00 %"],
-        ]
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Students").click()
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const studentTableContents = [
-          // time, students, passedfirsttry, eventuallypassed, pass%, neverpassed, fail%]
-          ["Total", 179, 168,	6, "97.21 %",	5,	"2.79 %"],
-          ["2017-18", 95, 86, 4, "94.74 %", 5, "5.26 %"],
-          ["2016-17", 53, 52, 1, "100.00 %", 0, "0.00 %"],
-          ["2015-16", 19, 18, 1, "100.00 %", 0, "0.00 %"],
-          ["2014-15", 6, 6, 0, "100.00 %", 0, "0.00 %"],
-        ]
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const studentTableContents = [
-          // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-          ["Total", 122, 98, 7,	"86.07 %", 17, "13.93 %"],
-          ["2017-18", 65, 49, 1, "76.92 %", 15, "23.08 %"],
-          ["2016-17", 13, 12, 0, "92.31 %", 1, "7.69 %"],
-          ["2015-16", 10, 9, 1, "100.00 %", 0, "0.00 %"],
-          ["2014-15", 11, 11, 0, "100.00 %", 0, "0.00 %"],
-        ]
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-
-      cy.contains("#CourseStatPanes a.item", "Table").click()
-      cy.contains("#CourseStatPanes a.item", "Attempts").click()
-      cy.get("[data-cy=gradeToggle]", { force: true }).click({ force: true })
-
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const gradesTableContents = [
-          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-          ["Total", 227, 12, 4,	7, 11, 18, 161,	14],
-          ["2017-18", 136, 10, 3, 4, 4, 4, 102, 9],
-          ["2016-17", 57, 1, 0, 1, 6, 8, 41, 0],
-          ["2015-16", 21, 1, 1, 1, 0, 5, 11, 2],
-          ["2014-15", 6, 0, 0, 0, 0, 0, 3, 3],
-          ["2013-14", 1, 0, 0, 0, 0, 0, 1, 0],
-        ]
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
-      })
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const gradesTableContents = [
-          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-          ["Total", 140, 30, 2,	5, 13, 20, 67, 3],
-          ["2017-18", 71, 19, 2, 4, 6, 13, 27, 0],
-          ["2016-17", 16, 1, 0, 0, 0, 3, 12, 0],
-          ["2015-16", 10, 1, 0, 0, 0, 0, 7, 2],
-          ["2014-15", 11, 0, 0, 0, 1, 0, 9, 1],
-          ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
-        ]
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr").eq(trIndex).within(() => {
-            values.forEach((value, tdIndex) => {
-              cy.get("td").eq(tdIndex).contains(value)
-            })
-          })
-        })
-        cy.get("tr").should('have.length', 14)
+        cy.contains('Show population').should('be.enabled')
       })
     })
   })
 
-  it("Some features of Course Statistics are hidden for courseStatistics-users", () => {
-    cy.contains("Users").click()
-    cy.contains("Oodikone users")
-    cy.contains("mocking").should('not.exist')
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains('.button', 'Edit').click()
-    })
-
-    cy.get("[data-cy=access-groups-form]").click().contains("courseStatistics").click()
-    cy.get("[data-cy=access-groups-save]").click()
-
-    cy.route('POST', '/api/superlogin/usermk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains("mocking as usermk")
-    cy.wait(1000)
-
-    cy.get("[data-cy=navbar-courseStatistics]").click()
-    cy.get("[data-cy=course-code-input]").type('TKT20003')
-    cy.contains("tr", "TKT20003").click()
-    cy.contains("Fetch statistics").should('be.enabled').click()
-    cy.contains("Filter statistics by study programmes").should('not.exist')
-    cy.contains("Faculty statistics").should('not.exist')
-    cy.contains("Show population").should('not.exist')
+  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+    cy.init('/coursestatistics', 'onlycoursestatistics')
+    cy.get('[data-cy=navbar-courseStatistics]').click()
+    cy.get('[data-cy=course-code-input]').type('TKT20003')
+    cy.contains('tr', 'TKT20003').click()
+    cy.contains('Fetch statistics').should('be.enabled').click()
+    cy.contains('Filter statistics by study programmes').should('not.exist')
+    cy.contains('Faculty statistics').should('not.exist')
+    cy.contains('Show population').should('not.exist')
 
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
-      ["Total*", 57, 48, 9, "84.21 %"],
-      ["2018-19", "5 or less students", "NA", "NA", "NA"],
-      ["2017-18", 39, 31, 8, "79.49 %"],
+      ['Total*', 288, 213, 75, '73.96 %'],
+      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      ['2019-2020', 164, 121, 43, '73.78 %'],
+      ['2018-2019', 85, 60, 25, '70.59 %'],
+      ['2017-2018', 39, 32, 7, '82.05 %'],
     ]
 
-    cy.contains("#CourseStatPanes a.item", "Attempts").click()
-    cy.get("#CourseStatPanes table>tbody").within(() => {
+    cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+    cy.get('#CourseStatPanes table>tbody').within(() => {
       attemptsTableContents.forEach((values, trIndex) => {
-        cy.get("tr").eq(trIndex).within(() => {
-          values.forEach((value, tdIndex) => {
-            cy.get("td").eq(tdIndex).contains(value)
+        cy.get('tr')
+          .eq(trIndex)
+          .within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get('td').eq(tdIndex).contains(value)
+            })
           })
-        })
       })
-      cy.get("tr").should('have.length', 10)
+      cy.get('tr').should('have.length', 7)
     })
-
-    cy.contains("Stop mocking").click()
-    cy.wait(1000)
-
-    cy.contains("Users").click()
-    cy.contains("Oodikone users")
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains('.button', 'Edit').click()
-    })
-    cy.contains("Access Groups").siblings().within(($row) => {
-      cy.get("i[class='dropdown icon clear']").click()
-    })
-    cy.get("[data-cy=access-groups-save]").click()
-
   })
 })

cypress/integration/Custom_population.js

@@ -1,226 +1,195 @@
 /// <reference types="Cypress" />
 
 const openCustomPopupForm = () => {
-  cy.get("button")
-    .contains("Custom population")
-    .click();
-};
+  cy.get('button').contains('Custom population').click()
+}
 
 const fillName = () => {
-  const name = `TEST-${new Date().getTime()}`;
-  cy.contains("Insert name for this custom population if you wish to save it")
+  const name = `TEST-${new Date().getTime()}`
+  cy.contains('Insert name for this custom population if you wish to save it')
     .siblings()
-    .get("input[placeholder=name]")
-    .type(name);
-  return name;
-};
+    .get('input[placeholder=name]')
+    .type(name)
+  return name
+}
 
 const save = () => {
-  cy.get("button")
-    .contains("Save")
-    .click();
-};
+  cy.get('button').contains('Save').click()
+}
 
 const selectSavedPopulation = name => {
-   cy.get('[data-cy="history-search"]').children().eq(0).type(name).type("{enter}");
-};
+  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+}
 
 const deleteAllSearches = () => {
-  cy.get('[data-cy="history-search"]').children().eq(0).click();
-  cy.contains("Saved populations")
+  cy.get('[data-cy="history-search"]').children().eq(0).click()
+  cy.contains('Saved populations')
     .parent()
     .parent()
     .parent()
     .parent()
-    .get(".dropdown")
+    .get('.dropdown')
     .then(d => {
-      const searchItems = d.find("div[role=option] > span[class=text]");
+      const searchItems = d.find('div[role=option] > span[class=text]')
       for (let i = 0; i < searchItems.length; i++) {
-        if (searchItems[i].textContent.includes("TEST-")) {
-          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type("{enter}");
-          cy.get("button")
-            .contains("Delete")
-            .click();
+        if (searchItems[i].textContent.includes('TEST-')) {
+          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('button').contains('Delete').click()
         }
       }
-    });
-};
+    })
+}
 
 const fillForm = content => {
-  cy.contains("Insert studentnumbers you wish to use for population here")
+  cy.contains('Insert studentnumbers you wish to use for population here')
     .siblings()
-    .get("textarea")
-    .type(content.join("\n"));
-};
+    .get('textarea')
+    .type(content.join('\n'))
+}
 
 const search = () => {
-  cy.get("button")
-    .contains("Search population")
-    .click();
-};
+  cy.get('button').contains('Search population').click()
+}
 
 const searchFor = studentnumbers => {
-  openCustomPopupForm();
-  fillForm(studentnumbers);
-  search();
-};
+  openCustomPopupForm()
+  fillForm(studentnumbers)
+  search()
+}
 
 const hasLanded = () => {
-  cy.contains("Credit accumulation");
-  cy.contains("Programme distribution");
-  cy.contains("Courses of population");
-  cy.contains("Students");
-};
+  cy.contains('Credit accumulation')
+  cy.contains('Programme distribution')
+  cy.contains('Courses of population')
+  cy.contains('Students')
+}
 
 const containsAmountOfStudents = (amount = 0) => {
-  cy.contains(`Credit accumulation (for ${amount} students)`);
-};
+  cy.contains(`Credit accumulation (for ${amount} students)`)
+}
 
 const containsSpecificStudents = (studentnumbers = []) => {
-  cy.contains(`Students (${studentnumbers.length})`).click();
+  cy.contains(`Students (${studentnumbers.length})`).click()
 
-  studentnumbers.forEach(s => cy.contains(s));
-};
+  studentnumbers.forEach(s => cy.contains(s))
+}
 
-const loginAs = username => {
-  cy.contains("Users").click();
-  cy.contains(username)
-    .siblings()
-    .contains("Edit")
-    .click();
-  cy.get(".spy").click();
-  cy.visit(`${Cypress.config().baseUrl}custompopulation`);
-  cy.url().should("include", "/custompopulation");
-  cy.contains("Custom population");
-};
-
-describe("Custom population tests", () => {
+describe('Custom population tests', () => {
+  const nonExistentStudentNumbers = ['123', 'X', '-', ' ']
   beforeEach(() => {
-    cy.init('custompopulation')
-    cy.url().should("include", "/custompopulation");
-    cy.contains("Custom population");
-  });
+    cy.init('/custompopulation')
+    cy.url().should('include', '/custompopulation')
+    cy.contains('Custom population')
+  })
 
   after(() => {
-    cy.visit(`${Cypress.config().baseUrl}custompopulation`);
-    cy.url().should("include", "/custompopulation");
-    cy.contains("Custom population");
-    cy.get("button")
-      .contains("Custom population")
-      .click();
-    deleteAllSearches();
-  });
+    cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+    cy.url().should('include', '/custompopulation')
+    cy.contains('Custom population')
+    cy.get('button').contains('Custom population').click()
+    deleteAllSearches()
+  })
 
-  describe("Custom population searching", () => {
-    it("Finds a proper population", () => {
-      const testStudentNumbers = ["010623419", "010942404", "010484975"];
-      searchFor(testStudentNumbers);
-      hasLanded();
-      containsAmountOfStudents(3);
-      containsSpecificStudents(testStudentNumbers);
-    });
+  describe('Custom population searching', () => {
+    it('Finds a proper population', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
+        const students = studentNumbersForCSStudentsSet1
+        searchFor(students)
+        hasLanded()
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
+      })
+    })
 
     it("Doesn't return non-existing students", () => {
-      const testStudentNumbers = [
-        "010623419",
-        "010942404",
-        "123",
-        "x",
-        "-",
-        " "
-      ];
-      searchFor(testStudentNumbers);
-      hasLanded();
-      containsAmountOfStudents(2);
-      containsSpecificStudents(["010623419", "010942404"]);
-    });
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
+        const students = [...studentNumbersForCSStudentsSet1, ...nonExistentStudentNumbers]
+        searchFor(students)
+        hasLanded()
+        containsAmountOfStudents(studentNumbersForCSStudentsSet1.length)
+        containsSpecificStudents(studentNumbersForCSStudentsSet1)
+      })
+    })
 
     it("Doesn't find empty custom population", () => {
-      const testStudentNumbers = ["1", "2", "3"];
-      searchFor(testStudentNumbers);
-      cy.contains("Credit accumulation").should("not.exist");
-      cy.contains("Programme distribution").should("not.exist");
-      cy.contains("Courses of population").should("not.exist");
-      cy.get("button").contains("Custom population");
-    });
+      searchFor(nonExistentStudentNumbers)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
 
-    it("Doesn't return students to whom the user doesn't have rights to", () => {
-      const kasvatusStudents = ["010102273", "010112616", "010116531"];
-      const kapistelyStudents = ["010623419", "010942404", "010484975"];
-      loginAs("Normaalikäyttäjä");
+    it("Doesn't return students user has no right to", () => {
+      // TODO: check that this doesn't create false positives, since this test goes
+      // through even if following students are not in anon sis-db.
+      const studentsForEduBachStudents = ['014990067', '013069465', '014853890']
+      searchFor(studentsForEduBachStudents)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
+  })
 
-      searchFor(kasvatusStudents);
-      cy.contains("Credit accumulation").should("not.exist");
-      cy.contains("Programme distribution").should("not.exist");
-      cy.contains("Courses of population").should("not.exist");
+  describe('Custom population search saving', () => {
+    it('Saves a custom population search', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
+        const students = studentNumbersForCSStudentsSet1
+        openCustomPopupForm()
+        const name = fillName()
+        fillForm(students)
+        save()
 
-      searchFor(kapistelyStudents.concat(kasvatusStudents));
-      hasLanded();
-      containsAmountOfStudents(3);
-      containsSpecificStudents(kapistelyStudents);
-    });
-  });
+        // Round 1
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
 
-  describe("Custom population search saving", () => {
-    it("Saves a custom population search", () => {
-      const testStudentNumbers = ["010623419", "010942404", "010484975"];
-      openCustomPopupForm();
-      const name = fillName();
-      fillForm(testStudentNumbers);
-      save();
+        // Round 2
+        cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+        cy.url().should('include', '/custompopulation')
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students.length)
+        containsSpecificStudents(students)
+      })
+    })
 
-      // Round 1
-      selectSavedPopulation(name);
-      search();
-      cy.contains(`Custom population "${name}"`);
-      containsAmountOfStudents(3);
-      containsSpecificStudents(testStudentNumbers);
+    it('Updates a custom population search', () => {
+      cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
+        const students1 = studentNumbersForCSStudentsSet1
+        const students2 = studentNumbersForCSStudentsSet2
+        openCustomPopupForm()
+        const name = fillName()
+        fillForm(students1)
+        save()
 
-      // Round 2
-      cy.visit(`${Cypress.config().baseUrl}custompopulation`);
-      cy.url().should("include", "/custompopulation");
-      cy.contains("Custom population");
-      cy.get("button")
-        .contains("Custom population")
-        .click();
-      selectSavedPopulation(name);
-      search();
-      cy.contains(`Custom population "${name}"`);
-      containsAmountOfStudents(3);
-      containsSpecificStudents(testStudentNumbers);
-    });
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students1.length)
+        containsSpecificStudents(students1)
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        fillForm([...students1, ...students2])
+        save()
 
-    it("Updates a custom population search", () => {
-      const testStudentNumbers = ["010623419", "010942404", "010484975"];
-      openCustomPopupForm();
-      const name = fillName();
-      fillForm(testStudentNumbers);
-      save();
-
-      selectSavedPopulation(name);
-      search();
-      cy.contains(`Custom population "${name}"`);
-      containsAmountOfStudents(3);
-      containsSpecificStudents(testStudentNumbers);
-      cy.contains("Custom population");
-      cy.get("button")
-        .contains("Custom population")
-        .click();
-      selectSavedPopulation(name);
-      fillForm(["", "011284954"]);
-      save();
-
-      cy.visit(`${Cypress.config().baseUrl}custompopulation`);
-      cy.url().should("include", "/custompopulation");
-      cy.contains("Custom population");
-      cy.get("button")
-        .contains("Custom population")
-        .click();
-      selectSavedPopulation(name);
-      search();
-      cy.contains(`Custom population "${name}"`);
-      containsAmountOfStudents(4);
-      containsSpecificStudents(testStudentNumbers.concat("011284954"));
-    });
-  });
-});
+        cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
+        cy.url().should('include', '/custompopulation')
+        cy.contains('Custom population')
+        cy.get('button').contains('Custom population').click()
+        selectSavedPopulation(name)
+        search()
+        cy.contains(`Custom population "${name}"`)
+        containsAmountOfStudents(students1.length + students2.length)
+        containsSpecificStudents([...students1, ...students2])
+      })
+    })
+  })
+})

cypress/integration/Filters.js

@@ -1,199 +1,296 @@
 /// <reference types="Cypress" />
 
-const checkFilteringResult = (studentCount) => {
-  cy.contains(`Students (${studentCount})`);
-  // I don't know why this got fkd but we don't have time to fix it now. Works acually, though.
-  // cy.cs("active-filter-count").should(noFiltering ? "not.exist" : "exist")
-};
+// const baseUrl = Cypress.config().baseUrl
+
+const checkFilteringResult = studentCount => {
+  cy.contains(`Students (${studentCount})`)
+}
+
+// Semantic UI doesn't allow injection of data-cy:s for single multiple dropdown selection.
+// This function tries to click "x" on all selections inside dropdown matching given attribute
+const clearSemanticUIMultipleDropDownSelection = dataCyAttribute => {
+  cy.cs(dataCyAttribute).find('i.delete').click()
+}
+
+const clearSingleDropdownSelection = dataCyAttribute => {
+  cy.cs(dataCyAttribute).find('i.clear').click()
+}
+
+// Helper tool to create pre and post steps for each filter step. Created to avoid copypasting clicking and checking
+// to every it-function. Reason behind using test function wrapper is that Cypresses internal beforeEach / afterEach
+// functions don't take any parameters and using global object for matching test step name seemed overcomplicated.
+const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
+  return (dataCyAttributeOfHeaderToClick, testStepFunctionToRun) => {
+    cy.cs(dataCyAttributeOfHeaderToClick).click()
+    checkFilteringResult(amountWithoutFiltering)
+    testStepFunctionToRun()
+    checkFilteringResult(amountWithoutFiltering)
+    cy.cs(dataCyAttributeOfHeaderToClick).click()
+  }
+}
+
+describe('Population Statistics', () => {
+  const pathToCSBach2018 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2018'
+  const defaultAmountOfStudents = 148
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
-describe("Population Statistics", () => {
   before(() => {
-    // Tests run considerably faster if we don't init() before each case.
-    // Drawback is that cases then rely on the tray being toggled open.
-    // TODO: Make smarter (if one test fails all subsequent tests fail, too)?
-    cy.init();
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-  });
-
-  // anon data doesn't work with current population statistic logic, so we need to skip
-  // these
-  it.skip("Graduation filter works", () => {
-    cy.cs("graduatedFromProgrammeFilter-header").click();
-    cy.selectFromDropdown("graduatedFromProgrammeFilter-dropdown", 0);
-    checkFilteringResult(193);
-    cy.selectFromDropdown("graduatedFromProgrammeFilter-dropdown", 1);
-    checkFilteringResult(1);
-    cy.cs("graduatedFromProgrammeFilter-clear").click();
-    checkFilteringResult(194, true);
-  });
-
-  it.skip("Transfer filter works", () => {
-    cy.cs("transferredToProgrammeFilter-header").click();
-    cy.cs("transferredToProgrammeFilter-have").click();
-    checkFilteringResult(25);
-    cy.cs("transferredToProgrammeFilter-havenot").click();
-    checkFilteringResult(194);
-    cy.cs("transferredToProgrammeFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it.skip("Enrollment filter works", () => {
-    cy.cs("enrollmentStatusFilter-header").click();
-    cy.selectFromDropdown("enrollmentStatusFilter-status", 0);
-    checkFilteringResult(219, true);
-    cy.selectFromDropdown("enrollmentStatusFilter-semesters", [0, 0]);
-    checkFilteringResult(202);
-    cy.selectFromDropdown("enrollmentStatusFilter-status", 1);
-    checkFilteringResult(13);
-    cy.cs("enrollmentStatusFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it.skip("Credit filter works", () => {
-    cy.cs("credit-filter-header").click();
-    cy.cs("credit-filter-min").click().type("50{enter}");
-    checkFilteringResult(98);
-    cy.cs("credit-filter-max").click().type("100{enter}");
-    checkFilteringResult(88);
-    cy.cs("credit-filter-min-clear").click();
-    checkFilteringResult(209);
-    cy.cs("credit-filter-max-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it.skip("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(48);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(171);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-    cy.cs("genderFilter-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it.skip("Starting year filter works", () => {
-    cy.cs("startYearAtUni-header").click();
-    cy.selectFromDropdown("startYearAtUni-dropdown", [16]);
-    checkFilteringResult(13);
-    cy.cs("startYearAtUni-clear").click();
-    checkFilteringResult(219, true);
-  });
-
-  it.skip("Courses filter works", () => {
-    cy.cs("courseFilter-header").click();
-    cy.cs("courseFilter-course-dropdown").click().contains("MAT11002").click();
-    checkFilteringResult(82);
-    cy.selectFromDropdown("courseFilter-MAT11002-dropdown", 1);
-    checkFilteringResult(75);
-    cy.cs("courseFilter-course-dropdown").click().contains("TKT20001").click();
-    checkFilteringResult(62);
-    cy.selectFromDropdown("courseFilter-TKT20001-dropdown", 6);
-    checkFilteringResult(24);
-    cy.cs("courseFilter-MAT11002-clear").click();
-    checkFilteringResult(125);
-    cy.cs("courseFilter-TKT20001-clear").click();
-    checkFilteringResult(219, true);
-  });
-});
-
-describe("Course Statistics", () => {
-  before(() => {
-    cy.init();
-    cy.cs("navbar-courseStatistics").click();
-    cy.cs("course-code-input").click().type("TKT20002");
-    cy.get(":nth-child(2) > .ten").click();
-    cy.cs("fetch-stats-button").click();
-    cy.get(":nth-child(2) > :nth-child(1) > div > .item > .level").click();
-  });
-
-  /*
-  it("Enrollment filter works", () => {
-    cy.cs("enrollmentStatusFilter-header").click()
-    cy.selectFromDropdown("enrollmentStatusFilter-status", 0)
-    checkFilteringResult(93, true)
-    cy.selectFromDropdown("enrollmentStatusFilter-semesters", [0, 0])
-    checkFilteringResult(92)
-    cy.selectFromDropdown("enrollmentStatusFilter-status", 1)
-    checkFilteringResult(0)
-    cy.cs("enrollmentStatusFilter-clear").click()
-    checkFilteringResult(93, true)
+    cy.init(pathToCSBach2018)
   })
-  */
 
-  it("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(19);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(74);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-  });
-
-  /*
-  it("Starting year filter works", () => {
-    cy.cs("startYearAtUni-header").click()
-    cy.selectFromDropdown("startYearAtUni-dropdown", [0])
-    checkFilteringResult(0)
-    cy.cs("startYearAtUni-clear").click()
-    checkFilteringResult(93, true)
+  it('Transfer filter is on not transferred by default', () => {
+    cy.contains('By default only students who have not transferred to this study programme are shown.')
+    cy.cs('transferredToProgrammeFilter-havenot').find('input').should('be.checked')
   })
-  */
 
-  // it("Grade filter works", () => {
-  //   cy.cs("gradeFilter-header").click()
-  //   cy.cs("grade-option-5").click()
-  //   checkFilteringResult(24)
-  //   cy.cs("grade-option-Hyl.").click()
-  //   checkFilteringResult(40)
-  // })
-});
+  it('Graduation filter works', () => {
+    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+      const graduated = 11
+      cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+      checkFilteringResult(graduated)
+      cy.cs('graduatedFromProgrammeFilter-graduated-false').click()
+      checkFilteringResult(defaultAmountOfStudents - graduated)
+      cy.cs('graduatedFromProgrammeFilter-all').click()
+    })
+  })
+
+  it('Transfer filter works', () => {
+    runTestStepWithPreAndPostParts('transferredToProgrammeFilter-header', () => {
+      const transferred = 1
+      cy.cs('transferredToProgrammeFilter-have').click()
+      checkFilteringResult(transferred)
+      cy.cs('transferredToProgrammeFilter-all').click()
+      checkFilteringResult(defaultAmountOfStudents + transferred)
+      cy.cs('transferredToProgrammeFilter-havenot').click()
+    })
+  })
+
+  it('Enrollment filter works', () => {
+    runTestStepWithPreAndPostParts('enrollmentStatusFilter-header', () => {
+      cy.selectFromDropdown('enrollmentStatusFilter-status', 0)
+      cy.selectFromDropdown('enrollmentStatusFilter-semesters', [0])
+      checkFilteringResult(137)
+      clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
+    })
+  })
+
+  it('Credit filter works', () => {
+    runTestStepWithPreAndPostParts('credit-filter-header', () => {
+      cy.cs('credit-filter-min').type('50')
+      cy.cs('credit-filter-max').type('150')
+      checkFilteringResult(116)
+      cy.cs('credit-filter-min').find('input').clear()
+      cy.cs('credit-filter-max').find('input').clear()
+    })
+  })
+
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('40')
+      checkFilteringResult(43)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
+
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('genderFilter-header', () => {
+      cy.selectFromDropdown('genderFilter-dropdown', 0)
+      checkFilteringResult(36)
+      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      checkFilteringResult(112)
+      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      checkFilteringResult(0)
+      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', 0)
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.selectFromDropdown('startYearAtUni-dropdown', 14)
+      checkFilteringResult(121)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+      checkFilteringResult(5)
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      checkFilteringResult(10)
+      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
+    })
+  })
+
+  // plz fix
+  it.skip('Courses filter works', () => {
+    // courses takes some time to load, wait for it to complete
+    cy.wait(10000)
+    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      cy.cs('courseFilter-course-dropdown').click()
+      const courses = ['TKT20001', 'MAT11002']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(140)
+      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      checkFilteringResult(131)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(56)
+      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 2)
+      checkFilteringResult(2)
+      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
+    })
+  })
+
+  it('Filter combinations work', () => {
+    runTestStepWithPreAndPostParts('graduatedFromProgrammeFilter-header', () => {
+      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+        cy.cs('graduatedFromProgrammeFilter-graduated-true').click()
+        cy.cs('ageFilter-min').type('20')
+        cy.cs('ageFilter-max').type('30')
+        checkFilteringResult(11)
+        cy.cs('ageFilter-min').find('input').clear()
+        cy.cs('ageFilter-max').find('input').clear()
+        cy.cs('graduatedFromProgrammeFilter-all').click()
+      })
+    })
+  })
+})
+
+describe('Course Statistics', () => {
+  const pathToDSAndAlgoSpring2019 =
+    '/coursepopulation?coursecodes=%5B%22TKT20001%22%2C%2258131%22%5D&from=138&separate=true&to=138&years=Kev%C3%A4t%202019'
+  const defaultAmountOfStudents = 118
+  const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
-describe("Custom Population Statistics", () => {
   before(() => {
-    cy.init("custompopulation");
-    cy.cs("custom-pop-search-button").click();
-    cy.cs("student-no-input").click().type(`
-      010182086
-      010211504
-      010275964
-      010328785
-      010331154
-    `);
-    cy.cs("search-button").click();
-  });
+    cy.init(pathToDSAndAlgoSpring2019)
+  })
 
-  it("Gender filter works", () => {
-    cy.cs("genderFilter-header").click();
-    cy.selectFromDropdown("genderFilter-dropdown", 0);
-    checkFilteringResult(2);
-    cy.selectFromDropdown("genderFilter-dropdown", 1);
-    checkFilteringResult(3);
-    cy.selectFromDropdown("genderFilter-dropdown", 2);
-    checkFilteringResult(0);
-  });
+  it('Grade filter works', () => {
+    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+      cy.cs('gradeFilter-5').click()
+      checkFilteringResult(20)
+      cy.cs('gradeFilter-3').click()
+      checkFilteringResult(38)
+      cy.cs('gradeFilter-3').click()
+      cy.cs('gradeFilter-5').click()
+    })
+  })
 
-  // it("Starting year filter works", () => {
-  //   cy.cs("startYearAtUni-header").click()
-  //   cy.selectFromDropdown("startYearAtUni-dropdown", [0])
-  //   checkFilteringResult(5)
-  // })
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('40')
+      checkFilteringResult(33)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
 
-  // it("Courses filter works", () => {
-  //   cy.cs("courseFilter-header").click()
-  //   cy.cs("courseFilter-course-dropdown").click().contains("MAT12003").click()
-  //   checkFilteringResult(3)
-  //   cy.selectFromDropdown("courseFilter-MAT12003-dropdown", 5)
-  //   checkFilteringResult(2)
-  //   cy.cs("courseFilter-course-dropdown").click().contains("TKT21014").click()
-  //   checkFilteringResult(1)
-  //   cy.selectFromDropdown("courseFilter-TKT21014-dropdown", 3)
-  //   checkFilteringResult(0)
-  //   cy.cs("courseFilter-MAT12003-clear").click()
-  //   checkFilteringResult(0)
-  //   cy.cs("courseFilter-TKT21014-clear").click()
-  //   checkFilteringResult(5, true)
-  // })
-});
+  it('Gender filter works', () => {
+    runTestStepWithPreAndPostParts('genderFilter-header', () => {
+      cy.selectFromDropdown('genderFilter-dropdown', 0)
+      checkFilteringResult(27)
+      cy.selectFromDropdown('genderFilter-dropdown', 1)
+      checkFilteringResult(91)
+      cy.selectFromDropdown('genderFilter-dropdown', 2)
+      checkFilteringResult(0)
+      cy.selectFromDropdown('genderFilter-dropdown', 3)
+      checkFilteringResult(0)
+      clearSingleDropdownSelection('genderFilter-dropdown')
+    })
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+      cy.selectFromDropdown('startYearAtUni-dropdown', 12)
+      checkFilteringResult(29)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Filter combinations work', () => {
+    runTestStepWithPreAndPostParts('gradeFilter-header', () => {
+      runTestStepWithPreAndPostParts('ageFilter-header', () => {
+        cy.cs('gradeFilter-3').click()
+        cy.cs('ageFilter-min').type('20')
+        cy.cs('ageFilter-max').type('30')
+        checkFilteringResult(1)
+        cy.cs('ageFilter-min').find('input').clear()
+        cy.cs('ageFilter-max').find('input').clear()
+        cy.cs('gradeFilter-3').click()
+      })
+    })
+  })
+})
+
+describe('Custom Population Statistics', () => {
+  let runTestStepWithPreAndPostParts
+
+  before(() => {
+    cy.init('/custompopulation')
+    cy.cs('custom-pop-search-button').click()
+    cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1, studentNumbersForCSStudentsSet2 }) => {
+      const students = [...studentNumbersForCSStudentsSet1, ...studentNumbersForCSStudentsSet2]
+      runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(students.length)
+      cy.cs('student-no-input').click().type(students.join('/n'))
+    })
+    cy.cs('search-button').click()
+  })
+
+  it('Age filter works', () => {
+    runTestStepWithPreAndPostParts('ageFilter-header', () => {
+      cy.cs('ageFilter-min').type('20')
+      cy.cs('ageFilter-max').type('30')
+      checkFilteringResult(1)
+      cy.cs('ageFilter-min').find('input').clear()
+      cy.cs('ageFilter-max').find('input').clear()
+    })
+  })
+
+  it('Gender filter works', () => {
+    cy.cs('genderFilter-header').click()
+    cy.selectFromDropdown('genderFilter-dropdown', 0)
+    checkFilteringResult(3)
+    cy.selectFromDropdown('genderFilter-dropdown', 1)
+    checkFilteringResult(2)
+    cy.selectFromDropdown('genderFilter-dropdown', 2)
+    checkFilteringResult(0)
+    clearSingleDropdownSelection('genderFilter-dropdown')
+    checkFilteringResult(5, true)
+  })
+
+  it('Starting year filter works', () => {
+    runTestStepWithPreAndPostParts('startYearAtUni-header', () => {
+      cy.selectFromDropdown('startYearAtUni-dropdown', [0])
+      checkFilteringResult(1)
+      clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
+    })
+  })
+
+  it('Courses filter works', () => {
+    runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      const courses = ['MAT11001', 'TKT20004']
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
+      checkFilteringResult(4)
+      cy.selectFromDropdown(`courseFilter-${courses[0]}-dropdown`, 1)
+      checkFilteringResult(4)
+      cy.cs('courseFilter-course-dropdown').click().contains(courses[1]).click()
+      checkFilteringResult(3)
+      cy.selectFromDropdown(`courseFilter-${courses[1]}-dropdown`, 3)
+      checkFilteringResult(1)
+      courses.forEach(course => cy.cs(`courseFilter-${course}-clear`).click())
+    })
+  })
+})

cypress/integration/Population_statistics.js

@@ -1,167 +1,167 @@
 /// <reference types="Cypress" />
 
 const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains("Advanced settings")
-    .siblings()
-    .get("[data-cy=advanced-toggle]")
-    .click();
-  includeSettings.forEach((setting) => {
-    cy.contains("Advanced settings")
-      .parent()
-      .siblings()
-      .contains(setting)
-      .click();
-  });
-  cy.get(".adv-stats-until > .form-control").click().clear().type(until);
-  cy.contains("Fetch population with new settings").click();
-  cy.contains("Advanced settings");
-};
+  cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
+  includeSettings.forEach(setting => {
+    cy.contains('Advanced settings').parent().siblings().contains(setting).click()
+  })
+  cy.get('.adv-stats-until > .form-control').click().clear().type(until)
+  cy.contains('Fetch population with new settings').click()
+  cy.contains('Advanced settings')
+}
 
-describe("Population Statistics tests", () => {
-  beforeEach(() => {
-    cy.init();
-  });
+describe('Population Statistics tests', () => {
+  const pathToCSBach2017 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  describe('when using basic user', () => {
+    beforeEach(() => {
+      cy.init('/populations')
+    })
 
-  const checkAmountOfStudents = (assertion) => {
-    let students = 0;
-    cy.contains("Credit accumulation")
-      .click()
-      .invoke("text")
-      .then((text) => {
-        students = Number(text.match(/\d+/g)[0]);
-        expect(students).to.equal(assertion);
-      });
-  };
+    it('Population statistics search form is usable', () => {
+      cy.contains('See population').should('be.disabled')
+      cy.contains('Search for population')
+      cy.contains('Class of')
+        .parent()
+        .within(() => {
+          cy.get('.form-control').as('enrollmentSelect')
+        })
 
-  it("Population statistics search form is usable", () => {
-    cy.cs("navbar-studyProgramme").click();
-    cy.cs("navbar-class").click();
-    cy.contains("See population").should("be.disabled");
-    cy.url().should("include", "/populations");
-    cy.contains("Search for population");
-    cy.contains("Class of")
-      .parent()
-      .within(() => {
-        cy.get(".form-control").as("enrollmentSelect");
-      });
+      cy.get('@enrollmentSelect')
+        .its(`${[0]}.value`)
+        .then(beforeVal => {
+          cy.get('@enrollmentSelect').click()
+          cy.get('.yearSelectInput .rdtPrev').click({ force: true })
+          cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
+          cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
+        })
 
-    cy.get("@enrollmentSelect")
-      .its(`${[0]}.value`)
-      .then((beforeVal) => {
-        cy.get("@enrollmentSelect").click();
-        // go back to 2010-2019
-        cy.get(".yearSelectInput .rdtPrev").click({ force: true });
-        cy.get(".yearSelectInput table")
-          .contains("2014-2015")
-          .click({ force: true });
-        cy.get("@enrollmentSelect").should("not.have.value", beforeVal);
-      });
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen maisteriohjelma')
+        .click()
+      cy.contains('See population').should('be.enabled')
+    })
 
-    cy.contains("Select study programme")
-      .click()
-      .siblings()
-      .contains("Tietojenkäsittelytieteen koulutusohjelma")
-      .click();
-    cy.contains("Select degree")
-      .click()
-      .siblings()
-      .contains("Luonnontieteiden kandidaatti");
-  });
+    it('Searching for population really shows population', () => {
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen kandiohjelma')
+        .click()
+      cy.contains('See population').click()
+      cy.contains('Population statistics')
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+    })
 
-  // FIXME: re-enable when CI has mandatory courses in the db
-  it.skip("Population statistics is usable on general level", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
-    setPopStatsUntil("September 2019");
+    it('Population statistics is usable on general level', () => {
+      cy.visit(pathToCSBach2017)
+      setPopStatsUntil('toukokuu 2020')
 
-    cy.get(".card").within(() => {
-      cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
-      cy.contains("Sample size: 29 students");
-      cy.contains("Excludes exchange students");
-      cy.contains("Excludes students who haven't enrolled present nor absent");
-    });
-    cy.contains("Courses of population").click({ force: true });
+      cy.get('.card').within(() => {
+        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.contains('Sample size: 149 students')
+        cy.contains('Excludes exchange students')
+        cy.contains("Excludes students who haven't enrolled present nor absent")
+        cy.contains('Excludes students with non-degree study right')
+        cy.contains('Excludes students who have transferred out of this programme')
+      })
+      cy.contains('Courses of population').click({ force: true })
 
-    cy.route("/api/v3/courseyearlystats**").as("coursePage");
-    cy.wait(150);
-    cy.cs("expand-CSM10000").click();
-    cy.cs("coursestats-link-CSM12101").click();
-    cy.wait("@coursePage");
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("CSM12101");
-  });
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=expand-TKT1]').click()
+      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+    })
 
-  it.skip("Student list checking works as intended", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
-    cy.contains("Students (16)").click();
-    cy.contains("010429464");
-    cy.contains("666666666").should("not.exist");
-    cy.contains("button", "Check studentnumbers").click();
-    cy.contains("Check for studentnumbers");
-    cy.get("textarea").type("010429464").type("{enter}").type("666666666");
-    cy.contains("button", "check students").click();
-    cy.contains("#checkstudentsresults", "Results").within((e) => {
-      cy.contains("Student numbers in list and in oodi").click();
-      cy.contains("#found", "010429464");
-      cy.contains("Student numbers in list but not in oodi").click();
-      cy.contains("#notfound", "666666666");
-      cy.contains("Student numbers in oodi but not in list").click();
-      cy.contains("#notsearched", "010533091");
-    });
-  });
+    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
+      cy.contains('No tags defined. You can define them here.').find('a').click()
+      cy.contains('Tags').click()
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Create new tag')
+    })
 
-  it.skip("Empty 'tags' tab has a link to the page where tags can be created", () => {
-    cy.selectStudyProgramme("Kielten kandiohjelma");
-    cy.contains("Students (5)").click();
-    cy.get("[data-cy=student-table-tabs]").contains("Tags").click();
-    cy.contains("No tags defined. You can define them here.").find("a").click();
-    cy.contains("Kielten kandiohjelma");
-    cy.contains("Create new tag");
-  });
+    // TODO: this fails in CI, but not locally. Investigate and fix
+    it.skip('Advanced settings work', () => {
+      cy.visit(pathToCSBach2017)
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.contains('Statistics until')
+      // only spring
+      cy.cs('toggle-fall').click()
+      cy.contains('Fetch population').click()
 
-  it.skip("Advanced settings work", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.contains("Statistics until");
-    // only spring
-    cy.cs("toggle-fall").click();
-    cy.contains("Fetch population").click();
+      // moar waiting hack
+      cy.wait(10000)
 
-    cy.contains("Credit accumulation (for 0 students)");
+      cy.contains('No statistics found for the given query.')
 
-    // only fall
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.cs("toggle-fall").click();
-    cy.cs("toggle-spring").click();
+      // only fall
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-fall').click()
+      cy.cs('toggle-spring').click()
 
-    cy.contains("Fetch population").click();
+      cy.contains('Fetch population').click()
 
-    // FIXME: This fails randomly both locally and on CI.
-    cy.contains("Credit accumulation (for 194 students)");
+      cy.contains('Credit accumulation (for 149 students)')
 
-    // spring + fall and include cancelled
-    cy.get("[data-cy=advanced-toggle]").click();
-    cy.cs("toggle-spring").click();
-    cy.cs("toggle-cancelled").click();
-    cy.contains("Fetch population").click();
+      // spring + fall and include cancelled
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-spring').click()
+      cy.cs('toggle-cancelled').click()
+      cy.contains('Fetch population').click()
 
-    cy.contains("Credit accumulation (for 202 students)");
-  });
+      cy.contains('Credit accumulation (for 170 students)')
+    })
 
-  it.skip("Credit Statistics, Statistics pane works", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
-    cy.contains("Credit statistics").click();
-    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click();
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
-    cy.get("[data-cy='credit-stats-table-name-header']").should(
-      "contain",
-      "Statistic for n = 194 Students"
-    );
-    cy.get("[data-cy='credit-stats-mean']").should("contain", "45.10");
-    cy.get("[data-cy='credit-stats-stdev']").should("contain", "30.61");
-    cy.get("[data-cy='credit-stats-min']").should("contain", "0");
-    cy.get("[data-cy='credit-stats-q1']").should("contain", "19");
-    cy.get("[data-cy='credit-stats-q2']").should("contain", "47");
-    cy.get("[data-cy='credit-stats-q3']").should("contain", "66");
-    cy.get("[data-cy='credit-stats-max']").should("contain", "137");
-  });
-});
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
+      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
+      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
+      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
+    })
+  })
+
+  describe('when using admin', () => {
+    beforeEach(() => {
+      cy.init(pathToCSBach2017, 'admin')
+    })
+    it('Student list checking works as intended', () => {
+      const existing = '010113437'
+      const nonExisting = '66666666'
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.contains(existing)
+      cy.contains(nonExisting).should('not.exist')
+      cy.contains('button', 'Check studentnumbers').click()
+      cy.contains('Check for studentnumbers')
+      cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
+      cy.contains('button', 'check students').click()
+      cy.contains('#checkstudentsresults', 'Results').within(() => {
+        cy.contains('Student numbers in list and in oodi').click()
+        cy.contains('#found', existing)
+        cy.contains('Student numbers in list but not in oodi').click()
+        cy.contains('#notfound', nonExisting)
+        cy.contains('Student numbers in oodi but not in list').click()
+        cy.contains('#notsearched', '010614509')
+      })
+    })
+  })
+})

cypress/integration/Student_statistics.js

@@ -1,75 +1,90 @@
 /// <reference types="Cypress" />
 
-describe("Student Statistics tests", () => {
+describe('Student Statistics tests', () => {
   beforeEach(() => {
-    cy.init();
-    cy.contains("Student statistics").click();
-    cy.contains("Show student names");
-  });
+    cy.init()
+    cy.contains('Student statistics').click()
+    cy.contains('Show student names')
+  })
 
-  it("Student statistics search form is usable", () => {
-    cy.contains("Show student names");
-    cy.url().should("include", "/students");
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("Student number");
-    cy.contains("Started");
-    cy.contains("Credits");
-    cy.contains("Heidi Eeva").should("not.exist");
+  const student = {
+    firstnames: 'Matti Jaakko',
+    lastname: 'Nieminen',
+    studentnumber: '010654019',
+  }
 
-    cy.get("label").click();
-    cy.contains("Heidi Eeva");
+  it.only('Student statistics search form is usable', () => {
+    cy.contains('Show student names')
+    cy.url().should('include', '/students')
+    cy.get('.prompt').type(student.lastname)
+    cy.contains('Student number')
+    cy.contains('Started')
+    cy.contains('Credits')
+    cy.contains(student.firstnames).should('not.exist')
 
-    cy.get("label").click();
-    cy.contains("Heidi Eeva").should("not.exist");
-  });
-  it("Can get student specific page by clicking student", () => {
-    cy.url().should("include", "/students");
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.contains("Updated at 02.10.2019");
-    cy.contains("Musiikin didaktiikka (EDUK4212)");
-    cy.contains("Oinonen").should("not.exist");
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.firstnames)
 
-    cy.get("label").click();
-    cy.contains("Oinonen");
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.firstnames).should('not.exist')
+  })
 
-    cy.get("label").click();
-    cy.contains("Oinonen").should("not.exist");
-  });
+  it('can search with studentnumber too', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber)
+  })
 
-  it("Can get back to search menu", () => {
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.get(".remove").click();
-    cy.contains("Student number").should("not.exist");
-    cy.contains("Credits").should("not.exist");
-  });
+  it('Can get student specific page by clicking student', () => {
+    cy.url().should('include', '/students')
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.contains('Started: 01.08.2011')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
+    cy.contains(student.lastname).should('not.exist')
+    cy.contains(student.firstnames).should('not.exist')
 
-  it("Can jump to course", () => {
-    cy.get(".prompt").type("Oinonen");
-    cy.contains("014473717").click();
-    cy.contains("Musiikin didaktiikka (EDUK4212)")
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.lastname)
+    cy.contains(student.firstnames)
+
+    cy.cs('toggleStudentNames').click()
+    cy.contains(student.lastname).should('not.exist')
+    cy.contains(student.firstnames).should('not.exist')
+  })
+
+  it('Can get back to search menu', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.get('.remove').click()
+    cy.contains('Student number').should('not.exist')
+    cy.contains('Credits').should('not.exist')
+  })
+
+  it('Can jump to course', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.contains('Introduction to Machine Learning (DATA11002)')
       .siblings()
       .within(() => {
-        cy.get(".level").click();
-      });
-    cy.url().should("include", "/coursestatistics");
-    cy.contains("Musiikin didaktiikka");
-  });
+        cy.get('.level').click()
+      })
+    cy.url().should('include', '/coursestatistics')
+    cy.contains('Introduction to Machine Learning')
+  })
 
-  it("Searching with bad inputs doesnt yield results", () => {
-    cy.get(".prompt").type("Oin");
-    cy.contains("Student number").should("not.exist");
+  it('Searching with bad inputs doesnt yield results', () => {
+    cy.get('.prompt').type('SWAG LITTINEN')
+    cy.contains('Student number').should('not.exist')
 
-    cy.get(".prompt").clear().type("01114");
-    cy.contains("Student number").should("not.exist");
-  });
+    cy.get('.prompt').clear().type('01114')
+    cy.contains('Student number').should('not.exist')
+  })
 
-  it("Can jump to population page", () => {
-    cy.get(".prompt").type("Veli-Matti");
-    cy.contains("014824094").click();
-    cy.get("i.level.up.alternate.icon").eq(0).click();
-    cy.contains("Population statistics");
-    cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
-  });
-});
+  it.only('Can jump to population page', () => {
+    cy.get('.prompt').type(student.studentnumber)
+    cy.contains(student.studentnumber).click()
+    cy.get('i.level.up.alternate.icon').eq(0).click()
+    cy.contains('Population statistics')
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+  })
+})

cypress/integration/Studyprogramme_overview.js

@@ -1,162 +1,228 @@
 /// <reference types="Cypress" />
 
-const deleteTag = (name) => {
-  cy.contains(name).siblings().contains("Delete").click();
-  cy.contains("Are you sure you want to delete tag");
-  cy.contains("Confirm").click();
-  cy.contains(name).should("not.exist");
-};
+const deleteTag = name => {
+  cy.contains(name).siblings().contains('Delete').click()
+  cy.contains('Are you sure you want to delete tag')
+  cy.contains('Confirm').click()
+  cy.contains(name).should('not.exist')
+}
 
-describe("Studyprogramme overview", () => {
-  beforeEach(() => {
-    cy.init();
-    cy.contains("Study programme")
-      .click()
+describe('Studyprogramme overview', () => {
+  // check is used by couple of tests
+  const checkProgressAndProductivity = (user = 'basic') => {
+    // Column amount depends on user. Note that with dev user, you'll see even more
+    // stuff than with admin or basic.
+    const CSBachPopulationProgress2018ForAdmin = [
+      '160',
+      '42 (26%)',
+      '118 (73%)',
+      '159 (99%)',
+      '159',
+      '11',
+      '11',
+      '1',
+      '0',
+      '150',
+      '123',
+      '76',
+      '41',
+      '16',
+    ]
+    const CSBachPopulationProgress2018ForBasic = [
+      '160',
+      '42 (26%)',
+      '118 (73%)',
+      '159 (99%)',
+      '159',
+      '11',
+      '1',
+      '150',
+      '123',
+      '76',
+      '41',
+      '16',
+    ] // cancelled and transferred from are missing
+
+    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
+    cy.contains('Population progress')
+    cy.contains('Yearly productivity')
+
+    cy.contains('2018-2019')
       .siblings()
-      .contains("Overview")
-      .click();
-    cy.contains("Study Programme", { timeout: 100000 });
-  });
-
-  // FIXME: re-enable when CI has mandatory courses in the db
-  it.skip("can search for course mappings", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Code Mapper").click();
-    cy.wait(150);
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.get("input").type("582219");
-    });
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.get(".results").contains("Käyttöjärjestelmät (582219)");
-    });
-    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
-      cy.contains("button", "Add");
-    });
-  });
-
-  it("can view course groups", () => {
-    cy.contains("Kasvatustieteiden kandiohjelma").click();
-    cy.contains("Course Groups").click();
-
-    cy.contains("tr", "Test course group").get("i.edit").click();
-    cy.contains("Edit group");
-    cy.get(".prompt").type("Professori Pekka");
-    cy.contains("Add teacher").parent().contains("000960").click();
-    cy.contains("Teachers in group").parent().contains("000960");
-
-    cy.get("i.reply.link.icon").click();
-    cy.contains("tr a", "Test course group").click();
-    cy.contains("Total teachers");
-    cy.get("i.reply.icon").click();
-  });
-
-  it("renders progress and productivity tables", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Admin").click();
-    cy.contains("productivity").click();
-    cy.contains("throughput").click();
-
-    cy.wait(1000);
-    cy.get(".attached > :nth-child(1)").click();
-    cy.get("table").should("have.length", 3);
-    cy.contains("Population progress");
-    cy.contains("Yearly productivity");
-    cy.contains("2017-2018")
-      .siblings()
-      .contains("228")
-      .siblings()
-      .contains("178");
-
-    cy.get("table")
+      .each((elem, index) => {
+        cy.wrap(elem).contains(population[index])
+      })
+    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    cy.get('table')
       .eq(1)
-      .contains("2018")
+      .contains('2019')
       .siblings()
-      .contains("3023.00")
-      .siblings()
-      .contains("2631.00");
-    cy.get("table")
-      .eq(1)
-      .contains("2017")
-      .siblings()
-      .contains("3159.00")
-      .siblings()
-      .contains("2914.00");
-  });
+      .each((elem, index) => {
+        cy.wrap(elem).contains(populationproductivity2019[index])
+      })
+  }
 
-  it("can open Thesis page", () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.contains("Thesis Courses").click();
-    cy.contains("Add thesis course").click();
-    cy.contains("No results");
-  });
+  describe('when opening programme page with admin user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005', 'admin')
+    })
 
-  it("can move to Population statistics page by clickin", () => {
-    cy.contains("Tietojenkäsittelytieteen maisteriohjelma").click();
-    cy.get("i.level.up.alternate.icon").eq(0).click();
-    // cy.contains("Credit accumulation (for 16 students)");
-  });
+    it('progress should not be recalculating when opened for the first time', () => {
+      cy.contains('Recalculating').should('not.exist')
+      cy.contains('Last updated').should('exist')
+    })
 
-  it("can create and delete tags for population", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
-    cy.contains("2018");
-    deleteTag(name);
-  });
+    // Following vars are used to compare values before and after stats calculation
+    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
+    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
+    let originalProgressCalculatedText
+    let originalProductivityCalculatedText
+    const doRecalculation = () => {
+      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
+      cy.wait(2000)
+      cy.get('.attached').contains('Admin').click()
+      cy.contains('recalculate productivity').click()
+      cy.contains('recalculate throughput').click()
+      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
+      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
+      cy.wait(5000)
+      cy.reload()
+    }
 
-  it("can create personal tags", () => {
-    const name = `tag-${new Date().getTime()}`;
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
+    it('renders progress and productivity tables with calculated status', () => {
+      doRecalculation()
 
-    cy.get(".ui > label").click();
-    cy.contains("Create new tag").click();
-    cy.get(".purple");
-    cy.contains(name);
-    deleteTag(name);
-  });
+      // check contains also study programme options for admin
+      cy.contains('Study programme options')
+      cy.get('table').should('have.length', 3)
 
-  it("can add tags to students", () => {
-    const name = `tag-${new Date().getTime()}`;
+      checkProgressAndProductivity('admin')
 
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
-    cy.get(".attached").contains("Tags").click();
-    cy.get(".tagNameSelectInput > .ui > input").type(name);
-    cy.get(".yearSelectInput").type("2018");
-    cy.contains("Create new tag").click();
-    cy.contains(name);
+      // Grab update dates to be compared later
+      cy.cs('throughputUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProgressCalculatedText = normalizeText(text)
+          expect(originalProgressCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
 
-    cy.contains("Add tags to students").click();
-    cy.get(".form > .field > .dropdown")
-      .click()
-      .get(".ui > .search")
-      .type(name)
-      .click();
+      cy.cs('productivityUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProductivityCalculatedText = normalizeText(text)
+          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+    })
 
-    cy.get(".form > .field > .dropdown > .visible").contains(name).click();
+    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
+      doRecalculation()
+      checkProgressAndProductivity('admin')
 
-    cy.get("textarea").type("010623419");
-    cy.get(".positive").click();
+      // Check new calculation statuses are reported
+      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
+      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
 
-    cy.contains("Student statistics").click();
-    cy.get(".prompt").type("010623419");
-    cy.contains("10").click();
-    cy.contains(name);
+      newProgressCalculatedTextElement.should(text => {
+        const newProgressCalculatedText = normalizeText(text)
+        expect(newProgressCalculatedText).not.to.contain('recalculating')
+        expect(newProgressCalculatedText).not.to.contain('refresh')
+        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
 
-    cy.go("back");
-    cy.go("back");
+      newProductivityCalculatedTextElement.should(text => {
+        const newProductivityCalculatedText = normalizeText(text)
+        expect(newProductivityCalculatedText).not.to.contain('recalculating')
+        expect(newProductivityCalculatedText).not.to.contain('refresh')
+        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+    })
+  })
 
-    deleteTag(name);
+  describe('when opening programme page with basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005')
+    })
 
-    cy.contains("Student statistics").click();
-    cy.get(".prompt").type("010623419");
-    cy.contains("10").click();
-    cy.contains(name).should("not.exist");
-  });
-});
+    it('renders progress and productivity tables with previosly calculated values', () => {
+      // check doesn't contain study programme options for admin
+      cy.contains('Study programme options').should('not.exist')
+      cy.get('table').should('have.length', 2)
+      checkProgressAndProductivity()
+    })
+
+    it('can open Thesis page', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('Thesis Courses').click()
+      cy.contains('Add thesis course').click()
+      cy.contains('No results')
+    })
+
+    it('can move to Population statistics page by clickin', () => {
+      cy.get('i.level.up.alternate.icon').eq(0).click()
+      cy.contains('Students (10)')
+    })
+
+    it('can create and delete tags for population', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      cy.contains('2018')
+      deleteTag(name)
+    })
+
+    it('can create personal tags', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+
+      cy.get('.ui > label').click()
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      deleteTag(name)
+    })
+
+    it('can add tags to students', () => {
+      const name = `tag-${new Date().getTime()}`
+
+      const student = '010113437'
+
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+
+      cy.contains('Add tags to students').click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+
+      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+
+      cy.get('textarea').type('010113437')
+      cy.get('.positive').click()
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name)
+
+      cy.go('back')
+      cy.go('back')
+
+      deleteTag(name)
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name).should('not.exist')
+    })
+  })
+})

cypress/integration/Tags.js

@@ -1,44 +0,0 @@
-/// <reference types="Cypress" />
-
-const deleteTag = (name) => {
-  cy.contains('Create tags for study programme')
-  cy.contains(name).siblings().contains('Delete').click()
-  cy.contains('Are you sure you want to delete tag')
-  cy.contains('Confirm').click()
-  cy.contains(name).should('not.exist')
-}
-
-describe('Tag tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains("Study programme").click()
-    cy.contains("Overview").click()
-    cy.contains("Datatieteen maisteriohjelma").click()
-    cy.contains('Tags').click()
-  })
-
-  it("Tagged population works", () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.get(':nth-child(1) > .field > .ui > input').type(name)
-    cy.get('.form-control').type('2016')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
-    cy.contains('2016')
-
-    cy.contains('Add tags to students').click()
-    cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
-
-    cy.get('.form > .field > .dropdown > .visible').contains(name).click()
-
-    cy.get('textarea').type('014022579\n011122249')
-    cy.get('.positive').click()
-    cy.contains(name).find('.level').click()
-
-    cy.contains('Credit accumulation (for 1 students)')
-    cy.contains("Students (1)").click()
-    cy.contains('014022579')
-
-    cy.go('back')
-    deleteTag(name)
-  })
-})

cypress/integration/Teachers.js

@@ -2,62 +2,51 @@
 
 describe('Teachers page tests', () => {
   beforeEach(() => {
-    cy.init()
-    cy.contains("Teachers").click()
-    cy.contains("Teacher statistics by course providers")
+    // login as admin = has teacher rights
+    cy.init('/teachers', 'admin')
   })
 
-  it("Check Statistics", () => {
+  const teacher1 = 'Landgraf Leo'
+  const teacher2 = 'Vinogradov Jussi Petteri'
+
+  it('Check Statistics', () => {
     cy.get(':nth-child(1) > .ui > .search').click()
-    cy.contains('Autumn 2017').click()
+    cy.contains('Autumn 2020').click()
     cy.cs('course-providers').click()
     cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
     cy.cs('course-providers').children('.icon').click()
     cy.get('.form > .fluid').click()
     cy.contains('Name')
-    cy.contains('Wu')
-    cy.contains('Jämsä')
-    cy.contains('Kuusinen')
+    cy.contains(teacher1)
+    cy.contains(teacher2)
   })
 
-  it("Teacher search works", () => {
+  it('Teacher search works', () => {
     cy.url().should('include', '/teachers')
     cy.get('.borderless > :nth-child(3)').click()
-    cy.get('.prompt').type('Pekka')
-    cy.contains('Professori Pekka')
-    cy.contains('Saren')
-    cy.contains('Jari')
-    cy.contains('Lopez')
+    cy.get('.prompt').type(teacher1.split()[0])
+    cy.contains(teacher1)
   })
 
-  it("Can check teacher page", () => {
+  it('Can check teacher page', () => {
     cy.url().should('include', '/teachers')
     cy.get('.borderless > :nth-child(3)').click()
-    cy.get('.prompt').type('Pekka')
-    cy.contains('Leinonen').click()
-    cy.contains('Academic and Professional Communication in English 1 & 2 (CEFR B2)')
+    cy.get('.prompt').type(teacher1.split()[0])
+    cy.contains(teacher1).click()
+    cy.contains('Aineopintojen harjoitustyö: Tietokantasovellus')
   })
 
-  it("Can check teacher page if teacher doesn't have courses", () => {
-    cy.url().should('include', '/teachers')
-    cy.get('.borderless > :nth-child(3)').click()
-    cy.get('.prompt').type('Pekka')
-    cy.contains('Professori').click()
-    cy.contains('Name').should('not.exist')
+  it('Check leaderboad works', () => {
+    cy.get('.borderless > :nth-child(2)').click({ force: true })
+    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
+    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
+    cy.contains('2019-2020').click({ force: true })
+    cy.contains('Recalculate this year').click({ force: true })
+    cy.wait(5000)
+    cy.reload()
+    cy.contains('2019-2020').click({ force: true })
+    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
+    cy.contains('Passed')
+    cy.contains('Ahokas Jarmo Ilkka')
   })
-
-  // it("Check leaderboad works", () => {
-  //   cy.get('.borderless > :nth-child(2)').click({ force: true })
-  //   cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-  //   cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-  //   cy.contains('2017-18').click({ force: true })
-  //   cy.contains("Recalculate this year").click({ force: true })
-  //   cy.wait(5000)
-  //   cy.reload()
-  //   cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-  //   cy.contains('2017-18').click()
-  //   cy.contains("Passed")
-  //   cy.contains("Antero")
-  //   cy.contains("Sassmannshausen")
-  // })
-})
+})

cypress/integration/Users.js

@@ -1,65 +1,62 @@
 /// <reference types="Cypress" />
 
 describe('Users tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains("Users").click()
-    cy.contains("Oodikone users")
+  describe('Using user with just grp-oodikone-user, no other rights', () => {
+    it('shows only frontpage, trends feedback', () => {
+      cy.init('', 'norights') // login with norights user
+      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('.navBar').contains('Give feedback').should('exist')
+
+      cy.get('.navBar').contains('Study programme').should('not.exist')
+      cy.get('.navBar').contains('Student statistics').should('not.exist')
+      cy.get('.navBar').contains('Course statistics').should('not.exist')
+      cy.get('.navBar').contains('Teachers').should('not.exist')
+      cy.get('.navBar').contains('Users').should('not.exist')
+      cy.get('.navBar').contains('Faculty').should('not.exist')
+      cy.get('.navBar').contains('Updater').should('not.exist')
+    })
   })
 
-  it("Admin mocking normal user shows only the mocked user's programmes", () => {
-    cy.contains("mocking").should('not.exist')
-    cy.contains("tr", "Normaalikäyttäjä").within(($row) => {
-      cy.contains('.button', 'Edit').click()
-    })
-    cy.contains("Access rights").siblings().within(($row) => {
-      cy.get("div[role='listitem']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")
+  // TODO: Add user right checks for onlycoursestats user (should be like above + should
+  // show course stats
+  //
+  // TODO: Add user right checks for basic user (should show also study programme,
+  // student stats, course stats
+
+  describe('Using as admin', () => {
+    beforeEach(() => {
+      cy.init('/users', 'admin')
     })
 
-    cy.route('POST', '/api/superlogin/normk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains("mocking as normk")
-    cy.wait(1000)
-    cy.contains("Study programme").click().siblings().contains("Search by class").click()
-    cy.contains("label", "Study programme")
-    cy.contains("label", "Study programme").siblings().within(($row) => {
-      cy.get("div[role='option']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")
+    it('should see more stuff than others', () => {
+      cy.get('.navBar').contains('Study programme').should('exist')
+      cy.get('.navBar').contains('Student statistics').should('exist')
+      cy.get('.navBar').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
+      cy.get('.navBar').contains('Users').should('exist')
+      cy.get('.navBar').contains('Trends').should('exist')
     })
 
-  })
-
-  it("Admin mocking courseStatistics-user can see only Trends and Course Statistics", () => {
-    cy.contains("mocking").should('not.exist')
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains('.button', 'Edit').click()
+    it("mocking normal user shows only the mocked user's programmes", () => {
+      cy.contains('mocking').should('not.exist')
+      cy.contains('tr', 'basic').within($row => {
+        cy.contains('.button', 'Edit').click()
+      })
+      cy.intercept({
+        method: 'POST',
+        path: '/api/superlogin/basic',
+      }).as('superlogin')
+      cy.get('i.spy').click()
+      cy.wait('@superlogin')
+      cy.contains('mocking as basic')
+      cy.wait(1000)
+      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('label', 'Study programme')
+      cy.contains('label', 'Study programme')
+        .siblings()
+        .within($row => {
+          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
+        })
     })
-
-    cy.get("[data-cy=access-groups-form]").click().contains("courseStatistics").click()
-    cy.get("[data-cy=access-groups-save]").click()
-
-    cy.route('POST', '/api/superlogin/usermk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains("mocking as usermk")
-    cy.wait(1000)
-
-    cy.get("[data-cy=navbar-studyProgramme").should('not.exist')
-    cy.get("[data-cy=navbar-students]").should('not.exist')
-    cy.get("[data-cy=navbar-teachers]").should('not.exist')
-    cy.get("[data-cy=navbar-users").should('not.exist')
-
-    cy.contains("Stop mocking").click()
-    cy.wait(1000)
-
-    cy.contains("Users").click()
-    cy.contains("Oodikone users")
-    cy.contains("tr", "User manager").within(($row) => {
-      cy.contains('.button', 'Edit').click()
-    })
-    cy.contains("Access Groups").siblings().within(($row) => {
-      cy.get("i[class='dropdown icon clear']").click()
-    })
-    cy.get("[data-cy=access-groups-save]").click()
   })
 })

cypress/plugins/index.js

@@ -1,3 +1,4 @@
+/// <reference types="cypress" />
 // ***********************************************************
 // This example plugins/index.js can be used to load plugins
 //
@@ -11,10 +12,11 @@
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 
-const smokify = require('cypress-smokify')
-
+/**
+ * @type {Cypress.PluginConfig}
+ */
+// eslint-disable-next-line no-unused-vars
 module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
-  on('file:preprocessor', smokify(config))
 }

cypress/support/commands.js

@@ -1,34 +1,65 @@
 /// <reference types="Cypress" />
+//
+// Note: here we need to set keys to be all lowercase, since
+// we're replacing headers after they've left browser / frontend.
+// All of these users are available in anon user-db
+const adminUserHeaders = {
+  uid: 'admin',
+  displayname: 'Admin User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member;employee;faculty',
+  mail: 'grp-toska+mockadminuser@helsinki.fi',
+}
+
+const basicUserHeaders = {
+  uid: 'basic',
+  displayname: 'Basic User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mockbasicuser@helsinki.fi',
+}
+
+const onlycoursestatisticsUserHeaders = {
+  uid: 'onlycoursestatistics',
+  displayName: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-session',
+  hyGroupCn: 'grp-oodikone-basic-users',
+  eduPersonAffiliation: 'member',
+  mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+}
+
+const norightsUserHeaders = {
+  uid: 'norights',
+  displayname: 'Norights User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mocknorightuser@helsinki.fi',
+}
+
+const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
 
 /**
- * Initialize headers and load the base URL or optional path.
+ Set up headers to login, set up correct user (admin / basic / etc.) and open given path.
  */
-Cypress.Commands.add("init", (path = '') => {
-  cy.server({
-    onAnyRequest: function (route, proxy) {
-      if (Cypress.config().baseUrl.includes("http://nginx/")) {
-        proxy.xhr.setRequestHeader('uid', 'tktl')
-        proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
-        proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
-        proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
-      }
-    }
+Cypress.Commands.add('init', (path = '', userId = 'basic') => {
+  cy.intercept('', req => {
+    const headersToUse = userHeaders.find(({ uid }) => uid === userId)
+    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
+    req.headers = headersToUse
   })
-
-  // Babel throws an error probably because of markdown files. I'm sorry about this :lul:.
-  cy.on('uncaught:exception', (err, runnable) => {
-    expect(err.message).to.include("Cannot read property 'helpers' of undefined")
-    done()
-    return false
-  })
-
-  cy.visit(Cypress.config().baseUrl.concat(path))
+  // Hide SIS warning flag to avoid screen size problems when running headlessly
+  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
+  const baseUrl = Cypress.config().baseUrl
+  cy.visit(baseUrl.concat(path))
 })
 
 /**
  * Shorthand for using "Cypress Selectors" (CS), i.e., `data-cy` attributes.
  */
-Cypress.Commands.add("cs", (name) => {
+Cypress.Commands.add('cs', name => {
   return cy.get(`[data-cy='${name}']`)
 })
 
@@ -36,15 +67,25 @@ Cypress.Commands.add("cs", (name) => {
  * Select item specified by `index` number from a semantic-ui dropdown with
  * `data-cy` attribute value `name`.
  */
-Cypress.Commands.add("selectFromDropdown", (name, index) => {
+Cypress.Commands.add('selectFromDropdown', (name, index) => {
   const indexes = Array.isArray(index) ? index : [index]
 
-  indexes.forEach((i) => {
-    cy.cs(name).click().children(".menu").children().eq(i).click({ force: true })
+  indexes.forEach(i => {
+    cy.cs(name).click().children('.menu').children().eq(i).click({ force: true })
   })
 
   // Close multiple selection so it does not block elements underneath it.
   if (Array.isArray(index)) {
-    cy.cs(name).children(".icon").click({ force: true })
+    cy.cs(name).children('.icon').click({ force: true })
   }
 })
+
+/**
+ * Move to page with studyProgramme population
+ */
+Cypress.Commands.add('selectStudyProgramme', name => {
+  cy.cs('navbar-studyProgramme').click()
+  cy.cs('navbar-class').click()
+  cy.cs('select-study-programme').click().children().contains(name).click()
+  cy.contains('See population').click()
+})

cypress/support/index.js

@@ -13,20 +13,4 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-// Import commands.js using ES2015 syntax:
 import './commands'
-import './populationStatistics'
-
-// Alternatively you can use CommonJS syntax:
-// require('./commands')
-
-// describe('Before all tests', () => {
-//     it('reset anon database', () => {
-//       cy.exec(
-//         'source ./scripts/scripts.sh && time reset_db_for_cypress',
-//         { timeout: 5*60*1000 }
-//       ).then(result => {
-//         console.log(result)
-//       })
-//     })
-// })

cypress/support/populationStatistics.js

@@ -1,8 +0,0 @@
-/// <reference types="Cypress" />
-
-Cypress.Commands.add("selectStudyProgramme", (name) => {
-  cy.cs("navbar-studyProgramme").click()
-  cy.cs("navbar-class").click()
-  cy.cs("select-study-programme").click().children().contains(name).click()
-  cy.contains("See population").click()
-})

dco.data.yml

@@ -1,77 +0,0 @@
-version: "3.2"
-
-services:
-  nats:
-    image: nats-streaming:0.15.1
-    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
-    expose:
-      - "4222"
-    ports:
-      - "8222:8222"
-      - "4222:4222"
-    container_name: nats
-    volumes:
-      - datastore:/datastore
-
-  db:
-    image: postgres:10-alpine
-    ports:
-      - "5421:5432"
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_HOST_AUTH_METHOD=trust
-
-  db_sis:
-    image: postgres:13.2
-    shm_size: 1g
-    ports:
-      - "5426:5432"
-    volumes:
-      - pgdata_sis:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_sis
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_HOST_AUTH_METHOD=trust
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - redis-data:/data
-    container_name: redis
-
-  # updater_writer:
-  #   build:
-  #     context: services/backend
-  #     dockerfile: updater_writer/Dockerfile
-  #   depends_on:
-  #     - nats
-  #     - db
-  #   environment:
-  #     NATS_URI: nats://nats:4222
-  #     DB_URL: postgres://postgres@db:5432/tkt_oodi
-  #     TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-  #     TEST_DB_SCHEMA: updater_test
-  #   container_name: updater_writer
-
-  sis-importer-db:
-    image: postgres:13.1
-    environment:
-      POSTGRES_HOST_AUTH_METHOD: trust
-    volumes:
-      - sis-importer-pgdata:/var/lib/postgresql/data
-    container_name: sis-importer-db
-
-volumes:
-  datastore:
-  pgdata:
-  pgdata_sis:
-  redis-data:
-  sis-importer-pgdata:

docker-compose.ci.yml

@@ -0,0 +1,76 @@
+version: "3.9"
+
+x-defaults: &defaults
+  restart: unless-stopped
+
+services:
+
+  backend:
+    <<: *defaults
+    container_name: backend
+    depends_on:
+      - kone-db
+      - redis
+      - sis-db
+    environment:
+      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      FRONT_URL: http://localhost:1337
+      REDIS: redis
+      SECRET_TOKEN: dev
+      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      TOKEN_SECRET: IWannaBeTheVeryBest
+      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      USERSERVICE_URL: http://userservice:4567
+    image: toska/oodikone2-backend:trunk
+
+  frontend:
+    <<: *defaults
+    container_name: frontend
+    image: toska/oodikone2-frontend:trunk
+
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    image: registry.toska.cs.helsinki.fi/kone-db
+
+  redis:
+    <<: *defaults
+    command: "redis-server --appendonly yes"
+    container_name: redis
+    image: redis
+    ports:
+      - 6379:6379
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    image: registry.toska.cs.helsinki.fi/sis-db
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    image: registry.toska.cs.helsinki.fi/user-db
+
+  userservice:
+    <<: *defaults
+    container_name: userservice
+    depends_on:
+      - user-db
+    environment:
+      DB_URL: postgres://postgres@user-db:5432/user-db
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest
+    image: toska/oodikone2-userservice:trunk
+
+  nginx:
+    image: nginx:latest
+    container_name: nginx
+    depends_on:
+      - frontend
+      - backend
+      - userservice
+    ports:
+      -  1337:80
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf

docker-compose.real.yml

@@ -0,0 +1,26 @@
+services:
+
+  backend:
+    environment:
+      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
+      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
+
+  redis:
+    volumes:
+      - redis-data-real:/data
+
+  sis-updater-scheduler:
+    environment:
+      SIS_IMPORTER_DATABASE: sis-importer-db-real
+
+  sis-updater-worker:
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db-real
+      SIS_IMPORTER_DATABASE: sis-importer-db-real
+
+  userservice:
+    environment:
+      DB_URL: postgres://postgres@user-db:5432/user-db-real
+
+volumes:
+  redis-data-real:

docker-compose.test.yml

@@ -0,0 +1,10 @@
+services:
+
+  userservice-test:
+    build:
+      context: services/userservice
+      target: test
+    container_name: userservice-test
+    environment:
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest

docker-compose.yml

@@ -0,0 +1,196 @@
+x-defaults: &defaults
+  restart: unless-stopped
+
+x-build_defaults: &build_defaults
+  target: development
+
+services:
+
+  adminer:
+    <<: *defaults
+    container_name: adminer
+    environment:
+      ADMINER_DESIGN: pepa-linha
+    image: adminer:4.8.0
+    ports:
+      - 5050:8080
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+
+  backend:
+    <<: *defaults
+    build:
+      <<: *build_defaults
+      context: services/backend
+    container_name: backend
+    depends_on:
+      - kone-db
+      - redis
+      - sis-db
+    environment:
+      DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
+      FRONT_URL: http://localhost:3000
+      REDIS: redis
+      SECRET_TOKEN: dev
+      SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      TOKEN_SECRET: IWannaBeTheVeryBest
+      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      USERSERVICE_URL: http://userservice:4567
+    ports:
+      - 8080:8080
+    profiles:
+      - oodikone
+    volumes:
+      - backend-nodemod:/usr/src/app/node_modules
+      - ./services/backend/:/usr/src/app
+
+  frontend:
+    <<: *defaults
+    build:
+      <<: *build_defaults
+      context: services/frontend
+    container_name: frontend
+    ports:
+      - 3000:3000
+    profiles:
+      - oodikone
+    volumes:
+      - frontend-nodemod:/usr/src/app/node_modules
+      - ./services/frontend/:/usr/src/app
+
+  kone-db:
+    <<: *defaults
+    container_name: kone-db
+    image: registry.toska.cs.helsinki.fi/kone-db
+    profiles:
+      - oodikone
+    volumes:
+      - kone-db-data:/data
+
+  redis:
+    <<: *defaults
+    command: "redis-server --appendonly yes"
+    container_name: redis
+    image: redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+
+  sis-db:
+    <<: *defaults
+    container_name: sis-db
+    image: registry.toska.cs.helsinki.fi/sis-db
+    volumes:
+      - sis-db-data:/data
+
+  sis-importer-db:
+    <<: *defaults
+    container_name: sis-importer-db
+    image: registry.toska.cs.helsinki.fi/sis-importer-db
+    profiles:
+      - updater
+    volumes:
+      - sis-importer-db-data:/data
+
+  sis-updater-nats:
+    <<: *defaults
+    command: "-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore"
+    container_name: sis-updater-nats
+    image: nats-streaming:0.16.2
+    profiles:
+      - updater
+
+  sis-updater-scheduler:
+    <<: *defaults
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-scheduler
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      REDIS_URI: //redis:6379
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+    volumes:
+      - sis-updater-scheduler-nodemod:/usr/src/app/node_modules
+      - ./updater/sis-updater-scheduler/:/usr/src/app
+
+  sis-updater-worker:
+    <<: *defaults
+    build:
+      <<: *build_defaults
+      context: updater/sis-updater-worker
+    depends_on:
+      - redis
+      - sis-db
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      DB_URL: postgres://postgres@sis-db:5432/sis-db
+      REDIS_URI: //redis:6379
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+    profiles:
+      - updater
+    volumes:
+      - sis-updater-worker-nodemod:/usr/src/app/node_modules
+      - ./updater/sis-updater-worker/:/usr/src/app
+
+  user-db:
+    <<: *defaults
+    container_name: user-db
+    image: registry.toska.cs.helsinki.fi/user-db
+    profiles:
+      - oodikone
+    volumes:
+      - user-db-data:/data
+
+  userservice:
+    <<: *defaults
+    build:
+      <<: *build_defaults
+      context: services/userservice
+    container_name: userservice
+    depends_on:
+      - user-db
+    environment:
+      DB_URL: postgres://postgres@user-db:5432/user-db
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest
+    ports:
+      - 4567:4567
+    profiles:
+      - oodikone
+    volumes:
+      - userservice-nodemod:/usr/src/app/node_modules
+      - ./services/userservice/:/usr/src/app
+
+volumes:
+  backend-nodemod:
+  frontend-nodemod:
+  kone-db-data:
+  redis-data:
+  sis-db-data:
+  sis-importer-db-data:
+  sis-updater-scheduler-nodemod:
+  sis-updater-worker-nodemod:
+  user-db-data:
+  userservice-nodemod:

docker/dco.cypress.yml

@@ -1,14 +0,0 @@
-version: "3.2"
-services:
-  cypress:
-    image: "cypress/included:3.8.3"
-    environment:
-      - CYPRESS_baseUrl=${CYPRESS_baseUrl}
-      - CYPRESS_videoCompression=false
-    working_dir: /e2e
-    volumes:
-      - ./cypress:/e2e/cypress
-      - ./cypress.json:/e2e/cypress.json
-      - ./node_modules:/node_modules
-    container_name: cypress
-    command: cypress run --browser chrome --headless

docker/dco.data.ci.yml

@@ -1,15 +0,0 @@
-version: '3.2'
-
-services:
-  db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-
-  db_sis:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_sis
-
-volumes:
-  datastore-real:
-  redis-data-real:
-  mongodata-real:

docker/dco.data.dev.real.yml

@@ -1,14 +0,0 @@
-version: '3.2'
-
-services:
-  nats:
-    volumes:
-      - datastore-real:/datastore
-
-  redis:
-    volumes:
-      - redis-data-real:/data
-
-volumes:
-  datastore-real:
-  redis-data-real:

docker/dco.data.dev.yml

@@ -1,28 +0,0 @@
-version: '3.2'
-
-services:
-  db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
-    restart: unless-stopped
-    shm_size: '2gb'
-
-  db_sis:
-    command:
-      - "postgres"
-      - "-c"
-      - "max_wal_size=8GB"
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_sis,db_sis_real
-    restart: unless-stopped
-
-  adminer:
-    image: adminer:4.8.0
-    environment:
-      - ADMINER_DESIGN=pepa-linha
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-    ports:
-      - 5050:8080
-    container_name: adminer
-    restart: unless-stopped

oodikone.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-if test $# -eq 0; then
-  # Run interactive CLI.
-  scripts/run.sh
-else
-  while test $# -gt 0; do
-    case "$1" in
-      -h|--help)
-        echo "Oodikone CLI"
-        echo " "
-        echo "./oodikone.sh [options]"
-        echo " "
-        echo "options:"
-        echo "-h, --help                Show help."
-        echo "-m, --morning             Run this every morning 🌞"
-        echo "                          Purges, updates, rebuilds and restarts the whole shebang."
-        exit 0
-        ;;
-      -m|--morning)
-        scripts/morning.sh
-        exit 0
-        ;;
-      *)
-        echo "Unknown option. See ./oodikone.sh --help."
-        break
-        ;;
-    esac
-  done
-fi

package.json

@@ -1,77 +1,73 @@
 {
   "name": "oodikone",
-  "version": "1.0.0",
-  "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the dco.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
-  "main": "./scripts/run.sh",
+  "description": "An application for analyzing university data.",
   "scripts": {
-    "cli": "./oodikone.sh",
-    "cypress": "cypress",
-    "cypress:open": "cypress open --config baseUrl=http://localhost:8081/",
-    "cypress:run": "CYPRESS_baseUrl=http://frontend:8081/ npm run docker:cypress",
-    "cypress:run:smoke": "CYPRESS_baseUrl=http://frontend:8081/ npm run docker:cypress -- --env smokify=0.6",
-    "cypress:ci": "CYPRESS_baseUrl=http://nginx/ npm run docker:ci:cypress",
     "concurrently": "concurrently",
-    "services": "./scripts/exec_command_for_services.sh",
-    "lint": "./scripts/exec_parallel_npm_in_services.sh -c lint",
-    "lintfix": "npm run services -- -c lintfix",
-    "test_services": "npm run services -- -e -c test",
-    "test": "./scripts/test.sh",
-    "test:smoke": "./scripts/test.sh -s",
-    "docker:updater:base": "docker-compose -f ./dco.data.yml -f ./updater/dco.updater.yml",
-    "docker:updater:dev": "npm run docker:updater:base -- -f ./docker/dco.data.dev.yml -f ./updater/docker/dco.dev.yml",
-    "docker:updater:real": "npm run docker:updater:dev -- -f ./docker/dco.data.dev.real.yml -f ./updater/docker/dco.dev.real.yml",
-    "docker:oodikone:base": "docker-compose -f ./dco.data.yml -f ./services/dco.oodikone.yml",
-    "docker:oodikone:dev": "npm run docker:oodikone:base -- -f ./docker/dco.data.dev.yml -f ./services/docker/dco.dev.yml",
-    "docker:oodikone:real": "npm run docker:oodikone:dev -- -f ./docker/dco.data.dev.real.yml -f ./services/docker/dco.dev.real.yml",
-    "docker:both:base": "docker-compose -f ./dco.data.yml -f ./updater/dco.updater.yml -f ./services/dco.oodikone.yml",
-    "docker:both:dev": "npm run docker:both:base -- -f ./docker/dco.data.dev.yml -f ./updater/docker/dco.dev.yml -f ./services/docker/dco.dev.yml",
-    "docker:both:real": "npm run docker:both:dev -- -f ./docker/dco.data.dev.real.yml  -f ./updater/docker/dco.dev.real.yml -f ./services/docker/dco.dev.real.yml",
-    "docker:cypress": "npm run docker:oodikone:dev -- -f ./docker/dco.cypress.yml run --rm cypress",
-    "docker:ci": "npm run docker:both:base -- -f ./docker/dco.data.ci.yml -f ./services/docker/dco.ci.yml -f ./updater/docker/dco.ci.yml",
-    "docker:ci:cypress": "npm run docker:ci -- -f ./docker/dco.cypress.yml run --rm cypress",
-    "docker:build": "npm run docker:oodikone:dev -- build",
-    "docker:mac:up": "npm run docker:oodikone:dev -- up -d backend && npm run docker:up",
-    "docker:mac:up:real": "npm run docker:oodikone:real -- up -d backend && npm run docker:up:real",
-    "docker:up": "npm run docker:oodikone:dev -- up -d",
-    "docker:up:real": "npm run docker:oodikone:real -- up -d",
-    "docker:down": "npm run docker:oodikone:dev -- down",
-    "docker:logs": "npm run docker:oodikone:dev -- logs -f",
-    "docker:logs:backend": "npm run docker:oodikone:dev -- logs -f backend",
-    "docker:restart": "npm run docker:oodikone:dev -- restart",
-    "docker:restart:backend": "npm run docker:oodikone:dev -- restart backend",
-    "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune -a",
-    "start": "npm run docker:up && npm run docker:logs",
-    "start:real": "npm run docker:oodikone:real -- up -d && npm run docker:oodikone:real -- logs -f",
-    "start:updater": "npm run docker:up -- --scale sis-updater-worker=4 sis-updater-scheduler sis-updater-worker db_sis sis-updater-nats db adminer redis && npm run docker:logs -- --tail=\"1000\"",
-    "updater:dev:up": "npm run docker:updater:real -- up",
-    "updater:dev:down": "npm run docker:updater:real -- down --remove-orphans",
+    "cypress": "cypress",
+    "eslint": "eslint",
+    "stylelint": "stylelint",
+    "cli": "./cli.sh",
+    "oodikone": "./run.sh oodikone anon up --detach",
+    "oodikone:real": "./run.sh oodikone real up --detach",
+    "updater": "./run.sh updater anon up --detach",
+    "updater:real": "./run.sh updater real up --detach",
+    "both": "./run.sh both anon up --detach",
+    "both:real": "./run.sh both real up --detach",
+    "docker:down": "./run.sh both down --remove-orphans",
     "diff:students": "docker exec backend node /usr/src/app/src/sisoodi_diff/students/index.js",
     "diff:populations": "docker exec backend node /usr/src/app/src/sisoodi_diff/populations.js",
     "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
     "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
-    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js"
+    "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
+    "prepare": "husky install",
+    "lint:docker-compose": "docker-compose config -q && :",
+    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
+    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
+    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "scripts/grep_only.sh",
-      "pre-push": "npm run lint"
-    }
+  "lint-staged": {
+    "{services,updater}/**/*.{js,jsx}": "eslint --fix",
+    "*.{js,json,md}": "prettier --write",
+    "*.css": "stylelint --fix",
+    ".github/workflows/*": "actionlint",
+    "Dockerfile": "hadolint",
+    "*.sh": "shellcheck -x",
+    "docker-compose.yml": "npm run lint:docker-compose",
+    "docker-compose.real.yml": "npm run lint:docker-compose:real",
+    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
+    "docker-compose.test.yml": "npm run lint:docker-compose:test"
   },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/UniversityOfHelsinkiCS/oodikone.git"
   },
-  "author": "",
   "license": "ISC",
   "bugs": {
     "url": "https://github.com/UniversityOfHelsinkiCS/oodikone/issues"
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "concurrently": "^6.2.0",
-    "cypress": "^7.4.0",
-    "cypress-smokify": "0.0.3",
-    "husky": "^6.0.0"
-  },
-  "dependencies": {}
+    "@babel/eslint-parser": "^7.15.0",
+    "@babel/plugin-syntax-jsx": "^7.14.5",
+    "concurrently": "^6.2.1",
+    "cypress": "^8.2.0",
+    "eslint": "^7.32.0",
+    "eslint-config-airbnb": "^18.2.1",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-babel": "^5.3.1",
+    "eslint-plugin-cypress": "^2.11.3",
+    "eslint-plugin-import": "^2.24.0",
+    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "eslint-plugin-prefer-object-spread": "^1.2.1",
+    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-react": "^7.24.0",
+    "eslint-plugin-react-hooks": "^4.2.0",
+    "husky": "^7.0.1",
+    "lint-staged": "^11.1.2",
+    "prettier": "^2.3.2",
+    "stylelint": "^13.13.1",
+    "stylelint-config-prettier": "^8.0.2",
+    "stylelint-config-standard": "^22.0.0",
+    "stylelint-prettier": "^1.2.0"
+  }
 }

run.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+# This script is used to run oodikone with different setups and is mainly used by
+# scripts in package.json.
+# Base for script: https://betterdev.blog/minimal-safe-bash-script-template/
+
+# === Config ===
+
+# Fail immediately if script fails, unbound variables are referenced
+# or command inside pipe fails. -E ensures cleanup trap fires in rare ERR cases.
+set -euoE pipefail
+
+# Set up constants
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+
+# Set up messages and exiting
+source "$PROJECT_ROOT"/scripts/utils.sh
+
+usage() {
+  cat <<EOF
+Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
+
+Parameters:
+* Option: oodikone/updater/both
+* Version: anon/real. Not necessary when running down/logs/pull.
+* Command: will be passed to docker-compose.
+EOF
+}
+
+# Parse parameters. If arguments are not correct, print usage and exit with error.
+parse_params() {
+  args=("$@")
+
+  [[ ${#args[@]} -eq 0 ]] && usage && die "Wrong number of arguments"
+  option=${args[0]}
+
+  #  parse arguments
+  [[ ("$option" != "oodikone" && "$option" != "updater" && "$option" != "both") ]] && \
+    usage && die "Wrong option: $option"
+
+  [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
+
+  # Some arguments can be passed without version. Otherwise parse version and then pass
+  # rest to compose
+  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" || "${args[1]}" == "pull" ]]; then
+    version=""
+    compose_command=${args[*]:1}
+  else
+    version=${args[1]}
+    [[ "$version" != "anon" && "$version" != "real" ]] && usage && \
+      die "Wrong version $version"
+    compose_command=${args[*]:2}
+  fi
+  return 0
+}
+
+# Set which profile to use to launch correct services
+parse_profiles() {
+  if [[ "$option" == "both" ]]; then
+    profiles="--profile oodikone --profile updater"
+  else
+    profiles="--profile $option"
+  fi
+  return 0
+}
+
+# Set which docker-compose files to use based on version. In ci, profiles are also
+# emptied, since they're passed from github actions.
+parse_env() {
+  env=""
+  if [[ "$version" == "real" ]]; then
+    env="--file docker-compose.yml --file docker-compose.real.yml"
+  fi
+  return 0
+}
+
+# === Run script ===
+
+parse_params "$@"
+
+# Create command that will be run. Empty command and "down" command will be handled
+# differently.
+parse_profiles
+parse_env
+if [[ "$compose_command" == "" ]]; then
+  final_command="docker-compose ${env}"
+elif [[ "$compose_command" == *"down"* ]]; then
+  final_command="docker-compose ${compose_command}"
+else
+  final_command="docker-compose ${env} ${profiles} ${compose_command}"
+fi
+
+msg "${BLUE}Running: ${final_command}${NOFORMAT}"
+eval "$final_command"

scripts/assets/instructions.txt

@@ -1,6 +0,0 @@
-Oodikone2 should now be running on your machine. Note that bringing up the containers may take upwards of 5 minutes on macOS. Have patience, young padawan.
-
-View Oodikone @ http://localhost:8081
-SQL viewer and editor @ http://localhost:5050/
-
-Happy developing!

scripts/assets/logo.txt

@@ -1,5 +1,3 @@
-
-
    ______                   __  __  __
   /      \                 |  \|  \|  \
  |  $$$$$$\  ______    ____| $$ \$$| $$   __   ______   _______    ______
@@ -10,4 +8,3 @@
   \$$    $$ \$$    $$ \$$    $$| $$| $$  \$$\ \$$    $$| $$  | $$ \$$     \
    \$$$$$$   \$$$$$$   \$$$$$$$ \$$ \$$   \$$  \$$$$$$  \$$   \$$  \$$$$$$$
        
-

scripts/assets/mopo.txt

@@ -1,40 +1,38 @@
-                                                                                                    
-             ...`````    `....---.:--:/`                                                            
-            .++++++++++++-       .: -:++                                                            
-            `-::::://///:          -osso++++++++++++o:`                                             
-                                    ://++++++++oooosyo.                                             
-                                      .-      `````..`                                              
-                          .`           .`                                                           
-                          -             -/.                              `````                      
-                         ``  -           `--.```         `````..........``   `.``````               
-                        `.  .-                 ``........```                         `.``           
- `+/:-.```            ```   +    `                                                      `.`         
-  ```..-:///-.` `+o:       .`  .-`                                                        ``        
-          `.-/++oooo/`   `     `   .``.                                                     .`      
-              .--/+os/-  `     `  `` ``                 `-.                 `:.              `.     
-             .`    `-::         `   `.``   `         `` -Nh                  Ny         `     ``    
-            -.`                  -. .`   ```          . .Mm                  md      `  `      .    
-           .::+          -+++/.` ..`.    `            ...MM----::://:-` ./osomN   -shhhy-      ``   
-          ./` `        .ys+//od+                      +ddNMdhhmMMdhyhddomm+/oNM``omy-..oo       .   
-    `.`.--:-           y+.....+d      `               ...hM+`-Nm:`   `+NMo`  oM/hd:`-/oss       .`  
-   ``   -`             //....-s/      `               `  sMo hM/       sMNdhhdMmMo:dho/:y       .`  
-        `               -:::-:.                       `  /My dM-       +M+--/yMo+mNN`  .d       .   
-   `                                                   ` -Md oMy`     `dMo` `/M+ .yNs/+hh       .   
-   `.`                                                 . `MN``yMh/-.-+dmddyooso`   --.``       ``   
-    `.....````                                         ```ymy++sso/::--.``                  ```.    
-           .````                                          ``                           ``````..     
-            .`                                                                       ``.  ``        
-            `..`                                      ``````                        ``:``-..`       
-             ..````                              ``-:++-``````                     ``-- `           
-            .-` `` ``                         ```:oyy/`      .`                  ``.`/    ---       
-            --.  ````                        ``.+yyy. .---.`  -`                ``.``-   `::-`.     
-            --..``  ```                       `/ssy. `    `    :             `````.` `    ```/:     
-            .--...    `````                  `-+ss/     ..`    :`    ````````     .`       ` .``    
-            `--.            ````````` ````````:.:-     -//o-   -`` ``              -`         `     
-             .-.                    ``````````-        -:--`   .                    .``     `..`    
-              .`                           `  -            .:```                     ``..  `        
-               ``                             `.          `/: .                                     
-               .`    ``                        ..            ```                                    
-                                               `.-`        `..                                      
-                                                `.-----.`.```                                       
-
+             ::---..```  `ohmMMMMMNNNdy.                                                            
+            /MMMMMMMMMMMMMMMMMMMMmyMMMMd                                                            
+            `sddmmmNNNNNMMMMMMMMMMMMMMMMmdddddhhhhhhy+`                                             
+                   ``````mMMMMMMMMMMMMMMNNMMMMMMMMMMMN-                                             
+                         .yMMMMMMMMMMMMo``````...----.                                              
+                          :MMMMMMMMMMMMm-                                                           
+                          hMMNNMMMMMMMMMNy-                             `.--.`                      
+                         -MMMdMMMMMMMMMMMMNdo:.``        ``..://ossyhdmmNNNNNNmhys+:.               
+  `                     -mMMmdMMMMMMMMMMMMMMMNNNmmmmddmmmNNNNNMMMMMMMMMMMMMMMMMMMMMMNmh/.           
+ .mds+:-.`        `  -+hNMMNsNNNNNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNy-         
+  .`.-/+syyy+:. -ddymNMMMMNmNNNNdNNNNNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNo`       
+         ``-oydmmMMMMMMMMNNNNNNNNNNmNNmNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMm:      
+              /NMMMMMMMMMNNMMNNNNNNmNNmNNNNMMMMMMMMMMMMMNdmMMMMMMMMMMMMMMMMMMhmMMMMMMMMMMMMMMN/     
+            `oNMMMMMMMMMMMMMMMNNNNNNmdNNNMMNNMMMMMMMMMMMd`/MMMMMMMMMMMMMMMMMM`+MMMMMMMMMMMMMMMN-    
+            yMMMMMMMMMMMMMMMMMNNNNmNdmNMMNNNMMMMMMMMMMMMN .MMMMMMMMMMMMMMMMMM.:MMMMMMNMMMMMMMMMh    
+           oMMMMMMMMMMMMMdyyyhmNNNmmdMMMMNNMMMMMMMMMMMNmm` ddddhhhyyhdNMmyo+o..MMMd+:::/mMMMMMMM:   
+          -NMMMMMMMMMMMm+sdmdy/hNNNNNMMMMMMMMMMMMMMMMMy-:` -::.  -///--o.-syo. NNo.+dmmosMMMMMMMm   
+    `/osyhmMMMMMMMMMMMM+mMMMMMN:MMMMMMNMMMMMMMMMMMMMMMNmN/ yNd`.hNMMMNs` oNMMs y:-hNdys+oMMMMMMMM`  
+   :mMMMhmMMMMMMMMMMMMMhmMMMMMhyMMMMMMNMMMMMMMMMMMMMMMMMMo sM: hMMMMMMMo .-::: . oh-:oyh+MMMMMMMM`  
+   NMMMMNMMMMMMMMMNNNMMMmmNmNmmMMMMMMMMMMMMMMMMMMMMMMMMMMy +M- mMMMMMMMy sddh+ os.`.NMMm:MMMMMMMM   
+   NMMMMMMMMMMMMMMMNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMd -Ms +NMMMMMN: sNMNy sMm/.oys:/MMMMMMMh   
+   +NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMN `NN+`:ydmdy:.+-+ss+oNMMMddmNNMMMMMMMN-   
+    .+yhdmdMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM/.+ss++oyhhddNMNMMMMMMMMMMMMMMMMMMMMM+    
+         ` hMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNMMMMMMMMMMMMMMMMMMMMMMMMMMMNmmNNMMy     
+           .NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmmMMMMMMm.    
+            /NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmyNMMMMMMMMh    
+             +MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMdhMMMMMMMMMMM`   
+            /NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmyMMMMMMMMMMMM.   
+            yMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMyMMMMMMMMMNMMM`   
+            hMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNNNhMMMMMMMMMMMMh    
+            yMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMhMMMMMMMMMMMMMMMMMMMMMMMNNmhys+/-...oMMMMMMMMMMMN:    
+            :MMMMMMMMMMMMMMMNddmNMMMMMMMMMMMMMhMMMMMMMMMNmmMMMMN++/++/-           `dMMMMMMMMMMs`    
+             hMMMMMMMMMMMMMN:   .:/+osyyhddmmmdMMMMMMMMMMMMMMMMy                   `omMMMMMMNmy-    
+              yMMMMMMMMMMMm-              `+-`mMMMMMMMMMMMMMMMM.         ````````.``.-osyo//`       
+              `sNMMMMMMMMo.``   ``` `````` ```+MMMMMMMMMMMMMMMy```  `````                           
+              `yy.shmMMmh-      `        ``. ``hMMMMMMMMMMMMN+-.``                                  
+                `.`...`     `...``.```....`.-://mMMMMMMMMMMhy/                                      
+                                               `+hNMMMNmhs:-.                                 

scripts/assets/mopo2.txt

@@ -1,41 +0,0 @@
-
-                                                                                                     
-             ::---..```  `ohmMMMMMNNNdy.                                                            
-            /MMMMMMMMMMMMMMMMMMMMmyMMMMd                                                            
-            `sddmmmNNNNNMMMMMMMMMMMMMMMMmdddddhhhhhhy+`                                             
-                   ``````mMMMMMMMMMMMMMMNNMMMMMMMMMMMN-                                             
-                         .yMMMMMMMMMMMMo``````...----.                                              
-                          :MMMMMMMMMMMMm-                                                           
-                          hMMNNMMMMMMMMMNy-                             `.--.`                      
-                         -MMMdMMMMMMMMMMMMNdo:.``        ``..://ossyhdmmNNNNNNmhys+:.               
-  `                     -mMMmdMMMMMMMMMMMMMMMNNNmmmmddmmmNNNNNMMMMMMMMMMMMMMMMMMMMMMNmh/.           
- .mds+:-.`        `  -+hNMMNsNNNNNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNy-         
-  .`.-/+syyy+:. -ddymNMMMMNmNNNNdNNNNNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNo`       
-         ``-oydmmMMMMMMMMNNNNNNNNNNmNNmNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMm:      
-              /NMMMMMMMMMNNMMNNNNNNmNNmNNNNMMMMMMMMMMMMMNdmMMMMMMMMMMMMMMMMMMhmMMMMMMMMMMMMMMN/     
-            `oNMMMMMMMMMMMMMMMNNNNNNmdNNNMMNNMMMMMMMMMMMd`/MMMMMMMMMMMMMMMMMM`+MMMMMMMMMMMMMMMN-    
-            yMMMMMMMMMMMMMMMMMNNNNmNdmNMMNNNMMMMMMMMMMMMN .MMMMMMMMMMMMMMMMMM.:MMMMMMNMMMMMMMMMh    
-           oMMMMMMMMMMMMMdyyyhmNNNmmdMMMMNNMMMMMMMMMMMNmm` ddddhhhyyhdNMmyo+o..MMMd+:::/mMMMMMMM:   
-          -NMMMMMMMMMMMm+sdmdy/hNNNNNMMMMMMMMMMMMMMMMMy-:` -::.  -///--o.-syo. NNo.+dmmosMMMMMMMm   
-    `/osyhmMMMMMMMMMMMM+mMMMMMN:MMMMMMNMMMMMMMMMMMMMMMNmN/ yNd`.hNMMMNs` oNMMs y:-hNdys+oMMMMMMMM`  
-   :mMMMhmMMMMMMMMMMMMMhmMMMMMhyMMMMMMNMMMMMMMMMMMMMMMMMMo sM: hMMMMMMMo .-::: . oh-:oyh+MMMMMMMM`  
-   NMMMMNMMMMMMMMMNNNMMMmmNmNmmMMMMMMMMMMMMMMMMMMMMMMMMMMy +M- mMMMMMMMy sddh+ os.`.NMMm:MMMMMMMM   
-   NMMMMMMMMMMMMMMMNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMd -Ms +NMMMMMN: sNMNy sMm/.oys:/MMMMMMMh   
-   +NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMN `NN+`:ydmdy:.+-+ss+oNMMMddmNNMMMMMMMN-   
-    .+yhdmdMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM/.+ss++oyhhddNMNMMMMMMMMMMMMMMMMMMMMM+    
-         ` hMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNMMMMMMMMMMMMMMMMMMMMMMMMMMMNmmNNMMy     
-           .NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmmMMMMMMm.    
-            /NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmyNMMMMMMMMh    
-             +MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMdhMMMMMMMMMMM`   
-            /NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmyMMMMMMMMMMMM.   
-            yMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMyMMMMMMMMMNMMM`   
-            hMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNNNhMMMMMMMMMMMMh    
-            yMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMhMMMMMMMMMMMMMMMMMMMMMMMNNmhys+/-...oMMMMMMMMMMMN:    
-            :MMMMMMMMMMMMMMMNddmNMMMMMMMMMMMMMhMMMMMMMMMNmmMMMMN++/++/-           `dMMMMMMMMMMs`    
-             hMMMMMMMMMMMMMN:   .:/+osyyhddmmmdMMMMMMMMMMMMMMMMy                   `omMMMMMMNmy-    
-              yMMMMMMMMMMMm-              `+-`mMMMMMMMMMMMMMMMM.         ````````.``.-osyo//`       
-              `sNMMMMMMMMo.``   ``` `````` ```+MMMMMMMMMMMMMMMy```  `````                           
-              `yy.shmMMmh-      `        ``. ``hMMMMMMMMMMMMN+-.``                                  
-                `.`...`     `...``.```....`.-://mMMMMMMMMMMhy/                                      
-                                               `+hNMMMNmhs:-.                                 
-

scripts/assets/welcome.txt

@@ -1,5 +0,0 @@
-Welcome to Oodikone CLI!
-
-This tool helps you in managing the project configuration. If you are new to
-Oodikone development, you should probably run "Set up oodikone"
-which will take care of setting up and starting Oodikone for you. 

scripts/diff_between_folders.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-parse_and_diff() {
-  source_file=`echo "$1" | awk -F'/' '{print $NF}'`
-  target_file_path="${target_folder}/${source_file}"
-  printf "\nDIFF $entry $target_file_path:\n\n"
-  [ ! -f $target_file_path ] || diff $1 $target_file_path
-}
-
-export source_folder=$1
-export target_folder=$2
-export N_P=`getconf _NPROCESSORS_ONLN`
-export -f parse_and_diff
-
-exec_parallel() {
-  parallel --group parse_and_diff ::: `find $source_folder -type f -print`
-}
-
-exec_serial() {
-  for entry in "$source_folder"/*
-  do
-    parse_and_diff $entry
-  done
-}
-
-command -v parallel >/dev/null 2>&1 && exec_parallel || exec_serial

scripts/docker-entrypoint-initdb.d/create-multiple-postgresql-databases.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-set -e
-set -u
-
-function create_user_and_database() {
-	local database=$1
-	echo "  Creating user and database '$database'"
-	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
-	    CREATE USER $database;
-	    CREATE DATABASE $database;
-	    GRANT ALL PRIVILEGES ON DATABASE $database TO $database;
-EOSQL
-}
-
-if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
-	echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
-	for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
-		create_user_and_database $db
-	done
-	echo "Multiple databases created"
-fi

scripts/exec_command_for_services.sh

@@ -1,60 +0,0 @@
-#!/bin/bash
-#
-# This script is used to execute npm commands inside multiple running docker services.
-#
-# Params:
-#   -c (required). The npm command to execute in the services.
-#     Example:
-#       ./exec_command_for_services.sh -c lint
-#
-#   -e (optional). Sets the script to stop on first error.
-#     Example:
-#       ./exec_command_for_services.sh -e -c test
-#
-#   -p (optional). Additional params for the npm command.
-#
-# You can also run f.ex. "./exec_command_for_services.sh -c lint backend frontend"
-# to run the specified command in some specific services (in this case, backend & frontend).
-
-while getopts 'ec:p::' flag; do
-  case "${flag}" in
-    # Stop on first error
-    e) set -e ;;
-    c) COMMAND=${OPTARG};;
-    p) PARAMS=${OPTARG};;
-  esac
-done
-
-# Check that command is given
-if [ -z $COMMAND ]; then
-  echo "-c (command) missing"
-  exit
-fi
-
-# Remove opts from argument list
-shift $(( OPTIND - 1 ))
-
-# Service names given as arguments
-WHITELISTED_SERVICES=${@}
-
-if [ ! -z "$WHITELISTED_SERVICES" ]; then
-  services=($WHITELISTED_SERVICES)
-else
-  services=(
-    backend
-    frontend
-    analytics
-    userservice
-  )
-fi
-
-RED='\033[0;31m'
-Green='\033[0;32m'
-NC='\033[0m'
-
-echo -e "Executing command ${RED}$COMMAND${NC} $([ -n "$PARAMS" ] && echo "with param(s) ${RED}${PARAMS[@]}${NC} ")to services ${Green}${services[@]}${NC}"
-
-# Execute the given npm command in given services
-for service in "${services[@]}";
-  do docker exec -t $service npm run "$COMMAND" --if-present -- "${PARAMS[@]}"
-done

scripts/exec_parallel_npm_in_services.sh

@@ -1,67 +0,0 @@
-#!/bin/bash
-
-# This script is used to run an npm command in all directories
-# in services/ which contain a package.json (excluding node_modules)
-
-# Params:
-#   -c (required). The npm command to execute in the services.
-#       Example:
-#           ./exec_services_npm.sh -c lint
-#
-#   -e (optional). Sets the script to stop on first error.
-#     Example:
-#       ./exec_command_for_services.sh -e -c test
-#
-#   -p (optional). Additional params for the npm command.
-#
-#   -t (optional). Amount of threads to use to run the commands in parallel.
-#                  Defaults to "getconf _NPROCESSORS_ONLN".
-
-if [[ ! -d 'services' ]]; then
-    echo 'Directory "services" not found in current directory'
-    echo 'Please make sure PWD is the root of the repository!'
-    exit 1
-fi
-
-while getopts 'ec:p::' flag; do
-  case "${flag}" in
-    # Stop on first error
-    e) set -e ;;
-    c) COMMAND=${OPTARG};;
-    p) PARAMS=${OPTARG};;
-    t) THREADS="${OPTARG}"
-  esac
-done
-
-# Check that command is given
-if [ -z $COMMAND ]; then
-  echo "-c (command) missing"
-  exit
-fi
-
-if [[ -z $THREADS ]]; then
-  THREADS="$(getconf _NPROCESSORS_ONLN)"
-fi
-
-if [[ ! -z $CI ]]; then
-  THREADS='2'
-fi
-
-# Remove opts from argument list
-shift $(( OPTIND - 1 ))
-
-directories="$(find ./services -path '**/node_modules' -prune -o -name 'package.json' -exec dirname {} \;)"
-
-RED='\033[0;31m'
-Green='\033[0;32m'
-NC='\033[0m'
-
-echo -e "Executing command ${RED}$COMMAND${NC} $([ -n "$PARAMS" ] && echo "with param(s) ${RED}${PARAMS[@]}${NC} ")in directories:\n${Green}${directories[@]}${NC}"
-echo ''
-
-if [[ -n "$PARAMS" ]]
-then
-    echo "$directories" | xargs -n 1 -P "$THREADS" -I{} bash -c "pushd '{}' > /dev/null ; set -o pipefail && npm run --if-present '$COMMAND' -- '${PARAMS[@]}' | sed -e \"s/^/\$(basename \$PWD): /;\" || exit 255 ; popd > /dev/null"
-else
-    echo "$directories" | xargs -n 1 -P "$THREADS" -I{} bash -c "pushd '{}' > /dev/null ; set -o pipefail && npm run --if-present '$COMMAND' | sed -e \"s/^/\$(basename \$PWD): /;\" || exit 255 ; popd > /dev/null"
-fi

scripts/functions_for_setup.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+
+# This file includes functions that are used to setup oodikone. File doesn't run anything
+# by itself and should be sourced from other script.
+
+# === Config ===
+
+# Set up constants
+## Folders
+PROJECT_ROOT="$(git rev-parse --show-toplevel)"
+DUMP_DIR="$PROJECT_ROOT/.databasedumps"
+USER_DATA_FILE="$DUMP_DIR/hyuserdata"
+
+## Following the naming convention in docker-compose, these are names for database
+## services. The real data databases inside service will have names suffixed by "-real".
+KONE_DB_NAME="kone-db"
+SIS_DB_NAME="sis-db"
+SIS_IMPORTER_DB_NAME="sis-importer-db"
+USER_DB_NAME="user-db"
+DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
+
+## Urls should be in same order as databases as both are iterated through by indexes.
+KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
+
+# Source utility functions
+source "$PROJECT_ROOT"/scripts/utils.sh
+
+# === Function ===
+
+draw_mopo() {
+  # Some hacks to print mopo as green, since colours from common config don't work when
+  # catting images. Please feel free to fix this. And tell otahontas how you did it!
+  local mopogreen
+  mopogreen=$(tput setaf 34)
+  local normal
+  normal=$(tput sgr0)
+  if [ "$(tput cols)" -gt "100" ]; then
+    while IFS="" read -r p || [ -n "$p" ]; do
+      printf '%40s\n' "${mopogreen}$p${normal}"
+    done < "$PROJECT_ROOT"/scripts/assets/mopo.txt
+  fi
+}
+
+retry() {
+  sleep 5
+  for i in {1..60}; do
+    "$@" && break || warningmsg "Retry attempt $i failed, waiting..." && sleep 5;
+  done
+}
+
+download_real_dump() {
+  local database=$1
+  local pannu_url=$2
+  local dump_destination="$DUMP_DIR/$database.sqz"
+  scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" "$username@$pannu_url" "$dump_destination"
+}
+
+check_if_postgres_is_ready() {
+  local container=$1
+  local database=$2
+  retry docker exec -u postgres "$container" pg_isready --dbname="$database"
+}
+
+reset_databases() {
+  local databases=("$@")
+  local database_dump_dir=$DUMP_DIR
+
+  infomsg "Restoring PostgreSQL dumps from backups. This might take a while."
+
+  docker-compose down
+  docker-compose up -d "${databases[@]}"
+
+  for database in "${databases[@]}"; do
+    local database_dump="$database_dump_dir/$database.sqz"
+    local database_container="$database"
+    local database_name="$database-real"
+
+    infomsg "Attempting to create database $database_name from dump $database_dump inside container $database_container"
+
+    infomsg "Trying to remove possibly existing previous version of database"
+    check_if_postgres_is_ready "$database_container" "$database_name"
+    docker exec -u postgres "$database_container" dropdb "$database_name" || warningmsg "This is okay, continuing"
+
+    infomsg "Trying to create new database"
+    check_if_postgres_is_ready "$database_container" "$database_name"
+    docker exec -u postgres "$database_container" createdb "$database_name" || warningmsg "This is okay, continuing"
+
+    infomsg "Restoring database from dump database"
+    msg "1. Copying dump..."
+    docker cp "$database_dump" "$database_container:/asd.sqz"
+    msg "2. Writing database..."
+    docker exec "$database_container" pg_restore -U postgres --no-owner -F c --dbname="$database_name" -j4 /asd.sqz
+    msg ""
+  done
+
+  successmsg "Database setup finished"
+}
+
+reset_all_real_data() {
+  infomsg "Downloading real data dumps, asking for pannu password when needed"
+  for i in ${!DATABASES[*]}; do
+    local database="${DATABASES[$i]}"
+    local url="${REAL_DUMP_URLS[$i]}"
+    download_real_dump "$database" "$url"
+  done
+  reset_databases "${DATABASES[@]}"
+}
+
+reset_sis_importer_data() {
+  infomsg "Downloading sis-importer-db dump"
+  local database=$SIS_IMPORTER_DB_NAME
+  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+  download_real_dump $database $url
+  reset_databases $database
+}
+
+set_up_oodikone_from_scratch() {
+  draw_mopo
+  folders_to_set_up=(
+    "$PROJECT_ROOT"
+    "$PROJECT_ROOT/services/frontend"
+    "$PROJECT_ROOT/services/userservice"
+    "$PROJECT_ROOT/services/backend"
+    "$PROJECT_ROOT/updater/sis-updater-scheduler"
+    "$PROJECT_ROOT/updater/sis-updater-worker"
+  )
+
+  infomsg "Cleaning up docker setup and node_modules from possible previous installations"
+  local compose_down_args="--remove-orphans --volumes --rmi all"
+  "$PROJECT_ROOT"/run.sh both down "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.test.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.ci.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"
+  done
+
+  infomsg "Installing npm packages locally to root and each subfolder to enable \
+linting and formatting"
+
+  for folder in "${folders_to_set_up[@]}"; do
+    cd "$folder" || die "Couldn't change directory to folder $folder"
+    ([[ -d node_modules ]] && die "$folder already has node_modules, exiting!") || npm ci
+  done
+  cd "$PROJECT_ROOT" || die "Couldn't change directory to project root"
+
+  infomsg "Setting up husky for pre-commit linting"
+  npm run prepare
+
+  infomsg "Pulling images"
+  "$PROJECT_ROOT"/run.sh oodikone anon pull
+
+  infomsg "Building images"
+  "$PROJECT_ROOT"/run.sh oodikone anon build
+
+  successmsg "Setup ready, oodikone can be started! See README for more info."
+}
+
+init_dirs() {
+  if [[ ! -d "$DUMP_DIR" ]]; then
+    infomsg "Creating directory for dumps"
+    mkdir "$DUMP_DIR"
+  fi
+}
+
+# If username is not set, get username from data file.
+# Ask user to provide username, if username was not found from data file.
+get_username() {
+  if [ ! -f "$USER_DATA_FILE" ]; then
+    warningmsg "University username is needed to get database dumps from toska servers, please enter it now:"
+    read -r username
+    echo "$username" > "$USER_DATA_FILE"
+    successmsg "Succesfully saved username for later usage."
+  fi
+  username=$(head -n 1 < "$USER_DATA_FILE")
+
+  infomsg "Using your university username - $username - for getting database dumps"
+}

scripts/grep_only.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-RESULT=$(git grep -HEn --cached "(it|describe)\.only" $(git diff --cached --name-only) ./cypress)
-if [ ! -z "$RESULT" ]; then
-    echo -e "\e[31mDo not commit .only-tests. Did you remember to git add?\e[0m"
-    echo -e "$RESULT"
-    exit 1
-else
-    echo "No .only-tests found"
-    exit 0
-fi

scripts/morning.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-git checkout trunk
-git pull
-npm run docker:down
-echo "y" | docker system prune -a
-npm run docker:build
-npm run docker:up

scripts/populate-db.sh

@@ -1,63 +0,0 @@
-#!/bin/bash
-set -e
-
-DIR_PATH=$(dirname $(dirname "$0"))
-SERVICES=(db_sis db)
-BACKUPS=$DIR_PATH/backups
-
-# DB SIS CONFIG
-DB_SIS=db_sis
-DB_SIS_CONTAINER=db_sis
-DB_SIS_BACKUP="$BACKUPS/sis-updater-staging.sqz"
-DB_SIS_ARGS="$DB_SIS $DB_SIS_CONTAINER $DB_SIS_BACKUP"
-
-# ANON DB CONFIG
-DB_ANON=tkt_oodi
-DB_ANON_CONTAINER=oodi_db
-DB_ANON_BACKUP="$BACKUPS/demo-anon-staging.sqz"
-DB_ANON_ARGS="$DB_ANON $DB_ANON_CONTAINER $DB_ANON_BACKUP"
-
-retry () {
-    for i in {1..60}
-    do
-        $@ && break || echo "Retry attempt $i failed, waiting..." && sleep 10;
-    done
-}
-
-drop_create_populate () {
-    DB=$1
-    CONTAINER=$2
-    BACKUP=$3
-
-    echo "Dropping $DB"
-    retry docker exec -u postgres $CONTAINER pg_isready --dbname=$DB
-    docker exec -u postgres $CONTAINER psql -c "DROP DATABASE \"$DB\"" || echo "container $CONTAINER DB $DB doesn't exists"
-
-    echo "Creating $DB"
-    retry docker exec -u postgres $CONTAINER pg_isready --dbname=$DB
-    docker exec -u postgres $CONTAINER psql -c "CREATE DATABASE \"$DB\"" || echo "container $CONTAINER DB $DB already exists"
-
-    echo "Populating $DB"
-    docker cp $BACKUP $CONTAINER:/asd.sqz
-    docker exec $CONTAINER pg_restore -U postgres --no-owner -F c --dbname="$DB" -j4 /asd.sqz
-}
-
-export -f retry
-export -f drop_create_populate
-mkdir -p backups
-
-
-echo "Enter your Uni Helsinki username:"
-read username
-echo "Fetching latest staging SIS data & OODI demo data"
-scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" $username@oodikone-staging:/home/tkt_oodi/backups/\{sis-updater-staging.sqz,demo-anon-staging.sqz\} $BACKUPS
-
-echo "Setting up services: ${SERVICES[@]}"
-npm run docker:down --prefix $DIR_PATH
-npm run docker:up --prefix $DIR_PATH -- ${SERVICES[@]}
-
-echo "Populating DB_SIS and DB_ANON"
-echo $DB_SIS_ARGS $DB_ANON_ARGS | xargs -n 3 -P 2 bash -c 'drop_create_populate $1 $2 $3' bash
-
-echo "Cleaning up"
-npm run docker:down --prefix $DIR_PATH

scripts/programme_module_dump.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-docker exec -it db_sis pg_dump -aO -U postgres -d db_sis -t programme_modules -t programme_module_children > programme_module_dump.sql

scripts/run.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# Fail fast if theres error when executing script
-set -Eeuo pipefail
-
-source ./scripts/scripts.sh
-
-PS3='Please enter your choice: '
-
-mopo () {
-    if [ "$(tput cols)" -gt "100" ]; then
-        cat scripts/assets/mopo2.txt
-    fi
-}
-
-logo () {
-    if [ "$(tput cols)" -gt "76" ]; then
-        cat scripts/assets/logo.txt
-    fi
-}
-
-logo
-cat scripts/assets/welcome.txt
-
-options=(
-    "Set up oodikone with real data."
-    "Download & reset all real data."
-    "Download & reset sis importer data."
-    "Download & reset old oodi data."
-    "Quit."
-)
-
-while true; do
-    select opt in "${options[@]}"; do
-        case $opt in
-            "Set up oodikone with real data.")
-                mopo
-                run_full_setup
-                ;;
-            "Download & reset all real data.")
-                run_full_real_data_reset
-                ;;
-            "Download & reset sis importer data.")
-                run_importer_data_reset
-                ;;
-            "Download & reset old oodi data.")
-                run_oodi_data_reset
-                ;;
-            "Quit.")
-                break 2
-                ;;
-            *) echo "Invalid option $REPLY";;
-        esac
-        break
-    done
-done

scripts/scripts.sh

@@ -1,184 +0,0 @@
-#!/bin/bash
-
-# Constant filenames and paths used in scripts
-USER_DATA_FILE_PATH="hyuserdata"
-BACKUP_DIR=backups
-
-PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
-KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz"
-USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
-ANALYTICS_REAL_DB_BACKUP="$BACKUP_DIR/latest-analytics-pg.sqz"
-SIS_REAL_DB_BACKUP="$BACKUP_DIR/latest-sis.sqz"
-
-# Remember username during runtime
-username=""
-
-### === GENERIC HELPER FUNCTIONS ===
-
-get_username() {
-  # Check if username has already been set
-  [ -z "$username" ]|| return 0
-
-  # Check if username is saved to data file and ask it if not
-  if [ ! -f "$USER_DATA_FILE_PATH" ]; then
-    echo ""
-    echo "!! No previous username data found. Will ask it now !!"
-    echo "Enter your Uni Helsinki username:"
-    read -r username
-    echo "$username" > "$USER_DATA_FILE_PATH"
-    echo "Succesfully saved username"
-    echo ""
-  fi
-
-  # Set username
-  username=$(head -n 1 < $USER_DATA_FILE_PATH)
-}
-
-retry () {
-  for i in {1..60}; do
-    "$@" && break || echo "Retry attempt $i failed, waiting..." && sleep 10;
-  done
-}
-
-docker-compose-dev () {
-    npm run docker:oodikone:dev -- "$@"
-}
-
-### === PSQL HELPER FUNCTIONS ===
-
-restore_psql_from_backup () {
-    echo ""
-    echo "Restoring database from backup $1 to container $2:"
-    echo "  1. Copying dump..."
-    docker cp "$1" "$2:/asd.sqz"
-    echo "  2. Writing database..."
-    docker exec "$2" pg_restore -U postgres --no-owner -F c --dbname="$3" -j4 /asd.sqz
-}
-
-ping_psql () {
-    drop_psql "$1" "$2"
-    echo "Creating psql in container $1 with db name $2"
-    retry docker exec -u postgres "$1" pg_isready --dbname="$2"
-    docker exec -u postgres "$1" createdb "$2" || echo "container $1 DB $2 already exists"
-}
-
-drop_psql () {
-    echo "Dropping psql in container $1 with db name $2"
-    retry docker exec -u postgres "$1" pg_isready --dbname="$2"
-    docker exec -u postgres "$1" dropdb "$2" || echo "container $1 DB $2 does not exist"
-}
-
-### === FUNCTIONS TO RUN CLI OPTIONS ===
-
-# Download & reset all real data
-run_full_real_data_reset () {
-    get_username
-    echo "Using your Uni Helsinki username: $username"
-
-    echo "Downloading user-db, analytics-db and kone-db dumps"
-    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" \
-    "$username@oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/*" "$BACKUP_DIR/"
-
-    echo "Downloading sis-db dump"
-    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" \
-    "$username@svm-96.cs.helsinki.fi:/home/updater_user/backups/*" "$BACKUP_DIR/"
-
-    echo "Downloading importer-db dump"
-    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" \
-    "$username@importer:/home/importer_user/importer-db/backup/importer-db.sqz" "$BACKUP_DIR/"
-
-    docker-compose-dev down
-    docker-compose-dev up -d user_db db_kone analytics_db db_sis sis-importer-db
-
-    echo "Restoring PostgreSQL from backup. This might take a while."
-
-    ping_psql "db_kone" "db_kone_real"
-    restore_psql_from_backup $KONE_REAL_DB_BACKUP db_kone db_kone_real
-    ping_psql "oodi_user_db" "user_db_real"
-    restore_psql_from_backup $USER_REAL_DB_BACKUP oodi_user_db user_db_real
-    ping_psql "oodi_analytics_db" "analytics_db_real"
-    restore_psql_from_backup $ANALYTICS_REAL_DB_BACKUP oodi_analytics_db analytics_db_real
-    ping_psql "db_sis" "db_sis_real"
-    restore_psql_from_backup $SIS_REAL_DB_BACKUP db_sis db_sis_real
-    ping_psql "sis-importer-db" "importer-db"
-    restore_psql_from_backup "$BACKUP_DIR/importer-db.sqz" sis-importer-db importer-db
-
-    echo "Database setup finished"
-
-    docker-compose-dev down
-}
-
-install_local_npm_packages () {
-    # These are required for linting and only installed if not already there
-    [[ -d node_modules ]] || npm ci
-    cd services/oodikone2-frontend || return
-    [[ -d node_modules ]] || npm ci
-    cd ../oodikone2-analytics || return
-    [[ -d node_modules ]] || npm ci
-    cd ../oodikone2-userservice || return
-    [[ -d node_modules ]] || npm ci
-    cd ../backend/oodikone2-backend || return
-    [[ -d node_modules ]] || npm ci
-    cd ../updater_writer || return
-    [[ -d node_modules ]] || npm ci
-
-    cd ../../../
-}
-
-init_dirs () {
-  mkdir -p $BACKUP_DIR nginx nginx/cache nginx/letsencrypt
-  touch nginx/error.log
-  touch nginx/log
-  chmod -R 755 scripts/docker-entrypoint-initdb.d
-}
-
-# Set up oodikone with real data
-run_full_setup () {
-    echo "=== Setup npm packages ==="
-    install_local_npm_packages
-    echo "=== Creating directories and ensuring rights are correct ==="
-    init_dirs
-    echo "=== Setting up needed databases ==="
-    run_full_real_data_reset
-    echo "=== Building images ==="
-    docker-compose-dev build
-    echo "=== Starting oodikone ==="
-    npm run docker:up:real
-    cat scripts/assets/instructions.txt
-}
-
-# Download & reset sis importer data
-run_importer_data_reset() {
-    get_username
-    echo "Using your Uni Helsinki username: $username"
-
-    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" \
-    "$username@importer:/home/importer_user/importer-db/backup/importer-db.sqz" "$BACKUP_DIR/"
-
-    docker-compose -f dco.data.yml up -d sis-importer-db
-    ping_psql "sis-importer-db" "importer-db"
-    restore_psql_from_backup "$BACKUP_DIR/importer-db.sqz" sis-importer-db importer-db
-    docker-compose -f dco.data.yml down
-}
-
-# Download & reset old oodi data
-run_oodi_data_reset () {
-    get_username
-    echo "Using your Uni Helsinki username: $username"
-
-    echo "Downloading oodi-db dump"
-    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" \
-    "$username@svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/*" "$BACKUP_DIR/"
-
-    docker-compose-dev down
-    docker-compose-dev up -d db
-
-    echo "Restoring PostgreSQL from backup. This might take a while."
-
-    ping_psql "oodi_db" "tkt_oodi_real"
-    restore_psql_from_backup $PSQL_REAL_DB_BACKUP oodi_db tkt_oodi_real
-
-    echo "Database setup finished"
-
-    docker-compose-dev down
-}

scripts/test.sh

@@ -1,19 +0,0 @@
-
-#!/bin/bash
-
-# stop on first error
-set -e
-
-while getopts 's' flag; do
-  case "${flag}" in
-    s) SMOKE=1;
-  esac
-done
-
-shift $(( OPTIND - 1 ))
-
-if [ ! -z $SMOKE ]; then
-  npm run test_services && npm run cypress:run:smoke
-else
-  npm run test_services && npm run cypress:run
-fi

scripts/utils.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# This file includes some utility functions that are used in oodikone scripts.
+# Functions are based on https://betterdev.blog/minimal-safe-bash-script-template/
+
+# Print messages and logs that are not script output to stderr
+msg() {
+    echo >&2 -e "${1-}"
+}
+
+# Some special types of messages with colours
+successmsg() {
+  msg "${GREEN}$1${NOFORMAT}
+  "
+}
+errormsg() {
+  msg "${RED}$1${NOFORMAT}
+  "
+}
+infomsg() {
+  msg "${BLUE}$1${NOFORMAT}
+  "
+}
+warningmsg() {
+  msg "${ORANGE}$1${NOFORMAT}
+  "
+}
+
+# Quit and exit with given message and error code
+# By default uses exit code 1, message must be given.
+die() {
+  local msg=$1
+  local code=${2-1}
+  msg "${RED}${msg}${NOFORMAT}"
+  exit "$code"
+}
+
+# Setup colors for messages if running interactive shell
+if [[ -t 2 ]] && [[ "${TERM-}" != "dumb" ]]; then
+  NOFORMAT='\033[0m' RED='\033[0;31m' GREEN='\033[0;32m' ORANGE='\033[0;33m' BLUE='\033[0;34m' PURPLE='\033[0;35m' CYAN='\033[0;36m' YELLOW='\033[1;33m'
+else
+  NOFORMAT='' RED='' GREEN='' ORANGE='' BLUE='' PURPLE='' CYAN='' YELLOW=''
+fi

scripts/v2_notifier.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-OLD=$(git diff --name-only --cached | grep -Ec "(\/services\/|\/routes\/)")
-NEW=$(git diff --name-only --cached | grep -Ec "(\/servicesV2\/|\/routesV2\/)")
-if (( $OLD > $NEW )); then
-    echo "Do not update old services or routes without updating SIS"
-    exit 1
-else 
-    exit 0
-fi

services/backend/.dockerignore

@@ -1,5 +1,6 @@
-**/node_modules
-updater_writer/**/models
-updater_writer/**/migrations
-oodikone2-backend/**/models
-oodikone2-backend/**/migrations
+.dockerignore
+Dockerfile
+node_modules/
+requests/
+scripts/
+*.log

services/backend/.gitignore

@@ -0,0 +1,3 @@
+# Student diff output
+/student_diff_*
+/diff_partial_att_data.json

services/backend/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:14.17-alpine3.13 as base
+
+WORKDIR /usr/src/app
+EXPOSE 8080
+
+FROM base as development
+ENV NODE_ENV=development
+COPY ./package* /usr/src/app/
+RUN npm ci
+COPY . /usr/src/app/
+CMD ["npm", "run", "dev"]
+
+FROM base as production
+ENV NODE_ENV=production
+ARG SENTRY_ENVIRONMENT
+ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
+ARG SENTRY_RELEASE
+ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+USER node
+CMD ["npm", "start"]

services/backend/index.js

@@ -1,4 +1,3 @@
-//require('./src/tester')
 require('./src/app')
 
 process.on('unhandledRejection', reason => {

services/backend/oodikone2-backend/.eslintignore

@@ -1 +0,0 @@
-*.sql

services/backend/oodikone2-backend/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

services/backend/oodikone2-backend/.gitignore

@@ -1,21 +0,0 @@
-node_modules
-.vscode
-.nyc_output
-src/tester.js
-update.sh
-.env
-logs/*
-run_update_script.sh
-student_numbers
-debug.log
-npm-debug.log
-
-studentnumbers.txt
-studentnumbers*.txt
-src/anonymized_API
-
-models
-migrations
-
-/student_diff_*
-/diff_partial_att_data.json

services/backend/oodikone2-backend/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

services/backend/oodikone2-backend/Dockerfile

@@ -1,20 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-EXPOSE 8080
-CMD npm start
-
-COPY ./oodikone2-backend/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-ARG TAG=local
-ENV TAG=${TAG}
-
-COPY ./oodikone2-backend /usr/src/app
-COPY ./shared/models /usr/src/app/src/models
-COPY ./shared/migrations /usr/src/app/src/database/migrations
-COPY ./shared/migrations_kone /usr/src/app/src/database/migrations_kone
-

services/backend/oodikone2-backend/README.md

@@ -1,259 +0,0 @@
-# oodikone2-backend
-
-## Dev
-
-### Getting started
-
-To install docker & docker-compose to run postgres and redis follow these steps:
-
-Install docker CE: https://docs.docker.com/engine/installation/ 
-
-Install docker-compose: https://docs.docker.com/compose/install/
-
-Create docker-compose.yml file containing following:
-
-```
-version: '3'
-
-services:
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-    container_name: oodi_db
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-```
-
-To start a redis and a database, run:
-
-```docker-compose up -d```
-
-Copy a backup dump from oodikone
-
-`scp oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/2018-08-01.bak <local folder>`
-
-Create database tkt_oodi:
-
-`docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi"`
-
-Fill database with data from backup file: 
-
-`cat 2018-08-01.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi`
-
-Check that you have all the data you ever wanted:
-
-`docker exec -it -u postgres oodi_db psql -d tkt_oodi`
-
-`\dt`
-
-Create test database:
-
-`docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_test"`
-
-Fill database with data from test.bak dump file: 
-
-`cat test.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi_test`
-
-### Running 
-
-#### Dev
-
-Run `npm run dev`
-
-#### Testing
-
-Run `npm test`
-
-## Deployment
-
-Always deploy first to staging! 
-
-When merging from trunk to master Travis builds a docker image for staging and pushes it automatically to dockerh-hub. Currently **staging** is also automatically deployed after the image is pushed.
-
-When a release (or tag) is created in master branch Travis builds a docker image for production and pushes it automatically.
-
-### To update: 
-
-Log into svm-59 (aka oodikone)
-
-`ssh {username}@oodikone.cs.helsinki.fi`
-
-Switch user to tkt_oodi
-
-`sudo su - tkt_oodi`
-
-Navigate to correct folder
-`cd oodikone.cs.helsinki.fi` or `cd oodikone.cs.helsinki.fi/staging`
-
-Use the update script to restart the software
-`./update.sh`
-
-Be amazed!
-
-## In case you mess up your DB
-
-Drop your current db
-
-`docker exec -u postgres oodi_db dropdb tkt_oodi`
-
-Copy a backup dump from oodikone
-
-`scp oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/2018-08-01.bak <local folder>`
-
-Create a new database and fill it with the backup dump following the instructions above
-
-## Accessing production/staging DB
-
-To access the database in oodikone.cs.helsinki.fi you can use `postgres_connect.sh` script in production/staging directory. Alternatively you can use commands: 
-
-`docker exec -it -u postgres db psql -d tkt_oodi`
-
-or
-
-`docker exec -it -u postgres db_staging psql -d tkt_oodi_staging`
-
-Production db backups are taken daily and can be found in /var/toska/tkt_oodi/backups/postgres/. Symlink to latest backup can always be found as /home/tkt_oodi/backups/latest-pg.bak.bz2.
-
-To create a dump of the production db manually, run: 
-
-``docker exec -u postgres db pg_dump tkt_oodi > `date +%Y-%m-%d`.bak``
-
-## Updater
-
-Oodikone2-backend includes the update-script that fetches student data from WebOodi. Script is split on two parts. Both production and staging environments have separate updater set up, in directories /oodikone.cs.helsinki.fi/updater and /oodikone.cs.helsinki.fi/staging/updater respectively. Both of updater directories have README file with summary of commands. Container names are production_updater and staging_updater, replace in examples below CONTAINER_NAME with correct one. Both containers have updater_files-directory that is mounted within container to /updater_files.
-
-Updater uses respective backend-image (production or staging, respectively).
-
-### Student list updater
-
-Finds out the valid student nubers. 
-
-Run in updater directory with _update_student_list.sh_
-
-Updater is run in separate detached container, see graylog or _debug.log_ to find out when it is finished.
-
-Environment variable _INCREMENT_ (currently not set) controls how many new student numbers are searched (from the currently known greatest student number).
-
-At the moment script uses the local postgres (see .env) to save student numbers.
-
-### Student info updater (database updater)
-
-To update your database run `docker-compose run -d CONTAINER_NAME npm run update_database` with optional args `file="<filename>"` and `index=<number>` with former telling the studentnumber filename to update and latter telling which row to start from.
-
-
-### Anonymized data & E2E environment
-
-Use [oodikone-cli](https://github.com/UniversityOfHelsinkiCS/oodikone) to setup you're environments.
-
-Before running cli make yourself a deployment key for Anonyymioodi private repository and put it in your cli folder as `private.key`. This allows you to download the anonymized dumps from the repository.
-
-For local development use with anonymized data use `2) Anon setup` (you can run e2e tests with this).
-
-`1) e2e setup` is designed for use in travis. But you can still try it locally if you want to.
-
-Run E2E tests in ToskaWatch with `npm run cypress:run` or `npm run cypress:open` for local tests.
-
-
-## Teacher leader board statistics
-
-The statistics for the teacher leader board have to be calculated separately with a script.
-
-```
-# Update top teachers for academic years 50 (1999-20) through 70 (2018-19).
-docker-compose run -d CONTAINER_NAME npm run update_top_teachers from=50 to=70
-
-# Omit the to–argument to update teachers just for a single academic year. 
-docker-compose run -d CONTAINER_NAME npm run update_top_teachers from=50
-```
-
-## DEFA report
-
-The npm script `create_defa_report` can quickly aggregate data about DEFA students and display it in neat csv format. Here's how you can use it on the Oodikone production server. It does not work if run locally.
-
-### Input files
-
-Input files should be placed in the `/home/tkt_oodi/oodikone.cs.helsinki.fi/defa` directory unless a different directory has been included into the backend docker container as `/usr/src/app/defa`.
-
-* `params.json`
-  * Defines the input/output files. File paths are for the container, so the working directory is `/usr/src/app/defa/`.
-  * Defines the timeframe to look at: from `params.in.timeframe.start` to `params.in.timeframe.end`. Studyattainments outside this timeframe will not be considered to be part of DEFA. Changing the timeframe is the main reason you'd want to make changes to this file.
-  * Example:
-```
-{
-  "out": {
-    "report": "/usr/src/app/defa/defa-report.csv",
-    "stats": "/usr/src/app/defa/defa-stats.csv"
-  },
-  "in": {
-    "course_ids": "/usr/src/app/defa/ids.txt",
-    "included_courses": "/usr/src/app/defa/included.txt",
-    "required_courses": "/usr/src/app/defa/required.txt",
-    "timeframe": {
-      "start": "2018-09-01T01:00:00.000Z",
-      "end": "2019-06-30T01:00:00.000Z"
-    }
-  }
-}
-```
-
-* `ids.txt` (or another name specified in `params.in.course_ids`)
-  * Holds the courseunitrealisation ids of DEFA courses to be included in the report.
-  * Example:
-```
-126240934
-126237007
-126238082
-126239325
-12699580
-```
-
-* `included.txt` (or another name specified in `params.in.included_courses`)
-  * Holds the learningopportunity ids of courses to be included in the report. Basically put all DEFA course codes here.
-  * One id per line.
-  * AY courses like `AYTKT0000` can be entered without the `AY` prefix to match both with and without it. e.g. `TKT0000` matches both `TKT0000` and `AYTKT0000`. `AYTKT0000` will only match `AYTKT0000`.
-  * Example:
-```
-TKT10002
-TKT10003
-MAT11001
-TKT10004
-TKT20001
-TKT20004
-TKT20009
-TKT21007
-MAT12001
-MAT12002
-```
-
-* `required.txt` (or another name specified in `params.in.required_courses`)
-  * Holds the learningopportunity ids of courses that are required to complete for a DEFA student to qualify.
-  * Same format as `included.txt`.
-  * All entries in this file should also be in `included.txt`.
-  * Example: same as `included.txt`
-
-### Running
-
-1. Make sure the backend docker container is running.
-2. run `docker exec <backend container> npm run create_defa_report defa/<parameter json file>` or use the ready made script `/home/tkt_oodi/oodikone.cs.helsinki.fi/defa/create_report.sh`.
-3. Check the output files.
-
-### Output files
-
-Output files will be created in the same directory where the input files are. Old output files will be overwritten unless they have been renamed/moved. 
-
-* `defa-report.csv` (or another name defined in `params.out.report`)
-  * Contains individual information about each DEFA student and individual inforamtion about each DEFA course.
-  * Contains aggregate information about all DEFA students.
-
-* `defa-stats.csv` (or another name defined in `params.out.stats`)
-  * Contains the credit amount distribution of DEFA students.

services/backend/oodikone2-backend/scripts/create_course_groups_kayttis.js

@@ -1,34 +0,0 @@
-const { createCourseGroup } = require('../src/services/courseGroups')
-
-const run = async () => {
-  await Promise.all(groups.map(name => createCourseGroup(name)))
-  process.exit(0)
-}
-
-const groups = [
-  'Yleinen ja aikuiskasvatustiede',
-  'Kasvatustiede, yleinen didaktiikka',
-  'Kasvatuspsykologia',
-  'Varhaiskasvatus (VO+VAKAMO)',
-  'Erityispedagogiikka',
-  'Kotitaloustiede',
-  'Käsityötiede',
-  'Aikuisopetus',
-  'Biologia ja maantiede',
-  'Historia ja yhteiskuntaoppi',
-  'Uskonto',
-  'Elämänkatsomustieto',
-  'Suomen kieli ja kirjallisuus',
-  'Vieraat kielet',
-  'Matematiikka',
-  'Fysiikka ja kemia',
-  'Musiikki',
-  'Kuvataide',
-  'Liikunta',
-  'Draama',
-  'Mediakasvatus',
-  'Tiedekeskuspedagogiikka',
-  'Svenska studier'
-]
-
-run()

services/backend/oodikone2-backend/src/anonymizer/anonDisciplines.js

@@ -1,115 +0,0 @@
-const disciplines = {
-  data: {
-    data: [
-      {
-        discipline_id: '0',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Witchcraft'
-          }
-        ]
-      },
-      {
-        discipline_id: '1',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Magic'
-          }
-        ]
-      },
-      {
-        discipline_id: '2',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Alchemy'
-          }
-        ]
-      },
-      {
-        discipline_id: '3',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Arithmancy'
-          }
-        ]
-      },
-      {
-        discipline_id: '4',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Runes'
-          }
-        ]
-      },
-      {
-        discipline_id: '5',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Creatures'
-          }
-        ]
-      },
-      {
-        discipline_id: '6',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Astronomy'
-          }
-        ]
-      },
-      {
-        discipline_id: '7',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'History'
-          }
-        ]
-      },
-      {
-        discipline_id: '8',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Charms'
-          }
-        ]
-      },
-      {
-        discipline_id: '9',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Flight'
-          }
-        ]
-      },
-      {
-        discipline_id: '10',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Necromancy'
-          }
-        ]
-      },
-      {
-        discipline_id: '11',
-        name: [
-          {
-            langcode: 'fi',
-            text: 'Pyromancy'
-          }
-        ]
-      }
-    ]
-  }
-}
-module.exports = { disciplines }

services/backend/oodikone2-backend/src/anonymizer/anonStudyRights.js

@@ -1,63 +0,0 @@
-const studyRightCodes = {
-  a: {
-    name: 'Transfiguration',
-    code: '666ABA'
-  },
-  b: {
-    name: 'Defence Against the Dark Arts',
-    code: '666ABB'
-  },
-  c: {
-    name: 'Charms',
-    code: '666ABC'
-  },
-  d: {
-    name: 'Potions',
-    code: '666ABD'
-  },
-  e: {
-    name: 'Astronomy',
-    code: '666ABE'
-  },
-  f: {
-    name: 'History of Magic',
-    code: '666ABF'
-  },
-  g: {
-    name: 'Herbology',
-    code: '666ABG'
-  },
-  h: {
-    name: 'Arithmancy',
-    code: '666ABH'
-  },
-  i: {
-    name: 'Muggle Studies',
-    code: '666ABI'
-  },
-  j: {
-    name: 'Divination',
-    code: '666ABJ'
-  },
-  k: {
-    name: 'Study of Ancient Runes',
-    code: '666ABK'
-  },
-  l: {
-    name: 'Care of Magical Creatures',
-    code: '666ABL'
-  },
-  m: {
-    name: 'Alchemy',
-    code: '666ABM'
-  },
-  bachelor: {
-    name: 'Bachelors Degree on Witchcraft and Wizardry',
-    code: '666bach'
-  },
-  master: {
-    name: 'Master of Witchcraft and Wizardry',
-    code: '666mast'
-  }
-}
-module.exports = { studyRightCodes }

services/backend/oodikone2-backend/src/anonymizer/anon_oodi_interface.js

@@ -1,104 +0,0 @@
-const axios = require('axios')
-const https = require('https')
-require('dotenv').config()
-
-const instance = axios.create({
-  httpsAgent: new https.Agent({
-    rejectUnauthorized: false
-  })
-})
-
-axios.defaults.auth = {
-  username: 'tktl'
-}
-
-instance.interceptors.request.use(config => {
-  config.params = { token: process.env.TOKEN }
-  return config
-})
-
-const getStudent = studentNumber => {
-  return instance
-    .get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)
-    .then(response => {
-      return response.data.data
-    })
-    .catch(error => {
-      return error
-    })
-}
-const getStudyAttainments = async studentNumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentNumber}/studyattainments`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getTeacher = async teacherId => {
-  const url = `${process.env.OODI_ADDR}/teachers/${teacherId}/info`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getStudyRights = async studentNumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentNumber}/studyrights`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const getCourseTypeCodes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/learningopportunities/types`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/studyattainments/statuses`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getSemesters = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/semesters`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${process.env.OODI_ADDR}/students/${studentnumber}/semesterenrollments`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${process.env.OODI_ADDR}/codes/courseunitrealisations/types`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${process.env.OODI_ADDR}/learningopportunities/${id}`
-  const response = await instance.get(url)
-  return response.data.data
-}
-const courseUnitRealisations = async () => {
-  const url = `${process.env.OODI_ADDR}/courseunitrealisations/changes/ids/0000-12-24`
-  const response = await instance.get(url)
-  return response.data.data
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${process.env.OODI_ADDR}/courseunitrealisations/${id}`
-  const response = await instance.get(url)
-  return response.data.data
-}
-module.exports = {
-  getStudyRights,
-  getStudent,
-  getTeacher,
-  getStudyAttainments,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisations,
-  getCourseUnitRealisation
-}

services/backend/oodikone2-backend/src/anonymizer/anonymizer.js

@@ -1,489 +0,0 @@
-const faker = require('faker')
-const moment = require('moment')
-const mkdirp = require('mkdirp')
-const getDirName = require('path').dirname
-const logger = require('../util/logger')
-const { disciplines } = require('./anonDisciplines')
-const { studyRightCodes } = require('./anonStudyRights')
-const oodi = require('./anon_oodi_interface')
-const fs = require('fs')
-const _ = require('lodash')
-const status = require('node-status')
-
-require('dotenv').config()
-const startTime = moment()
-
-let student_numbers = []
-
-// few objects to save connections between anonymized and real data to keep relations.
-let teacherRelations = {}
-let studentRelations = {}
-let learningOpportunityRelations = {}
-
-const startCodesStatusBar = () => {
-  status.start({ pattern: 'Running: {uptime.time} {spinner.monkey.blue} | {codes.bar} | codes anonymized: {codes}' })
-}
-const startStudentsStatusBar = () => {
-  status.start({
-    pattern: 'Running: {uptime.time} {spinner.hearts.magenta} | {students.bar} | students anonymized: {students}'
-  })
-}
-const startRealisationsStatusBar = () => {
-  status.start({
-    pattern: 'Running: {uptime.time} {spinner.clock.red} | {realisations.bar} | realisations anonymized: {realisations}'
-  })
-}
-const stopStatusBar = () => {
-  status.stamp()
-  status.stop()
-}
-
-const getAnonymizedTeacher = async teacherId => {
-  if (teacherRelations[teacherId]) {
-    return teacherRelations[teacherId]
-  }
-  let teacher = await oodi.getTeacher(teacherId)
-  const last_name = faker.name.lastName()
-  const first_names = faker.name.firstName()
-  teacher = {
-    ...teacher,
-    teacher_id: generateRandomId(),
-    userid: faker.internet.userName(),
-    last_name,
-    first_names,
-    calling_name: faker.name.firstName(),
-    full_name: `${last_name} ${first_names}`,
-    email: faker.internet.email(),
-    mobile_phone: faker.phone.phoneNumberFormat()
-  }
-  teacherRelations[teacherId] = teacher
-  return teacher
-}
-
-const generateRandomId = () => String(Math.floor(Math.random() * (999999999 - 100000000 + 1) + 100000000))
-
-const anonymizeStudentInfo = async id => {
-  let student = await oodi.getStudent(id)
-  let anonStudent = undefined
-  if (student) {
-    anonStudent = studentRelations[student.student_number]
-  }
-  if (!anonStudent) {
-    anonStudent = {
-      ...student,
-      first_names: faker.name.firstName(),
-      last_name: faker.name.lastName(),
-      birthdate: new Date(
-        moment(
-          moment('1940', 'YYYY').valueOf() +
-            Math.random() *
-              (moment()
-                .subtract(17, 'years')
-                .valueOf() -
-                moment('1940', 'YYYY').valueOf())
-        )
-      ),
-      email: faker.internet.email(),
-      mobile_phone: faker.phone.phoneNumberFormat(),
-      national_student_number: null,
-      phone: faker.phone.phoneNumberFormat(),
-      address1: faker.address.streetName(),
-      address2: faker.address.streetName(),
-      zipcode: faker.address.zipCode(),
-      student_number: `0${Math.floor(Math.random() * (1999999 - 1000000 + 1) + 1000000)}`,
-      person_id: generateRandomId(),
-      city: [
-        {
-          langcode: 'en',
-          text: 'Harlem'
-        },
-        {
-          langcode: 'fi',
-          text: 'Kajaani'
-        }
-      ],
-      country: [
-        {
-          langcode: 'en',
-          text: 'Canada'
-        },
-        {
-          langcode: 'fi',
-          text: 'Ruotsi'
-        }
-      ]
-    }
-  }
-  if (!student) {
-    return anonStudent
-  }
-  studentRelations[student.student_number] = anonStudent
-
-  return anonStudent
-}
-const getRandomAmount = attainments => {
-  const rate = Math.random() * (0.375 - 0.05) + 0.05
-  const amount = Math.floor(rate * attainments.length)
-  return amount
-}
-const dropRandomAttainments = attainments => {
-  if (student_numbers.length < 1) {
-    return attainments
-  }
-  const dropAmount = getRandomAmount(attainments)
-  let i = 0
-  while (i < dropAmount) {
-    let randomIndex = Math.floor(Math.random() * attainments.length)
-    attainments.splice(randomIndex, 1)
-    i++
-  }
-  return attainments
-}
-const addRandomAttainments = (attainments, studentnumber_attainments) => {
-  if (studentnumber_attainments.length < 1) {
-    return attainments
-  }
-  let studentnumbers = studentnumber_attainments.map(s => s.student_number)
-  const addAmount = getRandomAmount(attainments)
-  let i = 0
-  while (i < addAmount) {
-    let randomStudentNumber = _.sample(studentnumbers)
-    let randomAttainment = _.sample(
-      _.flattenDeep(
-        studentnumber_attainments.filter(s => s.student_number === randomStudentNumber).map(s => s.attainments)
-      )
-    )
-    if (!randomAttainment) {
-      continue
-    }
-    randomAttainment.semester_code = _.sample(attainments).semester_code // Make attainment happen at the same time as the students some other attainment.
-    attainments = attainments.concat(randomAttainment)
-    i++
-  }
-  return attainments
-}
-const populateAnonymizedTeachers = async teachers => {
-  let anonTeachers = []
-  for (let teacher of teachers) {
-    let anonTeacher = await getAnonymizedTeacher(teacher.teacher_id)
-    APIWriter(`./src/anonymized_API/teachers/${anonTeacher.teacher_id}/info`, { data: { data: anonTeacher } })
-    anonTeachers = anonTeachers.concat(anonTeacher)
-  }
-  return anonTeachers
-}
-
-const anonymizeStudentAttainments = async (id, studentInfo, studentnumber_attainments) => {
-  let attainments = await oodi.getStudyAttainments(id)
-  attainments = dropRandomAttainments(attainments)
-  attainments = addRandomAttainments(attainments, studentnumber_attainments)
-  let anonAttainments = []
-  for (let attainment of attainments) {
-    let anonLearningOpportunity = learningOpportunityRelations[attainment.learningopportunity_id]
-    if (!anonLearningOpportunity) {
-      const learningOpportunity = await oodi.getLearningOpportunity(attainment.learningopportunity_id)
-      if (learningOpportunity === null) {
-        continue
-      }
-      anonLearningOpportunity = {
-        ...learningOpportunity,
-        disciplines: [disciplines.data.data[Math.floor(Math.random() * 12)]],
-        learningopportunity_id: generateRandomId(),
-        names: [
-          {
-            langcode: 'en',
-            text: faker.company.bs()
-          },
-          {
-            langcode: 'fi',
-            text: faker.company.bs()
-          }
-        ],
-        descriptions: [],
-        materials: [],
-        organisations: [],
-        learningopportunity_internal: generateRandomId(),
-        start_date: moment(learningOpportunity.start_date)
-          .add(_.sample([-3, 0, 3]), 'months')
-          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days')),
-        end_date: moment(learningOpportunity.start_date)
-          .add(_.sample([-3, 0, 3]), 'months')
-          .add(_.sample(Math.floor(Math.random() * 100) - 50, 'days'))
-          .add(100, 'years')
-      }
-      learningOpportunityRelations[attainment.learningopportunity_id] = anonLearningOpportunity
-      APIWriter(`./src/anonymized_API/learningopportunities/${anonLearningOpportunity.learningopportunity_id}`, {
-        data: { data: anonLearningOpportunity }
-      })
-    }
-    let teachers = []
-    teachers = await populateAnonymizedTeachers(attainment.teachers)
-
-    // semester code is calculated as "code * 6 months since 1950-1-1", Im bucketing all study attainments in 3 month periods by semester code abstracting real study attainment dates
-    const attainment_date = new Date(
-      moment(moment('1950', 'YYYY').add(attainment.semester_code * 6 + _.sample([-3, 0, 3]), 'months'))
-    )
-    attainment = {
-      ...attainment,
-      studyattainment_id: generateRandomId(),
-      person_id: studentInfo.person_id,
-      attainment_date,
-      teachers,
-      learningopportunity_id: anonLearningOpportunity.learningopportunity_id,
-      learningopportunity_name: anonLearningOpportunity.names
-    }
-    anonAttainments = anonAttainments.concat(attainment)
-  }
-  return anonAttainments
-}
-
-const anonymizeStudentStudyRights = async (id, studentInfo) => {
-  let studyRights = await oodi.getStudyRights(id)
-  let anonStudyRights = []
-  for (let studyright of studyRights) {
-    let program = _.sample(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
-    let degree = 'bachelor'
-    if (studyright.elements.some(element => element.name.some(name => name.text.includes('Master')))) {
-      program = _.sample(['i', 'j', 'k', 'l', 'm', 'h'])
-      degree = 'master'
-    }
-    const elements = studyright.elements
-      .filter(element => element.element_id === 10 || element.element_id === 20)
-      .map(element => {
-        if (element.element_id === 10) {
-          return {
-            ...element,
-            code: studyRightCodes[degree].code,
-            name: [
-              {
-                langcode: 'en',
-                text: studyRightCodes[degree].name
-              },
-              {
-                langcode: 'fi',
-                text: studyRightCodes[degree].name
-              }
-            ]
-          }
-        } else if (element.element_id === 20) {
-          return {
-            ...element,
-            code: studyRightCodes[program].code,
-            name: [
-              {
-                langcode: 'en',
-                text: studyRightCodes[program].name
-              },
-              {
-                langcode: 'fi',
-                text: studyRightCodes[program].name
-              }
-            ]
-          }
-        }
-        return
-      })
-
-    let anonStudyright = {
-      ...studyright,
-      studyright_id: generateRandomId(),
-      person_id: studentInfo.person_id,
-      elements,
-      organisation_name: [
-        {
-          langcode: 'en',
-          text: 'Hogwarts School of Witchcraft and Wizardry'
-        },
-        {
-          langcode: 'fi',
-          text: 'Tylypahkan noitien ja velhojen koulu'
-        }
-      ]
-    }
-    anonStudyRights = anonStudyRights.concat(anonStudyright)
-  }
-  return anonStudyRights
-}
-const getElapsedTime = () =>
-  `${moment().diff(startTime, 'minutes')} m, ${moment().diff(startTime, 'seconds') /
-    (moment().diff(startTime, 'minutes') * 60)} s.`
-const anonymizeSemesterEnrollments = async id => {
-  const data = await oodi.getSemesterEnrollments(id)
-  let anonymizedData = []
-  for (const enrollment of data) {
-    anonymizedData = anonymizedData.concat({
-      ...enrollment,
-      yths_code: null,
-      absence_reason_code: null,
-      payment_amount: 100,
-      yths_description: null,
-      semester_enrollment_date: moment(moment(enrollment.semester_enrollment_date).format('YYYY-MM'), 'YYYY-MM'),
-      payment_date: moment(moment(enrollment.payment_date).format('YYYY-MM'), 'YYYY-MM')
-    })
-  }
-  return anonymizedData
-}
-const anonymizeStudent = async (id, student_numbers) => {
-  const info = await anonymizeStudentInfo(id)
-
-  const attainments = await anonymizeStudentAttainments(id, info, studentnumber_attainments)
-  studentnumber_attainments = studentnumber_attainments.concat({
-    student_number: info.student_number,
-    attainments: attainments
-  })
-  info.studyattainments = attainments
-    .filter(
-      attainment => ['Hyv.', '1', '2', '3', '4', '5'].includes(attainment.grade[0].text) && attainment.credits < 25
-    )
-    .map(attainment => attainment.credits)
-    .reduce((acc, curr) => acc + curr, 0)
-
-  const studyrights = await anonymizeStudentStudyRights(id, info)
-
-  const semesterEnrollments = await anonymizeSemesterEnrollments(id)
-  const infoPath = `./src/anonymized_API/students/${info.student_number}/info`
-  const semesterEnrollmentPath = `./src/anonymized_API/students/${info.student_number}/semesterenrollments`
-  const courseEnrollmentPath = `./src/anonymized_API/students/${info.student_number}/enrollments`
-  const attainmentPath = `./src/anonymized_API/students/${info.student_number}/studyattainments`
-  const studyRightPath = `./src/anonymized_API/students/${info.student_number}/studyrights`
-  student_numbers = student_numbers.concat(info.student_number)
-
-  APIWriter(infoPath, { data: { data: info } })
-  APIWriter(studyRightPath, { data: { data: studyrights } })
-  APIWriter(courseEnrollmentPath, { data: { data: [] } })
-  APIWriter(attainmentPath, { data: { data: attainments } })
-  APIWriter(semesterEnrollmentPath, { data: { data: semesterEnrollments } })
-  return student_numbers
-}
-const anonymizeCourseUnitRealisations = async () => {
-  const courseUnitRealisations = await oodi.courseUnitRealisations()
-  const realisationsCounter = status.addItem('realisations', { max: courseUnitRealisations.length })
-  startRealisationsStatusBar()
-  let anonRealisations = []
-  for (const realisation of courseUnitRealisations) {
-    continue
-    realisationsCounter.inc() // eslint-disable-line
-    if (!learningOpportunityRelations[realisation.learningopportunity_id]) {
-      continue
-    }
-    const anonLearningId = learningOpportunityRelations[realisation.learningopportunity_id].learningopportunity_id
-    const anonRealisation = {
-      ...realisation,
-      course_id: generateRandomId(),
-      learningopportunity_id: anonLearningId
-    }
-    anonRealisations = anonRealisations.concat(anonRealisation)
-    const courseUnitRealisation = await oodi.getCourseUnitRealisation(realisation.course_id)
-    if (!courseUnitRealisation) {
-      continue
-    }
-    let anonStudents = []
-    for (const student of courseUnitRealisation.students) {
-      if (student) {
-        anonStudents = anonStudents.concat(await anonymizeStudentInfo(student.student_number))
-      }
-    }
-
-    let anonTeachers = []
-    for (const teacher of courseUnitRealisation.teachers) {
-      anonTeachers = anonTeachers.concat(await getAnonymizedTeacher(teacher.teacher_id))
-    }
-
-    const anonCourseUnitRealisation = {
-      ...courseUnitRealisation,
-      realisation_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      students: anonStudents,
-      providing_organisation: [],
-      instruction_locality: [],
-      events: [],
-      officials: [],
-      child_ids: [],
-      courses_extra_organisations: [],
-      teachers: anonTeachers, // note that new teachers created here are not populated in the API! (for future possible issues) regards: Saus 31.7.2018
-      organisations: [],
-      course_id: anonRealisation.course_id,
-      providing_locality: [],
-      learningopportunity_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      realisation_root_name: learningOpportunityRelations[realisation.learningopportunity_id].names,
-      learningopportunity_id: learningOpportunityRelations[realisation.learningopportunity_id].learningopportunity_id,
-      descriptions: [],
-      languages: []
-    }
-    await APIWriter(`src/anonymized_API/courseunitrealisations/${anonCourseUnitRealisation.course_id}`, {
-      data: { data: anonCourseUnitRealisation }
-    })
-  }
-  await APIWriter('src/anonymized_API/courseunitrealisations/changes/ids/0000-12-24', {
-    data: { data: anonRealisations }
-  })
-  stopStatusBar()
-  return
-}
-
-const APIWriter = async (path, data) => {
-  mkdirp(getDirName(path), async () => {
-    fs.writeFile(path, JSON.stringify(data, null, 4), error => {
-      if (error) {
-        console.log(error)
-      }
-    })
-    return 'succeeee'
-  })
-}
-
-let studentnumber_attainments = []
-
-const anonymize = async () => {
-  logger.verbose(`starting time: ${startTime.format('HH:mm:ss')}`)
-  const filename = './studentnumbers.txt'
-  const readStudentNumbersFromFile = async filename => {
-    let studentnumbers = fs
-      .readFileSync(filename, 'utf-8')
-      .split('\n')
-      .map(s => s.replace(' ', ''))
-    const randomCopies = _.sampleSize(
-      studentnumbers,
-      Math.floor((Math.random() * (1 - 0.33) + 0.33) * studentnumbers.length)
-    )
-    studentnumbers = studentnumbers.concat(randomCopies)
-    studentnumbers = _.shuffle(studentnumbers)
-    return studentnumbers.filter(studentnumber => !!studentnumber)
-  }
-  const codeCounter = status.addItem('codes', { max: 5 })
-  startCodesStatusBar()
-  const semesters = { data: { data: await oodi.getSemesters() } }
-  codeCounter.inc()
-  const courseUnitRealisationsTypes = { data: { data: await oodi.getCourseRealisationTypes() } }
-  codeCounter.inc()
-  const creditTypeCodes = { data: { data: await oodi.getStudyattainmentStatusCodes() } }
-  codeCounter.inc()
-  const courseTypeCodes = { data: { data: await oodi.getCourseTypeCodes() } }
-  codeCounter.inc()
-  const courseDisciplines = disciplines
-  codeCounter.inc()
-  await APIWriter('./src/anonymized_API/codes/semesters', semesters)
-  await APIWriter('./src/anonymized_API/codes/courseunitrealisations/types', courseUnitRealisationsTypes)
-  await APIWriter('./src/anonymized_API/codes/studyattainments/statuses', creditTypeCodes)
-  await APIWriter('./src/anonymized_API/codes/learningopportunities/types', courseTypeCodes)
-  await APIWriter('./src/anonymized_API/codes/learningopportunities/disciplines', courseDisciplines)
-  const numberList = await readStudentNumbersFromFile(filename)
-  const studentCounter = status.addItem('students', { max: numberList.length })
-  status.stamp()
-  startStudentsStatusBar()
-  let student_numbers = []
-  for (let id of numberList) {
-    studentCounter.inc()
-    try {
-      student_numbers = await anonymizeStudent(id, student_numbers)
-    } catch (error) {
-      logger.verbose(`FAILED ${id}, ${error}  elapsed time:  ${getElapsedTime()}`)
-      process.exit(1)
-    }
-  }
-  status.stamp()
-  await anonymizeCourseUnitRealisations()
-  console.log(student_numbers)
-  await APIWriter('./studentnumbersN.txt', student_numbers)
-  logger.verbose(`end time: ${moment()}, total run-time:  ${getElapsedTime()}`)
-}
-
-anonymize()

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -1,28 +0,0 @@
-const fs = require('fs')
-const { Migration } = require('../models/index')
-const { MigrationKone } = require('../models/models_kone')
-
-const DEFAULT_PATH = `${process.cwd()}/src/database/migrations`
-const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`
-
-const saveMigrationsToDatabase = Migration => async filenames =>
-  Promise.all(filenames.map(name => Migration.upsert({ name })))
-
-const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(Migration)(filenames)
-}
-
-const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(MigrationKone)(filenames)
-}
-
-const seedAllMigrations = async () => {
-  await seedMigrations()
-  await seedMigrationsKone()
-}
-
-module.exports = {
-  seedAllMigrations
-}

services/backend/oodikone2-backend/src/databaseV2/connection.js

@@ -1,44 +0,0 @@
-const Sequelize = require('sequelize')
-const EventEmitter = require('events')
-const { SIS_DB_URL } = process.env
-
-class DbConnection extends EventEmitter {
-  constructor() {
-    super()
-    this.RETRY_ATTEMPTS = 15
-    this.established = false
-
-    this.sequelize = new Sequelize(SIS_DB_URL, {
-      dialect: 'postgres',
-      pool: {
-        max: 25,
-        min: 0,
-        acquire: 10000,
-        idle: 300000000
-      },
-      logging: false
-    })
-  }
-
-  async connect(attempt = 1) {
-    try {
-      await this.sequelize.authenticate()
-      this.emit('connect')
-      this.established = true
-    } catch (e) {
-      if (attempt > this.RETRY_ATTEMPTS) {
-        this.emit('error', e)
-        return
-      }
-      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
-      console.log(e)
-      setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
-    }
-  }
-}
-
-const dbConnections = new DbConnection()
-dbConnections
-module.exports = {
-  dbConnections
-}

services/backend/oodikone2-backend/src/routes/coolDataScience.js

@@ -1,81 +0,0 @@
-const router = require('express').Router()
-const {
-  withErr,
-  getProtoC,
-  getProtoCProgramme,
-  getStatus,
-  getUber,
-  getStartYears,
-  getGraduatedStatus
-} = require('../services/coolDataScience')
-const useSisRouter = require('../util/useSisRouter')
-const trendsV2 = require('../routesV2/trends')
-
-router.get('/start-years', async (req, res) => {
-  const years = await getStartYears()
-  res.json(years.map(({ studystartdate }) => studystartdate))
-})
-
-router.get(
-  '/uber-data',
-  withErr(async (req, res) => {
-    const data = await getUber(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data-programme',
-  withErr(async (req, res) => {
-    const data = await getProtoCProgramme(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/proto-c-data',
-  withErr(async (req, res) => {
-    const data = await getProtoC(req.query)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status-graduated',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const data = await getGraduatedStatus(date.getTime(), showByYear)
-    res.json(data)
-  })
-)
-
-router.get(
-  '/status',
-  withErr(async (req, res) => {
-    const { date: unixMillis, showByYear } = req.query
-    const date = new Date(Number(unixMillis))
-    const endOfToday = new Date()
-    endOfToday.setHours(23, 59, 59, 999)
-
-    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
-      return res.status(400).json({ error: 'Invalid date' })
-    }
-
-    // End of day
-    date.setHours(23, 59, 59, 999)
-    const status = await getStatus(date.getTime(), showByYear)
-    res.json(status)
-  })
-)
-
-module.exports = useSisRouter(trendsV2, router)

services/backend/oodikone2-backend/src/routes/courseGroups.js

@@ -1,61 +0,0 @@
-const router = require('express').Router()
-const CourseGroupService = require('../services/courseGroups')
-
-router.get('/programme/:programmeId/:force?', async (req, res) => {
-  const { programmeId, force } = req.params
-  const { rights, roles } = req
-  const semesterCode = req.query.semester
-  if (rights.includes(programmeId) || (roles && roles.includes('admin'))) {
-    const courseGroups = await CourseGroupService.getCourseGroupsWithTotals(
-      programmeId,
-      semesterCode,
-      force === 'force'
-    )
-    return res.json(courseGroups)
-  }
-  return res.status(403).json({ error: 'No right to view programme data' })
-})
-
-router.get('/academic-years', async (req, res) => {
-  const academicYears = await CourseGroupService.getAcademicYears()
-  return res.json(academicYears)
-})
-
-router.get('/courses', async (req, res) => {
-  const { teacherIds: ids, semester } = req.query
-  const teacherIds = ids && JSON.parse(ids)
-
-  if (!teacherIds || !Array.isArray(teacherIds)) {
-    return res.sendStatus(400)
-  }
-
-  if (teacherIds.length === 0) return res.send([])
-
-  const courses = await CourseGroupService.getCoursesByTeachers(teacherIds, semester)
-
-  return courses ? res.send(courses) : res.sendStatus(404)
-})
-
-router.get('/:id', async (req, res) => {
-  const { id } = req.params
-  const { semester } = req.query
-  const groupData = await CourseGroupService.getCourseGroup(Number(id), Number(semester))
-
-  return groupData ? res.send(groupData) : res.sendStatus(404)
-})
-
-router.post('/:id/add/:teacherid', async (req, res) => {
-  const { id, teacherid } = req.params
-  const success = await CourseGroupService.addTeacher(Number(id), teacherid)
-
-  return success ? res.status(200).json('added') : res.sendStatus(404)
-})
-
-router.post('/:id/remove/:teacherid', async (req, res) => {
-  const { id, teacherid } = req.params
-  const success = await CourseGroupService.removeTeacher(Number(id), teacherid)
-
-  return success ? res.status(200).json('removed') : res.sendStatus(404)
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/courses.js

@@ -1,106 +0,0 @@
-const router = require('express').Router()
-const crypto = require('crypto')
-const Course = require('../services/courses')
-const { validateParamLength } = require('../util')
-const logger = require('../util/logger')
-const coursesV2 = require('../routesV2/courses')
-const useSisRouter = require('../util/useSisRouter')
-
-router.get('/courses', async (req, res) => {
-  let results = []
-  if (req.query.name) {
-    results = await Course.bySearchTerm(req.query.name, req.query.language)
-  }
-
-  res.json(results)
-})
-
-router.get('/coursesmulti', async (req, res) => {
-  let results = []
-  if (req.query.name || req.query.discipline || req.query.type) {
-    results = await Course.bySearchTermTypeAndDiscipline(
-      req.query.name,
-      req.query.type,
-      req.query.discipline,
-      req.query.language
-    ) // eslint-disable-line
-  }
-  res.json(results)
-})
-
-router.get('/v2/coursesmulti', async (req, res) => {
-  let results = { courses: [], groups: {}, groupMeta: {} }
-  const { name, code } = req.query
-
-  if (!(validateParamLength(name, 5) || validateParamLength(code, 2))) {
-    return res.status(400).json({ error: 'name or code invalid' })
-  }
-
-  results = await Course.byNameAndOrCodeLike(name, code)
-  res.json(results)
-})
-
-router.get('/coursetypes', async (req, res) => {
-  const coursetypes = await Course.getAllCourseTypes()
-  res.json(coursetypes)
-})
-
-router.get('/coursedisciplines', async (req, res) => {
-  const courseDisciplines = await Course.getAllDisciplines()
-  res.json(courseDisciplines)
-})
-
-router.get('/v3/courseyearlystats', async (req, res) => {
-  try {
-    const { rights, roles } = req
-
-    const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
-
-    // If user has rights to see at least one programme, then they are allowed
-    // to see all of them
-    if (!allowedRoles && rights.length < 1) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
-
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
-
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      // Studentnumbers should be obfuscated to all other users except admins
-      // and users with rights to any specific study programmes
-      const anonymize = allowedRoles !== 'admin' && rights.length < 1
-      const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
-      res.json(results)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
-  }
-})
-
-router.get('/courses/duplicatecodes/:programme', async (req, res) => {
-  // const { programme } = req.params
-  const results = await Course.getMainCourseToCourseMap()
-  return res.json(results)
-})
-
-router.post('/courses/duplicatecodes/:code1/:code2', async (req, res) => {
-  const { code1, code2 } = req.params
-  await Course.setDuplicateCode(code1, code2)
-  const results = await Course.getMainCourseToCourseMap()
-  res.status(200).json(results)
-})
-
-router.delete('/courses/duplicatecodes/:code', async (req, res) => {
-  const { code } = req.params
-  await Course.deleteDuplicateCode(code)
-  const results = await Course.getMainCourseToCourseMap()
-  res.status(200).json(results)
-})
-
-module.exports = useSisRouter(coursesV2, router)

services/backend/oodikone2-backend/src/routes/diff.js

@@ -1,33 +0,0 @@
-const router = require('express').Router()
-const SisCourse = require('../servicesV2/courses')
-const OodiCourse = require('../services/courses')
-const diff = require('../services/diff')
-const logger = require('../util/logger')
-
-router.get('/diff/courseyearlystats', async (req, res) => {
-  try {
-    const { roles } = req
-    const admin = roles && roles.includes('admin')
-    if (!admin) {
-      return res.status(403).json({ error: 'No access to course stats diff' })
-    }
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
-
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      const oodiResults = await OodiCourse.courseYearlyStats(codes, separate, unifyOpenUniCourses)
-      const sisResults = await SisCourse.courseYearlyStats(codes, separate, unifyOpenUniCourses)
-      const result = diff.getCourseYearlyStatsDiff(sisResults, oodiResults)
-      res.json(result)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -1,269 +0,0 @@
-const router = require('express').Router()
-const { getStudentsUserCanAccess } = require('../services/userService')
-const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const MandatoryCourses = require('../services/mandatoryCourses')
-const { productivityStatsForCode, throughputStatsForCode, optionData } = require('../services/studyprogramme')
-const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-  getNonGraduatedStudents,
-  ping
-} = require('../services/analyticsService')
-const elementDetailsV2 = require('../routesV2/elementdetails')
-const useSisRouter = require('../util/useSisRouter')
-
-const programmeStatsSince = new Date('2017-07-31')
-
-router.get('/elementdetails/all', async (req, res) => {
-  try {
-    const elementdetails = await getAllElementDetails()
-    res.json(elementdetails)
-  } catch (e) {
-    res.status(500).json(e)
-  }
-})
-
-router.get('/studyprogrammes', async (req, res) => {
-  try {
-    const studyrights = await getAllDegreesAndProgrammes()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
-  try {
-    const {
-      roles,
-      decodedToken,
-      params: { id }
-    } = req
-    if (!id) return res.status(400).json({ error: 'programme id missing' })
-
-    const nonGraduatedStudents = await getNonGraduatedStudents(id)
-    if (!nonGraduatedStudents) return res.status(200).json({})
-    const {
-      data: { formattedData, studentnumbers }
-    } = nonGraduatedStudents
-
-    const filteredData = { ...formattedData }
-    const studentsUserCanAccess = await getStudentsUserCanAccess(studentnumbers, roles, decodedToken.userId)
-    Object.keys(filteredData).forEach(year => {
-      filteredData[year] = filteredData[year].filter(({ studentNumber }) => studentsUserCanAccess.has(studentNumber))
-    })
-
-    res.json(filteredData)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
-  if (req.params.id) {
-    const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
-    res.json(codes)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/ping', async (req, res) => {
-  try {
-    const result = await ping()
-    res.json(result)
-  } catch (e) {
-    res.status(500).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    if (code.includes('MH')) {
-      const data = await optionData(new Date('2017-07-31'), new Date(), code, 'MSC')
-      return res.json(data)
-    } else if (code.includes('KH')) {
-      const data = await optionData(new Date('2017-07-31'), new Date(), code, 'BSC')
-      return res.json(data)
-    } else {
-      return res.json([])
-    }
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      console.error(e)
-    }
-
-    if (!data) {
-      try {
-        const stats = await productivityStatsForCode(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        console.error(e)
-      }
-    }
-
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  console.log('Productivity stats recalculation starting')
-  const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
-  try {
-    await patchProductivity(
-      codes.reduce((acc, c) => {
-        acc[c] = { status: 'RECALCULATING' }
-        return acc
-      }, {})
-    )
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
-
-  let ready = 0
-  for (const code of codes) {
-    try {
-      if (code.includes('MH') || code.includes('KH')) {
-        const data = await productivityStatsForCode(code, new Date('2017-07-31'))
-        await setProductivity(data)
-      } else {
-        const data = await productivityStatsForCode(code, new Date('2000-07-31'))
-        await setProductivity(data)
-      }
-    } catch (e) {
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' }
-        })
-      } catch (e) {
-        console.error(e)
-        return
-      }
-      console.error(e)
-      console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-    }
-    ready += 1
-    console.log(`Productivity stats recalculation ${ready}/${codes.length} done`)
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      console.error(e)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForCode(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        console.error(e)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  console.log('Throughput stats recalculation starting')
-  const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
-  try {
-    const data = codes.reduce((acc, id) => ({ ...acc, [id]: { status: 'RECALCULATING' } }), {})
-    await patchThroughput(data)
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
-
-  let ready = 0
-  for (const code of codes) {
-    try {
-      if (code.includes('MH') || code.includes('KH')) {
-        const data = await throughputStatsForCode(code, 2017)
-        await setThroughput(data)
-      } else {
-        const data = await throughputStatsForCode(code, 2000)
-        await setThroughput(data)
-      }
-    } catch (e) {
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        console.error(e)
-        return
-      }
-      console.error(e)
-      console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-    }
-    ready += 1
-    console.log(`Throughput stats recalculation ${ready}/${codes.length} done`)
-  }
-})
-
-router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  if (id) {
-    const thesis = await findProgrammeTheses(id)
-    res.json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
-  const { id } = req.params
-  const { course, thesisType } = req.body
-  if (id && course && thesisType) {
-    const thesis = await createThesisCourse(id, course, thesisType)
-    res.status(201).json(thesis)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
-  const { id, course } = req.params
-  if (id && course) {
-    const deleted = await deleteThesisCourse(id, course)
-    res.status(204).json(deleted)
-  } else {
-    res.status(422).end()
-  }
-})
-
-module.exports = useSisRouter(elementDetailsV2, router)

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -1,9 +0,0 @@
-const router = require('express').Router()
-const { all } = require('../services/organisations')
-
-router.get('/', async (req, res) => {
-  const faculties = await all()
-  res.json(faculties)
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/facultyStats.js

@@ -1,34 +0,0 @@
-const router = require('express').Router()
-const { all } = require('../services/organisations')
-const { byType: getElementDetailsByType } = require('../services/elementdetails')
-const { getFacultyYearlyStats } = require('../services/analyticsService')
-
-router.get('/faculties', async (req, res) => {
-  const { roles, faculties } = req
-  const allFaculties = await all()
-  res.json(roles.includes('admin') ? allFaculties : allFaculties.filter(({ code }) => faculties.has(code)))
-})
-
-router.get('/yearlystats', async (req, res) => {
-  try {
-    const { roles, faculties } = req
-    const facultyYearlyStats = await getFacultyYearlyStats()
-    res.json(roles.includes('admin') ? facultyYearlyStats : facultyYearlyStats.filter(({ id }) => faculties.has(id)))
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-router.get('/programmes', async (req, res) => {
-  try {
-    const { roles } = req
-    const isAdmin = roles.includes('admin')
-    const facultyProgrammes = await getElementDetailsByType(20)
-    const userRightsSet = new Set(req.rights)
-    res.json(facultyProgrammes.filter(({ code }) => userRightsSet.has(code) || isAdmin))
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/oodi.js

@@ -1,50 +0,0 @@
-const axios = require('axios')
-const https = require('https')
-const fs = require('fs')
-const { wrapper, router } = require('./router').routerWithWrapper()
-const { KEY_PATH, CERT_PATH, OODI_ADDR, OODI_SECRET, OODI_SECRET_HEADER_KEY } = require('../conf-backend')
-const logger = require('../util/logger')
-
-const agent =
-  KEY_PATH && CERT_PATH
-    ? new https.Agent({
-        cert: fs.readFileSync(CERT_PATH, 'utf8'),
-        key: fs.readFileSync(KEY_PATH, 'utf8')
-      })
-    : new https.Agent({
-        rejectUnauthorized: false
-      })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = instance.get
-
-const getOodiApi = async relative => {
-  const route = OODI_ADDR + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const validOodApiRequest = request => {
-  const secret = request.headers[OODI_SECRET_HEADER_KEY]
-  return !!OODI_SECRET && OODI_SECRET === secret
-}
-
-wrapper.post('', async (req, res) => {
-  const valid = validOodApiRequest(req)
-  if (!valid) {
-    return res.status(401).send()
-  }
-  const { route } = req.body
-  try {
-    const oodiApiResponse = await getOodiApi(route)
-    res.json(oodiApiResponse.data.data)
-  } catch (error) {
-    logger.error(error)
-    res.status(500).send()
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/semesters.js

@@ -1,11 +0,0 @@
-const router = require('express').Router()
-const { getSemestersAndYears } = require('../services/semesters')
-const semestersV2 = require('../routesV2/semesters')
-const useSisRouter = require('../util/useSisRouter')
-
-router.get('/semesters/codes', async (req, res) => {
-  const providers = await getSemestersAndYears()
-  res.json(providers)
-})
-
-module.exports = useSisRouter(semestersV2, router)

services/backend/oodikone2-backend/src/routes/tags.js

@@ -1,201 +0,0 @@
-const router = require('express').Router()
-const Tags = require('../services/tags')
-const TagStudent = require('../services/tagstudent')
-const Students = require('../services/students')
-const { difference } = require('lodash')
-const tagsV2 = require('../routesV2/tags')
-const useSisRouter = require('../util/useSisRouter')
-
-const filterRelevantTags = (tags, userId) => {
-  return tags.filter(tag => !tag.personal_user_id || tag.personal_user_id === userId)
-}
-
-const filterRelevantStudentTags = (studentTags, userId) => {
-  return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
-}
-
-router.get('/tags', async (req, res) => {
-  try {
-    const tags = await Tags.findTags()
-    res.status(200).json(tags)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/tags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.post('/tags', async (req, res) => {
-  try {
-    const {
-      tag: { studytrack, tagname, year, personal_user_id }
-    } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err.message)
-  }
-})
-
-router.delete('/tags', async (req, res) => {
-  try {
-    const { tag } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (
-      !rights.includes(tag.studytrack) &&
-      !(roles && roles.includes('admin')) &&
-      !(tag.personal_user_id === decodedToken.id)
-    )
-      return res.status(403).end()
-
-    await Tags.deleteTag(tag)
-    const t = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(filterRelevantTags(t, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/studenttags', async (req, res) => {
-  try {
-    const result = await TagStudent.getStudentTags()
-    res.status(200).json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/studenttags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.get('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { studentnumber } = req.params
-    const { decodedToken } = req
-    const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.post('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { tag, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    await TagStudent.createStudentTag(tag)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.post('/studenttags', async (req, res) => {
-  try {
-    const { tags, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const existingtags = await Tags.findTagsByStudytrack(studytrack)
-    const existingTagids = existingtags.map(t => t.tag_id)
-    const tagids = [...new Set(tags.map(t => t.tag_id))]
-    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
-
-    const studentnumbers = tags.map(t => t.studentnumber)
-    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
-    const missingStudents = difference(studentnumbers, students)
-    if (missingStudents.length !== 0)
-      return res
-        .status(400)
-        .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
-
-    await TagStudent.createMultipleStudentTags(tags)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.delete('/studenttags/delete_one', async (req, res) => {
-  try {
-    const { tag_id, studentnumber, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
-
-    await TagStudent.deleteStudentTag(studentnumber, tag_id)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
-})
-
-router.delete('/studenttags/delete_many', async (req, res) => {
-  try {
-    const { tagId, studentnumbers, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
-
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
-
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
-
-    await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).json(err)
-  }
-})
-
-module.exports = useSisRouter(tagsV2, router)

services/backend/oodikone2-backend/src/routes/tasks.js

@@ -1,20 +0,0 @@
-const router = require('express').Router()
-const { getTeacherUpdateStatus, startTopTeacherUpdate } = require('../services/tasks')
-
-router.get('/tasks/topteachers', (_, res) => {
-  const status = getTeacherUpdateStatus()
-  res.json(status)
-})
-
-router.post('/tasks/topteachers', async (req, res) => {
-  const { from = 50, to = 70 } = req.body
-  const { computing } = getTeacherUpdateStatus()
-  if (computing === true) {
-    res.status(409).send('Task already exists')
-  } else {
-    startTopTeacherUpdate(from, to)
-    res.status(201).send()
-  }
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -1,72 +0,0 @@
-const teachers = require('../services/teachers')
-const topteachers = require('../services/topteachers')
-const r = require('./router').routerWithWrapper()
-const { FEATURES } = require('../conf-backend')
-const { mapToProviders } = require('../util/utils')
-const teacherV2 = require('../routesV2/teachers')
-const useSisRouter = require('../util/useSisRouter')
-
-const router = FEATURES.ERROR_HANDLER ? r.wrapper : r.router
-
-router.get('/', async (req, res) => {
-  const { searchTerm } = req.query
-  if (!searchTerm) return res.status(400).json({ error: 'searchTerm missing' })
-
-  const trimmedSearchTerm = searchTerm.trim()
-  if (
-    !teachers.splitByEmptySpace(trimmedSearchTerm).find(t => t.length >= 4) ||
-    (Number(trimmedSearchTerm) && trimmedSearchTerm.length < 6)
-  ) {
-    return res.status(400).json({ error: 'invalid searchTerm' })
-  }
-
-  const result = await teachers.bySearchTerm(trimmedSearchTerm)
-  res.json(result)
-})
-
-router.get('/top', async (req, res) => {
-  const { yearcode, category = topteachers.ID.ALL } = req.query
-  if (!yearcode) {
-    return res.status(422).send('Missing required yearcode query param')
-  }
-  const result = await topteachers.getTeacherStats(category, yearcode)
-  res.json(result)
-})
-
-router.post('/top', async (req, res) => {
-  const { startyearcode, endyearcode } = req.body
-  res.status(200).end()
-  await topteachers.findAndSaveTeachers(startyearcode, endyearcode)
-})
-
-router.get('/top/categories', async (req, res) => {
-  const result = await topteachers.getCategoriesAndYears()
-  res.json(result)
-})
-
-router.get('/stats', async (req, res) => {
-  const { rights, roles } = req
-
-  const { providers, semesterStart, semesterEnd } = req.query
-  if (!providers || !semesterStart) {
-    return res.status(422).send('Missing required query parameters.')
-  }
-  const providerRights = mapToProviders(rights)
-
-  if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
-    return res.status(403).send('You do not have rights to see this data')
-  }
-  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd || semesterStart + 1)
-  res.json(result)
-})
-
-router.get('/:id', async (req, res) => {
-  const { id } = req.params
-  const result = await teachers.teacherStats(id)
-  if (!result) {
-    return res.status(404).send()
-  }
-  res.json(result)
-})
-
-module.exports = useSisRouter(teacherV2, r.router)

services/backend/oodikone2-backend/src/routes/updater.js

@@ -1,286 +0,0 @@
-const router = require('express').Router()
-const {
-  updateOldestStudents,
-  getStatuses,
-  updateActiveStudents,
-  updateAllStudents,
-  updateAttainments,
-  updateMetadata,
-  updateStudentlist,
-  rescheduleScheduled,
-  rescheduleFetched,
-  updateNoStudents,
-  updateDaily
-} = require('../services/updaterService')
-const {
-  updateSISMetadata,
-  updateSISStudents,
-  updateSISStudentsByProgramme,
-  updateSISProgrammes,
-  updateSISRedisCache,
-  updateStudentsByStudentNumber,
-  abort,
-  updateCoursesByCourseCode
-} = require('../services/sisUpdaterService')
-const { refreshStatistics, refreshStatisticsV2 } = require('../events')
-
-router.post('/update/oldest', async (req, res) => {
-  const { amount } = req.body
-  if (!Number.isInteger(Number(amount))) {
-    res.status(400).end()
-    return
-  }
-  try {
-    const response = await updateOldestStudents(amount)
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/all', async (req, res) => {
-  try {
-    const response = await updateAllStudents()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/active', async (req, res) => {
-  try {
-    const response = await updateActiveStudents()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/no_student', async (req, res) => {
-  try {
-    const response = await updateNoStudents()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/attainment', async (req, res) => {
-  try {
-    const response = await updateAttainments()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/meta', async (req, res) => {
-  try {
-    const response = await updateMetadata()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/studentlist', async (req, res) => {
-  try {
-    const response = await updateStudentlist()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.get('/status', async (req, res) => {
-  try {
-    const statuses = await getStatuses()
-    res.status(200).json(statuses)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json([{ label: 'request status', value: 'request failed' }])
-  }
-})
-
-router.post('/reschedule/scheduled', async (req, res) => {
-  try {
-    const response = await rescheduleScheduled()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/reschedule/fetched', async (req, res) => {
-  try {
-    const response = await rescheduleFetched()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/update/daily', async (req, res) => {
-  try {
-    const response = await updateDaily()
-    if (response) {
-      res.status(200).json({ ...response })
-    }
-  } catch (err) {
-    res.status(500).json(err)
-  }
-})
-
-router.post('/refresh_statistics', async (req, res) => {
-  try {
-    refreshStatistics()
-    res.status(200).json('Refreshing statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/update/v2/meta', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISMetadata(req)
-    if (response) {
-      res.status(200).json('Update SIS meta scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISStudents()
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateStudentsByStudentNumber(req.body)
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/update/v2/courses', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateCoursesByCourseCode(req.body)
-    if (response) {
-      res.status(200).json('Update SIS courses scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/update/v2/programmes', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISProgrammes()
-    if (response) {
-      res.status(200).json('Update SIS programmes scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/update/v2/students_by_programme', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-  try {
-    const response = await updateSISStudentsByProgramme(req.body)
-    if (response) {
-      res.status(200).json(`Update for ${req.body.programme} scheduled`)
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/refresh_redis_cache', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISRedisCache()
-    if (response) {
-      res.status(200).json('Refreshing SIS redis cache scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.post('/refresh_statistic_v2', async (req, res) => {
-  try {
-    refreshStatisticsV2()
-    res.status(200).json('Refreshing sis statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-router.get('/abort', async (req, res) => {
-  try {
-    await abort()
-    res.status(200).json()
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
-module.exports = router