Merge pull request #3157 from UniversityOfHelsinkiCS/trunk

Add some new special programmes for users with faculty rights

.github/workflows/release-oodikone-staging.yaml

@@ -48,12 +48,16 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: "Build and push"
         uses: docker/build-push-action@v2
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
             SENTRY_ENVIRONMENT=staging
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
             type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
@@ -79,6 +83,7 @@ jobs:
           set_commits: 'skip'
           sourcemaps: './build/static/js'
           url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
 
   build_and_release_userservice:
     name: "Build and release userservice"

.github/workflows/release-oodikone.yaml

@@ -49,6 +49,9 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: "Extract release version"
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
@@ -61,6 +64,7 @@ jobs:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
             type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
@@ -86,6 +90,7 @@ jobs:
           set_commits: 'skip'
           sourcemaps: './build/static/js'
           url_prefix: '~/oodikone/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
 
   build_and_release_userservice:
     name: "Build and release userservice"

.github/workflows/test-oodikone.yaml

@@ -142,46 +142,6 @@ jobs:
 
   # === End-to-end tests ===
 
-  test_course_population:
-    name: "Test course population"
-    needs:
-      - build_backend
-      - build_frontend
-      - build_userservice
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: "Download images"
-        uses: actions/download-artifact@v2
-        with:
-          path: /tmp/
-      - name: "Load built images"
-        run: |
-          docker load --input /tmp/artifact/backend.tar
-          docker load --input /tmp/artifact/frontend.tar
-          docker load --input /tmp/artifact/userservice.tar
-      - name: "Login to toska docker registry"
-        uses: docker/login-action@v1
-        with:
-          registry: registry.toska.cs.helsinki.fi
-          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
-          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
-      - name: "Run services, pull rest of the needed images"
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate
-      - name: "Run cypress"
-        uses: cypress-io/github-action@v2
-        with:
-          browser: chrome
-          spec: cypress/integration/Course_population.js
-        env:
-          CYPRESS_baseUrl: http://localhost:1337
-      - name: "Upload video if tests fail"
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Cypress videos
-          path: ./cypress/videos/*
-
   test_course_statistics:
     name: "Test course statistics"
     needs:
@@ -221,6 +181,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_custom_population:
     name: "Test custom population"
@@ -261,6 +224,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_population_statistics:
     name: "Test population statistics"
@@ -301,6 +267,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_filters:
     name: "Test filters"
@@ -341,6 +310,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_student_statistics:
     name: "Test student statistics"
@@ -381,6 +353,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_studyprogramme_overview:
     name: "Test studyprogramme overview"
@@ -421,6 +396,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   test_teachers_and_users:
     name: "Test teachers and users"
@@ -463,7 +441,9 @@ jobs:
         with:
           name: Cypress videos
           path: ./cypress/videos/*
-
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   # === Other tests ===
 
@@ -500,6 +480,9 @@ jobs:
             echo 'Some process had nonzero exit code';
             exit 1;
           fi
+      - name: "Print logs on failure"
+        if: failure()
+        run: docker-compose --file docker-compose.ci.yml logs
 
   push_to_dockerhub:
     name: "Push images to docker hub for caching"
@@ -508,7 +491,6 @@ jobs:
     - lint_frontend
     - lint_userservice
     - run_userservice_integration_tests
-    - test_course_population
     - test_course_statistics
     - test_custom_population
     - test_filters
@@ -545,7 +527,6 @@ jobs:
     - lint_frontend
     - lint_userservice
     - run_userservice_integration_tests
-    - test_course_population
     - test_course_statistics
     - test_custom_population
     - test_filters
@@ -563,5 +544,3 @@ jobs:
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: test-failure
-      - name: "Print logs on failure"
-        run: docker-compose --file docker-compose.ci.yml logs

README.md

@@ -7,12 +7,10 @@ An application for analyzing university data, running at [https://oodikone.cs.he
 To run Oodikone locally, you will need the following:
 
 1. Applications:
-
-- [Docker](https://docs.docker.com/install/) (version 20.10+),
-- [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
-- [npm](https://docs.npmjs.com/cli/v7) (version 7+)
-
-2. Access to Toska Docker Hub, see Toska Gitlab for more information on how to login
+   - [Docker](https://docs.docker.com/install/) (version 20.10+),
+   - [Docker Compose](https://docs.docker.com/compose/install/)(version 0.29+) and
+   - [npm](https://docs.npmjs.com/cli/v7) (version 7+)
+2. Access to Toska Docker Hub, see Toska Gitlabs password document for more information on how to login
 3. (Optional) For real data setup, access to Toskas production servers.
 
 ## 🚀 Installation
@@ -28,26 +26,18 @@ _Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary
 What different CLI options do:
 
 1. Set up oodikone from scratch:
-
-- Cleans up any previous installations
-- Installs all needed npm packages locally
-- Sets up pre-commit linting hooks.
-- Pulls and builds all needed Docker images and sets up dockerized development environment
-- **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
-
+   - Cleans up any previous installations
+   - Installs all needed npm packages locally
+   - Sets up pre-commit linting hooks.
+   - Pulls and builds all needed Docker images and sets up dockerized development environment.
+   - Setup happens with anonymized data, which developers should use by default.
+   - **Note:** Running this option cleans up all real data too, so please don't run option 2 before this.
 2. Reset all real data:
-
-- Cleans up any previous real data databases
-- Downloads needed database dumps from production servers.
-- Creates real data databases and populates them with downloaded dumps.
-
+   - Cleans up any previous real data databases
+   - Downloads needed database dumps from production servers.
+   - Creates real data databases and populates them with downloaded dumps.
 3. Reset sis-importer data
-
-- Like option 2, but only for sis-importer database. Useful when developing updater microservice.
-
-4. Reset old oodi data:
-
-- Like option 2, but for old oodi data, which is not refreshed anymore. Is only needed when setting up real databases after running option 1.
+   - Like option 2, but only for sis-importer database. Useful when developing updater microservice.
 
 ## ⌨️ Development
 
@@ -57,7 +47,7 @@ The development environment is entirely configured in the `docker-compose.yml` f
 
 Running oodikone with real data requires separated databases and redis, which are defined in `docker-compose.real.yml` file. Otherwise real data development environment uses the configuration as anonymized data development environment.
 
-Some useful commands are predefined in `package.json` and can be run with `npm run <command>` as follows:
+Some useful commands are defined in `package.json` and can be run with `npm run <command>` as follows:
 
 - `npm run oodikone`: starts oodikone with anonymized data
 - `npm run oodikone:real`: starts oodikone with real data
@@ -67,11 +57,7 @@ Some useful commands are predefined in `package.json` and can be run with `npm r
 - `npm run both:real`: starts oodikone and updater with real data
 - `npm run docker:down`: stops the whole environment
 
-If you're first timer, just run `npm run oodikone` after setting up oodikone according to [installation](#installation). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
-
-### Anon / real data users
-
-Some notes about mluukkai / tkl user here.
+Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
 ### Run.sh script
 
@@ -89,41 +75,45 @@ It is recommended to spend some time to become familiar with `docker` and `docke
 - `docker exec -it userservice sh`: open bash terminal inside userservice container
 - `docker exec -it sis-db psql -U postgres sis-db-real`: open psql client to investigate sis real data database.
 
-### Linting and tests
+### User types for development
 
-Linting and other stuff here
+By default, you're login as `mluukkai` dev user when running oodikone in development mode. If you want to debug with certain type of user, you can use mocking: go to "Users" -page, click on "edit" on user you want to use and then click icon on the right corner of user card. Another way is to mock user headers - see frontend's api configuration for how this is done.
 
-The test suite is run in CI on every push to `trunk`. Take advantage of this as running the tests can take upwards of 15 minutes.
+Anon data contains some preset user types, corresponding to most usual user types in real data. These are used in testing too (see "Testing" below for more info).
 
-### Updater stuff
+### Linting
 
-Stop with HTTP GET
+Cli script sets up pre-commit hooks that are used to lint and fix files before committing. If your `package.json` file doesn't include "lint-staged" -key, please set hooks manually with `npm run prepare`.
 
-```bash
-http://localhost:8082/v1/abort?token=dev
-```
+For more information on how files are tested, take a look at "lint-staged" in `package.json`. Some files (e.g. github action files, dockerfiles, shell scripts) are checked with external tools and may require you to install those tools in case you're modifying files in question.
 
-Schedule students with HTTP POST
+There's no separate "lintfix" / "format" / "lint" command available, since pre-commit hooks will fix formatting and basic linting errors for you. In case pre-commit fails, you should fix the erroring code manually.
 
-```bash
-http://localhost:8082/v1/students?token=dev
-```
+However, if you want to run linting / formatting manually from command line, `package.json` defines `npm run eslint`, `npm run prettier` and `npm run stylelint` which you can use as an entrypoint.
 
-body
+## 🔨 Testing & CI
 
-```bash
-{
-    "studentnumbers": [
-        "014598456"
-    ]
-}
-```
+There are three types of tests in this project: static tests with eslint/prettier/other tools, jest integration tests for single service and cypress end-to-end tests. Linting is described above and other do work as follows:
 
-## 📖 Documentation
+- Cypress
+  - can be launched in interactive mode with `npm run cypress open`. `package.json` defines entrypoint `npm run cypress` so you can basically run cypress with any arguments you want
+  - are defined in cypress -folder and cypress.json
+  - whole cypress test stack takes about 15 to 20 mins to run. Since tests are ran in our Github actions CI pipe, you're encouraged to take advantage of this instead of running all tests locally.
+  - There are some different user types and cypress commands defined for testing. Take a look at these when debugging tests.
+- Jest
+  - are ran with `docker-compose.test.yml`
+  - test command and environment themselves are in Dockerfile for service in question
+  - can be debugged by overriding default command in `docker-compose.test.yml` and running docker container in interactive mode
 
-Testing + CI docs link here.
+Continuous integration (CI) works with Github actions and is defined in workflow files in `.github/workflows` folder:
 
-##❓FAQ
+- oodikone setup for cypress and other tests in CI is defined in `docker-compose.ci.yml`. Take a look at this too if debugging github action workflows.
+- tests are run on every push to branches other than `master`. Pull requests can't be merged to master unless tests are ok.
+- after successful merge to `master`, oodikone is deployed to staging
+- after creating a release, oodikone is deployed to production
+- updater is deployed to production on every push to `trunk`.
+
+## ❓FAQ
 
 ### Modules are missing after updating package.json
 
@@ -133,4 +123,8 @@ You should always install the dependencies **inside** the container to have the
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
 
-## Maintainers and contribution
+## ✌🏼 Maintainers and contribution
+
+Toska of course.
+
+University of Helsinki

adminer-plugins/login-servers.php

@@ -8,7 +8,6 @@ $driver = "pgsql";
 return new AdminerLoginServers(
     $servers = array(
         "kone-db" => array("server" => "kone-db", "driver" => $driver),
-        "oodi-db" => array("server" => "oodi-db", "driver" => $driver),
         "sis-db" => array("server" => "sis-db", "driver" => $driver),
         "sis-importer-db" => array("server" => "sis-importer-db", "driver" => $driver),
         "user-db" => array("server" => "user-db", "driver" => $driver)

cli.sh

@@ -51,7 +51,6 @@ options=(
   "Set up oodikone from scratch."
   "Reset all real data."
   "Reset sis-importer data."
-  "Reset old oodi data."
   "Quit."
 )
 
@@ -69,8 +68,6 @@ while true; do
         reset_all_real_data;;
       "Reset sis-importer data.")
         reset_sis_importer_data;;
-      "Reset old oodi data.")
-        reset_old_oodi_data;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress.json

@@ -1,9 +1,7 @@
 {
   "projectId": "c3jsph",
   "baseUrl": "http://localhost:3000",
-  "defaultCommandTimeout": 60000,
-  "requestTimeout": 60000,
-  "responseTimeout": 60000,
-  "viewportHeight": 1080,
-  "viewportWidth": 1920
-}
+  "defaultCommandTimeout": 30000,
+  "requestTimeout": 30000,
+  "videoCompression": false
+}

cypress/fixtures/example.json

@@ -1,5 +0,0 @@
-{
-  "name": "Using fixtures to represent data",
-  "email": "hello@cypress.io",
-  "body": "Fixtures are a great way to mock data for responses to routes"
-}

cypress/integration/Course_population.js

@@ -1,29 +0,0 @@
-/// <reference types="Cypress" />
-
-describe('Course population tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Course statistics').click()
-    cy.contains('Search for courses')
-  })
-
-  it('Can find course population', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains('tr', 'TKT20003').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-
-    cy.contains('Käyttöjärjestelmät')
-    cy.contains('TKT20003')
-
-    cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
-    cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
-    cy.contains('TKT20003')
-    // cy.contains("Students (all=24)");
-
-    cy.contains('Students (127)').click()
-    cy.contains('010135486')
-    cy.contains('010431753')
-  })
-})

cypress/integration/Course_statistics.js

@@ -1,695 +1,328 @@
 /// <reference types="Cypress" />
 
 describe('Course Statistics tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Course statistics').click()
-    cy.contains('Search for courses')
-  })
-
-  it('Searching single course having substitution mappings shows course statistics', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-    // Click uni course, not avoin
-    cy.contains('td', /^TKT20001/).click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-
-    cy.contains('Tietorakenteet ja algoritmit')
-    cy.contains('TKT20001')
-    cy.contains('58131') // old mapped code
-
-    cy.contains('.tabular.menu a', 'Table').click()
-    cy.contains('All')
-    cy.contains('.modeSelectorRow a', 'Attempts').click()
-    cy.contains('.modeSelectorRow a', 'Students').click()
-
-    cy.contains('.tabular.menu a', 'Pass rate chart').click()
-    cy.get('div.modeSelectorContainer').click()
-    cy.contains('svg', 'Pass rate chart')
-
-    cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-    cy.get('div.modeSelectorContainer').click()
-    cy.contains('svg', 'Grades')
-
-    cy.contains('a', 'New query').click()
-    cy.contains('Search for courses')
-  })
-
-  it('Searching multiple courses having substitution mappings shows course statistics', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT')
-    cy.contains('td', /^TKT20001/).click()
-    cy.contains('td', /^TKT10002/).click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-
-    cy.contains('.courseNameCell', 'Tietorakenteet ja algoritmit').contains('TKT20001').click()
-    cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').should('not.exist')
-    cy.contains('TKT20001')
-    cy.contains('58131') // old mapped code
-    cy.contains('Summary').click()
-
-    cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').contains('TKT10002').click()
-    cy.contains('.courseNameCell', 'Käyttöjärjestelmät').should('not.exist')
-    cy.contains('TKT10002')
-    cy.contains('581325') // old mapped code
-    cy.contains('Summary').click()
-
-    cy.contains('a', 'New query').click()
-    cy.contains('Search for courses')
-  })
-
-  it('On consecutive searches should not crash and search should work', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
-    cy.contains('td', 'TKT20003').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Show population')
-    cy.contains('TKT20003')
-
-    cy.contains('Course statistics').click()
-    cy.contains('Search for courses')
-    //cy.wait(500)
-    cy.get("input[placeholder='Search by a course code']").type('TKT20001')
-    cy.contains('td', 'TKT20001').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Show population')
-    cy.contains('TKT20001')
-  })
-
-  it('Shows statistics when separating by semester', () => {
-    const courseCode = 'DATA11002'
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by a course code']").type(courseCode)
-    cy.contains('td', courseCode).click()
-    cy.contains('Separate statistics for Spring and Fall semesters').click()
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Syksy 2019')
-    cy.contains('Syksy 2017')
-    cy.get('tbody').contains('19')
-    cy.contains(courseCode)
-  })
-
-  it('Searching course by name displays right courses', () => {
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('Search for courses')
-    cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-
-    cy.contains('Tietokantojen perusteet')
-    cy.contains('Avoin yo: Tietokantojen perusteet')
-    cy.contains('A581328')
-    cy.contains('AYTKT10004')
-    cy.contains('TKT10004, 581328')
-    cy.contains('td', /^TKT10004/).click()
-
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-
-    cy.contains('TKT10004, 581328 Tietokantojen perusteet')
-    cy.get('.right').click()
-    cy.contains('No results')
-
-    cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
-    cy.contains('td', /^AYTKT10004/).click()
-
-    cy.contains('Fetch statistics').should('be.enabled').click()
-    cy.contains('Search for courses').should('not.exist')
-    cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
-  })
-
-  describe('When searching unified course stats', () => {
+  describe('when using basic user', () => {
     beforeEach(() => {
+      cy.init('/coursestatistics')
+    })
+    it('Searching single course having substitution mappings shows course statistics', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')
-      cy.get("input[name='unifyOpenUniCourses']").parent().click()
-      cy.get("input[placeholder='Search by a course code']").type('TKT10002')
-      cy.contains('td', 'TKT10002, AYTKT10002').click()
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      // Click uni course, not avoin
+      cy.contains('td', /^TKT20001/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
       cy.contains('Search for courses').should('not.exist')
-      cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('TKT20001')
+      cy.contains('58131') // old mapped code
+
+      cy.contains('.tabular.menu a', 'Table').click()
+      cy.contains('All')
+      cy.contains('.modeSelectorRow a', 'Attempts').click()
+      cy.contains('.modeSelectorRow a', 'Students').click()
+
+      cy.contains('.tabular.menu a', 'Pass rate chart').click()
+      cy.get('div.modeSelectorContainer').click()
+      cy.contains('svg', 'Pass rate chart')
+
+      cy.contains('.tabular.menu a', 'Grade distribution chart').click()
+      cy.get('div.modeSelectorContainer').click()
+      cy.contains('svg', 'Grades')
+
+      cy.contains('a', 'New query').click()
+      cy.contains('Search for courses')
     })
 
-    // Statistics
-    const yearRange = { from: '2000-2001', to: '2020-2021' }
-    const attemptsTableContents = [
-      // [time, passed, failed, passrate]
-      ['Total', 511, 486, 25],
-    ]
-    /*
-    const studentTableContents = [
-      // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-      ['Total', 289, 254, 13, '92.39 %', 22, '7.61 %'],
-      ['2017-18', 157, 132, 5, '87.26 %', 20, '12.74 %'],
-      ['2016-17', 62, 60, 1, '98.39 %', 1, '1.61 %'],
-      ['2015-16', 27, 25, 2, '100.00 %', 0, '0.00 %'],
-      ['2014-15', 16, 16, 0, '100.00 %', 0, '0.00 %'],
-      ['2013-14', 5, 3, 2, '100.00 %', 0, '0.00 %'],
-    ]
-    */
+    it('Searching multiple courses having substitution mappings shows course statistics', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT')
+      cy.contains('td', /^TKT20001/).click()
+      cy.contains('td', /^TKT10002/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
 
-    const gradesTableContents = [
-      // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-      ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
-      ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
-      ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
-      ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
-      ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
-      ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
-      ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
-    ]
+      cy.contains('.courseNameCell', 'Tietorakenteet ja algoritmit').contains('TKT20001').click()
+      cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').should('not.exist')
+      cy.contains('TKT20001')
+      cy.contains('58131') // old mapped code
+      cy.contains('Summary').click()
 
-    it('shows stats', () => {
-      // Time range
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', yearRange.to)
-        cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', '2000-2001')
-        cy.get("div[role='option']").should('have.length', 21)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', '2020-2021')
-        cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', yearRange.from)
-        cy.get("div[role='option']").should('have.length', 21)
-      })
-      cy.contains('Show population').should('not.be.enabled')
+      cy.contains('.courseNameCell', 'Ohjelmoinnin perusteet').contains('TKT10002').click()
+      cy.contains('.courseNameCell', 'Käyttöjärjestelmät').should('not.exist')
+      cy.contains('TKT10002')
+      cy.contains('581325') // old mapped code
+      cy.contains('Summary').click()
 
-      /*
-      const timeRangeFilter =
-        timeRange =>
-        ([time]) => {
-          return (timeRange.from <= time && time <= timeRange.to) || time === 'Total'
-        }
-*/
-      cy.contains('#CourseStatPanes a.item', 'Table').click()
-      cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-      cy.get('#CourseStatPanes table>tbody').within(() => {
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get('tr')
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get('td').eq(tdIndex).contains(value)
-              })
-            })
-        })
-        cy.get('tr').should('have.length', 17)
-      })
-      cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
-      cy.get('#CourseStatPanes table>tbody').within(() => {
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get('tr')
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get('td').eq(tdIndex).contains(value)
-              })
-            })
-        })
-        cy.get('tr').should('have.length', 17)
-      })
+      cy.contains('a', 'New query').click()
+      cy.contains('Search for courses')
     })
 
-    it('after changing time range shows same stats', () => {
-      const newYearRange = { from: '2016-2017', to: '2019-2020' }
-      cy.get("div[name='fromYear']")
-        .click()
-        .within(() => {
-          cy.contains(newYearRange.from).click()
-        })
-      cy.get("div[name='toYear']")
-        .click()
-        .within(() => {
-          cy.contains(newYearRange.to).click()
-        })
+    it('On consecutive searches should not crash and search should work', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
+      cy.contains('td', 'TKT20003').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Show population')
+      cy.contains('TKT20003')
 
-      // Time range
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', newYearRange.to)
-        cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', '2000-2001')
-        cy.get("div[role='option']").should('have.length', 20)
-      })
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should('have.text', '2020-2021')
-        cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
-        cy.get("div[role='option']").last().should('have.text', newYearRange.from)
-        cy.get("div[role='option']").should('have.length', 5)
-      })
-      cy.contains('Show population').should('be.enabled')
+      cy.contains('Course statistics').click()
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      cy.contains('td', 'TKT20001').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Show population')
+      cy.contains('TKT20001')
     })
-    /*
-      // Filters
-      // Data currently has only "Muu" so we can't test these yet! See
-      // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3058
-      // const primaryContents = [
-      //   // [studentcount, label, code]
-      //   [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-      //   [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-      //   [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-      //   [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-      //   [9, "Matematiikan koulutusohjelma", "520070"],
-      // ];
-      // const totalStudyProgrammes = 20;
-      // cy.get("div[name='primary']").within(() => {
-      //   primaryContents.forEach(([count, label, code], i) => {
-      //     cy.get("div[role='option']")
-      //       .eq(i)
-      //       .within(() => {
-      //         cy.contains(label);
-      //         cy.contains("div>div:last-child>div.label", code);
-      //         cy.contains("div>div:first-child>div.label", count);
-      //       });
-      //   });
-      //   cy.get("div[role='option']")
-      //     .eq(10)
-      //     .within(() => {
-      //       cy.contains("Muu");
-      //       cy.contains("div>div:last-child>div.label", "OTHER");
-      //       cy.contains("div>div:first-child>div.label", "2");
-      //     });
-      //   cy.get("div[role='option']").should(
-      //     "have.length",
-      //     totalStudyProgrammes
-      //   );
-      // });
-      // const comparisonContents = [[289, "All", "ALL"], ...primaryContents];
-      // cy.get("div[name='comparison']").within(() => {
-      //   comparisonContents.forEach(([count, label, code], i) => {
-      //     cy.get("div[role='option']")
-      //       .eq(i)
-      //       .within(() => {
-      //         cy.contains(label);
-      //         cy.contains("div>div:last-child>div.label", code);
-      //         cy.contains("div>div:first-child>div.label", count);
-      //       });
-      //   });
-      //   cy.get("div[role='option']")
-      //     .eq(11)
-      //     .within(() => {
-      //       cy.contains("Muu");
-      //       cy.contains("div>div:last-child>div.label", "OTHER");
-      //       cy.contains("div>div:first-child>div.label", "2");
-      //     });
-      //   cy.get("div[role='option']").should(
-      //     "have.length",
-      //     totalStudyProgrammes + 1
-      //   ); // +1 comes from already selected option
-      // });
-      // cy.contains("Select excluded study programmes").should("be.disabled");
 
-        
+    it('Shows statistics when separating by semester', () => {
+      const courseCode = 'DATA11002'
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type(courseCode)
+      cy.contains('td', courseCode).click()
+      cy.contains('Separate statistics for Spring and Fall semesters').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Syksy 2019')
+      cy.contains('Syksy 2017')
+      cy.get('tbody').contains('19')
+      cy.contains(courseCode)
+    })
 
-      // Filters
-      // Data currently has only "Muu" so we can't test these yet! See
-      // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3058
-      //const primaryContentsAfterTimeChange = [
-      //  // [studentcount, label, code]
-      //  [79, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-      //  [15, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-      //  [10, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-      //  [7, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-      //];
-      //const totalStudyProgrammesAfterTimeChange = 13;
-      //cy.get("div[name='primary']").within(() => {
-      //  primaryContentsAfterTimeChange.forEach(([count, label, code], i) => {
-      //    cy.get("div[role='option']")
-      //      .eq(i)
-      //      .within(() => {
-      //        cy.contains(label);
-      //        cy.contains("div>div:last-child>div.label", code);
-      //        cy.contains("div>div:first-child>div.label", count);
-      //      });
-      //  });
-      //  cy.get("div[role='option']")
-      //    .eq(5)
-      //    .within(() => {
-      //      cy.contains("Muu");
-      //      cy.contains("div>div:last-child>div.label", "OTHER");
-      //      cy.contains("div>div:first-child>div.label", "2");
-      //    });
-      //  cy.get("div[role='option']").should(
-      //    "have.length",
-      //    totalStudyProgrammesAfterTimeChange
-      //  );
-      //});
-      //const comparisonContentsAfterTimeChange = [
-      //  [107, "All", "ALL"],
-      //  ...primaryContentsAfterTimeChange,
-      //];
-      //cy.get("div[name='comparison']").within(() => {
-      //  comparisonContentsAfterTimeChange.forEach(([count, label, code], i) => {
-      //    cy.get("div[role='option']")
-      //      .eq(i)
-      //      .within(() => {
-      //        cy.contains(label);
-      //        cy.contains("div>div:last-child>div.label", code);
-      //        cy.contains("div>div:first-child>div.label", count);
-      //      });
-      //  });
-      //  cy.get("div[role='option']")
-      //    .eq(6)
-      //    .within(() => {
-      //      cy.contains("Muu");
-      //      cy.contains("div>div:last-child>div.label", "OTHER");
-      //      cy.contains("div>div:first-child>div.label", "2");
-      //    });
-      //  cy.get("div[role='option']").should(
-      //    "have.length",
-      //    totalStudyProgrammesAfterTimeChange + 1
-      //  ); // +1 comes from already selected option
-      //});
-      //cy.contains("Select excluded study programmes").should("be.disabled");
+    it('Searching course by name displays right courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
 
-      const timeAttemptsTableContents = [
-        // [time, passed, failed, passrate]
-        ["Total", 113, 109, 4, "96.46 %"],
-        ["2017-18", 204, 175, 29, "85.78 %"],
-        ["2016-17", 68, 66, 2, "97.06 %"],
-        ["2015-16", 29, 27, 2, "93.10 %"],
-        ["2014-15", 16, 16, 0, "100.00 %"],
-        ["2013-14", 8, 6, 2, "75.00 %"],
-      ];
+      cy.contains('Tietokantojen perusteet')
+      cy.contains('Avoin yo: Tietokantojen perusteet')
+      cy.contains('A581328')
+      cy.contains('AYTKT10004')
+      cy.contains('TKT10004, 581328')
+      cy.contains('td', /^TKT10004/).click()
 
-      const timeStudentTableContents = [
-        // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-        ["Total", 105, 101, 3, "99.05 %", 1, "0.95 %"],
-        ["2017-18", 195, 131, 43, "89.23 %", 1, "10.77 %"],
-        ["2016-17", 62, 60, 1, "98.39 %", 1, "1.61 %"],
-        ["2015-16", 27, 25, 2, "100.00 %", 0, "0.00 %"],
-        ["2014-15", 16, 16, 0, "100.00 %", 0, "0.00 %"],
-        ["2013-14", 5, 2, 3, "100.00 %", 0, "0.00 %"],
-      ];
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
 
-      const timeGradesTableContents = [
-        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-        ["Total", 113, 4, 1, 2, 7, 15, 76, 8],
-        ["2017-18", 204, 29, 5, 8, 10, 16, 127, 9],
-        ["2016-17", 68, 2, 0, 1, 6, 10, 49, 0],
-        ["2015-16", 29, 2, 1, 1, 0, 5, 16, 4],
-        ["2014-15", 16, 0, 0, 0, 1, 0, 11, 4],
-        ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
-      ];
+      cy.contains('TKT10004, 581328 Tietokantojen perusteet')
+      cy.get('.right').click()
+      cy.contains('No results')
 
-    
+      cy.get("input[placeholder='Search by entering a course name']").type('tietokantojen perusteet')
+      cy.contains('td', /^AYTKT10004/).click()
 
-    // Not working since only "Muu" students
-    it("after changing filters shows different stats", () => {
-      cy.get("div[name='primary']")
-        .click()
-        .within(() => {
-          cy.contains(
-            "div[role='option']",
-            "Tietojenkäsittelytieteen kandiohjelma"
-          ).click();
-        });
-      cy.get("body").type("{esc}"); // close dropdown
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
+    })
 
-      // Time range (should not have changed)
-      cy.get("div[name='fromYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", yearRange.to);
-        cy.contains("div[role='option']", yearRange.from).should(
-          "have.class",
-          "selected"
-        );
-        cy.get("div[role='option']").last().should("have.text", "2000-2001");
-        cy.get("div[role='option']").should("have.length", 21);
-      });
-      cy.get("div[name='toYear']").within(() => {
-        cy.get("div[role='option']").first().should("have.text", "2020-2021");
-        cy.contains("div[role='option']", yearRange.to).should(
-          "have.class",
-          "selected"
-        );
-        cy.get("div[role='option']").last().should("have.text", yearRange.from);
-        cy.get("div[role='option']").should("have.length", );
-      });
-      cy.contains("Show population").should("be.enabled");
+    it('Searching course by name displays right courses, 10 credit courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
 
-      // Filters
-      // Data currently has only "Muu" so we can't test these yet! See
-      // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3058
-      const primaryContentsAfterFilterChange = [
-        // [studentcount, label, code]
-        [289, "All", "ALL"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ];
-      const totalStudyProgrammesAfterFilterChange = 20;
-      cy.get("div[name='primary']").within(() => {
-        primaryContentsAfterFilterChange.forEach(([count, label, code], i) => {
-          cy.get("div[role='option']")
-            .eq(i)
-            .within(() => {
-              cy.contains(label);
-              cy.contains("div>div:last-child>div.label", code);
-              cy.contains("div>div:first-child>div.label", count);
-            });
-        });
-        cy.get("div[role='option']").should(
-          "have.length",
-          totalStudyProgrammesAfterFilterChange
-        );
-      });
-      const comparisonContentsAfterFilterChange = [
-        // [studentcount, label, code]
-        [289, "All", "ALL"],
-        [179, "Tietojenkäsittelytieteen kandiohjelma", "KH50_005"],
-        [122, "Excluded", "EXCLUDED"],
-        [64, "Matemaattisten tieteiden kandiohjelma", "KH50_001"],
-        [24, "Tietojenkäsittelytieteen koulutusohjelma", "520080"],
-        [16, "Tietojenkäsittelytieteen maisteriohjelma", "MH50_009"],
-        [9, "Matematiikan koulutusohjelma", "520070"],
-      ];
-      cy.get("div[name='comparison']").within(() => {
-        comparisonContentsAfterFilterChange.forEach(
-          ([count, label, code], i) => {
-            cy.get("div[role='option']")
-              .eq(i)
-              .within(() => {
-                cy.contains(label);
-                cy.contains("div>div:last-child>div.label", code);
-                cy.contains("div>div:first-child>div.label", count);
-              });
-          }
-        );
-        cy.get("div[role='option']").should(
-          "have.length",
-          totalStudyProgrammesAfterFilterChange + 2
-        ); // +2 comes from EXCLUDED and the filtered programme
-      });
-      cy.contains("Select excluded study programmes")
-        .should("be.enabled")
-        .click();
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
+      cy.contains('AYTKT20001')
+      cy.contains('TKT20001, 58131')
+      cy.contains('td', /^TKT20001/).click()
 
-      // Statistics
-      cy.contains("#CourseStatPanes a.item", "Table").click();
-      cy.contains("#CourseStatPanes a.item", "Attempts").click();
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const attemptsTableContents = [
-          // [time, passed, failed, passrate]
-          ["Total", 227, 215, 12, "94.71 %"],
-          ["2017-18", 136, 126, 10, "92.65 %"],
-          ["2016-17", 57, 56, 1, "98.25 %"],
-          ["2015-16", 21, 20, 1, "95.24 %"],
-          ["2014-15", 6, 6, 0, "100.00 %"],
-        ];
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const attemptsTableContents = [
-          // [time, passed, failed, passrate]
-          ["Total", 140, 110, 30, "78.57 %"],
-          ["2017-18", 71, 52, 19, "73.24 %"],
-          ["2016-17", 16, 15, 1, "93.75 %"],
-          ["2015-16", 10, 9, 1, "90.00 %"],
-          ["2014-15", 11, 11, 0, "100.00 %"],
-        ];
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
 
-      cy.contains("#CourseStatPanes a.item", "Table").click();
-      cy.contains("#CourseStatPanes a.item", "Students").click();
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const studentTableContents = [
-          // time, students, passedfirsttry, eventuallypassed, pass%, neverpassed, fail%]
-          ["Total", 179, 168, 6, "97.21 %", 5, "2.79 %"],
-          ["2017-18", 95, 86, 4, "94.74 %", 5, "5.26 %"],
-          ["2016-17", 53, 52, 1, "100.00 %", 0, "0.00 %"],
-          ["2015-16", 19, 18, 1, "100.00 %", 0, "0.00 %"],
-          ["2014-15", 6, 6, 0, "100.00 %", 0, "0.00 %"],
-        ];
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const studentTableContents = [
-          // time, students, passedfirsttry, passedeventually, pass%, neverpassed, fail%]
-          ["Total", 122, 98, 7, "86.07 %", 17, "13.93 %"],
-          ["2017-18", 65, 49, 1, "76.92 %", 15, "23.08 %"],
-          ["2016-17", 13, 12, 0, "92.31 %", 1, "7.69 %"],
-          ["2015-16", 10, 9, 1, "100.00 %", 0, "0.00 %"],
-          ["2014-15", 11, 11, 0, "100.00 %", 0, "0.00 %"],
-        ];
-        studentTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get('.right').click()
+      cy.contains('No results')
 
-      cy.contains("#CourseStatPanes a.item", "Table").click();
-      cy.contains("#CourseStatPanes a.item", "Attempts").click();
-      cy.get("[data-cy=gradeToggle]", { force: true }).click({ force: true });
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', /^AYTKT20001/).click()
 
-      cy.get("#PrimaryDataTable table>tbody").within(() => {
-        const gradesTableContents = [
-          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-          ["Total", 227, 12, 4, 7, 11, 18, 161, 14],
-          ["2017-18", 136, 10, 3, 4, 4, 4, 102, 9],
-          ["2016-17", 57, 1, 0, 1, 6, 8, 41, 0],
-          ["2015-16", 21, 1, 1, 1, 0, 5, 11, 2],
-          ["2014-15", 6, 0, 0, 0, 0, 0, 3, 3],
-          ["2013-14", 1, 0, 0, 0, 0, 0, 1, 0],
-        ];
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-      cy.get("#ComparisonDataTable table>tbody").within(() => {
-        const gradesTableContents = [
-          // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
-          ["Total", 140, 30, 2, 5, 13, 20, 67, 3],
-          ["2017-18", 71, 19, 2, 4, 6, 13, 27, 0],
-          ["2016-17", 16, 1, 0, 0, 0, 3, 12, 0],
-          ["2015-16", 10, 1, 0, 0, 0, 0, 7, 2],
-          ["2014-15", 11, 0, 0, 0, 1, 0, 9, 1],
-          ["2013-14", 8, 2, 0, 0, 2, 0, 4, 0],
-        ];
-        gradesTableContents.forEach((values, trIndex) => {
-          cy.get("tr")
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get("td").eq(tdIndex).contains(value);
-              });
-            });
-        });
-        cy.get("tr").should("have.length", 14);
-      });
-    });
-  });
-*/
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
 
-    it('Some features of Course Statistics are hidden for courseStatistics-users', () => {
-      cy.contains('Users').click()
-      cy.contains('Oodikone users')
-      cy.contains('mocking').should('not.exist')
-      cy.contains('tr', 'User manager').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
+      cy.get('.right').click()
+      cy.contains('No results')
+      cy.get("input[name='unifyOpenUniCourses']").parent().click()
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
 
-      cy.get('[data-cy=access-groups-form]').click().contains('courseStatistics').click()
-      cy.get('[data-cy=access-groups-save]').click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+    })
 
-      cy.route('POST', '/api/superlogin/usermk').as('superlogin')
-      cy.get('i.spy').click()
-      cy.wait('@superlogin')
-      cy.contains('mocking as usermk')
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(1000)
-
-      cy.get('[data-cy=navbar-courseStatistics]').click()
-      cy.get('[data-cy=course-code-input]').type('TKT20003')
+    it('Can find course population', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains('tr', 'TKT20003').click()
       cy.contains('Fetch statistics').should('be.enabled').click()
-      cy.contains('Filter statistics by study programmes').should('not.exist')
-      cy.contains('Faculty statistics').should('not.exist')
-      cy.contains('Show population').should('not.exist')
 
+      cy.contains('Käyttöjärjestelmät')
+      cy.contains('TKT20003')
+
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
+      cy.contains('TKT20003')
+
+      cy.contains('Students (127)').click()
+      cy.contains('010135486')
+      cy.contains('010431753')
+    })
+
+    it('Population of course shows grades for each student', () => {
+      cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+      cy.contains(/^TKT20001$/).click()
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
+      cy.contains('Population of course Tietorakenteet ja algoritmit 2018-2019')
+      cy.contains('Students (197)').click()
+      cy.contains('td', '010262566').siblings().contains('4')
+      cy.contains('td', '010674989').siblings().contains('1')
+    })
+
+    describe('When searching unified course stats', () => {
+      beforeEach(() => {
+        cy.url().should('include', '/coursestatistics')
+        cy.contains('Search for courses')
+        cy.get("input[name='unifyOpenUniCourses']").parent().click()
+        cy.get("input[placeholder='Search by a course code']").type('TKT10002')
+        cy.contains('td', 'TKT10002, AYTKT10002').click()
+        cy.contains('Fetch statistics').should('be.enabled').click()
+        cy.contains('Search for courses').should('not.exist')
+        cy.contains('TKT10002, 581325, AYTKT10002, A581325 Ohjelmoinnin perusteet')
+      })
+
+      // Statistics
+      const yearRange = { from: '2000-2001', to: '2020-2021' }
       const attemptsTableContents = [
         // [time, passed, failed, passrate]
-        ['Total*', 288, 213, 75, '73.96 %'],
-        ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
-        ['2019-2020', 164, 121, 43, '73.78 %'],
-        ['2018-2019', 85, 60, 25, '70.59 %'],
-        ['2017-2018', 39, 32, 7, '82.05 %'],
+        ['Total', 511, 486, 25],
       ]
 
-      cy.contains('#CourseStatPanes a.item', 'Attempts').click()
-      cy.get('#CourseStatPanes table>tbody').within(() => {
-        attemptsTableContents.forEach((values, trIndex) => {
-          cy.get('tr')
-            .eq(trIndex)
-            .within(() => {
-              values.forEach((value, tdIndex) => {
-                cy.get('td').eq(tdIndex).contains(value)
+      const gradesTableContents = [
+        // [time, attempts, 0, 1, 2, 3, 4, 5, other passed]
+        ['Total', 511, 25, 11, 17, 21, 59, 365, 13],
+        ['2020-2021', 1, 0, 0, 0, 0, 0, 1, 0],
+        ['2019-2020', 54, 4, 3, 1, 3, 4, 39, 0],
+        ['2018-2019', 154, 7, 2, 6, 5, 15, 119, 0],
+        ['2017-2018', 186, 9, 3, 5, 6, 16, 139, 8],
+        ['2016-2017', 71, 2, 1, 2, 5, 13, 48, 0],
+        ['2015-2016', 21, 1, 1, 1, 0, 7, 8, 3],
+      ]
+
+      it('shows stats', () => {
+        // Time range
+        cy.get("div[name='fromYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', yearRange.to)
+          cy.contains("div[role='option']", yearRange.from).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', '2000-2001')
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", yearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', yearRange.from)
+          cy.get("div[role='option']").should('have.length', 21)
+        })
+        cy.contains('Show population').should('not.be.enabled')
+
+        cy.contains('#CourseStatPanes a.item', 'Table').click()
+        cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          attemptsTableContents.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
               })
+          })
+          cy.get('tr').should('have.length', 17)
+        })
+        cy.get('[data-cy=gradeToggle]', { force: true }).click({ force: true })
+        cy.get('#CourseStatPanes table>tbody').within(() => {
+          gradesTableContents.forEach((values, trIndex) => {
+            cy.get('tr')
+              .eq(trIndex)
+              .within(() => {
+                values.forEach((value, tdIndex) => {
+                  cy.get('td').eq(tdIndex).contains(value)
+                })
+              })
+          })
+          cy.get('tr').should('have.length', 17)
+        })
+      })
+
+      it('after changing time range shows same stats', () => {
+        const newYearRange = { from: '2016-2017', to: '2019-2020' }
+        cy.get("div[name='fromYear']")
+          .click()
+          .within(() => {
+            cy.contains(newYearRange.from).click()
+          })
+        cy.get("div[name='toYear']")
+          .click()
+          .within(() => {
+            cy.contains(newYearRange.to).click()
+          })
+
+        // Time range
+        cy.get("div[name='fromYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', newYearRange.to)
+          cy.contains("div[role='option']", newYearRange.from).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', '2000-2001')
+          cy.get("div[role='option']").should('have.length', 20)
+        })
+        cy.get("div[name='toYear']").within(() => {
+          cy.get("div[role='option']").first().should('have.text', '2020-2021')
+          cy.contains("div[role='option']", newYearRange.to).should('have.class', 'selected')
+          cy.get("div[role='option']").last().should('have.text', newYearRange.from)
+          cy.get("div[role='option']").should('have.length', 5)
+        })
+        cy.contains('Show population').should('be.enabled')
+      })
+    })
+  })
+
+  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+    cy.init('/coursestatistics', 'onlycoursestatistics')
+    cy.get('[data-cy=navbar-courseStatistics]').click()
+    cy.get('[data-cy=course-code-input]').type('TKT20003')
+    cy.contains('tr', 'TKT20003').click()
+    cy.contains('Fetch statistics').should('be.enabled').click()
+    cy.contains('Filter statistics by study programmes').should('not.exist')
+    cy.contains('Faculty statistics').should('not.exist')
+    cy.contains('Show population').should('not.exist')
+
+    const attemptsTableContents = [
+      // [time, passed, failed, passrate]
+      ['Total*', 288, 213, 75, '73.96 %'],
+      ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      ['2019-2020', 164, 121, 43, '73.78 %'],
+      ['2018-2019', 85, 60, 25, '70.59 %'],
+      ['2017-2018', 39, 32, 7, '82.05 %'],
+    ]
+
+    cy.contains('#CourseStatPanes a.item', 'Attempts').click()
+    cy.get('#CourseStatPanes table>tbody').within(() => {
+      attemptsTableContents.forEach((values, trIndex) => {
+        cy.get('tr')
+          .eq(trIndex)
+          .within(() => {
+            values.forEach((value, tdIndex) => {
+              cy.get('td').eq(tdIndex).contains(value)
             })
-        })
-        cy.get('tr').should('have.length', 7)
+          })
       })
-
-      cy.contains('Stop mocking').click()
-      // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(1000)
-
-      cy.contains('Users').click()
-      cy.contains('Oodikone users')
-      cy.contains('tr', 'User manager').within(() => {
-        cy.contains('.button', 'Edit').click()
-      })
-      cy.contains('Access Groups')
-        .siblings()
-        .within(() => {
-          cy.get("i[class='dropdown icon clear']").click()
-        })
-      cy.get('[data-cy=access-groups-save]').click()
+      cy.get('tr').should('have.length', 7)
     })
   })
 })

cypress/integration/Custom_population.js

@@ -74,15 +74,6 @@ const containsSpecificStudents = (studentnumbers = []) => {
   studentnumbers.forEach(s => cy.contains(s))
 }
 
-const loginAs = username => {
-  cy.contains('Users').click()
-  cy.contains(username).siblings().contains('Edit').click()
-  cy.get('.spy').click()
-  cy.visit(`${Cypress.config().baseUrl}/custompopulation`)
-  cy.url().should('include', '/custompopulation')
-  cy.contains('Custom population')
-}
-
 describe('Custom population tests', () => {
   const nonExistentStudentNumbers = ['123', 'X', '-', ' ']
   beforeEach(() => {
@@ -128,22 +119,16 @@ describe('Custom population tests', () => {
       cy.get('button').contains('Custom population')
     })
 
-    // Check login as when anon user database rights have been corrected
-    // it("Doesn't return students to whom the user doesn't have rights to", () => {
-    //   const kasvatusStudents = ["010102273", "010112616", "010116531"];
-    //   const kapistelyStudents = ["010623419", "010942404", "010484975"];
-    //   loginAs("Normaalikäyttäjä");
-
-    //   searchFor(kasvatusStudents);
-    //   cy.contains("Credit accumulation").should("not.exist");
-    //   cy.contains("Programme distribution").should("not.exist");
-    //   cy.contains("Courses of population").should("not.exist");
-
-    //   searchFor(kapistelyStudents.concat(kasvatusStudents));
-    //   hasLanded();
-    //   containsAmountOfStudents(3);
-    //   containsSpecificStudents(kapistelyStudents);
-    // });
+    it("Doesn't return students user has no right to", () => {
+      // TODO: check that this doesn't create false positives, since this test goes
+      // through even if following students are not in anon sis-db.
+      const studentsForEduBachStudents = ['014990067', '013069465', '014853890']
+      searchFor(studentsForEduBachStudents)
+      cy.contains('Credit accumulation').should('not.exist')
+      cy.contains('Programme distribution').should('not.exist')
+      cy.contains('Courses of population').should('not.exist')
+      cy.get('button').contains('Custom population')
+    })
   })
 
   describe('Custom population search saving', () => {

cypress/integration/Filters.js

@@ -1,6 +1,6 @@
 /// <reference types="Cypress" />
 
-const baseUrl = Cypress.config().baseUrl
+// const baseUrl = Cypress.config().baseUrl
 
 const checkFilteringResult = studentCount => {
   cy.contains(`Students (${studentCount})`)
@@ -36,8 +36,7 @@ describe('Population Statistics', () => {
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
-    cy.init()
-    cy.visit(baseUrl.concat(pathToCSBach2018))
+    cy.init(pathToCSBach2018)
   })
 
   it('Transfer filter is on not transferred by default', () => {
@@ -121,8 +120,22 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Courses filter works', () => {
+  it('Admission type filter works', () => {
+    runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
+      checkFilteringResult(5)
+      cy.selectFromDropdown('admissionTypeFilter-dropdown', 2)
+      checkFilteringResult(10)
+      clearSingleDropdownSelection('admissionTypeFilter-dropdown')
+    })
+  })
+
+  // plz fix
+  it.skip('Courses filter works', () => {
+    // courses takes some time to load, wait for it to complete
+    cy.wait(10000)
     runTestStepWithPreAndPostParts('courseFilter-header', () => {
+      cy.cs('courseFilter-course-dropdown').click()
       const courses = ['TKT20001', 'MAT11002']
       cy.cs('courseFilter-course-dropdown').click().contains(courses[0]).click()
       checkFilteringResult(140)
@@ -158,8 +171,7 @@ describe('Course Statistics', () => {
   const runTestStepWithPreAndPostParts = createRunTestStepWithPreAndPostPartsFunction(defaultAmountOfStudents)
 
   before(() => {
-    cy.init()
-    cy.visit(baseUrl.concat(pathToDSAndAlgoSpring2019))
+    cy.init(pathToDSAndAlgoSpring2019)
   })
 
   it('Grade filter works', () => {

cypress/integration/Population_statistics.js

@@ -11,167 +11,157 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 }
 
 describe('Population Statistics tests', () => {
-  beforeEach(() => {
-    cy.init()
-  })
+  const pathToCSBach2017 =
+    '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  describe('when using basic user', () => {
+    beforeEach(() => {
+      cy.init('/populations')
+    })
 
-  /*
-  const checkAmountOfStudents = assertion => {
-    let students = 0
-    cy.contains('Credit accumulation')
-      .click()
-      .invoke('text')
-      .then(text => {
-        students = Number(text.match(/\d+/g)[0])
-        expect(students).to.equal(assertion)
-      })
-  }
-  */
+    it('Population statistics search form is usable', () => {
+      cy.contains('See population').should('be.disabled')
+      cy.contains('Search for population')
+      cy.contains('Class of')
+        .parent()
+        .within(() => {
+          cy.get('.form-control').as('enrollmentSelect')
+        })
 
-  it('Population statistics search form is usable', () => {
-    cy.cs('navbar-studyProgramme').click()
-    cy.cs('navbar-class').click()
-    cy.contains('See population').should('be.disabled')
-    cy.url().should('include', '/populations')
-    cy.contains('Search for population')
-    cy.contains('Class of')
-      .parent()
-      .within(() => {
-        cy.get('.form-control').as('enrollmentSelect')
-      })
+      cy.get('@enrollmentSelect')
+        .its(`${[0]}.value`)
+        .then(beforeVal => {
+          cy.get('@enrollmentSelect').click()
+          cy.get('.yearSelectInput .rdtPrev').click({ force: true })
+          cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
+          cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
+        })
 
-    cy.get('@enrollmentSelect')
-      .its(`${[0]}.value`)
-      .then(beforeVal => {
-        cy.get('@enrollmentSelect').click()
-        cy.get('.yearSelectInput .rdtPrev').click({ force: true })
-        cy.get('.yearSelectInput table').contains('2018-2019').click({ force: true })
-        cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
-      })
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen maisteriohjelma')
+        .click()
+      cy.contains('See population').should('be.enabled')
+    })
 
-    cy.contains('Select study programme')
-    cy.get('[data-cy=select-study-programme]')
-      .click()
-      .children()
-      .contains('Tietojenkäsittelytieteen maisteriohjelma')
-      .click()
-  })
-
-  it('Population statistics is usable on general level', () => {
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    setPopStatsUntil('toukokuu 2020')
-
-    cy.get('.card').within(() => {
+    it('Searching for population really shows population', () => {
+      cy.contains('Select study programme')
+      cy.get('[data-cy=select-study-programme]')
+        .click()
+        .children()
+        .contains('Tietojenkäsittelytieteen kandiohjelma')
+        .click()
+      cy.contains('See population').click()
+      cy.contains('Population statistics')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-      cy.contains('Sample size: 149 students')
-      cy.contains('Excludes exchange students')
-      cy.contains("Excludes students who haven't enrolled present nor absent")
-      cy.contains('Excludes students with non-degree study right')
-      cy.contains('Excludes students who have transferred out of this programme')
     })
-    cy.contains('Courses of population').click({ force: true })
 
-    cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
-    // eslint-disable-next-line cypress/no-unnecessary-waiting
-    cy.wait(3000) // a bit hacky way, wait until ui is ready
-    cy.cs('expand-TKT1').click()
-    cy.cs('coursestats-link-TKT10002').click()
-    cy.wait('@coursePage')
-    cy.url().should('include', '/coursestatistics')
-    cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
-  })
+    it('Population statistics is usable on general level', () => {
+      cy.visit(pathToCSBach2017)
+      setPopStatsUntil('toukokuu 2020')
 
-  it('Student list checking works as intended', () => {
-    const existing = '010113437'
-    const nonExisting = '66666666'
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    cy.contains('Students (149)').click()
-    cy.contains(existing)
-    cy.contains(nonExisting).should('not.exist')
-    cy.contains('button', 'Check studentnumbers').click()
-    cy.contains('Check for studentnumbers')
-    cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
-    cy.contains('button', 'check students').click()
-    cy.contains('#checkstudentsresults', 'Results').within(() => {
-      cy.contains('Student numbers in list and in oodi').click()
-      cy.contains('#found', existing)
-      cy.contains('Student numbers in list but not in oodi').click()
-      cy.contains('#notfound', nonExisting)
-      cy.contains('Student numbers in oodi but not in list').click()
-      cy.contains('#notsearched', '010614509')
+      cy.get('.card').within(() => {
+        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+        cy.contains('Sample size: 149 students')
+        cy.contains('Excludes exchange students')
+        cy.contains("Excludes students who haven't enrolled present nor absent")
+        cy.contains('Excludes students with non-degree study right')
+        cy.contains('Excludes students who have transferred out of this programme')
+      })
+      cy.contains('Courses of population').click({ force: true })
+
+      cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.get('[data-cy=expand-TKT1]').click()
+      cy.get('[data-cy=coursestats-link-TKT10002]').click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('TKT10002, 581325 Ohjelmoinnin perusteet')
+    })
+
+    it("Empty 'tags' tab has a link to the page where tags can be created", () => {
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
+      cy.contains('No tags defined. You can define them here.').find('a').click()
+      cy.contains('Tags').click()
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Create new tag')
+    })
+
+    // TODO: this fails in CI, but not locally. Investigate and fix
+    it.skip('Advanced settings work', () => {
+      cy.visit(pathToCSBach2017)
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.contains('Statistics until')
+      // only spring
+      cy.cs('toggle-fall').click()
+      cy.contains('Fetch population').click()
+
+      // moar waiting hack
+      cy.wait(10000)
+
+      cy.contains('No statistics found for the given query.')
+
+      // only fall
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-fall').click()
+      cy.cs('toggle-spring').click()
+
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 149 students)')
+
+      // spring + fall and include cancelled
+      cy.get('[data-cy=advanced-toggle]').click()
+      cy.cs('toggle-spring').click()
+      cy.cs('toggle-cancelled').click()
+      cy.contains('Fetch population').click()
+
+      cy.contains('Credit accumulation (for 170 students)')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
+
+      cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
+      cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
+      cy.get("[data-cy='credit-stats-min']").should('contain', '11')
+      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
+      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
+      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
+      cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })
 
-  it("Empty 'tags' tab has a link to the page where tags can be created", () => {
-    cy.cs('navbar-studyProgramme').click()
-    cy.cs('navbar-class').click()
-    cy.contains('See population').should('be.disabled')
-    cy.url().should('include', '/populations')
-    cy.contains('Search for population')
-    cy.contains('Class of')
-      .parent()
-      .within(() => {
-        cy.get('.form-control').as('enrollmentSelect')
+  describe('when using admin', () => {
+    beforeEach(() => {
+      cy.init(pathToCSBach2017, 'admin')
+    })
+    it('Student list checking works as intended', () => {
+      const existing = '010113437'
+      const nonExisting = '66666666'
+      cy.visit(pathToCSBach2017)
+      cy.contains('Students (149)').click()
+      cy.contains(existing)
+      cy.contains(nonExisting).should('not.exist')
+      cy.contains('button', 'Check studentnumbers').click()
+      cy.contains('Check for studentnumbers')
+      cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
+      cy.contains('button', 'check students').click()
+      cy.contains('#checkstudentsresults', 'Results').within(() => {
+        cy.contains('Student numbers in list and in oodi').click()
+        cy.contains('#found', existing)
+        cy.contains('Student numbers in list but not in oodi').click()
+        cy.contains('#notfound', nonExisting)
+        cy.contains('Student numbers in oodi but not in list').click()
+        cy.contains('#notsearched', '010614509')
       })
-
-    cy.get('@enrollmentSelect')
-      .its(`${[0]}.value`)
-      .then(beforeVal => {
-        cy.get('@enrollmentSelect').click()
-        cy.get('.yearSelectInput .rdtPrev').click({ force: true })
-        cy.get('.yearSelectInput table').contains('2019-2020').click({ force: true })
-        cy.get('@enrollmentSelect').should('not.have.value', beforeVal)
-      })
-    cy.selectStudyProgramme('Datatieteen maisteriohjelma')
-    cy.contains('Students (13)').click()
-    cy.get('[data-cy=student-table-tabs]').contains('Tags').click()
-    cy.contains('No tags defined. You can define them here.').find('a').click()
-    cy.contains('Tags').click()
-    cy.contains('Datatieteen maisteriohjelma')
-    cy.contains('Create new tag')
-  })
-
-  it.skip('Advanced settings work', () => {
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.contains('Statistics until')
-    // only spring
-    cy.cs('toggle-fall').click()
-    cy.contains('Fetch population').click()
-
-    cy.contains('Credit accumulation (for 0 students)')
-
-    // only fall
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.cs('toggle-fall').click()
-    cy.cs('toggle-spring').click()
-
-    cy.contains('Fetch population').click()
-
-    // FIXME: This fails randomly both locally and on CI.
-    cy.contains('Credit accumulation (for 194 students)')
-
-    // spring + fall and include cancelled
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.cs('toggle-spring').click()
-    cy.cs('toggle-cancelled').click()
-    cy.contains('Fetch population').click()
-
-    cy.contains('Credit accumulation (for 202 students)')
-  })
-
-  it('Credit Statistics, Statistics pane works', () => {
-    cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
-    cy.contains('Credit statistics').click()
-    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
-
-    cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
-    cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
-    cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
-    cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-    cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-    cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-    cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
-    cy.get("[data-cy='credit-stats-max']").should('contain', '314')
+    })
   })
 })

cypress/integration/Studyprogramme_overview.js

@@ -8,53 +8,51 @@ const deleteTag = name => {
 }
 
 describe('Studyprogramme overview', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Study programme').click().siblings().contains('Overview').click()
-    cy.contains('Study Programme', { timeout: 100000 })
-  })
+  // check is used by couple of tests
+  const checkProgressAndProductivity = (user = 'basic') => {
+    // Column amount depends on user. Note that with dev user, you'll see even more
+    // stuff than with admin or basic.
+    const CSBachPopulationProgress2018ForAdmin = [
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
+      '1',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
+      '16',
+    ]
+    const CSBachPopulationProgress2018ForBasic = [
+      '162',
+      '44 (27%)',
+      '118 (72%)',
+      '161 (99%)',
+      '161',
+      '1',
+      '11',
+      '11',
+      '152',
+      '125',
+      '78',
+      '42',
+      '16',
+    ] // cancelled and transferred from are missing
 
-  // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-  const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-  let originalProgressCalculatedText
-  let originalProductivityCalculatedText
-
-  // Following function used by two tests
-  const testProgressAndProductivity = () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.contains('Admin').click()
-    cy.contains('productivity').click()
-    cy.contains('throughput').click()
-
-    cy.wait(1000)
-    cy.get('.attached > :nth-child(1)').click()
-    cy.get('table').should('have.length', 3)
+    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
     cy.contains('Population progress')
     cy.contains('Yearly productivity')
 
-    // Graduation feature is shown for only for devs, these are the values for normal user
-    const populationprogress2018 = [
-      '160',
-      '42 (26%)',
-      '118 (73%)',
-      '159 (99%)',
-      '159',
-      '11',
-      '11',
-      '1',
-      '0',
-      '150',
-      '123',
-      '76',
-      '41',
-      '16',
-    ]
     cy.contains('2018-2019')
       .siblings()
       .each((elem, index) => {
-        cy.wrap(elem).contains(populationprogress2018[index])
+        cy.wrap(elem).contains(population[index])
       })
-    const populationproductivity2019 = ['7388.0', '17', '7329.00', '59.00', '349.00']
+    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
     cy.get('table')
       .eq(1)
       .contains('2019')
@@ -64,130 +62,167 @@ describe('Studyprogramme overview', () => {
       })
   }
 
-  it('progress should not be recalculating when opened for the first time', () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.wait(1000)
-    cy.contains('Recalculating').should('not.exist')
-  })
-
-  it('renders progress and productivity tables with calculated status', () => {
-    testProgressAndProductivity()
-    // Wait to "recalculating" to disappear
-    cy.wait(1000)
-    // Grab update dates to be compared later
-    cy.cs('throughputUpdateStatus')
-      .invoke('text')
-      .then(text => {
-        originalProgressCalculatedText = normalizeText(text)
-        expect(originalProgressCalculatedText).not.to.contain('recalculating')
-        expect(originalProgressCalculatedText).not.to.contain('refresh')
-      })
-
-    cy.cs('productivityUpdateStatus')
-      .invoke('text')
-      .then(text => {
-        originalProductivityCalculatedText = normalizeText(text)
-        expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-        expect(originalProgressCalculatedText).not.to.contain('refresh')
-      })
-  })
-
-  it('can open Thesis page', () => {
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.contains('Thesis Courses').click()
-    cy.contains('Add thesis course').click()
-    cy.contains('No results')
-  })
-
-  it('can move to Population statistics page by clickin', () => {
-    cy.contains('Lääketieteen koulutusohjelma').click()
-    cy.get('i.level.up.alternate.icon').eq(0).click()
-    cy.contains('Students (121)')
-  })
-
-  it('can create and delete tags for population', () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached').contains('Tags').click()
-    cy.get('.tagNameSelectInput > .ui > input').type(name)
-    cy.get('.yearSelectInput').type('2018')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
-    cy.contains('2018')
-    deleteTag(name)
-  })
-
-  it('can create personal tags', () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached').contains('Tags').click()
-    cy.get('.tagNameSelectInput > .ui > input').type(name)
-    cy.get('.yearSelectInput').type('2018')
-
-    cy.get('.ui > label').click()
-    cy.contains('Create new tag').click()
-    cy.get('.purple')
-    cy.contains(name)
-    deleteTag(name)
-  })
-
-  it('can add tags to students', () => {
-    const name = `tag-${new Date().getTime()}`
-
-    const student = '010113437'
-
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached').contains('Tags').click()
-    cy.get('.tagNameSelectInput > .ui > input').type(name)
-    cy.get('.yearSelectInput').type('2018')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
-
-    cy.contains('Add tags to students').click()
-    cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
-
-    cy.get('.form > .field > .dropdown > .visible').contains(name).click()
-
-    cy.get('textarea').type('010113437')
-    cy.get('.positive').click()
-
-    cy.contains('Student statistics').click()
-    cy.get('.prompt').type(student)
-    cy.contains(student).click()
-    cy.contains(name)
-
-    cy.go('back')
-    cy.go('back')
-
-    deleteTag(name)
-
-    cy.contains('Student statistics').click()
-    cy.get('.prompt').type(student)
-    cy.contains(student).click()
-    cy.contains(name).should('not.exist')
-  })
-
-  it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-    testProgressAndProductivity()
-
-    // Wait to "recalculating" to disappear
-    cy.wait(1000)
-    // Check new calculation statuses are reported
-    const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-    const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
-
-    newProgressCalculatedTextElement.should(text => {
-      const newProgressCalculatedText = normalizeText(text)
-      expect(newProgressCalculatedText).not.to.contain('recalculating')
-      expect(newProgressCalculatedText).not.to.contain('refresh')
-      expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+  describe('when opening programme page with admin user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005', 'admin')
     })
 
-    newProductivityCalculatedTextElement.should(text => {
-      const newProductivityCalculatedText = normalizeText(text)
-      expect(newProductivityCalculatedText).not.to.contain('recalculating')
-      expect(newProductivityCalculatedText).not.to.contain('refresh')
-      expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+    it('progress should not be recalculating when opened for the first time', () => {
+      cy.contains('Recalculating').should('not.exist')
+      cy.contains('Last updated').should('exist')
+    })
+
+    // Following vars are used to compare values before and after stats calculation
+    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
+    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
+    let originalProgressCalculatedText
+    let originalProductivityCalculatedText
+    const doRecalculation = () => {
+      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
+      cy.wait(2000)
+      cy.get('.attached').contains('Admin').click()
+      cy.contains('recalculate productivity').click()
+      cy.contains('recalculate throughput').click()
+      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
+      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
+      cy.wait(5000)
+      cy.reload()
+    }
+
+    it('renders progress and productivity tables with calculated status', () => {
+      doRecalculation()
+
+      // check contains also study programme options for admin
+      cy.contains('Study programme options')
+      cy.get('table').should('have.length', 3)
+
+      checkProgressAndProductivity('admin')
+
+      // Grab update dates to be compared later
+      cy.cs('throughputUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProgressCalculatedText = normalizeText(text)
+          expect(originalProgressCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+
+      cy.cs('productivityUpdateStatus')
+        .invoke('text')
+        .then(text => {
+          originalProductivityCalculatedText = normalizeText(text)
+          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
+          expect(originalProgressCalculatedText).not.to.contain('refresh')
+        })
+    })
+
+    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
+      doRecalculation()
+      checkProgressAndProductivity('admin')
+
+      // Check new calculation statuses are reported
+      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
+      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
+
+      newProgressCalculatedTextElement.should(text => {
+        const newProgressCalculatedText = normalizeText(text)
+        expect(newProgressCalculatedText).not.to.contain('recalculating')
+        expect(newProgressCalculatedText).not.to.contain('refresh')
+        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+
+      newProductivityCalculatedTextElement.should(text => {
+        const newProductivityCalculatedText = normalizeText(text)
+        expect(newProductivityCalculatedText).not.to.contain('recalculating')
+        expect(newProductivityCalculatedText).not.to.contain('refresh')
+        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
+      })
+    })
+  })
+
+  describe('when opening programme page with basic user', () => {
+    beforeEach(() => {
+      cy.init('/study-programme/KH50_005')
+    })
+
+    it('renders progress and productivity tables with previosly calculated values', () => {
+      // check doesn't contain study programme options for admin
+      cy.contains('Study programme options').should('not.exist')
+      cy.get('table').should('have.length', 2)
+      checkProgressAndProductivity()
+    })
+
+    it('can open Thesis page', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.contains('Thesis Courses').click()
+      cy.contains('Add thesis course').click()
+      cy.contains('No results')
+    })
+
+    it('can move to Population statistics page by clickin', () => {
+      cy.get('i.level.up.alternate.icon').eq(0).click()
+      cy.contains('Students (10)')
+    })
+
+    it('can create and delete tags for population', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      cy.contains('2018')
+      deleteTag(name)
+    })
+
+    it('can create personal tags', () => {
+      const name = `tag-${new Date().getTime()}`
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+
+      cy.get('.ui > label').click()
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+      deleteTag(name)
+    })
+
+    it('can add tags to students', () => {
+      const name = `tag-${new Date().getTime()}`
+
+      const student = '010113437'
+
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
+      cy.get('.tagNameSelectInput > .ui > input').type(name)
+      cy.get('.yearSelectInput').type('2018')
+      cy.contains('Create new tag').click()
+      cy.contains(name)
+
+      cy.contains('Add tags to students').click()
+      cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
+
+      cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+
+      cy.get('textarea').type('010113437')
+      cy.get('.positive').click()
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name)
+
+      cy.go('back')
+      cy.go('back')
+
+      deleteTag(name)
+
+      cy.contains('Student statistics').click()
+      cy.get('.prompt').type(student)
+      cy.contains(student).click()
+      cy.contains(name).should('not.exist')
     })
   })
 })

cypress/integration/Teachers.js

@@ -2,9 +2,8 @@
 
 describe('Teachers page tests', () => {
   beforeEach(() => {
-    cy.init()
-    cy.contains('Teachers').click()
-    cy.contains('Teacher statistics by course providers')
+    // login as admin = has teacher rights
+    cy.init('/teachers', 'admin')
   })
 
   const teacher1 = 'Landgraf Leo'

cypress/integration/Users.js

@@ -1,70 +1,62 @@
 /// <reference types="Cypress" />
 
 describe('Users tests', () => {
-  beforeEach(() => {
-    cy.init()
-    cy.contains('Users').click()
-    cy.contains('Oodikone users')
+  describe('Using user with just grp-oodikone-user, no other rights', () => {
+    it('shows only frontpage, trends feedback', () => {
+      cy.init('', 'norights') // login with norights user
+      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('.navBar').contains('Give feedback').should('exist')
+
+      cy.get('.navBar').contains('Study programme').should('not.exist')
+      cy.get('.navBar').contains('Student statistics').should('not.exist')
+      cy.get('.navBar').contains('Course statistics').should('not.exist')
+      cy.get('.navBar').contains('Teachers').should('not.exist')
+      cy.get('.navBar').contains('Users').should('not.exist')
+      cy.get('.navBar').contains('Faculty').should('not.exist')
+      cy.get('.navBar').contains('Updater').should('not.exist')
+    })
   })
 
-  it("Admin mocking normal user shows only the mocked user's programmes", () => {
-    cy.contains('mocking').should('not.exist')
-    cy.contains('tr', 'Normaalikäyttäjä').within($row => {
-      cy.contains('.button', 'Edit').click()
-    })
-    cy.contains('Access rights')
-      .siblings()
-      .within($row => {
-        cy.get("div[role='listitem']").should('have.length', 1).contains('Tietojenkäsittelytieteen kandiohjelma')
-      })
+  // TODO: Add user right checks for onlycoursestats user (should be like above + should
+  // show course stats
+  //
+  // TODO: Add user right checks for basic user (should show also study programme,
+  // student stats, course stats
 
-    cy.route('POST', '/api/superlogin/normk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains('mocking as normk')
-    cy.wait(1000)
-    cy.contains('Study programme').click().siblings().contains('Search by class').click()
-    cy.contains('label', 'Study programme')
-    cy.contains('label', 'Study programme')
-      .siblings()
-      .within($row => {
-        cy.get("div[role='option']").should('have.length', 1).contains('Tietojenkäsittelytieteen kandiohjelma')
-      })
-  })
-
-  it('Admin mocking courseStatistics-user can see only Trends and Course Statistics', () => {
-    cy.contains('mocking').should('not.exist')
-    cy.contains('tr', 'User manager').within($row => {
-      cy.contains('.button', 'Edit').click()
+  describe('Using as admin', () => {
+    beforeEach(() => {
+      cy.init('/users', 'admin')
     })
 
-    cy.get('[data-cy=access-groups-form]').click().contains('courseStatistics').click()
-    cy.get('[data-cy=access-groups-save]').click()
-
-    cy.route('POST', '/api/superlogin/usermk').as('superlogin')
-    cy.get('i.spy').click()
-    cy.wait('@superlogin')
-    cy.contains('mocking as usermk')
-    cy.wait(1000)
-
-    cy.get('[data-cy=navbar-studyProgramme').should('not.exist')
-    cy.get('[data-cy=navbar-students]').should('not.exist')
-    cy.get('[data-cy=navbar-teachers]').should('not.exist')
-    cy.get('[data-cy=navbar-users').should('not.exist')
-
-    cy.contains('Stop mocking').click()
-    cy.wait(1000)
-
-    cy.contains('Users').click()
-    cy.contains('Oodikone users')
-    cy.contains('tr', 'User manager').within($row => {
-      cy.contains('.button', 'Edit').click()
+    it('should see more stuff than others', () => {
+      cy.get('.navBar').contains('Study programme').should('exist')
+      cy.get('.navBar').contains('Student statistics').should('exist')
+      cy.get('.navBar').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
+      cy.get('.navBar').contains('Users').should('exist')
+      cy.get('.navBar').contains('Trends').should('exist')
     })
-    cy.contains('Access Groups')
-      .siblings()
-      .within($row => {
-        cy.get("i[class='dropdown icon clear']").click()
+
+    it("mocking normal user shows only the mocked user's programmes", () => {
+      cy.contains('mocking').should('not.exist')
+      cy.contains('tr', 'basic').within($row => {
+        cy.contains('.button', 'Edit').click()
       })
-    cy.get('[data-cy=access-groups-save]').click()
+      cy.intercept({
+        method: 'POST',
+        path: '/api/superlogin/basic',
+      }).as('superlogin')
+      cy.get('i.spy').click()
+      cy.wait('@superlogin')
+      cy.contains('mocking as basic')
+      cy.wait(1000)
+      cy.contains('Study programme').click().siblings().contains('Search by class').click()
+      cy.contains('label', 'Study programme')
+      cy.contains('label', 'Study programme')
+        .siblings()
+        .within($row => {
+          cy.get("div[role='option']").should('have.length', 2).contains('Tietojenkäsittelytieteen kandiohjelma')
+        })
+    })
   })
 })

cypress/support/commands.js

@@ -1,25 +1,57 @@
 /// <reference types="Cypress" />
+//
+// Note: here we need to set keys to be all lowercase, since
+// we're replacing headers after they've left browser / frontend.
+// All of these users are available in anon user-db
+const adminUserHeaders = {
+  uid: 'admin',
+  displayname: 'Admin User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member;employee;faculty',
+  mail: 'grp-toska+mockadminuser@helsinki.fi',
+}
+
+const basicUserHeaders = {
+  uid: 'basic',
+  displayname: 'Basic User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mockbasicuser@helsinki.fi',
+}
+
+const onlycoursestatisticsUserHeaders = {
+  uid: 'onlycoursestatistics',
+  displayName: 'Onlycoursestatistics User',
+  'shib-session-id': 'mock-session',
+  hyGroupCn: 'grp-oodikone-basic-users',
+  eduPersonAffiliation: 'member',
+  mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+}
+
+const norightsUserHeaders = {
+  uid: 'norights',
+  displayname: 'Norights User',
+  'shib-session-id': 'mock-cypress-session',
+  hygroupcn: 'grp-oodikone-users',
+  edupersonaffiliation: 'member',
+  mail: 'grp-toska+mocknorightuser@helsinki.fi',
+}
+
+const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]
 
 /**
- * Initialize headers and load the base URL or optional path.
+ Set up headers to login, set up correct user (admin / basic / etc.) and open given path.
  */
-Cypress.Commands.add('init', (path = '') => {
-  cy.server({
-    onAnyRequest: function (route, proxy) {
-      proxy.xhr.setRequestHeader('uid', 'tktl')
-      proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
-      proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
-      proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
-    },
+Cypress.Commands.add('init', (path = '', userId = 'basic') => {
+  cy.intercept('', req => {
+    const headersToUse = userHeaders.find(({ uid }) => uid === userId)
+    if (!headersToUse) throw Error(`${userId} is not valid user id!`)
+    req.headers = headersToUse
   })
-
-  // Babel throws an error probably because of markdown files. I'm sorry about this :lul:.
-  cy.on('uncaught:exception', (err, runnable) => {
-    expect(err.message).to.include("Cannot read property 'helpers' of undefined")
-    done()
-    return false
-  })
-
+  // Hide SIS warning flag to avoid screen size problems when running headlessly
+  window.localStorage.setItem('SIS_WARNING_FLAG', 'true')
   const baseUrl = Cypress.config().baseUrl
   cy.visit(baseUrl.concat(path))
 })

docker-compose.ci.yml

@@ -10,11 +10,9 @@ services:
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:1337
       REDIS: redis
@@ -29,8 +27,6 @@ services:
   frontend:
     <<: *defaults
     container_name: frontend
-    environment:
-      BACKEND_ADDR: backend
     image: toska/oodikone2-frontend:trunk
 
   kone-db:
@@ -38,11 +34,6 @@ services:
     container_name: kone-db
     image: registry.toska.cs.helsinki.fi/kone-db
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-
   redis:
     <<: *defaults
     command: "redis-server --appendonly yes"

docker-compose.real.yml

@@ -2,7 +2,6 @@ services:
 
   backend:
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db-real
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db-real
       SIS_DB_URL: postgres://postgres@sis-db:5432/sis-db-real
 

docker-compose.yml

@@ -25,11 +25,9 @@ services:
     container_name: backend
     depends_on:
       - kone-db
-      - oodi-db
       - redis
       - sis-db
     environment:
-      DB_URL: postgres://postgres@oodi-db:5432/oodi-db
       DB_URL_KONE: postgres://postgres@kone-db:5432/kone-db
       FRONT_URL: http://localhost:3000
       REDIS: redis
@@ -53,13 +51,6 @@ services:
       <<: *build_defaults
       context: services/frontend
     container_name: frontend
-    environment:
-      ADMINER_URL: http://localhost:5050/?pgsql=oodi-db&username=postgres
-      BACKEND_ADDR: backend
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=kone-db&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=sis-db&username=postgres
-      SIS_IMPORTER_ADMINER_URL: http://localhost:5050/?pgsql=sis-importer-db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user-db&username=postgres
     ports:
       - 3000:3000
     profiles:
@@ -77,15 +68,6 @@ services:
     volumes:
       - kone-db-data:/data
 
-  oodi-db: # TODO: Remove when oodi stuff has been removed from oodikone
-    <<: *defaults
-    container_name: oodi-db
-    image: registry.toska.cs.helsinki.fi/oodi-db
-    profiles:
-      - oodikone
-    volumes:
-      - oodi-db-data:/data
-
   redis:
     <<: *defaults
     command: "redis-server --appendonly yes"
@@ -205,7 +187,6 @@ volumes:
   backend-nodemod:
   frontend-nodemod:
   kone-db-data:
-  oodi-db-data: # TODO: Remove when oodi stuff has been removed from oodikone
   redis-data:
   sis-db-data:
   sis-importer-db-data:

old_oodikone_stuff/old-backend-tests/course_stats_counter.test.js

@@ -1,197 +0,0 @@
-const { CourseStatsCounter } = require('./course_stats_counter')
-
-describe('CourseStatsCounter tests', () => {
-  const studentnumbers = ['1', '2', '3', '4', '5']
-  const coursecode = 'COURSE_01'
-  const coursename = 'COURSE NAME'
-
-  const createCounter = () => new CourseStatsCounter(coursecode, coursename, studentnumbers.length)
-
-  describe('counter constructor should set course values correctly', () => {
-    const counter = createCounter()
-
-    test('course name should be set after creation', () => {
-      expect(counter.course.name).toBe(coursename)
-    })
-
-    test('course code should be set after creation', () => {
-      expect(counter.course.code).toBe(coursecode)
-    })
-  })
-
-  describe('mark passed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '5'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, true, false, false)
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit for student', () => {
-    const student = studentnumbers[0]
-    const grade = '0'
-    const counter = createCounter()
-
-    counter.markCredit(student, grade, false, true, false)
-
-    const { all, failed, passed } = counter.students
-
-    test('all should have studentnumber as key', () => {
-      expect(all).toHaveProperty(student)
-    })
-
-    test('failed students should have studentnumber as key', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber as key', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then passed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '5', true, false, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber as key', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber as key', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, retryPassed } = counter.students
-
-    test('failed students should not have studentnumber as key', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('retryPassed should have student', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark failed credit then failed credit for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { failed, passed, failedMany } = counter.students
-
-    test('failed students should have studentnumber', () => {
-      expect(failed).toHaveProperty(student)
-    })
-
-    test('passed students should not have studentnumber', () => {
-      expect(passed).not.toHaveProperty(student)
-    })
-
-    test('failed many students should have students', () => {
-      expect(failedMany).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then failed then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '5', true, false, false)
-    counter.markCredit(student, '0', false, true, false)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, failed, failedMany, retryPassed } = counter.students
-
-    test('passed students should have studentnumber', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('failed students should not have studentnumber', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-
-    test('failedMany should not have studentnumber', () => {
-      expect(failedMany).not.toHaveProperty(student)
-    })
-
-    test('retryPassed should have studentnumber', () => {
-      expect(retryPassed).toHaveProperty(student)
-    })
-  })
-
-  describe('mark passed then improved grade for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', true, false, false)
-    counter.markCredit(student, '5', false, false, true)
-
-    const { passed, improvedPassedGrade } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should have student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-  })
-
-  describe('mark improved grade then failed for student', () => {
-    const student = studentnumbers[0]
-    const counter = createCounter()
-
-    counter.markCredit(student, '3', false, false, true)
-    counter.markCredit(student, '0', false, true, false)
-
-    const { passed, improvedPassedGrade, failed } = counter.students
-
-    test('passed should contain student', () => {
-      expect(passed).toHaveProperty(student)
-    })
-
-    test('improvedPassedGrade should contain student', () => {
-      expect(improvedPassedGrade).toHaveProperty(student)
-    })
-
-    test('failed should not contain student', () => {
-      expect(failed).not.toHaveProperty(student)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/globals.js

@@ -1,17 +0,0 @@
-const { sequelize } = require('../../src/models/index')
-const { sequelizeKone } = require('../../src/models/models_kone')
-const { redisClient } = require('../../src/services/redis')
-const { forceSyncDatabase } = require('../../src/database/connection')
-const { seedAllMigrations } = require('../../src/database/seed_migrations')
-
-beforeAll(async () => {
-  await forceSyncDatabase()
-  await seedAllMigrations()
-})
-
-afterAll(async () => {
-  await sequelize.close()
-  await sequelizeKone.close()
-  // https://stackoverflow.com/a/54560610
-  await new Promise(res => redisClient.quit(() => setImmediate(res)))
-})

old_oodikone_stuff/old-backend-tests/services/population.test.js

@@ -1,227 +0,0 @@
-const {
-  Student,
-  Course,
-  ElementDetails,
-  StudyrightElement,
-  Studyright,
-  Credit,
-  StudyrightExtent,
-  Semester,
-  SemesterEnrollment
-} = require('../../../src/models/index')
-const { optimizedStatisticsOf } = require('../../../src/services/populations')
-jest.mock('../../../src/services/semesters')
-const semesterService = require('../../../src/services/semesters')
-jest.setTimeout(10000)
-
-const langify = name => ({
-  en: `${name}_en`,
-  fi: `${name}_fi`,
-  sv: `${name}_sv`
-})
-
-const SEMESTER = { SPRING: 'SPRING', FALL: 'FALL' }
-
-const elementdetails = {
-  bsc: {
-    code: 'Element_BSC',
-    type: 10,
-    name: langify('Bachelor of Science')
-  },
-  maths: {
-    code: 'Element_MATH',
-    type: 20,
-    name: langify('Mathematics')
-  },
-  cs: {
-    code: 'ELEMENT_CS',
-    type: 20,
-    name: langify('Computer Science')
-  }
-}
-
-const courses = {
-  elements_of_ai: {
-    code: 'COURSE-1',
-    name: langify('ELEMENTS_OF_AI')
-  }
-}
-
-const semesters = {
-  fall: {
-    semestercode: 123,
-    name: { en: 'Autumn 2011', fi: 'Syksy 2011', sv: 'Hösten 2011' },
-    startdate: new Date('2011-07-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  spring: {
-    semestercode: 124,
-    name: { en: 'Spring 2012', fi: 'Kevät 2012', sv: 'Våren 2012' },
-    startdate: new Date('2011-12-31'),
-    enddate: new Date('2012-07-30'),
-    yearcode: 62,
-    yearname: '2011-12'
-  },
-  // used for mocking current
-  current: {
-    semestercode: 125,
-    name: { en: 'Autumn 2012', fi: 'Syksy 2012', sv: 'Hösten 2012' },
-    startdate: new Date('2012-08-01'),
-    enddate: new Date('2012-12-31'),
-    yearcode: 63,
-    yearname: '2012-13'
-  }
-}
-
-const createQueryObject = (year, semester, codes, months) => ({
-  studyRights: codes,
-  year,
-  semesters: [semester],
-  months
-})
-
-afterAll(async () => {})
-
-describe('optimizedStatisticsOf tests', () => {
-  const courseinstanceFall = {
-    id: 1,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2011-09-31 21:00:00+00')
-  }
-
-  const courseinstanceSpring = {
-    id: 2,
-    course_code: courses.elements_of_ai.code,
-    coursedate: new Date('2012-02-31 21:00:00+00')
-  }
-
-  const student = {
-    studentnumber: '1234'
-  }
-
-  const studyrightextents = {
-    bachelors: {
-      extentcode: 1,
-      name: langify("bachelor's degree extent")
-    }
-  }
-
-  const studyright = {
-    studyrightid: 1,
-    studystartdate: new Date('2011-07-31 21:00:00+00'),
-    student_studentnumber: student.studentnumber,
-    extentcode: studyrightextents.bachelors.extentcode
-  }
-
-  const creditFall = {
-    id: 'CREDIT-1',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceFall.course_code,
-    attainment_date: courseinstanceFall.coursedate,
-    semestercode: 123
-  }
-
-  const creditSpring = {
-    id: 'CREDIT-2',
-    student_studentnumber: student.studentnumber,
-    course_code: courseinstanceSpring.course_code,
-    attainment_date: courseinstanceSpring.coursedate,
-    semestercode: 124
-  }
-
-  const semesterEnrollments = {
-    fall: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.fall.semestercode,
-      enrollment_date: semesters.fall.startdate
-    },
-    spring: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.spring.semestercode,
-      enrollment_date: semesters.spring.startdate
-    },
-    current: {
-      enrollmenttype: 1,
-      studentnumber: student.studentnumber,
-      semestercode: semesters.current.semestercode,
-      enrollment_date: semesters.current.startdate
-    }
-  }
-
-  const studyrightelements = {
-    bsc: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.bsc.code,
-      studentnumber: student.studentnumber
-    },
-    maths: {
-      startdate: new Date('2011-07-31 21:00:00+00'),
-      enddate: Date('2016-12-20 22:00:00+00'),
-      studyrightid: studyright.studyrightid,
-      code: elementdetails.maths.code,
-      studentnumber: student.studentnumber
-    }
-  }
-
-  describe(`
-    Student with:
-    - BSc studyright (2011-07-31 – 2016-12-20),
-    - Mathematics studyright (2011-07-31 – 2016-12-20),
-    - Two credits in 2011-09-31 and 2012-02-31.
-    `, () => {
-    beforeAll(async () => {
-      await Semester.bulkCreate([semesters.fall, semesters.spring, semesters.current])
-      await Student.create(student)
-      await Course.create(courses.elements_of_ai)
-      await Credit.bulkCreate([creditFall, creditSpring])
-      await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
-      await StudyrightExtent.create(studyrightextents.bachelors)
-      await Studyright.create(studyright)
-      await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
-      await SemesterEnrollment.bulkCreate([
-        semesterEnrollments.fall,
-        semesterEnrollments.spring,
-        semesterEnrollments.current
-      ])
-      // Mock current semester implementation
-      semesterService.getCurrentSemester.mockImplementation(async () => {
-        const currentSemester = await Semester.findOne({
-          where: {
-            semestercode: semesters.current.semestercode
-          }
-        })
-        return currentSemester
-      })
-    })
-
-    test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-
-    test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
-    })
-
-    test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
-      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/services/test_data.js

@@ -1,42 +0,0 @@
-const faculties = [
-  {
-    code: 'FAC1',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 1 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 1 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 1 SV'
-      }
-    ]
-  },
-  {
-    code: 'FAC2',
-    organisation_type_code: 2,
-    name: [
-      {
-        langcode: 'fi',
-        text: 'FAC 2 FIN'
-      },
-      {
-        langcode: 'sv',
-        text: 'FAC 2 SV'
-      },
-      {
-        langcode: 'en',
-        text: 'FAC 3 SV'
-      }
-    ]
-  }
-]
-
-module.exports = {
-  faculties
-}

old_oodikone_stuff/old-backend-tests/studyprogramme.test.js

@@ -1,264 +0,0 @@
-const {
-  getCreditsForProvider,
-  productivityStatsFromCredits,
-  productivityStatsForProvider,
-  findGraduated,
-  graduatedStatsFromStudyrights,
-  combineStatistics,
-  productivityStatsForCode,
-  findProgrammeThesisCredits,
-  thesisProductivityFromCredits,
-  thesisProductivityForStudytrack
-} = require('./studyprogramme')
-const { sequelize } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { readFileSync } = require('fs')
-const { mapToProviders } = require('../util/utils')
-
-const { MASTER, BACHELOR } = ThesisTypeEnums
-
-const provider = '500-M010'
-const studytrack = 'MH50_010'
-
-const thesisCourse = {
-  // from studyprogramme.test.sql
-  programmeCode: 'MH50_010',
-  courseCode: 'THESIS_01',
-  thesisType: MASTER
-}
-
-beforeAll(async () => {
-  const query = readFileSync('./src/services/studyprogramme.test.sql', 'utf8')
-  await sequelize.query(query)
-  await ThesisCourse.create(thesisCourse)
-})
-
-test('Credits for provider should contain passed course credit', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({ id: 'CREDIT_01' })
-})
-
-test('Credits for provider should return object in correct format', async () => {
-  const credits = await getCreditsForProvider(provider, '2001-01-01')
-  const credit = await credits.find(cr => cr.id === 'CREDIT_01')
-  expect(credit).toMatchObject({
-    credits: 5,
-    year: 2016,
-    id: 'CREDIT_01'
-  })
-})
-
-test('Credits for provider should not contain passed credits from other providers', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_04')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit with failed credit typecode 10', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_02')
-  expect(credit).toBeFalsy()
-})
-
-test('Credits for provider should not contain credit for passed study module', async () => {
-  const credits = await getCreditsForProvider(provider)
-  const credit = await credits.find(cr => cr.id === 'CREDIT_03')
-  expect(credit).toBeFalsy()
-})
-
-test('KH50_005 track code maps to 500-K005 provider code', () => {
-  const track = 'KH50_005'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-K005')
-})
-
-test('MH50_010 track code maps to 500-M010 provider code', () => {
-  const track = 'MH50_010'
-  const provider = mapToProviders([track])[0]
-  expect(provider).toBe('500-M010')
-})
-
-test('productivityStatsFromCredits calculates stats correctly', () => {
-  const credits = [
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2016, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 5 },
-    { year: 2015, credits: 10 }
-  ]
-  const stats = productivityStatsFromCredits(credits)
-  expect(stats).toMatchObject({
-    2016: { year: 2016, credits: 15 },
-    2015: { year: 2015, credits: 20 }
-  })
-})
-
-test('productivityStats integrates correctly', async () => {
-  const stats = await productivityStatsForProvider(provider, '2001-01-01')
-  expect(stats).toMatchObject({
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 }
-  })
-})
-
-test('findGraduated finds graduated studyright', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.studyrightid === '10')
-  expect(match).toBeTruthy()
-})
-
-test('findGraduated formats graduated studyrights correctly', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  expect(studyrights).toContainEqual({ studyrightid: '10', year: 2016, timeToGraduation: NaN })
-})
-
-test('findGraduated does not return studyrights that are not graduated', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '11')
-  expect(match).toBeFalsy()
-})
-
-test('findGraduated does return graduated studyrights from other studytracks', async () => {
-  const studyrights = await findGraduated(studytrack, '2001-01-01')
-  const match = studyrights.find(sr => sr.id === '12')
-  expect(match).toBeFalsy()
-})
-
-test('graduatedStatsFromStudyrights calculates stats correctly', () => {
-  const studyrights = [{ year: 2015 }, { year: 2015 }, { year: 2014 }]
-  const stats = graduatedStatsFromStudyrights(studyrights)
-  expect(stats).toMatchObject({
-    2015: { graduated: 2 },
-    2014: { graduated: 1 }
-  })
-})
-
-test('combineStatistics returns correctly formatted array', () => {
-  const creditStats = {
-    2015: { year: 2015, credits: 40 },
-    2016: { year: 2016, credits: 5 },
-    2014: { year: 2014, credits: 20 }
-  }
-  const studyrightStats = {
-    2015: { graduated: 2 },
-    2016: { graduated: 1 }
-  }
-  const thesisStats = {
-    2014: { mThesis: 1 },
-    2015: { mThesis: 2, bThesis: 1 }
-  }
-  const creditsForMajors = {
-    2014: 10,
-    2015: 22
-  }
-  const transferredCredits = {
-    2014: 2,
-    2015: 4
-  }
-
-  const stats = combineStatistics(creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits)
-  expect(stats).toContainEqual({
-    year: 2015,
-    mThesis: 2,
-    bThesis: 1,
-    credits: 40,
-    graduated: 2,
-    creditsForMajors: 22,
-    transferredCredits: 4
-  })
-  expect(stats).toContainEqual({
-    year: 2014,
-    mThesis: 1,
-    bThesis: 0,
-    credits: 20,
-    graduated: 0,
-    creditsForMajors: 10,
-    transferredCredits: 2
-  })
-  expect(stats).toContainEqual({
-    year: 2016,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    graduated: 1,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('productivityStatsForCode integrates', async () => {
-  const stats = await productivityStatsForCode(studytrack, '2001-01-01')
-  expect(stats.data).toContainEqual({
-    year: 2015,
-    graduated: 0,
-    bThesis: 0,
-    mThesis: 1,
-    credits: 40,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-  expect(stats.data).toContainEqual({
-    year: 2016,
-    graduated: 1,
-    mThesis: 0,
-    bThesis: 0,
-    credits: 5,
-    creditsForMajors: 0,
-    transferredCredits: 0
-  })
-})
-
-test('findProgrammeThesisCredits returns correct credit', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toBeTruthy()
-  expect(credits.length).toBe(1)
-  expect(credits.find(c => c.id === 'CREDIT_07')).toBeTruthy()
-})
-
-test('findProgrammeThesisCredits format credit correctly', async () => {
-  const credits = await findProgrammeThesisCredits(studytrack)
-  expect(credits).toContainEqual({
-    id: 'CREDIT_07',
-    code: 'THESIS_01',
-    type: MASTER,
-    year: 2015
-  })
-})
-
-test('thesisProductivityFromCredits', async () => {
-  const credits = [
-    { type: MASTER, year: 2015 },
-    { type: MASTER, year: 2015 },
-    { type: BACHELOR, year: 2015 },
-    { type: BACHELOR, year: 2014 },
-    { type: MASTER, year: 2013 }
-  ]
-  const stats = thesisProductivityFromCredits(credits)
-  expect(stats).toHaveProperty('2015', '2014', '2013')
-  expect(stats).toMatchObject({
-    '2015': {
-      mThesis: 2,
-      bThesis: 1
-    },
-    '2014': {
-      mThesis: 0,
-      bThesis: 1
-    },
-    '2013': {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})
-
-test('thesisProductivityForStudytrack integrates', async () => {
-  const stats = await thesisProductivityForStudytrack(studytrack)
-  expect(stats).toMatchObject({
-    2015: {
-      mThesis: 1,
-      bThesis: 0
-    }
-  })
-})

old_oodikone_stuff/old-backend-tests/studyprogramme.test.sql

@@ -1,93 +0,0 @@
-DO $$
-DECLARE
-  ts TIMESTAMP = CURRENT_TIMESTAMP;
-  studytrk_01 VARCHAR(50) = 'MH50_010';
-  studytrk_02 VARCHAR(50) = 'KH50_005';
-  provider_01 VARCHAR(50) = '500-M010';
-  provider_02 VARCHAR(50) = '500-K005';
-  course_01 VARCHAR(50) = 'M010_COURSE';
-  module_01 VARCHAR(50) = 'M010_MODULE';
-  course_02 VARCHAR(50) = 'K005_COURSE';
-  thesis VARCHAR(50) = 'THESIS_01';
-  date_2016 TIMESTAMPTZ = '2016-09-28 00:00:00+00';
-  date_2015 TIMESTAMPTZ = '2015-09-28 00:00:00+00';
-BEGIN
-  INSERT INTO semesters ("semestercode", "createdAt", "updatedAt")
-  VALUES
-  (5, ts, ts)
-  ;
-
-  INSERT INTO course
-  ("code", "name", "is_study_module")
-  VALUES
-  (course_01, '{ "en": "COURSE_EN" }', 'false'),
-  (module_01, '{ "en": "MODULE_EN" }', 'true'),
-  (course_02, '{ "en": "COURSE_EN" }', 'false'),
-  (thesis, '{ "en": "Bachelor thesis" }', 'false')
-  ;
-
-  INSERT INTO student
-  ("studentnumber", "createdAt", "updatedAt")
-  VALUES
-  ('012111517', ts, ts)
-  ;
-
-  INSERT INTO credit_types
-  ("credittypecode", "createdAt", "updatedAt")
-  VALUES
-  (10, ts, ts),
-  (4, ts, ts)
-  ;
-
-  INSERT INTO credit
-  ("id", "createddate", "lastmodifieddate", "semestercode", "course_code", "credittypecode", "attainment_date", "credits", "student_studentnumber")
-  VALUES
-  ('CREDIT_01', ts, ts, 5, course_01, 4, date_2016, 5, '012111517'),
-  ('CREDIT_02', ts, ts, 5, course_01, 10, date_2016, 5, '012111517'),
-  ('CREDIT_03', ts, ts, 5, module_01, 4, date_2016, 20, '012111517'),
-  ('CREDIT_04', ts, ts, 5, course_02, 4, date_2016, 5, '012111517'),
-  ('CREDIT_05', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_06', ts, ts, 5, course_01, 4, date_2015, 5, '012111517'),
-  ('CREDIT_07', ts, ts, 5, thesis, 4, date_2015, 30, '012111517')
-  ;
-
-  INSERT INTO providers
-  ("providercode", "createdAt", "updatedAt")
-  VALUES
-  (provider_01, ts, ts),
-  (provider_02, ts, ts)
-  ;
-
-  INSERT INTO course_providers
-  ("coursecode", "providercode", "createdAt", "updatedAt")
-  VALUES
-  (course_01, provider_01, ts, ts),
-  (module_01, provider_01, ts, ts),
-  (course_02, provider_02, ts, ts),
-  (thesis, provider_01, ts, ts)
-  ;
-
-  INSERT INTO element_details
-  ("code", "createdAt", "updatedAt")
-  VALUES
-  (studytrk_01, ts, ts),
-  (studytrk_02, ts, ts)
-  ;
-
-  INSERT INTO studyright
-  ("studyrightid", "graduated", "enddate")
-  VALUES
-  (10, 1, date_2016),
-  (11, 0, date_2016),
-  (12, 1, date_2016)
-  ;
-
-  INSERT INTO studyright_elements
-  ("studyrightid", "code", "createdAt", "updatedAt", "studentnumber")
-  VALUES
-  (10, studytrk_01, ts, ts, '012111517'),
-  (11, studytrk_01, ts, ts, '012111517'),
-  (12, studytrk_02, ts, ts, '012111517')
-  ;
-
-END $$;

old_oodikone_stuff/old-backend-tests/teachers.test.js

@@ -1,53 +0,0 @@
-const { Teacher } = require('../models/index')
-const service = require('./teachers')
-
-const teacher = {
-  id: 'TID_001',
-  code: 'usercode',
-  name: 'Lastname Firstname Middlename'
-}
-
-beforeAll(async () => {
-  await Teacher.create(teacher)
-})
-
-describe('teacher service tests', () => {
-  describe('teacher by search term tests', () => {
-    const teacherIsInResults = (id, matches) => matches.some(teacher => teacher.id === 'TID_001')
-
-    test('Search by `usercode` finds teacher', async () => {
-      const matches = await service.bySearchTerm('usercode')
-      expect(teacherIsInResults('TID_001', matches)).toBe(true)
-    })
-
-    test('Search by `usercodeEXTRA` does not find teacher', async () => {
-      const matches = await service.bySearchTerm('usercodeEXTRA')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(false)
-    })
-
-    test('Search by just `Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Lastname Firstname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Lastname Firstname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `Firstname Lastname` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('Firstname Lastname')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-
-    test('Search by `id` should find the teacher', async () => {
-      const matches = await service.bySearchTerm('TID_001')
-      expect(teacherIsInResults(teacher.id, matches)).toBe(true)
-    })
-  })
-})

old_oodikone_stuff/old-backend-tests/thesis.test.js

@@ -1,85 +0,0 @@
-const { forceSyncDatabase } = require('../database/connection')
-const { Course, ElementDetails } = require('../models')
-const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
-const { createThesisCourse, deleteThesisCourse, findProgrammeTheses } = require('./thesis')
-
-const courseOne = {
-  code: 'COURSE',
-  name: { en: 'Course_EN' }
-}
-
-const courseTwo = {
-  code: 'COURSE_2',
-  name: { en: 'Course_EN_2' }
-}
-
-const progOne = {
-  code: 'CS_BSC',
-  name: { en: 'CS_BSC' }
-}
-
-const progTwo = {
-  code: 'CS_MSC',
-  name: { en: 'CS_MSC' }
-}
-
-beforeEach(async () => {
-  await forceSyncDatabase()
-  await Course.create(courseOne)
-  await Course.create(courseTwo)
-  await ElementDetails.create(progOne)
-  await ElementDetails.create(progTwo)
-})
-
-test('createThesisCourse should save thesis correctly', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-})
-
-test('createThesisCourse should create separate bachelor and thesis for same programme and course', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  const courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  const master = courses.find(c => c.thesisType === 'MASTER')
-  const bachelor = courses.find(c => c.thesisType === 'BACHELOR')
-  expect(master).toBeTruthy()
-  expect(master).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE', thesisType: 'MASTER' })
-  expect(bachelor).toBeTruthy()
-  expect(bachelor).toMatchObject({ programmeCode: 'CS_BSC', courseCode: 'COURSE_2', thesisType: 'BACHELOR' })
-})
-
-test('createThesisCourse should error for creating a thesis for existing (course, prog) pair', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const promise = createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.BACHELOR)
-  await expect(promise).rejects.toThrow('Validation error')
-})
-
-test('deleteThesisCourse removes thesis from table', async () => {
-  let courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(0)
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(2)
-  await deleteThesisCourse(progOne.code, courseOne.code)
-  courses = await ThesisCourse.findAll()
-  expect(courses.length).toBe(1)
-  expect(courses[0]).toMatchObject({ thesisType: 'BACHELOR' })
-})
-
-test('findProgrammeTheses returns correct courses', async () => {
-  await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  await createThesisCourse(progOne.code, courseTwo.code, ThesisTypeEnums.BACHELOR)
-  await createThesisCourse(progTwo.code, courseOne.code, ThesisTypeEnums.MASTER)
-  const theses = await findProgrammeTheses(progOne.code)
-  expect(theses.length).toBe(2)
-})
-
-test('createThesisCourse should return created thesis', async () => {
-  const thesis = await createThesisCourse(progOne.code, courseOne.code, ThesisTypeEnums.MASTER)
-  expect(thesis).toBeTruthy()
-  expect(thesis).toMatchObject({ programmeCode: progOne.code, courseCode: courseOne.code })
-})

old_oodikone_stuff/old-backend-tests/utils.js

@@ -1,248 +0,0 @@
-const faker = require('faker')
-
-const numberFromTo = (from, to) => Math.round(Math.random() * (to - from)) + from
-const daysAgo = daysAgo => new Date(new Date() - 1000 * 60 * 60 * 24 * daysAgo)
-const idFromTwoIds = (a, b) => (1 / 2) * (a + b) * (a + b + 1) + b
-
-const gradeArray = [
-  '2',
-  'CL',
-  'Eisa',
-  'L',
-  'Hyv.',
-  'TT',
-  'LUB',
-  'HT',
-  'MCLA',
-  '4',
-  '0',
-  'A',
-  'NSLA',
-  'Luop',
-  '5',
-  '3',
-  'Hyl.',
-  'ECLA',
-  '1',
-  'KH'
-]
-const statusArray = [
-  { status: 'Suoritettu', statuscode: 4 },
-  { status: 'Korotettu', statuscode: 7 },
-  { status: 'Hyväksiluettu', statuscode: 9 },
-  { status: 'Hylätty', statuscode: 10 }
-]
-const teacherRoleArray = ['Responsible', 'Approver', 'Teacher']
-
-const generateCourses = async amount => {
-  const number = amount || numberFromTo(10, 100)
-  const courses = []
-  for (let i = 0; i < number; i++) {
-    courses.push({
-      code: i.toString(10),
-      name: faker.lorem.words(3)
-    })
-  }
-  return courses
-}
-
-const generateCourseInstances = async (courses, amount) => {
-  const instances = []
-  let id = 0
-  courses.forEach(course => {
-    const number = amount || numberFromTo(5, 40)
-    for (let i = 0; i < number; i++) {
-      instances.push({
-        id,
-        coursedate: daysAgo(numberFromTo(10, 365)).toUTCString(),
-        course_code: course.code
-      })
-      id++
-    }
-  })
-  return instances
-}
-
-const generateStudents = async amount => {
-  const students = []
-  const number = amount || numberFromTo(5, 40)
-
-  const now = new Date()
-
-  for (let i = 0; i < number; i++) {
-    const birthdate = daysAgo(numberFromTo(365 * 18, 365 * 70))
-    const dateoffirstcredit = faker.date.between(birthdate, now)
-    const dateoflastcredit = faker.date.between(dateoffirstcredit, now)
-    const dateofuniversityenrollment = faker.date.between(birthdate, dateoflastcredit)
-    const lastname = faker.name.lastName()
-    const firstnames = faker.name.firstName(2)
-    students.push({
-      studentnumber: `0${numberFromTo(10000000, 15000000)}`,
-      lastname,
-      firstnames,
-      abbreviatedname: `${lastname} ${firstnames}`,
-      birthdate,
-      communicationlanguage: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      country: faker.address.country(),
-      creditcount: numberFromTo(0, 600),
-      dateoffirstcredit,
-      dateoflastcredit,
-      dateofuniversityenrollment,
-      gradestudent: `${numberFromTo(0, 1)}`,
-      nationalities: ['fi', 'sv', 'en'][numberFromTo(0, 2)],
-      semesterenrollmenttypecode: [null, '1.0', '2.0'][numberFromTo(0, 2)],
-      sex: ['female', 'male'][numberFromTo(0, 1)],
-      studentstatuscode: numberFromTo(2, 8)
-    })
-  }
-
-  return students
-}
-
-const generateCredits = async (courseInstances, students, amount) => {
-  const credits = []
-  students.forEach((student, studentIdx) => {
-    courseInstances.forEach(courseInstance => {
-      const number = amount || numberFromTo(5, 40)
-      for (let i = 0; i < number; i++) {
-        const statusObject = statusArray[numberFromTo(0, statusArray.length - 1)]
-        credits.push({
-          id: idFromTwoIds(idFromTwoIds(studentIdx, courseInstance.id), i),
-          grade: gradeArray[numberFromTo(0, gradeArray.length - 1)],
-          student_studentnumber: student.studentnumber,
-          credits: [5, 10][numberFromTo(0, 1)],
-          ordering: faker.date.between(student.dateoffirstcredit, student.dateoflastcredit),
-          status: statusObject.status,
-          statuscode: statusObject.statuscode,
-          courseinstance_id: courseInstance.id
-        })
-      }
-    })
-  })
-  return credits
-}
-
-const generateTeachers = async amount => {
-  const teachers = []
-  const number = amount || numberFromTo(5, 40)
-
-  for (let i = 0; i < number; i++) {
-    teachers.push({
-      id: i,
-      code: faker.lorem.word(),
-      name: `${faker.name.lastName()} ${faker.name.firstName()}`
-    })
-  }
-
-  return teachers
-}
-
-const generateCourseTeachers = async (courseInstances, teachers, maximumAmount) => {
-  const courseTeachers = []
-  courseInstances.forEach((courseInstance, index) => {
-    const number = numberFromTo(0, maximumAmount)
-    for (let i = 0; i < number; i++) {
-      const teacher = teachers[numberFromTo(0, teachers.length - 1)]
-      courseTeachers.push({
-        id: idFromTwoIds(index, i),
-        teacherrole: teacherRoleArray[numberFromTo(0, teacherRoleArray.length - 1)],
-        courseinstance_id: courseInstance.id,
-        teacher_id: teacher.id
-      })
-    }
-  })
-
-  return courseTeachers
-}
-
-const generateOrganizations = async amount => {
-  const organizations = []
-
-  const number = amount || numberFromTo(5, 40)
-  for (let i = 0; i < number; i++) {
-    organizations.push({
-      code: faker.lorem.words(3),
-      name: faker.company.companyName()
-    })
-  }
-  return organizations
-}
-
-const generateStudyrights = async (students, organization, amount, date, prioritycode) => {
-  const studyrights = []
-  const possibleStudyrightNames = []
-  if (amount) {
-    for (let i = 0; i < amount; i++) {
-      possibleStudyrightNames.push(faker.company.catchPhrase())
-    }
-  }
-  students.forEach(student => {
-    studyrights.push({
-      studyrightid: student.studentnumber,
-      canceldate: null,
-      cancelorganisation: null,
-      enddate: new Date().toUTCString(),
-      extentcode: numberFromTo(2, 99),
-      givendate: daysAgo(365 * numberFromTo(1, 100)),
-      graduated: null,
-      prioritycode: prioritycode || numberFromTo(1, 30),
-      startdate: date,
-      studystartdate: date,
-      organization_code: organization.code,
-      student_studentnumber: student.studentnumber
-    })
-  })
-  return studyrights
-}
-
-const generateUsers = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const users = []
-  for (let i = 0; i < number; i++) {
-    users.push({
-      full_name: `${faker.name.lastName()}, ${faker.name.firstName(2)}`,
-      username: faker.lorem.word(),
-      is_enabled: numberFromTo(0, 1) === 1
-    })
-  }
-  return users
-}
-
-const generateTags = amount => {
-  const number = amount || numberFromTo(10, 100)
-  const tags = []
-  for (let i = 0; i < number; i++) {
-    tags.push({
-      tagname: `${faker.company.catchPhraseAdjective()} ${i}` //tagname should be unique, add i just in case
-    })
-  }
-  return tags
-}
-
-// adds all tags to all students
-const generateTagStudents = (students, tags) => {
-  let tag_students = []
-  students.forEach(student => {
-    tags.forEach(tag => {
-      tag_students.push({
-        taggedstudents_studentnumber: student.studentnumber,
-        tags_tagname: tag.tagname
-      })
-    })
-  })
-  return tag_students
-}
-
-module.exports = {
-  generateCourses,
-  generateCourseInstances,
-  generateStudents,
-  generateStudyrights,
-  generateCredits,
-  generateTeachers,
-  generateCourseTeachers,
-  generateOrganizations,
-  generateUsers,
-  generateTags,
-  generateTagStudents
-}

old_oodikone_stuff/old-backend-tests/utils/semester.test.js

@@ -1,14 +0,0 @@
-const moment = require('moment')
-const semesterUtils = require('../../../src/util/semester')
-
-describe('Semester utils tests', () => {
-  test('Get passing semester', () => {
-    expect(semesterUtils.getPassingSemester(2018, moment('2017-09-09'))).toBe('BEFORE')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-09-09'))).toBe('0-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-05-01'))).toBe('0-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2019-09-01'))).toBe('1-FALL')
-    expect(semesterUtils.getPassingSemester(2018, moment('2020-05-01'))).toBe('1-SPRING')
-    expect(semesterUtils.getPassingSemester(2018, moment('2026-05-01'))).toBe('LATER')
-    expect(semesterUtils.getPassingSemester(2018, moment('2018-05-01'))).toBe('BEFORE')
-  })
-})

old_oodikone_stuff/updater_api/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_api/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_api/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_api/.gitignore

@@ -1 +0,0 @@
-node_modules/

old_oodikone_stuff/updater_api/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_api/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_api/doo_api_database_updater/db_studentnumbers_to_file.js

@@ -1,15 +0,0 @@
-const fs = require('fs')
-const { Student } = require('../../models/index')
-
-const getStudentNumbers = async (writefilepath = './studentnumbers.txt') => {
-  const file = fs.openSync(writefilepath, 'w')
-  const students = await Student.findAll()
-  for (let student of students) {
-    const data = `${student.studentnumber} \n`
-    fs.appendFileSync(file, data)
-  }
-  fs.closeSync(file)
-  process.exit(0)
-}
-
-getStudentNumbers()

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.js

@@ -1,350 +0,0 @@
-const moment = require('moment')
-const _ = require('lodash')
-
-const getStudyRightIdStrings = data => data['data'].map(elements => elements[0])
-
-const getTextsByLanguage = texts => {
-  const names = {}
-  texts.forEach(text => (names[text.langcode] = text.text))
-  return { fi: null, sv: null, en: null, ...names }
-}
-
-const defaultNameFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names.fi || names.en || names.sv
-}
-
-const jsonNamesFromTexts = texts => {
-  const names = getTextsByLanguage(texts)
-  return names
-}
-
-const universityEnrollmentDateFromStudyRights = studyRightArray => {
-  return _.sortBy(studyRightArray.map(s => s.start_date), n => moment(n).valueOf())[0]
-}
-
-const getStudentFromData = (student, studyrights) => {
-  const city = getTextsByLanguage(student.city)
-  const language = getTextsByLanguage(student.language)
-  const country = getTextsByLanguage(student.country)
-  const gender = getTextsByLanguage(student.gender)
-  const home_country = getTextsByLanguage(student.home_country)
-  return {
-    studentnumber: student.student_number,
-    email: student.email,
-    phone: student.phone,
-    city_fi: city.fi,
-    city_sv: city.sv,
-    national_student_number: student.national_student_number,
-    zipcode: student.zipcode,
-    address: student.address1,
-    creditcount: student.studyattainments,
-    address2: student.address2,
-    birthdate: getDate(student.birth_date),
-    language_fi: language.fi,
-    language_sv: language.sv,
-    language_en: language.en,
-    age: student.age_years,
-    lastname: student.last_name,
-    mobile: student.mobile_phone,
-    home_county_id: student.home_county_id,
-    country_fi: country.fi,
-    country_sv: country.sv,
-    country_en: country.en,
-    home_country_fi: home_country.fi,
-    home_country_sv: home_country.sv,
-    home_country_en: home_country.en,
-    firstnames: student.first_names,
-    communicationlanguage: language.en || language.fi || language.sv,
-    dateofuniversityenrollment: universityEnrollmentDateFromStudyRights(studyrights),
-    abbreviatedname: [student.last_name, student.first_names].join(' '),
-    gender_code: student.gender_code,
-    gender_fi: gender.fi,
-    gender_sv: gender.sv,
-    gender_en: gender.en
-  }
-}
-
-const getSemesterCode = attainment_date => moment(attainment_date).diff(moment('1950', 'YYYY'), 'months') / 6
-
-const getDate = (date, format = 'DD.MM.YYYY') => {
-  if (!date) return null
-  return moment(date, format).format('YYYY-MM-DD')
-}
-
-const getOrganisationFromData = ({ name, code }) => {
-  return {
-    code,
-    name: jsonNamesFromTexts(name)
-  }
-}
-
-const attainmentDataToCredit = (attainment, studentnumber) => {
-  return {
-    id: String(attainment.studyattainment_id),
-    grade: defaultNameFromTexts(attainment.grade),
-    credits: attainment.credits,
-    ordering: getDate(attainment.attainment_date, null),
-    credittypecode: attainment.attainment_status_code,
-    student_studentnumber: studentnumber,
-    attainment_date: parseDate(attainment.attainment_date),
-    course_code: `${attainment.learningopportunity_id}`,
-    semestercode: attainment.semester_code,
-    isStudyModule: attainment.is_leaf === 'false'
-  }
-}
-
-const attainmentDataToCourse = attainment => {
-  const { learningopportunity_name, attainment_date } = attainment
-  return {
-    code: attainment.learningopportunity_id,
-    name: jsonNamesFromTexts(learningopportunity_name),
-    latest_instance_date: parseDate(attainment_date),
-    is_study_module: attainment.is_leaf === 'false'
-  }
-}
-
-const getTeacherFromData = teacher => {
-  if (teacher) {
-    return {
-      id: teacher.teacher_id,
-      code: teacher.userid,
-      name: teacher.full_name
-    }
-  }
-}
-
-const parseDate = (date, format = 'YYYY-MM-DD HH:mm') => date && moment.utc(date, format).toDate()
-
-const getStudyRightFromData = (data, studentNumber) => {
-  return {
-    studyrightid: `${data.studyright_id}`,
-    canceldate: data.cancel_date,
-    cancelorganisation: data.organisation_code,
-    enddate: parseDate(data.degree_date) || parseDate(data.end_date),
-    extentcode: data.extent_code,
-    givendate: parseDate(data.admission_date),
-    graduated: Number(!!data.degree_date),
-    graduation_date: data.degree_date,
-    prioritycode: data.priority,
-    startdate: parseDate(data.start_date),
-    studystartdate: parseDate(data.study_start_date),
-    organization_code: data.organisation_code,
-    student_studentnumber: studentNumber,
-    faculty_code: data.faculty_code
-  }
-}
-
-const elementDetailFromData = element => {
-  const { code, name, element_id } = element
-  const names = getTextsByLanguage(name)
-  return {
-    code,
-    name: names,
-    type: element_id
-  }
-}
-
-const studyrightElementFromData = (element, studyrightid, studentnumber, degree_date) => {
-  const { start_date, end_date, code } = element
-  return {
-    code,
-    studyrightid,
-    studentnumber,
-    startdate: start_date,
-    enddate: degree_date || end_date
-  }
-}
-
-const attainmentDataToTeachers = data =>
-  data.teachers.map(teacherdata => ({
-    id: teacherdata.teacher_id
-  }))
-
-const studyrightDataToExtent = data => ({
-  extentcode: data.extent_code,
-  name: getTextsByLanguage(data.extent)
-})
-
-const courseTypeFromData = data => ({
-  coursetypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const disciplineFromData = data => ({
-  discipline_id: data.discipline_id,
-  name: getTextsByLanguage(data.name)
-})
-
-const learningOpportunityDataToCourseDisciplines = data => {
-  if (data) {
-    return data.disciplines.map(discipline => ({
-      discipline_id: discipline.discipline_id,
-      course_id: data.learningopportunity_id
-    }))
-  }
-}
-
-const learningOpportunityDataToCourse = data => {
-  if (data) {
-    return {
-      code: data.learningopportunity_id,
-      coursetypecode: data.learningopportunity_type_code,
-      disciplines: data.disciplines,
-      name: getTextsByLanguage(data.names),
-      startdate: parseDate(data.start_date),
-      enddate: parseDate(data.end_date)
-    }
-  }
-}
-
-const learningOpportunityDataToCourseProviders = data => {
-  if (!data) {
-    return { providers: [], courseproviders: [] }
-  }
-  const providers = data.organisations.map(organisation => ({
-    providercode: organisation.code,
-    name: getTextsByLanguage(organisation.name)
-  }))
-  const courseproviders = providers.map(provider => ({
-    providercode: provider.providercode,
-    coursecode: data.learningopportunity_id
-  }))
-  return {
-    providers,
-    courseproviders
-  }
-}
-
-const studyattainmentStatusCodeToCreditType = data => ({
-  credittypecode: data.code,
-  name: getTextsByLanguage(data.name)
-})
-
-const semesterFromData = data => ({
-  semestercode: data.semester_code,
-  name: getTextsByLanguage(data.name),
-  startdate: parseDate(data.start_date),
-  enddate: parseDate(data.end_date),
-  yearcode: data.year_code,
-  yearname: data.year_name
-})
-
-const semesterEnrollmentFromData = (data, studentnumber) => ({
-  enrollmenttype: data.semester_enrollment_type_code,
-  semestercode: data.semester_code,
-  enrollment_date: parseDate(data.semester_enrollment_date),
-  studentnumber
-})
-
-const getTransfersFromData = (data, studentnumber) => {
-  const studytracks = data.elements.filter(element => element.element_id === 20)
-  const sorted = _.sortBy(studytracks, 'end_date')
-  let transfers = []
-  let i = 0
-  while (i < sorted.length) {
-    if (i === 0) {
-      i++
-      continue
-    }
-    let target = sorted[i].code
-    let source = sorted[i - 1].code
-    let transferdate = sorted[i - 1].end_date
-
-    transfers = transfers.concat({
-      sourcecode: source,
-      targetcode: target,
-      transferdate,
-      studentnumber,
-      studyrightid: `${data.studyright_id}`
-    })
-    i++
-  }
-  return transfers
-}
-
-const courseRealisationTypeFromData = data => ({
-  realisationtypecode: `${data.code}`,
-  name: getTextsByLanguage(data.name)
-})
-
-const studentEnrollmentToModels = (data, studentnumber) => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.name),
-    realisationtypecode: `${data.type_code}`,
-    parent: `${data.parent_id}`,
-    root: `${data.root_id}`
-  }
-  const courseenrollment = {
-    studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }
-  const course = {
-    code: data.learningopportunity_id
-  }
-  return {
-    courserealisation,
-    courseenrollment,
-    course
-  }
-}
-
-const courseUnitRealisationDataToModels = data => {
-  const courserealisation = {
-    courserealisation_id: `${data.course_id}`,
-    startdate: parseDate(data.start_date),
-    enddate: parseDate(data.end_date),
-    coursecode: `${data.learningopportunity_id}`,
-    name: getTextsByLanguage(data.realisation_name),
-    realisationtypecode: `${data.realisation_type_code}`,
-    parent: `${data.parent_id}`
-  }
-  const course = {
-    code: courserealisation.coursecode
-  }
-  const students = data.students.map(({ student_number }) => ({
-    studentnumber: student_number
-  }))
-  const courseenrollments = students.map(student => ({
-    studentnumber: student.studentnumber,
-    courserealisation_id: courserealisation.courserealisation_id
-  }))
-  return {
-    courserealisation,
-    course,
-    students,
-    courseenrollments
-  }
-}
-
-module.exports = {
-  getStudentFromData,
-  getStudyRightIdStrings,
-  getStudyRightFromData,
-  getDate,
-  getTeacherFromData,
-  elementDetailFromData,
-  studyrightElementFromData,
-  attainmentDataToCourse,
-  attainmentDataToCredit,
-  getOrganisationFromData,
-  attainmentDataToTeachers,
-  studyrightDataToExtent,
-  courseTypeFromData,
-  learningOpportunityDataToCourse,
-  studyattainmentStatusCodeToCreditType,
-  disciplineFromData,
-  learningOpportunityDataToCourseDisciplines,
-  semesterFromData,
-  semesterEnrollmentFromData,
-  learningOpportunityDataToCourseProviders,
-  getTransfersFromData,
-  courseRealisationTypeFromData,
-  studentEnrollmentToModels,
-  courseUnitRealisationDataToModels,
-  getSemesterCode
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_data_mapper.test.js

@@ -1,199 +0,0 @@
-const mapper = require('./oodi_data_mapper')
-
-describe('studyattainment data mapping tests', () => {
-  const data = {
-    attainment_status_code: 4,
-    attainment_date: '2018-05-30T21:00:00.000Z',
-    studyattainment_id: 123456789,
-    attainment_lang: null,
-    person_id: 1000002,
-    textual_assessment: null,
-    teachers: [
-      {
-        teacher_id: 'TID_01',
-        person_id: 1000001,
-        short_name: 'Matti Meikäläinen',
-        last_names: 'Meikäläinen',
-        first_names: 'Matti Teppo'
-      }
-    ],
-    studyright_id: null,
-    semester_code: 136,
-    grade: [
-      {
-        langcode: 'fi',
-        text: 'Hyv.'
-      },
-      {
-        langcode: 'sv',
-        text: 'Godk'
-      },
-      {
-        langcode: 'en',
-        text: 'Pass'
-      }
-    ],
-    credits: 10,
-    expiry_date: null,
-    learningopportunity_name: [
-      {
-        langcode: 'fi',
-        text: 'learningopportunity_name FI'
-      },
-      {
-        langcode: 'sv',
-        text: 'learningopportunity_name SV'
-      },
-      {
-        langcode: 'en',
-        text: 'learningopportunity_name EN'
-      }
-    ],
-    learningopportunity_id: 'LID_001'
-  }
-
-  describe('credit mapping tests', () => {
-    const studentnumber = '0876654321'
-    const credit = mapper.attainmentDataToCredit(data, studentnumber)
-
-    describe('studyattainment date tests', () => {
-      const { attainment_date } = credit
-      const MAY = 4
-
-      test('attainment date is truthy', () => {
-        expect(attainment_date).toBeTruthy()
-      })
-
-      test('attainment date is instance of Date', () => {
-        expect(attainment_date instanceof Date).toBe(true)
-      })
-
-      test('attainment date has correct month, year and date', () => {
-        expect(attainment_date.getMonth()).toBe(MAY)
-        expect(attainment_date.getFullYear()).toBe(2018)
-        expect(attainment_date.getDate()).toBe(30)
-      })
-    })
-
-    test('course code matches learningopportunity_id', () => {
-      expect(credit.course_code).toBe('LID_001')
-    })
-
-    test('credit id should attainment id as string', () => {
-      expect(credit.id).toBe('123456789')
-    })
-
-    test('studentnumber is correct', () => {
-      expect(credit.student_studentnumber).toBe(studentnumber)
-    })
-
-    test('semestercode is correct', () => {
-      expect(credit.semestercode).toBe(136)
-    })
-  })
-
-  describe('teacher mapping tests', () => {
-    const teachers = mapper.attainmentDataToTeachers(data)
-
-    test('amount of mapped teachers is correct (1)', () => {
-      expect(teachers.length).toBe(1)
-    })
-
-    test('teacher should have correct id', () => {
-      expect(teachers[0].id).toBe('TID_01')
-    })
-  })
-})
-
-describe('semester enrollment data mapping tests', () => {
-  const studentnumber = '12345678'
-
-  const data = {
-    full_time_student: 'true',
-    yths_code: null,
-    semester_enrollment_type_code: 1,
-    absence_reason_code: null,
-    payment_amount: null,
-    yths_description: null,
-    student_union_member: 'true',
-    semester_enrollment_date: '2011-08-30T21:00:00.000Z',
-    payment_date: null,
-    semester_code: 123,
-    doctoral_current_year_estimate: null,
-    doctoral_previous_year: null
-  }
-
-  describe('mapping of enrolled data', () => {
-    const mapping = mapper.semesterEnrollmentFromData(data, studentnumber)
-
-    test('mapped semestercode should be integer 123', () => {
-      expect(mapping.semestercode).toBe(123)
-    })
-
-    test('mapped enrollmenttype code should be integer 1', () => {
-      expect(mapping.enrollmenttype).toBe(1)
-    })
-
-    test('mapped studentnumber should be correct', () => {
-      expect(mapping.studentnumber).toBe(studentnumber)
-    })
-
-    test('mapped enrollment_date should be instance of Date', () => {
-      expect(mapping.enrollment_date instanceof Date).toBe(true)
-    })
-
-    test('mapped enrollment_date should be equal to Date object with same day, month and year', () => {
-      expect(mapping.enrollment_date).toEqual(new Date('2011-08-30'))
-    })
-  })
-})
-
-describe('semester code data mapping tests', () => {
-  const data = {
-    teaching_start_date: '1950-09-14T22:00:00.000Z',
-    name: [
-      {
-        langcode: 'fi',
-        text: 'Syksy 1950'
-      },
-      {
-        langcode: 'sv',
-        text: 'Hösten 1950'
-      },
-      {
-        langcode: 'en',
-        text: 'Autumn 1950'
-      }
-    ],
-    year_name: '1950-51',
-    enroll_end_date: '1950-09-29T22:00:00.000Z',
-    enroll_start_date: '1950-05-31T22:00:00.000Z',
-    end_date: '1950-12-30T22:00:00.000Z',
-    semester_code: 1,
-    start_date: '1900-07-31T22:20:11.000Z',
-    teaching_end_date: '1950-12-20T22:00:00.000Z',
-    year_code: 1
-  }
-
-  const semester = mapper.semesterFromData(data)
-
-  test('semester code should be correct', () => {
-    expect(semester.semestercode).toBe(1)
-  })
-
-  test('semester name should have all language keys set as properties', () => {
-    expect(semester.name).toHaveProperty('fi')
-    expect(semester.name).toHaveProperty('en')
-    expect(semester.name).toHaveProperty('sv')
-  })
-
-  test('semester start and end date should be instances of Date', () => {
-    expect(semester.startdate instanceof Date).toBe(true)
-    expect(semester.enddate instanceof Date).toBe(true)
-  })
-
-  test('semester yearcode and yearname should be correct', () => {
-    expect(semester.yearcode).toBe(1)
-    expect(semester.yearname).toBe('1950-51')
-  })
-})

old_oodikone_stuff/updater_api/doo_api_database_updater/oodi_interface.js

@@ -1,253 +0,0 @@
-require('dotenv').config()
-const axios = require('axios')
-const LRU = require('lru-cache')
-const https = require('https')
-const fs = require('fs')
-
-const { OODI_ADDR, KEY_PATH, CERT_PATH } = process.env
-const base_url = OODI_ADDR
-
-const requestCache = new LRU({
-  max: 25000,
-  length: () => 1,
-  maxAge: 60 * 60 * 1000
-})
-
-const agent =
-  KEY_PATH && CERT_PATH
-    ? new https.Agent({
-        cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-        key: fs.readFileSync(process.env.KEY_PATH, 'utf8')
-      })
-    : new https.Agent({
-        rejectUnauthorized: false
-      })
-
-const instance = axios.create({
-  httpsAgent: agent
-})
-
-const getUrl = async url => {
-  const cacheHit = requestCache.get(url)
-  if (cacheHit) return cacheHit
-
-  const res = await instance.get(url)
-
-  // Sometimes oodi seems to randomly return a response with status 200
-  // that in reality contains an error. Check this out:
-  // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550167612
-  if (res.data.exception) {
-    throw new Error(res.data.exception.message)
-  }
-
-  if (!url.includes('/students/')) requestCache.set(url, res.data.data)
-  return res.data.data
-}
-
-const getOodiApi = async relative => {
-  const route = base_url + relative
-  const stuff = await getUrl(route)
-  return stuff
-}
-
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-const attemptGetFor = async (url, attempts = 6) => {
-  for (let attempt = 1; attempt <= attempts; ++attempt) {
-    try {
-      return await getUrl(url)
-    } catch (error) {
-      if (attempt === attempts) {
-        console.log('ATTEMPT GET FOR FAILURE')
-        throw error
-      }
-      await sleep(attempt * 5000)
-    }
-  }
-}
-
-if (process.env.NODE_ENV === 'development') {
-  // axios.defaults.auth = {
-  //   username: 'tktl',
-  //   password: process.env.OODI_PW
-  // }
-  instance.interceptors.request.use(config => {
-    config.params = { token: process.env.TOKEN }
-    return config
-  })
-}
-console.log(process.env.NODE_ENV)
-
-const getStudent = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDENT FAILED')
-    throw e
-  }
-}
-
-const getStudentStudyRights = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyrights`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STUDYRIGHT FAIELD')
-    throw e
-  }
-}
-
-const getFaculties = async () => {
-  const url = `${base_url}/codes/faculties`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET FACULTIE S FAIELD')
-    throw e
-  }
-}
-
-const getStudyAttainments = async studentNumber => {
-  const url = `${base_url}/students/${studentNumber}/studyattainments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET STSDU ATTAINEMNT FIALE')
-    throw e
-  }
-}
-
-const getTeacherInfo = async id => {
-  const url = `${base_url}/teachers/${id}/info`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET TCHR FIAIELd', id)
-    throw e
-  }
-}
-
-const getCourseTypeCodes = async () => {
-  const url = `${base_url}/codes/learningopportunities/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('CORUSE TTYPE CODE FAIEL')
-    throw e
-  }
-}
-
-const getLearningOpportunity = async id => {
-  const url = `${base_url}/learningopportunities/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET LEARNIGN OEPPRITUNCITY FAIELD', id)
-    throw e
-  }
-}
-
-const getStudyattainmentStatusCodes = async () => {
-  const url = `${base_url}/codes/studyattainments/statuses`
-  return await attemptGetFor(url)
-}
-
-const getCourseDisciplines = async () => {
-  const url = `${base_url}/codes/learningopportunities/disciplines`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log(' GET COURS EDISCIIPLINES FIALED')
-    throw e
-  }
-}
-const getSemesters = async () => {
-  const url = `${base_url}/codes/semesters`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET SEMESTERS FAIELD')
-    throw e
-  }
-}
-
-const getSemesterEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/semesterenrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GEST EMESTER ENROLLMENTS FAIELD')
-    throw e
-  }
-}
-
-const getCourseEnrollments = async studentnumber => {
-  const url = `${base_url}/students/${studentnumber}/enrollments`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EENREOLMETNS FAIELD')
-    throw e
-  }
-}
-
-const getCourseRealisationTypes = async () => {
-  const url = `${base_url}/codes/courseunitrealisations/types`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURSE REALSITASITON TYPES')
-    throw e
-  }
-}
-
-const courseUnitRealisationsSince = async sinceDate => {
-  const url = `${base_url}/courseunitrealisations/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('COURSEU UNTI REALISTIATION')
-    throw e
-  }
-}
-
-const learningOpportunitiesSince = async (sinceDate = '0000-01-01') => {
-  const url = `${base_url}/learningopportunities/changes/ids/${sinceDate}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('LRNG OPRRTN SINCE')
-    throw e
-  }
-}
-
-const getCourseUnitRealisation = async id => {
-  const url = `${base_url}/courseunitrealisations/${id}`
-  try {
-    return await attemptGetFor(url)
-  } catch (e) {
-    console.log('GET COURS EUNTI GREALSISTAITOn', id)
-    throw e
-  }
-}
-
-module.exports = {
-  getStudentStudyRights,
-  getStudent,
-  getFaculties,
-  getStudyAttainments,
-  getTeacherInfo,
-  getCourseTypeCodes,
-  getLearningOpportunity,
-  getStudyattainmentStatusCodes,
-  getCourseDisciplines,
-  getSemesters,
-  getSemesterEnrollments,
-  getCourseEnrollments,
-  getCourseRealisationTypes,
-  courseUnitRealisationsSince,
-  getCourseUnitRealisation,
-  learningOpportunitiesSince,
-  getOodiApi
-}

old_oodikone_stuff/updater_api/doo_api_database_updater/updater_formatter.js

@@ -1,158 +0,0 @@
-const Oodi = require('./oodi_interface')
-const mapper = require('./oodi_data_mapper')
-
-const getAllStudentInformationFromApi = async studentnumber => {
-  const [student, studyrights, studyattainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-    Oodi.getStudent(studentnumber),
-    Oodi.getStudentStudyRights(studentnumber),
-    Oodi.getStudyAttainments(studentnumber),
-    Oodi.getSemesterEnrollments(studentnumber),
-    Oodi.getCourseEnrollments(studentnumber)
-  ])
-  return {
-    student,
-    studyrights,
-    studyattainments,
-    studentnumber,
-    semesterEnrollments,
-    courseEnrollments
-  }
-}
-
-const formatStudyrights = async (api, studentnumber) => {
-  if (api.studyrights.length === 0) {
-    console.log(`No studyrights for ${studentnumber}`)
-    return []
-  }
-  return api.studyrights.map(studyrightData => {
-    const studyRightExtent = mapper.studyrightDataToExtent(studyrightData)
-    const studyright = mapper.getStudyRightFromData(studyrightData, studentnumber)
-
-    const elementDetails = studyrightData.elements.map(element => mapper.elementDetailFromData(element))
-    const studyRightElements = studyrightData.elements.map(element =>
-      mapper.studyrightElementFromData(element, studyright.studyrightid, studentnumber, studyrightData.degree_date)
-    )
-
-    const transfers = mapper.getTransfersFromData(studyrightData, studentnumber)
-    return { studyRightExtent, studyright, elementDetails, studyRightElements, transfers }
-  })
-}
-
-const formatCourseEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.courseEnrollments.map(enrollment => mapper.studentEnrollmentToModels(enrollment, studentnumber))
-  )
-
-const parseAttainmentData = (data, studentnumber) => {
-  return {
-    credit: mapper.attainmentDataToCredit(data, studentnumber),
-    teachers: mapper.attainmentDataToTeachers(data),
-    course: mapper.attainmentDataToCourse(data)
-  }
-}
-
-const createCreditTeachers = async (credit, teachers) => {
-  const creditTeachers = teachers.map(teacher => ({
-    credit_id: credit.id,
-    teacher_id: teacher.id
-  }))
-  return creditTeachers
-}
-
-const formatStudyattainments = async (api, studentnumber) => {
-  let studyAttainments = []
-  for (let data of api.studyattainments) {
-    const { credit, teachers, course } = parseAttainmentData(data, studentnumber)
-    let learningOpportunity
-    // For some learning opportunities oodi returns an SQL error.
-    // F.ex. /learningopportunities/590202B. Therefore, until the
-    // bug is fixed on Oodi's side, this is what we need to do.
-    try {
-      learningOpportunity = await Oodi.getLearningOpportunity(course.code)
-    } catch (e) {
-      if (e.message && e.message.includes('PreparedStatementCallback')) continue
-      throw e
-    }
-    const { providers, courseproviders } = mapper.learningOpportunityDataToCourseProviders(learningOpportunity)
-
-    studyAttainments = [
-      ...studyAttainments,
-      {
-        credit: credit.semestercode
-          ? credit
-          : { ...credit, semestercode: mapper.getSemesterCode(credit.attainment_date) },
-        creditTeachers: await createCreditTeachers(credit, teachers),
-        teachers: (await Promise.all(
-          teachers.map(async t => {
-            const teacher = await Oodi.getTeacherInfo(t.id)
-            // https://github.com/UniversityOfHelsinkiCS/oodikone/issues/1547#issuecomment-550184131
-            // Sometimes Oodi returns a null teacher... We need to figure out why.
-            if (!teacher) {
-              console.log('TEACHER MISSING!', t.id)
-              return null
-            }
-            return mapper.getTeacherFromData(teacher)
-          })
-        )).filter(t => t),
-        course: {
-          ...course,
-          ...mapper.learningOpportunityDataToCourse(learningOpportunity),
-          disciplines: mapper.learningOpportunityDataToCourseDisciplines(learningOpportunity),
-          providers,
-          courseproviders
-        }
-      }
-    ]
-  }
-  return studyAttainments
-}
-const formatSemesterEnrollments = async (apidata, studentnumber) =>
-  await Promise.all(
-    apidata.semesterEnrollments.map(apiEnrollment => mapper.semesterEnrollmentFromData(apiEnrollment, studentnumber))
-  )
-
-const getStudent = async studentnumber => {
-  const api = await getAllStudentInformationFromApi(studentnumber)
-  if (api.student == null) {
-    const error = new Error(`API returned ${api.student} for studentnumber ${studentnumber}.`)
-    error.name = 'NO_STUDENT'
-    throw error
-  }
-  try {
-    const studentInfo = await mapper.getStudentFromData(api.student, api.studyrights)
-    const [studyRights, studyAttainments, semesterEnrollments, courseEnrollments] = await Promise.all([
-      formatStudyrights(api, studentnumber),
-      formatStudyattainments(api, studentnumber),
-      formatSemesterEnrollments(api, studentnumber),
-      formatCourseEnrollments(api, studentnumber)
-    ])
-    return { studentInfo, studyRights, studyAttainments, semesterEnrollments, courseEnrollments }
-  } catch (e) {
-    console.error('getStudent failed', api)
-    throw e
-  }
-}
-
-const getMeta = async () => {
-  const [
-    faculties,
-    courseRealisationsTypes,
-    semesters,
-    creditTypeCodes,
-    courseTypeCodes,
-    disciplines
-  ] = await Promise.all([
-    (await Oodi.getFaculties()).map(mapper.getOrganisationFromData),
-    (await Oodi.getCourseRealisationTypes()).map(mapper.courseRealisationTypeFromData),
-    (await Oodi.getSemesters()).map(mapper.semesterFromData),
-    (await Oodi.getStudyattainmentStatusCodes()).map(mapper.studyattainmentStatusCodeToCreditType),
-    (await Oodi.getCourseTypeCodes()).map(mapper.courseTypeFromData),
-    (await Oodi.getCourseDisciplines()).map(mapper.disciplineFromData)
-  ])
-  return { faculties, courseRealisationsTypes, semesters, creditTypeCodes, courseTypeCodes, disciplines }
-}
-
-module.exports = {
-  getStudent,
-  getMeta
-}

old_oodikone_stuff/updater_api/index.js

@@ -1,72 +0,0 @@
-const { stan, opts } = require('./stan')
-const { getStudent, getMeta } = require('./doo_api_database_updater/updater_formatter')
-const logger = require('./logger')
-const moment = require('moment')
-
-const hasActiveEnrollment = semesterEnrollments => {
-  if (semesterEnrollments) {
-    const currentSemester = Math.floor(moment().diff(moment('1950', 'YYYY'), 'months') / 6)
-    const active = semesterEnrollments.find(e => e.semestercode === currentSemester && e.enrollmenttype === 1)
-    return active ? true : false
-  }
-  return false
-}
-
-const fetchData = async (priority, msg) => {
-  try {
-    const task = JSON.parse(msg.getData())
-    if (task.task === 'meta') {
-      const data = await getMeta()
-      stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', priority }), err => {
-        if (err) console.log('STATUS PUBLISH FAILED', err)
-      })
-      stan.publish(priority ? 'PriorityWrite' : 'UpdateWrite', JSON.stringify({ task: task.task, data }))
-    } else {
-      try {
-        const data = await getStudent(task.task)
-        // TODO: check that data is properly structured(?)
-        const active = hasActiveEnrollment(data.semesterEnrollments)
-        stan.publish('status', JSON.stringify({ task: task.task, status: 'FETCHED', active, priority }), err => {
-          if (err) console.log('STATUS PUBLISH FAIELD', err)
-        })
-        stan.publish(
-          priority ? 'PriorityWrite' : 'UpdateWrite',
-          JSON.stringify({ task: task.task, data, active }),
-          err => {
-            if (err) console.log('STATUS PUBLISH FAILED', err)
-          }
-        )
-      } catch (e) {
-        if (e.name === 'NO_STUDENT') {
-          stan.publish(
-            'status',
-            JSON.stringify({ task: task.task, status: 'NO_STUDENT', active: false, priority }),
-            err => {
-              if (err) console.log('STATUS PUBLISH FAILED', err)
-            }
-          )
-          return
-        } else {
-          console.error('Task failed', task)
-          throw e
-        }
-      }
-    }
-  } catch (e) {
-    console.log(e)
-    logger.info('failure', { service: 'API' })
-  }
-}
-
-const handleMessage = priority => async msg => {
-  await fetchData(priority, msg)
-  msg.ack()
-}
-
-stan.on('connect', async () => {
-  const sub = stan.subscribe('UpdateApi', 'updater.workers', opts)
-  const prioSub = stan.subscribe('PriorityApi', 'updater.workers.prio', opts)
-
-  sub.on('message', handleMessage(false))
-  prioSub.on('message', handleMessage(true))
-})

old_oodikone_stuff/updater_api/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_api/package.json

@@ -1,34 +0,0 @@
-{
-  "name": "updater_api",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "axios": "^0.21.1",
-    "dotenv": "^7.0.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^5.1.1",
-    "moment": "^2.29.1",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_api/stan.js

@@ -1,13 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-
-const opts = stan.subscriptionOptions()
-opts.setManualAckMode(true)
-opts.setAckWait(10 * 60 * 1000)
-opts.setMaxInFlight(1)
-
-module.exports = {
-  stan,
-  opts
-}

old_oodikone_stuff/updater_api/test_producer.js

@@ -1,32 +0,0 @@
-var stan = require('node-nats-streaming').connect('updaterNATS', 'producer', process.env.NATS_URI)
-
-stan.on('connect', () => {
-  stan.publish('UpdateApi', 'meta', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '011120775', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014441008', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-  stan.publish('UpdateApi', '014272112', (err, guid) => {
-    if (err) {
-      console.log('publish failed')
-    } else {
-      console.log('published', guid)
-    }
-  })
-})

old_oodikone_stuff/updater_scheduler/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_scheduler/.eslintignore

@@ -1 +0,0 @@
-*.sql

old_oodikone_stuff/updater_scheduler/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_scheduler/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_scheduler/Dockerfile

@@ -1,13 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-COPY package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY . .

old_oodikone_stuff/updater_scheduler/index.js

@@ -1,126 +0,0 @@
-const { stan } = require('./src/nats_connection')
-const { CronJob } = require('cron')
-const Schedule = require('./models')
-const logger = require('./logger')
-const {
-  scheduleDaily,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  scheduleMeta
-} = require('./src/schedule_students')
-const { isValidStudentId } = require('./src/util')
-require('./src/api')
-
-const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
-
-let updatedCount = 0
-let scheduledCount = 0
-let fetchedCount = 0
-
-const updateTask = async ({ task, status, type, updatetime, active }) => {
-  const doc = { task, status, type }
-  if (updatetime) doc.updatedAt = new Date()
-  if (active != null) doc.active = active
-  await Schedule.findOneAndUpdate({ task }, doc, { upsert: true })
-}
-
-stan.on('connect', async () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 0 * * 6', async () => {
-      // Every Friday-Saturday midnight
-      try {
-        console.log('SCHEDULING ALL STUDENTS')
-        await scheduleAllStudents()
-      } catch (err) {
-        console.log('SCHEDULING ALL STUDENTS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 6 * * 1-5', async () => {
-      // Every Monday through Friday at 06:00
-      try {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        await scheduleMeta()
-        await scheduleAttainmentUpdate()
-      } catch (err) {
-        console.log('SCHEDULING META AND ATTAINMENT')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-
-    schedule('0 20 * * 1-4', async () => {
-      // Every Monday through Thursday at 20:00
-      try {
-        console.log('SCHEDULING DAILY TASKS')
-        await scheduleDaily()
-      } catch (err) {
-        console.log('SCHEDULING DAILY TASKS FAILED')
-        console.log(err)
-        logger.info('failure', { service: 'SCHEDULER' })
-      }
-    })
-  }
-
-  schedule('0 * * * *', async () => {
-    // Every hour
-    console.log(
-      `${updatedCount} TASKS DONE IN LAST HOUR. ${scheduledCount} TASKS SCHEDULED IN LAST HOUR. ${fetchedCount} TASKS FETCHED FROM API IN LAST HOUR`
-    )
-    updatedCount = 0
-    fetchedCount = 0
-    scheduledCount = 0
-  })
-
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  // opts.setDeliverAllAvailable()
-  // opts.setDurableName('durable')
-  opts.setMaxInFlight(20)
-
-  const statusSub = stan.subscribe('status', opts)
-
-  const handleStatusMessage = async msg => {
-    const data = JSON.parse(msg.getData())
-    const { task, status, timems, active, priority } = data
-    let updatetime = false
-    switch (status) {
-      case 'DONE':
-      case 'NO_STUDENT':
-        updatedCount = updatedCount + 1
-        updatetime = true
-        break
-      case 'FETCHED':
-        fetchedCount = fetchedCount + 1
-        break
-      case 'SCHEDULED':
-        scheduledCount = scheduledCount + 1
-        break
-      default:
-        throw 'unknown status: ' + msg.getData()
-    }
-    const isStudent = !!isValidStudentId(task)
-    if (status === 'DONE')
-      logger.info(`Status changed for ${task} to ${status}`, {
-        task,
-        status,
-        student: isStudent,
-        timems,
-        active,
-        priority
-      })
-    await updateTask({ task, status, type: isStudent ? 'student' : 'other', updatetime, active })
-  }
-  statusSub.on('message', async msg => {
-    try {
-      await handleStatusMessage(msg)
-    } catch (err) {
-      console.log(err)
-      logger.info('failure', { service: 'SCHEDULER' })
-    }
-    msg.ack()
-  })
-})

old_oodikone_stuff/updater_scheduler/logger.js

@@ -1,29 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-transports.push(new winston.transports.Console({ level: 'debug' }))
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_scheduler/models.js

@@ -1,18 +0,0 @@
-const mongoose = require('mongoose')
-mongoose.connect(process.env.MONGO_URI, { useNewUrlParser: false, useFindAndModify: false })
-
-const scheduleSchema = new mongoose.Schema({
-  task: {
-    type: String,
-    unique: true
-  },
-  status: String,
-  type: String,
-  active: Boolean,
-  updatedAt: {
-    type: Date,
-    index: true
-  }
-})
-
-module.exports = mongoose.model('Schedule', scheduleSchema)

old_oodikone_stuff/updater_scheduler/package.json

@@ -1,35 +0,0 @@
-{
-  "name": "updater_scheduler",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node --max_old_space_size=4096 index.js",
-    "dev": "nodemon --max_old_space_size=4096 index.js",
-    "scheduleActiveStudents": "NATS_CLIENTID=manualActiveSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleActiveStudents()'",
-    "scheduleMeta": "NATS_CLIENTID=manualAllSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleMeta()'",
-    "getOldestTasks": "node --max_old_space_size=4096 -e 'require(\"./src/SchedulingStatistics.js\").getOldestTasks()'"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "cron": "^1.8.2",
-    "express": "^4.17.1",
-    "mongoose": "^5.12.3",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_scheduler/src/SchedulingStatistics.js

@@ -1,48 +0,0 @@
-const Schedule = require('../models')
-
-const getOldestTasks = async () => {
-  const oldestTask = (await Schedule.find({ type: 'student' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestActiveTask = (await Schedule.find({ type: 'student', active: true })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestMetaTask = (await Schedule.find({ type: 'other', task: 'meta' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-  const oldestAttainmentTask = (await Schedule.find({ type: 'other', task: 'attainment' })
-    .sort({ updatedAt: 1 })
-    .limit(1))[0]
-
-  return {
-    oldestChangedStatus: {
-      studentnumber: oldestTask ? oldestTask.task : null,
-      updatedAt: oldestTask ? oldestTask.updatedAt : null
-    },
-    oldestActiveStudentDone: {
-      studentnumber: oldestActiveTask ? oldestActiveTask.task : null,
-      updatedAt: oldestActiveTask ? oldestActiveTask.updatedAt : null
-    },
-    oldestMetaTask: {
-      updatedAt: oldestMetaTask ? oldestMetaTask.updatedAt : null,
-      status: oldestMetaTask ? oldestMetaTask.status : null
-    },
-    oldestAttainmentTask: {
-      updatedAt: oldestAttainmentTask ? oldestAttainmentTask.updatedAt : null,
-      status: oldestAttainmentTask ? oldestAttainmentTask.status : null
-    }
-  }
-}
-
-const getCurrentStatus = async () => {
-  const allTasksScheduled = await Schedule.count({ type: 'student', status: 'SCHEDULED' })
-  const allTasksFetched = await Schedule.count({ type: 'student', status: 'FETCHED' })
-  const allTasksDone = await Schedule.count({ type: 'student', status: 'DONE' })
-  const allTasksCreated = await Schedule.count({ type: 'student', status: 'CREATED' })
-  const allTasksNoStudent = await Schedule.count({ type: 'student', status: 'NO_STUDENT' })
-  const allTasksActive = await Schedule.count({ type: 'student', active: true })
-
-  return { allTasksScheduled, allTasksFetched, allTasksDone, allTasksCreated, allTasksActive, allTasksNoStudent }
-}
-
-module.exports = { getOldestTasks, getCurrentStatus }

old_oodikone_stuff/updater_scheduler/src/api.js

@@ -1,127 +0,0 @@
-const express = require('express')
-const app = express()
-const port = process.env.PORT
-const bodyParser = require('body-parser')
-const {
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleAllStudents,
-  scheduleActiveStudents,
-  scheduleMeta,
-  scheduleAttainmentUpdate,
-  scheduleStudentCheck,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily
-} = require('./schedule_students')
-const { getOldestTasks, getCurrentStatus } = require('./SchedulingStatistics')
-const { createTasks } = require('./student_list_updater')
-
-app.use(bodyParser.json())
-
-app.get('/ping', (req, res) => res.json({ message: 'pong ' }))
-
-app.post('/update', async (req, res) => {
-  await createTasks(req.body)
-  const scheduled = await scheduleStudentsByArray(req.body)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/oldest', async (req, res) => {
-  const { amount } = req.body
-  const scheduled = await scheduleOldestNStudents(amount)
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/all', async (req, res) => {
-  const scheduled = await scheduleAllStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/active', async (req, res) => {
-  const scheduled = await scheduleActiveStudents()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/attainment', async (req, res) => {
-  scheduleAttainmentUpdate()
-  res.json({ message: `scheduled attainments` })
-})
-
-app.post('/update/meta', async (req, res) => {
-  scheduleMeta()
-  res.json({ message: 'scheduled meta' })
-})
-
-app.post('/update/no_student', async (req, res) => {
-  const scheduled = await scheduleStudentCheck()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/update/studentlist', async (req, res) => {
-  createTasks()
-  res.json({ message: `tasks created` })
-})
-
-app.post('/update/daily', async (req, res) => {
-  const scheduled = await scheduleDaily()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/created', async (req, res) => {
-  const scheduled = await rescheduleCreated()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/scheduled', async (req, res) => {
-  const scheduled = await rescheduleScheduled()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.post('/reschedule/fetched', async (req, res) => {
-  const scheduled = await rescheduleFetched()
-  res.json({ message: `scheduled ${scheduled} updates` })
-})
-
-app.get('/statuses', async (req, res) => {
-  const statuses = []
-
-  const {
-    oldestChangedStatus: { studentnumber, updatedAt },
-    oldestActiveStudentDone: { studentnumber: studentnumberActive, updatedAt: updatedAtActive },
-    oldestMetaTask,
-    oldestAttainmentTask
-  } = await getOldestTasks()
-
-  const {
-    allTasksScheduled,
-    allTasksFetched,
-    allTasksDone,
-    allTasksCreated,
-    allTasksActive,
-    allTasksNoStudent
-  } = await getCurrentStatus()
-
-  statuses.push(
-    { label: 'CREATED', value: allTasksCreated },
-    { label: 'SCHEDULED', value: allTasksScheduled },
-    { label: 'FETCHED', value: allTasksFetched },
-    { label: 'DONE', value: allTasksDone },
-    { label: 'NO_STUDENT', value: allTasksNoStudent },
-    { label: 'ACTIVE', value: allTasksActive },
-    { label: 'oldest student studentnumber', value: studentnumber },
-    { label: 'oldest student timestamp', value: updatedAt },
-    { label: 'oldest active student studentnumber', value: studentnumberActive },
-    { label: 'oldest active student timestamp', value: updatedAtActive },
-    { label: 'oldest metadata update', value: oldestMetaTask.updatedAt },
-    { label: 'current metadata status', value: oldestMetaTask.status },
-    { label: 'oldest attainment update', value: oldestAttainmentTask.updatedAt },
-    { label: 'current attainment status', value: oldestAttainmentTask.status }
-  )
-
-  res.json(statuses)
-})
-
-app.listen(port, () => console.log(`listening on port ${port}!`))
-module.exports = { app }

old_oodikone_stuff/updater_scheduler/src/nats_connection.js

@@ -1,4 +0,0 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.NATS_CLIENTID || 'scheduler', {
-  url: process.env.NATS_URI
-})
-module.exports = { stan }

old_oodikone_stuff/updater_scheduler/src/schedule_students.js

@@ -1,181 +0,0 @@
-const { stan } = require('./nats_connection')
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-stan.on('connect', async () => {
-  const taskspermin = 300
-  const opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  opts.setAckWait(5 * 60 * 1000)
-  opts.setMaxInFlight(taskspermin)
-
-  const scheduleSub = stan.subscribe('schedule', opts)
-  scheduleSub.on('message', async msg => {
-    try {
-      const { task, priority } = JSON.parse(msg.getData())
-      stan.publish('status', JSON.stringify({ task, status: 'SCHEDULED', priority }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      stan.publish(priority ? 'PriorityApi' : 'UpdateApi', JSON.stringify({ task }), err => {
-        if (err) {
-          console.log('publish failed', err)
-        }
-      })
-      await sleep(60 * 1000)
-    } catch (err) {
-      console.log(err)
-    }
-    msg.ack()
-  })
-})
-
-const publishSingle = async ({ priority, task }) => {
-  const promise = new Promise(resolve => {
-    stan.publish('schedule', JSON.stringify({ task: task.task, priority }), async err => {
-      if (err) {
-        console.log('publish failed, waiting 10 seconds and republishing', err)
-        await sleep(10 * 1000)
-        resolve(await publishSingle({ priority, task }))
-      } else {
-        resolve(true)
-      }
-    })
-  })
-  return await promise
-}
-
-const publishAll = async (tasks, priority = false) => {
-  for (const task of tasks) {
-    await publishSingle({ task, priority })
-  }
-}
-
-const scheduleActiveStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleAllStudents = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleMeta = async () => {
-  await publishAll([{ task: 'meta' }], true)
-}
-
-const scheduleAttainmentUpdate = async () => {
-  stan.publish('status', JSON.stringify({ task: 'attainment', status: 'SCHEDULED' }), err => {
-    if (err) {
-      console.log('publish failed')
-    }
-  })
-  stan.publish('UpdateAttainmentDates', null, err => {
-    if (err) {
-      console.log('publish failed', 'UpdateAttainmentDates')
-    }
-  })
-}
-
-const scheduleStudentCheck = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'NO_STUDENT' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleStudentsByArray = async studentNumbers => {
-  try {
-    const tasks = await Schedule.find({ type: 'student', task: { $in: studentNumbers } })
-    publishAll(tasks, true)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const scheduleOldestNStudents = async amount => {
-  try {
-    const tasks = [
-      ...(await Schedule.find({ type: 'student' })
-        .sort({ updatedAt: 1 })
-        .limit(Number(amount)))
-    ]
-    publishAll(tasks)
-    return tasks.length
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const rescheduleCreated = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'CREATED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleScheduled = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'SCHEDULED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const rescheduleFetched = async () => {
-  const tasks = [...(await Schedule.find({ type: 'student', status: 'FETCHED' }).sort({ updatedAt: 1 }))]
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-const scheduleDaily = async () => {
-  const findStudentsRange = 10000
-  const highestStudentNumber = (await Schedule.find({ type: 'student', status: { $in: ['DONE', 'FETCHED'] } })
-    .sort({ task: -1 })
-    .limit(1))[0]
-  const regexResult = highestStudentNumber ? highestStudentNumber.task.match(/0(\d{7})\d/) : null
-  const minstudentnumber = 1000000
-  const studentNumberMid = (regexResult && Number(regexResult[1])) || minstudentnumber + findStudentsRange
-  const min = Math.max(studentNumberMid - findStudentsRange, minstudentnumber)
-  const max = Math.min(studentNumberMid + findStudentsRange, 9999999)
-
-  console.log({ findStudentsRange, minstudentnumber, studentNumberMid, min, max })
-
-  const activeStudents = [...(await Schedule.find({ type: 'student', active: true }).sort({ updatedAt: 1 }))]
-  const studentnumbers = activeStudents.map(t => t.task)
-  for (let i = min; i <= max; ++i) {
-    studentnumbers.push(`0${i}${getStudentNumberChecksum(i)}`)
-  }
-  const oldestStudents = [
-    ...(await Schedule.find({ type: 'student' })
-      .sort({ updatedAt: 1 })
-      .limit(10000))
-  ]
-  studentnumbers.push(...oldestStudents.map(t => t.task))
-
-  const tasks = [...new Set(studentnumbers)].map(sn => ({ task: sn }))
-  console.log(tasks.length, 'tasks to schedule')
-  publishAll(tasks)
-  return tasks.length
-}
-
-module.exports = {
-  scheduleActiveStudents,
-  scheduleStudentsByArray,
-  scheduleOldestNStudents,
-  scheduleMeta,
-  scheduleAllStudents,
-  scheduleAttainmentUpdate,
-  rescheduleScheduled,
-  rescheduleFetched,
-  rescheduleCreated,
-  scheduleDaily,
-  scheduleStudentCheck
-}

old_oodikone_stuff/updater_scheduler/src/student_list_updater.js

@@ -1,62 +0,0 @@
-const Schedule = require('../models')
-const { sleep, getStudentNumberChecksum } = require('./util')
-
-const createTasks = async (fromStudents = []) => {
-  console.log('Creating studentnumbers to DB...')
-
-  const writeStudents = async studentsToAdd => {
-    const tasks = studentsToAdd.map(student => ({ task: student, status: 'CREATED', type: 'student', active: true }))
-    const bulkOperation = Schedule.collection.initializeUnorderedBulkOp()
-    for (const task of tasks) {
-      bulkOperation.insert(task)
-    }
-    await bulkOperation.execute()
-  }
-
-  const writeToDB = async studentnumbers => {
-    try {
-      await writeStudents(studentnumbers)
-    } catch (err) {
-      const BulkWriteResult = err.result.result
-      const duplicateIndexError = 11000
-      const realErrors = BulkWriteResult.writeErrors.filter(e => e.err.code !== duplicateIndexError)
-      if (realErrors.length > 0 || BulkWriteResult.writeConcernErrors.length > 0) {
-        console.log(err)
-        return false
-      }
-    }
-    return true
-  }
-
-  const mongoWriteBatchSize = 1000
-  let studentnumbers = []
-  if (fromStudents.length) {
-    for (let i = 0; i < fromStudents.length; i++) {
-      studentnumbers.push(fromStudents[i])
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === fromStudents.length - 1) {
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        console.log(`Created tasks from ${studentnumbers}`)
-        studentnumbers = []
-      }
-    }
-  } else {
-    const minStudentNumber = 1010000
-    const maxStudentNumber = 1530000
-    for (let i = minStudentNumber; i <= maxStudentNumber; ++i) {
-      const studentNumber = '0' + i + getStudentNumberChecksum(i)
-      studentnumbers.push(studentNumber)
-      if (studentnumbers.length % mongoWriteBatchSize === 0 || i === maxStudentNumber) {
-        console.log('current studentnumber', i)
-        while (!(await writeToDB(studentnumbers))) {
-          await sleep(60 * 1000)
-        }
-        studentnumbers = []
-      }
-    }
-  }
-  console.log('Studentnumbers created to DB')
-}
-
-module.exports = { createTasks }

old_oodikone_stuff/updater_scheduler/src/util.js

@@ -1,43 +0,0 @@
-const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-
-// Shuffle in place, returns same array
-const shuffleArray = a => {
-  for (let i = a.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1))
-    const x = a[i]
-    a[i] = a[j]
-    a[j] = x
-  }
-  return a
-}
-
-const isValidStudentId = id => {
-  if (/^0\d{8}$/.test(id)) {
-    // is a 9 digit number
-    const multipliers = [7, 1, 3, 7, 1, 3, 7]
-    const checksum = id
-      .substring(1, 8)
-      .split('')
-      .reduce((sum, curr, index) => {
-        return (sum + curr * multipliers[index]) % 10
-      }, 0)
-    return (10 - checksum) % 10 == id[8]
-  }
-  return false
-}
-
-const getStudentNumberChecksum = studentNumber => {
-  const studentNumberString = String(studentNumber)
-  let checksumNumbers = [7, 3, 1]
-  let checksum = 0
-
-  for (let i = 0; i < studentNumberString.length; i++) {
-    // go from end t start
-    let currentNumber = studentNumberString[studentNumberString.length - (i + 1)]
-    checksum += currentNumber * checksumNumbers[i % checksumNumbers.length]
-  }
-
-  return (10 - (checksum % 10)) % 10
-}
-
-module.exports = { sleep, isValidStudentId, getStudentNumberChecksum, shuffleArray }

old_oodikone_stuff/updater_writer/.dockerignore

@@ -1 +0,0 @@
-node_modules

old_oodikone_stuff/updater_writer/.eslintignore

@@ -1,2 +0,0 @@
-*.sql
-updater/test_assets

old_oodikone_stuff/updater_writer/.eslintrc.json

@@ -1,13 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true,
-    "jest": true
-  },
-  "parser": "babel-eslint",
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "prettier/babel"],
-  "rules": {
-    "no-console": 0,
-    "prettier/prettier": "error"
-  }
-}

old_oodikone_stuff/updater_writer/.gitignore

@@ -1,4 +0,0 @@
-node_modules
-
-models
-migrations

old_oodikone_stuff/updater_writer/.prettierrc

@@ -1,9 +0,0 @@
-{
-  "semi": false,
-  "arrowParens": "avoid",
-  "singleQuote": true,
-  "printWidth": 120,
-  "tabWidth": 2,
-  "useTabs": false,
-  "jsxSingleQuote": false
-}

old_oodikone_stuff/updater_writer/Dockerfile

@@ -1,18 +0,0 @@
-FROM node:14-alpine
-
-WORKDIR /usr/src/app
-CMD npm start
-
-RUN apk add postgresql-client
-
-COPY ./updater_writer/package* ./
-
-RUN npm ci
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-
-COPY ./updater_writer /usr/src/app
-COPY ./shared/models /usr/src/app/models
-COPY ./shared/migrations /usr/src/app/database/migrations
-COPY ./shared/migrations_kone /usr/src/app/database/migrations_kone

old_oodikone_stuff/updater_writer/conf-backend.js

@@ -1,77 +0,0 @@
-require('dotenv').config()
-
-const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
-let DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const isTest = NODE_ENV === 'test'
-if (isTest) {
-  DB_URL = process.env.TEST_DB
-  DB_SCHEMA = process.env.TEST_DB_SCHEMA
-} else if (NODE_ENV === 'anon') {
-  DB_URL = process.env.ANON_DB
-}
-const frontend_addr = process.env.FRONT_URL
-const redis = process.env.REDIS
-const TOKEN_SECRET = process.env.TOKEN_SECRET
-const CERT_PATH = process.env.CERT_PATH // production/staging only
-const KEY_PATH = process.env.KEY_PATH // production/staging only
-const USERSERVICE_URL = process.env.USERSERVICE_URL
-const ANALYTICS_URL = process.env.ANALYTICS_URL
-const PORT = isTest ? 8079 : 8080
-const OODI_SECRET = process.env.OODI_SECRET
-
-const FEATURES = {
-  ERROR_HANDLER: false
-}
-
-const formatURL = url => {
-  return !!url && !url.startsWith('http') ? `http://${url}` : url
-}
-
-if (process.env.NODE_ENV === 'dev' && process.env.FEATURES) {
-  const toggled = process.env.FEATURES.split(',')
-  toggled
-    .map(toggle => toggle.trim())
-    .forEach(feature => {
-      if (FEATURES[feature] !== undefined) {
-        FEATURES[feature] = true
-      }
-    })
-}
-
-const OODI = {
-  test: 'http://localhost',
-  anon: process.env.OODI_ADDR_ANON
-}
-
-const OODI_ADDR = OODI[process.env.NODE_ENV] || process.env.OODI_ADDR
-const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-const OODI_SECRET_HEADER_KEY = 'x-oodi-secret'
-
-let requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
-if (process.env.NODE_ENV === 'staging') {
-  requiredGroup = ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-}
-if (process.env.NODE_ENV === 'dev' || isTest) {
-  requiredGroup = null
-}
-
-module.exports = {
-  frontend_addr,
-  DB_URL,
-  redis,
-  TOKEN_SECRET,
-  DB_SCHEMA,
-  OODI_ADDR,
-  CERT_PATH,
-  KEY_PATH,
-  FEATURES,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL),
-  ACCESS_TOKEN_HEADER_KEY,
-  PORT,
-  ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  requiredGroup,
-  OODI_SECRET,
-  OODI_SECRET_HEADER_KEY,
-  isTest
-}

old_oodikone_stuff/updater_writer/database/connection.js

@@ -1,69 +0,0 @@
-const Sequelize = require('sequelize')
-const Umzug = require('umzug')
-
-const conf = require('../conf-backend')
-
-const sequelize = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA,
-  searchPath: conf.DB_SCHEMA,
-  logging: false
-})
-sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
-
-const initializeDatabaseConnection = async () => {
-  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
-  const waitSeconds = 60
-  for (let i = 1; i <= waitSeconds; i++) {
-    try {
-      await sequelize.authenticate()
-      break
-    } catch (e) {
-      if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
-        throw e
-      }
-      console.log('.')
-      await sleep(1000)
-    }
-  }
-  if (!conf.isTest) {
-    try {
-      const migrator = new Umzug({
-        storage: 'sequelize',
-        storageOptions: {
-          sequelize: sequelize,
-          tableName: 'migrations'
-        },
-        logging: console.log,
-        migrations: {
-          params: [sequelize.getQueryInterface(), Sequelize],
-          path: `${process.cwd()}/database/migrations`,
-          pattern: /\.js$/,
-          schema: conf.DB_SCHEMA
-        }
-      })
-      const migrations = await migrator.up()
-
-      console.log('Migrations up to date', migrations)
-    } catch (e) {
-      console.log('Kone Migration error')
-      throw e
-    }
-  }
-}
-
-const forceSyncDatabase = async () => {
-  try {
-    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
-  } catch (e) {
-    // console.log(e)
-  }
-  console.log('OODI SYNC')
-  await sequelize.sync({ force: true })
-}
-
-module.exports = {
-  sequelize,
-  initializeDatabaseConnection,
-  forceSyncDatabase
-}

old_oodikone_stuff/updater_writer/database/dump_database.js

@@ -1,27 +0,0 @@
-const spawn = require('child_process').spawn
-const fs = require('fs')
-const { DB_SCHEMA } = require('../conf-backend')
-
-const dumpDatabase = async () => {
-  const { DB_URL } = process.env
-  const dumpOptions = ['-Fc', `-n${DB_SCHEMA}`, `${DB_URL}`]
-  console.log(dumpOptions)
-  const dumpProcess = spawn('pg_dump', dumpOptions, {
-    stdio: ['ignore', 'pipe', 'inherit']
-  })
-  dumpProcess.on('error', err => {
-    console.log(' dump process error:', err)
-  })
-  const writeStream = fs.createWriteStream(`./dumps/oodidb_prod-${new Date().toISOString()}`)
-
-  dumpProcess.stdout.pipe(writeStream)
-
-  dumpProcess.on('exit', async code => {
-    if (code !== 0) {
-      throw new Error('pg_dump: Bad exit code (' + code + ')')
-    } else {
-      console.log('database dump created')
-    }
-  })
-}
-module.exports = { dumpDatabase }

old_oodikone_stuff/updater_writer/database/force_sync_database.js

@@ -1,17 +0,0 @@
-const { seedAllMigrations } = require('./seed_migrations')
-const { forceSyncDatabase } = require('./connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-//sync()
-module.exports = { sync }

old_oodikone_stuff/updater_writer/database/seed_migrations.js

@@ -1,28 +0,0 @@
-const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
-
-const DEFAULT_PATH = `${process.cwd()}/database/migrations`
-const DEFAULT_PATH_KONE = `${process.cwd()}/database/migrations_kone`
-
-const saveMigrationsToDatabase = Migration => async filenames =>
-  Promise.all(filenames.map(name => Migration.upsert({ name })))
-
-const seedMigrations = async (migrationfilepath = DEFAULT_PATH) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(Migration)(filenames)
-}
-
-const seedMigrationsKone = async (migrationfilepath = DEFAULT_PATH_KONE) => {
-  const filenames = fs.readdirSync(migrationfilepath)
-  await saveMigrationsToDatabase(MigrationKone)(filenames)
-}
-
-const seedAllMigrations = async () => {
-  await seedMigrations()
-  await seedMigrationsKone()
-}
-
-module.exports = {
-  seedMigrations,
-  seedAllMigrations
-}

old_oodikone_stuff/updater_writer/index.js

@@ -1,69 +0,0 @@
-const natsStreaming = require('node-nats-streaming')
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./updater/database_updater')
-const logger = require('./logger')
-const { initializeDatabaseConnection } = require('./database/connection')
-
-initializeDatabaseConnection()
-  .then(() => {
-    const stan = natsStreaming.connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
-
-    console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-    const opts = stan.subscriptionOptions()
-    opts.setManualAckMode(true)
-    opts.setAckWait(10 * 60 * 1000) // some students have taken over 5 min to write!
-    // opts.setDeliverAllAvailable()
-    // opts.setDurableName('durable')
-    opts.setMaxInFlight(1)
-
-    stan.on('connect', function() {
-      const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
-      const attSub = stan.subscribe('UpdateAttainmentDates', 'updater.workers.attainments', opts)
-      const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
-
-      const writeStudent = priority => async msg => {
-        let data = null
-        try {
-          const start = new Date()
-          data = JSON.parse(msg.getData())
-          if (data.task === 'meta') {
-            await updateMeta(data.data)
-          } else {
-            await updateStudent(data.data)
-          }
-          stan.publish(
-            'status',
-            JSON.stringify({ task: data.task, status: 'DONE', timems: new Date() - start }, priority),
-            err => {
-              if (err) console.log(err)
-            }
-          )
-        } catch (err) {
-          console.log('update failed', data.task, err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      }
-
-      sub.on('message', writeStudent(false))
-      prioSub.on('message', writeStudent(true))
-
-      attSub.on('message', async msg => {
-        try {
-          await updateAttainmentMeta()
-          stan.publish('status', JSON.stringify({ task: 'attainment', status: 'DONE' }), err => {
-            if (err) {
-              console.log('publish failed')
-            }
-          })
-        } catch (err) {
-          console.log('attainment meta update failed', err)
-          logger.info('failure', { service: 'WRITER' })
-        }
-        msg.ack()
-      })
-    })
-  })
-  .catch(e => {
-    process.exitCode = 1
-    console.log(e)
-  })

old_oodikone_stuff/updater_writer/logger.js

@@ -1,27 +0,0 @@
-const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
-
-const transports = []
-
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
-  transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'UPDATER',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: process.env.NODE_ENV,
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf'
-      }
-    })
-  )
-}
-
-if (process.env.NODE_ENV !== 'test') {
-  transports.push(new winston.transports.File({ filename: 'debug.log' }))
-}
-
-const logger = winston.createLogger({ transports })
-
-module.exports = logger

old_oodikone_stuff/updater_writer/package.json

@@ -1,39 +0,0 @@
-{
-  "name": "updater_writer",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run jest",
-    "jest": "NODE_ENV=test jest --runInBand --verbose",
-    "test_docker": "docker exec -t updater_writer /bin/bash -c 'npm test'",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js",
-    "force_sync": "node -e \"require('./database/force_sync_database').sync()\"",
-    "dump": "node -e \"require('./database/dump_database').dumpDatabase()\""
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "dotenv": "^8.0.0",
-    "jest": "^24.8.0",
-    "lodash": "^4.17.19",
-    "node-nats-streaming": "^0.2.6",
-    "pg": "^8.6.0",
-    "sequelize": "^6.6.2",
-    "umzug": "^2.2.0",
-    "winston": "^3.2.1",
-    "winston-log2gelf": "^2.3.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.0.3",
-    "eslint": "^6.6.0",
-    "eslint-config-prettier": "^6.3.0",
-    "eslint-plugin-prettier": "^3.1.0",
-    "nodemon": "^1.19.4",
-    "prettier": "^1.18.2",
-    "prettier-eslint": "^9.0.0"
-  }
-}

old_oodikone_stuff/updater_writer/updater/database_updater.js

@@ -1,217 +0,0 @@
-const { notIn } = require('sequelize').Op
-const { sequelize } = require('../database/connection')
-const { sortBy, flatMap, uniqBy, sortedUniqBy } = require('lodash')
-
-const {
-  Student,
-  Credit,
-  Course,
-  CreditTeacher,
-  Teacher,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { updateAttainmentDates } = require('./update_attainment_dates')
-
-const getColumnsToUpdate = arr => (arr[0] ? Object.keys(arr[0]) : [])
-
-const updateAttainments = async (studyAttainments, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const courses = sortedUniqBy(sortBy(studyAttainments.map(e => e.course), 'code'), 'code')
-  const disciplines = sortBy(flatMap(studyAttainments, e => e.course.disciplines || []), 'discipline_id', 'course_id')
-  const providers = sortedUniqBy(
-    sortBy(flatMap(studyAttainments, e => e.course.providers), 'providercode'),
-    'providercode'
-  )
-  const courseproviders = sortBy(flatMap(studyAttainments, e => e.course.courseproviders), 'coursecode', 'providercode')
-  const credits = sortedUniqBy(sortBy(studyAttainments.map(e => e.credit), 'id'), 'id')
-  const teachers = sortedUniqBy(sortBy(flatMap(studyAttainments, e => e.teachers || []), 'id'), 'id')
-  const creditTeachers = sortBy(flatMap(studyAttainments, e => e.creditTeachers), 'teacher_id', 'credit_id')
-
-  await Promise.all([
-    Course.bulkCreate(courses, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(courses)
-    }),
-    Provider.bulkCreate(providers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(providers)
-    }),
-    Credit.bulkCreate(credits, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(credits)
-    }),
-    Teacher.bulkCreate(teachers, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(teachers)
-    })
-  ])
-
-  await Promise.all([
-    // must be after courses
-    CourseDisciplines.bulkCreate(disciplines, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after providers
-    CourseProvider.bulkCreate(courseproviders, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-    // must be after teachers
-    CreditTeacher.bulkCreate(creditTeachers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudyRights = async (studyRights, studentnumber, transaction) => {
-  // Sort data to avoid deadlocks. If there are duplicate primary keys in the same array, then bulkCreate won't work.
-  const studyRightExtents = sortedUniqBy(sortBy(studyRights.map(e => e.studyRightExtent), 'extentcode'), 'extentcode')
-  const studyrights = sortedUniqBy(sortBy(studyRights.map(e => e.studyright), 'studyrightid'), 'studyrightid')
-  const elementDetails = sortedUniqBy(sortBy(flatMap(studyRights, e => e.elementDetails), 'code'), 'code')
-  const studyRightElements = uniqBy(
-    sortBy(flatMap(studyRights, e => e.studyRightElements), 'startdate', 'enddate', 'studyrightid', 'code'),
-    s => `${s.startdate}${s.enddate}${s.studyrightid}${s.code}`
-  )
-  const transfers = sortBy(
-    flatMap(studyRights, e => e.transfers),
-    'transferdate',
-    'studentnumber',
-    'studyrightid',
-    'sourcecode',
-    'targetcode'
-  )
-
-  const studyrightids = studyrights.map(s => s.studyrightid)
-  await Studyright.destroy(
-    { where: { student_studentnumber: studentnumber, studyrightid: { [notIn]: studyrightids } } },
-    { transaction }
-  )
-  await StudyrightElement.destroy({ where: { studentnumber } }, { transaction })
-
-  await Promise.all([
-    StudyrightExtent.bulkCreate(studyRightExtents, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyRightExtents)
-    }),
-
-    Studyright.bulkCreate(studyrights, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(studyrights)
-    }),
-
-    ElementDetails.bulkCreate(elementDetails, {
-      transaction,
-      updateOnDuplicate: getColumnsToUpdate(elementDetails)
-    }),
-
-    StudyrightElement.bulkCreate(studyRightElements, {
-      transaction,
-      ignoreDuplicates: true
-    }),
-
-    Transfers.bulkCreate(transfers, {
-      transaction,
-      ignoreDuplicates: true
-    })
-  ])
-}
-
-const updateStudent = async student => {
-  let { studentInfo, studyAttainments, semesterEnrollments, studyRights } = student
-
-  // sort data to avoid deadlocks
-  semesterEnrollments = uniqBy(
-    sortBy(semesterEnrollments, 'studentnumber', 'semestercode'),
-    sE => `${sE.semestercode}${sE.studentnumber}`
-  )
-
-  const transaction = await sequelize.transaction()
-  try {
-    await Student.upsert(studentInfo, { transaction })
-
-    // Change this to bulkCreate after https://github.com/sequelize/sequelize/issues/11569
-    // is merged and released
-    for (let i = 0; i < semesterEnrollments.length; i++) {
-      await SemesterEnrollment.upsert(semesterEnrollments[i], { transaction })
-    }
-
-    if (studyAttainments) await updateAttainments(studyAttainments, transaction)
-
-    if (studyRights) await updateStudyRights(studyRights, studentInfo.studentnumber, transaction)
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-const updateAttainmentMeta = async () => {
-  await updateAttainmentDates()
-}
-
-const updateMeta = async ({
-  faculties,
-  courseRealisationsTypes,
-  semesters,
-  creditTypeCodes,
-  courseTypeCodes,
-  disciplines
-}) => {
-  const transaction = await sequelize.transaction()
-
-  try {
-    await Promise.all([
-      CourseType.bulkCreate(uniqBy(courseTypeCodes, 'coursetypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseTypeCodes)
-      }),
-      Organisation.bulkCreate(uniqBy(faculties, 'code'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(faculties)
-      }),
-      CourseRealisationType.bulkCreate(uniqBy(courseRealisationsTypes, 'realisationtypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(courseRealisationsTypes)
-      }),
-      Semester.bulkCreate(uniqBy(semesters, 'semestercode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(semesters)
-      }),
-      CreditType.bulkCreate(uniqBy(creditTypeCodes, 'credittypecode'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(creditTypeCodes)
-      }),
-      Discipline.bulkCreate(uniqBy(disciplines, 'discipline_id'), {
-        transaction,
-        updateOnDuplicate: getColumnsToUpdate(disciplines)
-      })
-    ])
-
-    await transaction.commit()
-  } catch (err) {
-    await transaction.rollback()
-    throw err
-  }
-}
-
-module.exports = {
-  updateStudent,
-  updateMeta,
-  updateAttainmentMeta
-}

old_oodikone_stuff/updater_writer/updater/database_updater.test.js

@@ -1,518 +0,0 @@
-const { updateStudent, updateMeta, updateAttainmentMeta } = require('./database_updater')
-const {
-  Student,
-  Credit,
-  Course,
-  Organisation,
-  CourseRealisationType,
-  Semester,
-  CreditType,
-  CourseType,
-  Discipline,
-  CourseDisciplines,
-  Teacher,
-  CreditTeacher,
-  SemesterEnrollment,
-  Provider,
-  CourseProvider,
-  Studyright,
-  StudyrightExtent,
-  ElementDetails,
-  StudyrightElement,
-  Transfers
-} = require('../models/index')
-const { students } = require('./test_assets/test_students')
-const meta = require('./test_assets/meta')
-const { sequelize, forceSyncDatabase } = require('../database/connection')
-const { seedMigrations } = require('../database/seed_migrations')
-jest.setTimeout(10000)
-describe('Updater works', () => {
-  beforeAll(async () => {
-    await forceSyncDatabase()
-    await seedMigrations()
-
-    await updateMeta(meta)
-    await updateStudent(students[0])
-    await updateStudent(students[1])
-    await updateStudent(students[2])
-    await updateAttainmentMeta()
-  })
-
-  afterAll(async () => {
-    await sequelize.close()
-  })
-
-  describe('All metadata is written', () => {
-    test('Organisations are written to database', async () => {
-      const organizations = await Organisation.findAll()
-      expect(organizations.length).toBe(17)
-      expect(organizations.find(_ => _.code === 'H50').name.find(n => n.langcode === 'fi').text).toBe(
-        'Matemaattis-luonnontieteellinen tiedekunta'
-      )
-    })
-
-    test('Course realisation types are written to database', async () => {
-      const courseRealisationTypes = await CourseRealisationType.findAll()
-      expect(courseRealisationTypes.length).toBe(18)
-      expect(courseRealisationTypes.map(_ => _.realisationtypecode)).not.toContain(null)
-      expect(courseRealisationTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Semesters are written to database', async () => {
-      const semesters = await Semester.findAll()
-      expect(semesters.length).toBe(160)
-      expect(semesters.map(_ => _.semestercode)).toEqual(expect.arrayContaining([...Array(160).keys()].map(_ => ++_)))
-      expect(semesters.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-      expect(semesters.map(_ => _.startdate)).not.toContain(null)
-      expect(semesters.map(_ => _.enddate)).not.toContain(null)
-      expect(semesters.map(_ => _.createdAt)).not.toContain(null)
-      expect(semesters.map(_ => _.updatedAt)).not.toContain(null)
-      expect(new Set(semesters.map(_ => _.yearcode)).size).toBe(80)
-      expect(semesters.find(_ => _.yearcode === 1).yearname).toBe('1950-51')
-    })
-
-    test('Credit types are written to database', async () => {
-      const creditTypes = await CreditType.findAll()
-      expect(creditTypes.length).toBe(11)
-      expect(creditTypes.map(_ => _.credittypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(creditTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Course types are written to database', async () => {
-      const courseTypes = await CourseType.findAll()
-      expect(courseTypes.length).toBe(35)
-      expect(courseTypes.map(_ => _.coursetypecode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-      expect(courseTypes.map(_ => _.name)[0]).toMatchObject({
-        en: expect.any(String),
-        fi: expect.any(String),
-        sv: expect.any(String)
-      })
-    })
-
-    test('Disciplines are written to database', async () => {
-      const disciplines = await Discipline.findAll()
-      expect(disciplines.length).toBe(430)
-      expect(disciplines.map(_ => _.discipline_id)).toEqual(expect.arrayContaining([expect.any(String)]))
-      expect(disciplines.map(_ => _.name)[0]).toMatchObject({
-        fi: expect.any(String)
-      })
-    })
-  })
-  describe('Updater writes students the right way', () => {
-    test('Student info is written to database', async () => {
-      const students = await Student.findAll({
-        order: [['studentnumber', 'DESC']]
-      })
-      expect(students.length).toBe(3)
-      const {
-        studentnumber,
-        lastname,
-        firstnames,
-        abbreviatedname,
-        birthdate,
-        communicationlanguage,
-        creditcount,
-        dateofuniversityenrollment,
-        city_fi,
-        city_sv,
-        gender_code,
-        gender_fi,
-        createdAt,
-        updatedAt
-      } = students[0]
-
-      expect(studentnumber).toBe('014441008')
-      expect(lastname).toBe('Orttenvuori')
-      expect(firstnames).toBe('Oona')
-      expect(abbreviatedname).toBe('Orttenvuori Oona')
-      expect(birthdate.toISOString()).toBe(new Date('1966-04-24 00:00:00+00').toISOString())
-      expect(communicationlanguage).toBe('Finnish')
-      expect(creditcount).toBe(34)
-      expect(dateofuniversityenrollment).toBe(null)
-      expect(city_fi).toBe('HELSINKI')
-      expect(city_sv).toBe('HELSINGFORS')
-      expect(gender_code).toBe(2)
-      expect(gender_fi).toBe('Nainen')
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-
-      const { creditcount: creditcount1, dateofuniversityenrollment: dofue } = students[1]
-
-      expect(creditcount1).toBe(380)
-      expect(dofue.toISOString()).toBe(new Date('2003-07-31 21:00:00+00').toISOString())
-    })
-
-    test('Courses are written to database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const {
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        startdate,
-        enddate,
-        max_attainment_date,
-        min_attainment_date,
-        coursetypecode
-      } = courses[0]
-
-      expect(courses.length).toBe(147)
-      expect(code).toBe('00031')
-      expect(name.en).toBe('Master of Science (Agriculture and Forestry)')
-      expect(latest_instance_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(is_study_module).toBeTruthy()
-      expect(startdate.toISOString()).toBe(new Date('1899-12-31 00:00:00+00').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2112-12-20 00:00:00+00').toISOString())
-      expect(max_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-      expect(min_attainment_date.toISOString()).toBe(new Date('2010-03-16 00:00:00+00').toISOString())
-
-      expect(coursetypecode).toBe(8)
-
-      const { code: code1, is_study_module: ism, coursetypecode: ctc } = courses[6]
-
-      expect(code1).toBe('402512')
-      expect(ism).toBe(false)
-      expect(ctc).toBe(2)
-    })
-
-    test('Credits are written to database', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const {
-        id,
-        grade,
-        student_studentnumber,
-        credits,
-        ordering,
-        attainment_date,
-        isStudyModule,
-        lastmodifieddate,
-        course_code,
-        credittypecode /*, semestercode*/
-      } = allCredits[0]
-
-      expect(allCredits.length).toBe(150)
-      expect(id).toBe('2340421')
-      expect(grade).toBe('4')
-      expect(allCredits.map(_ => _.grade)).not.toContain(null)
-      expect(student_studentnumber).toBe('014441008')
-      expect(credits).toBe(2)
-      expect(allCredits.map(_ => _.credits)).not.toContain(null)
-      expect(ordering).toBe('1997-10-21')
-      expect(attainment_date.toISOString()).toBe(new Date('1997-10-21 00:00:00+00').toISOString())
-      expect(isStudyModule).toBe(false)
-      expect(lastmodifieddate).not.toBe(null)
-      expect(course_code).toBe('65002')
-      expect(allCredits.map(_ => _.course_code)).not.toContain(null)
-      expect(credittypecode).toBe(4)
-      expect(allCredits.map(_ => _.semestercode)).toEqual(expect.arrayContaining([expect.any(Number)]))
-    })
-
-    test('Teachers are written to database', async () => {
-      const teachers = await Teacher.findAll({ order: [['name', 'ASC']] })
-      const { id, code, name } = teachers[0]
-
-      expect(teachers.length).toBe(51)
-      expect(id).toBe('9016947')
-      expect(code).toBe('hpyrhone')
-      expect(name).toBe('Ahvenniemi Emilia')
-    })
-
-    test('Credit Teachers are written to database', async () => {
-      const creditteachers = await CreditTeacher.findAll()
-
-      expect(creditteachers.length).toBe(119)
-      expect(creditteachers.map(_ => _.credit_id)).not.toContain(null)
-      expect(creditteachers.map(_ => _.teacher_id)).not.toContain(null)
-    })
-
-    test('Semester enrollments are written to database', async () => {
-      const semesterEnrollments = await SemesterEnrollment.findAll({
-        order: [['semestercode', 'ASC']]
-      })
-      const { id, enrollmenttype, enrollment_date, studentnumber, semestercode } = semesterEnrollments[0]
-
-      expect(semesterEnrollments.length).toBe(28)
-      expect(id).toEqual(expect.any(String))
-      expect(enrollmenttype).toBe(1)
-      expect(enrollment_date.toISOString()).toBe(new Date('2003-08-17 00:00:00+00').toISOString())
-      expect(studentnumber).toBe('014272112')
-      expect(semestercode).toBe(107)
-    })
-
-    test('Providers are written to database', async () => {
-      const providers = await Provider.findAll({
-        order: [['providercode', 'DESC']]
-      })
-      const { providercode, name } = providers[0]
-
-      expect(providers.length).toBe(20)
-      expect(providercode).toBe('H906')
-      expect(name).toMatchObject({
-        en: 'Language Centre',
-        fi: 'Kielikeskus',
-        sv: 'Språkcentrum'
-      })
-      expect(providers.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course providers are written to database', async () => {
-      const courseproviders = await CourseProvider.findAll()
-
-      expect(courseproviders.length).toBe(123)
-      expect(courseproviders.map(_ => _.coursecode)).not.toContain(null)
-      expect(courseproviders.map(_ => _.providercode)).not.toContain(null)
-    })
-    test('Course disciplines are written to database', async () => {
-      const courseDisciplines = await CourseDisciplines.findAll()
-      expect(courseDisciplines.length).toBe(40)
-      expect(courseDisciplines.map(_ => _.discipline_id)).not.toContain(null)
-      expect(courseDisciplines.map(_ => _.course_id)).not.toContain(null)
-    })
-
-    test('Study right extent are written to database', async () => {
-      const studyrightextent = await StudyrightExtent.findAll()
-      const { extentcode, name, createdAt, updatedAt } = studyrightextent[0]
-
-      expect(studyrightextent.length).toBe(2)
-      expect(extentcode).toBe(1)
-      expect(name.en).toBe("Bachelor's Degree")
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('Study rights are written to database', async () => {
-      const studyrights = await Studyright.findAll({
-        order: [['studyrightid', 'DESC']]
-      })
-      const {
-        studyrightid /*, canceldate*/,
-        cancelorganisation,
-        enddate,
-        givendate,
-        graduated,
-        prioritycode,
-        startdate,
-        studystartdate,
-        organization_code,
-        student_studentnumber,
-        extentcode
-      } = studyrights[0]
-
-      expect(studyrights.length).toBe(4)
-      expect(studyrightid).toBe('67628057')
-      expect(cancelorganisation).toBe('4325')
-      expect(enddate.toISOString()).toBe(new Date('2010-04-12 00:00:00+00').toISOString())
-      expect(givendate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(graduated).toBe(1)
-      expect(prioritycode).toBe(30)
-      expect(startdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(studystartdate.toISOString()).toBe(new Date('2004-07-31 00:00:00+00').toISOString())
-      expect(organization_code).toBe('4325')
-      expect(student_studentnumber).toBe('011120775')
-      expect(extentcode).toBe(1)
-    })
-
-    test('element details are written to database', async () => {
-      const elementDetails = await ElementDetails.findAll({
-        order: [['code', 'DESC']]
-      })
-      const { code, name, type, createdAt, updatedAt } = elementDetails[0]
-
-      expect(elementDetails.length).toBe(12)
-      expect(code).toBe('A2004')
-      expect(name.en).toBe('Government Decree (794/2004) on University Degrees')
-      expect(type).toBe(15)
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-    })
-
-    test('study right elements are written to database', async () => {
-      const studyright_elements = await StudyrightElement.findAll({
-        order: ['startdate', 'enddate', 'studyrightid', 'code']
-      })
-      const { id, startdate, enddate, createdAt, updatedAt, studyrightid, code, studentnumber } = studyright_elements[0]
-
-      expect(studyright_elements.length).toBe(21)
-      expect(id).toBe(11)
-      expect(startdate.toISOString()).toBe(new Date('2003-07-31T21:00:00.000Z').toISOString())
-      expect(enddate.toISOString()).toBe(new Date('2007-08-24T21:00:00.000Z').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studyrightid).toBe('45853818')
-      expect(code).toBe('A1995MAAT')
-      expect(studentnumber).toBe('014272112')
-    })
-    test('transfers are written to database', async () => {
-      const transfers = await Transfers.findAll()
-      const {
-        id,
-        transferdate,
-        createdAt,
-        updatedAt,
-        studentnumber,
-        studyrightid,
-        sourcecode,
-        targetcode
-      } = transfers[0]
-
-      expect(transfers.length).toBe(1)
-      expect(id).toBe('1')
-      expect(transferdate.toISOString()).toBe(new Date('2009-07-30 21:00:00+00').toISOString())
-      expect(createdAt).not.toBe(undefined)
-      expect(updatedAt).not.toBe(undefined)
-      expect(studentnumber).toBe('014272112')
-      expect(studyrightid).toBe('62346253')
-      expect(sourcecode).toBe('820016')
-      expect(targetcode).toBe('820042')
-    })
-  })
-  describe('Updater updates existing students correctly', () => {
-    beforeAll(async () => {
-      const { studentInfo, studyRights, studyAttainments } = students[0]
-      students[0].studentInfo = {
-        ...studentInfo,
-        email: 'Calgary.Flames@NHLonJätettä.fi',
-        mobile: '0400521981',
-        address: 'Yliopistonkatu 4',
-        creditcount: 322,
-        age: 41,
-        lastname: 'Tauriainen',
-        gender_code: 1,
-        abbreviatedname: 'Tauriainen Maria Helena'
-      }
-      const newCredit = {
-        id: 'ayyyyylmao',
-        grade: '5',
-        credits: 5,
-        ordering: '2019.05.04',
-        credittypecode: 1,
-        student_studentnumber: '011120775',
-        attainment_date: new Date(),
-        course_code: '420_666',
-        semestercode: 138,
-        isStudyModule: false
-      }
-      const newCourse = {
-        code: '420_666',
-        name: {
-          fi: 'Kaljan juonti',
-          en: 'Beer drinking',
-          sv: 'dryckade öl'
-        },
-        latest_instance_date: new Date(),
-        is_study_module: false,
-        coursetypecode: 1,
-        disciplines: [],
-        startdate: new Date(),
-        enddate: new Date(),
-        providers: [],
-        courseproviders: []
-      }
-
-      const newCreditTeachers = [
-        {
-          credit_id: 'ayyyyylmao',
-          teacher_id: '9016417'
-        }
-      ]
-      const newAttainment = {
-        credit: newCredit,
-        creditTeachers: newCreditTeachers,
-        course: newCourse
-      }
-      students[0].studyAttainments = [
-        ...studyAttainments,
-        newAttainment,
-        {
-          ...students[1].studyAttainments[2],
-          creditTeachers: [],
-          credit: {
-            ...students[1].studyAttainments[2].credit,
-            id: 'Kvas',
-            student_studentnumber: '011120775'
-          }
-        }
-      ]
-      students[0].studyAttainments[2] = {
-        ...students[0].studyAttainments[2],
-        credit: {
-          ...students[0].studyAttainments[2].credit,
-          grade: '0'
-        }
-      }
-      const newStudyRight = {
-        ...studyRights[0],
-        studyright: {
-          ...studyRights[0].studyright
-        }
-      }
-      students[0].studyRights = [newStudyRight]
-
-      await updateStudent(students[0])
-    })
-
-    test('StudentInfo for existing student is updated correclty', async () => {
-      const updatedStudent = await Student.findOne({
-        where: {
-          studentnumber: '011120775'
-        }
-      })
-      const {
-        studentnumber,
-        lastname,
-        abbreviatedname,
-        gender_code,
-        email,
-        age,
-        gender_fi,
-        address,
-        mobile
-      } = updatedStudent
-
-      expect(studentnumber).toBe('011120775')
-      expect(lastname).toBe('Tauriainen')
-      expect(abbreviatedname).toBe('Tauriainen Maria Helena')
-      expect(email).toBe('Calgary.Flames@NHLonJätettä.fi')
-      expect(gender_code).toBe(1)
-      expect(gender_fi).toBe('Nainen')
-      expect(age).toBe(41)
-      expect(address).toBe('Yliopistonkatu 4')
-      expect(mobile).toBe('0400521981')
-    })
-    test('Existing courses are updated (not duplicated) in database', async () => {
-      const courses = await Course.findAll({ order: [['code', 'ASC']] })
-      const newCourse = courses.find(_ => _.code === '420_666')
-      expect(courses.length).toBe(148)
-
-      expect(newCourse).not.toBe(undefined)
-    })
-
-    test('Updating credits works', async () => {
-      const allCredits = await Credit.findAll({ order: [['id', 'ASC']] })
-      const updatedCredit = allCredits.find(_ => _.id === students[0].studyAttainments[2].credit.id)
-      expect(allCredits.length).toBe(152)
-      expect(updatedCredit.grade).toBe('0')
-    })
-
-    test('Studyrights are rewritten', async () => {
-      const updatedStudyRights = await Studyright.findAll({
-        where: {
-          student_studentnumber: '011120775'
-        }
-      })
-      expect(updatedStudyRights.length).toBe(1)
-      // theres really not a lot to test here since everything related to studyrights are just rewritten
-    })
-  })
-})

old_oodikone_stuff/updater_writer/updater/update_attainment_dates.js

@@ -1,28 +0,0 @@
-const { sequelize } = require('../database/connection')
-
-const updateAttainmentDates = async () => {
-  const { schema } = sequelize.options
-  const transaction = await sequelize.transaction()
-  try {
-    await sequelize.query(
-      `
-      -- Block writing to table to avoid deadlocks:
-      LOCK TABLE ONLY ${schema}.course IN EXCLUSIVE MODE;
-
-      UPDATE  ${schema}.course
-      SET max_attainment_date = cr.max, min_attainment_date = cr.min
-      FROM (select course_code, max(attainment_date), min(attainment_date) from ${schema}.credit group by course_code) cr
-      WHERE course.code=cr.course_code;
-     `,
-      { transaction }
-    )
-    await transaction.commit()
-  } catch (e) {
-    await transaction.rollback()
-    throw e
-  }
-}
-
-module.exports = {
-  updateAttainmentDates
-}

package-lock.json

@@ -7,23 +7,23 @@
     "": {
       "license": "ISC",
       "devDependencies": {
-        "@babel/eslint-parser": "^7.14.7",
+        "@babel/eslint-parser": "^7.15.0",
         "@babel/plugin-syntax-jsx": "^7.14.5",
-        "concurrently": "^6.2.0",
-        "cypress": "^7.7.0",
-        "eslint": "^7.30.0",
+        "concurrently": "^6.2.1",
+        "cypress": "^8.2.0",
+        "eslint": "^7.32.0",
         "eslint-config-airbnb": "^18.2.1",
         "eslint-config-prettier": "^8.3.0",
         "eslint-plugin-babel": "^5.3.1",
         "eslint-plugin-cypress": "^2.11.3",
-        "eslint-plugin-import": "^2.23.4",
+        "eslint-plugin-import": "^2.24.0",
         "eslint-plugin-jsx-a11y": "^6.4.1",
         "eslint-plugin-prefer-object-spread": "^1.2.1",
         "eslint-plugin-prettier": "^3.4.0",
         "eslint-plugin-react": "^7.24.0",
         "eslint-plugin-react-hooks": "^4.2.0",
         "husky": "^7.0.1",
-        "lint-staged": "^11.0.1",
+        "lint-staged": "^11.1.2",
         "prettier": "^2.3.2",
         "stylelint": "^13.13.1",
         "stylelint-config-prettier": "^8.0.2",
@@ -101,9 +101,9 @@
       }
     },
     "node_modules/@babel/eslint-parser": {
-      "version": "7.14.7",
-      "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-7.14.7.tgz",
-      "integrity": "sha512-6WPwZqO5priAGIwV6msJcdc9TsEPzYeYdS/Xuoap+/ihkgN6dzHp2bcAAwyWZ5bLzk0vvjDmKvRwkqNaiJ8BiQ==",
+      "version": "7.15.0",
+      "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-7.15.0.tgz",
+      "integrity": "sha512-+gSPtjSBxOZz4Uh8Ggqu7HbfpB8cT1LwW0DnVVLZEJvzXauiD0Di3zszcBkRmfGGrLdYeHUwcflG7i3tr9kQlw==",
       "dev": true,
       "dependencies": {
         "eslint-scope": "^5.1.1",
@@ -585,9 +585,9 @@
       }
     },
     "node_modules/@eslint/eslintrc": {
-      "version": "0.4.2",
-      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.2.tgz",
-      "integrity": "sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg==",
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
+      "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==",
       "dev": true,
       "dependencies": {
         "ajv": "^6.12.4",
@@ -1446,9 +1446,9 @@
       "dev": true
     },
     "node_modules/concurrently": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-6.2.0.tgz",
-      "integrity": "sha512-v9I4Y3wFoXCSY2L73yYgwA9ESrQMpRn80jMcqMgHx720Hecz2GZAvTI6bREVST6lkddNypDKRN22qhK0X8Y00g==",
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-6.2.1.tgz",
+      "integrity": "sha512-emgwhH+ezkuYKSHZQ+AkgEpoUZZlbpPVYCVv7YZx0r+T7fny1H03r2nYRebpi2DudHR4n1Rgbo2YTxKOxVJ4+g==",
       "dev": true,
       "dependencies": {
         "chalk": "^4.1.0",
@@ -1580,9 +1580,9 @@
       }
     },
     "node_modules/cypress": {
-      "version": "7.7.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-7.7.0.tgz",
-      "integrity": "sha512-uYBYXNoI5ym0UxROwhQXWTi8JbUEjpC6l/bzoGZNxoKGsLrC1SDPgIDJMgLX/MeEdPL0UInXLDUWN/rSyZUCjQ==",
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
+      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
@@ -1719,12 +1719,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/dedent": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
-      "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=",
-      "dev": true
-    },
     "node_modules/deep-is": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
@@ -1965,13 +1959,13 @@
       }
     },
     "node_modules/eslint": {
-      "version": "7.30.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz",
-      "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==",
+      "version": "7.32.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz",
+      "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==",
       "dev": true,
       "dependencies": {
         "@babel/code-frame": "7.12.11",
-        "@eslint/eslintrc": "^0.4.2",
+        "@eslint/eslintrc": "^0.4.3",
         "@humanwhocodes/config-array": "^0.5.0",
         "ajv": "^6.10.0",
         "chalk": "^4.0.0",
@@ -2073,34 +2067,28 @@
       }
     },
     "node_modules/eslint-import-resolver-node": {
-      "version": "0.3.4",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz",
-      "integrity": "sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==",
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
+      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
       "dev": true,
       "dependencies": {
-        "debug": "^2.6.9",
-        "resolve": "^1.13.1"
+        "debug": "^3.2.7",
+        "resolve": "^1.20.0"
       }
     },
     "node_modules/eslint-import-resolver-node/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
       "dev": true,
       "dependencies": {
-        "ms": "2.0.0"
+        "ms": "^2.1.1"
       }
     },
-    "node_modules/eslint-import-resolver-node/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
-      "dev": true
-    },
     "node_modules/eslint-module-utils": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.1.tgz",
-      "integrity": "sha512-ZXI9B8cxAJIH4nfkhTwcRTEAnrVfobYqwjWy/QMCZ8rHkZHFjf9yO4BzpiF9kCSfNlMG54eKigISHpX0+AaT4A==",
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz",
+      "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==",
       "dev": true,
       "dependencies": {
         "debug": "^3.2.7",
@@ -2156,17 +2144,17 @@
       }
     },
     "node_modules/eslint-plugin-import": {
-      "version": "2.23.4",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.23.4.tgz",
-      "integrity": "sha512-6/wP8zZRsnQFiR3iaPFgh5ImVRM1WN5NUWfTIRqwOdeiGJlBcSk82o1FEVq8yXmy4lkIzTo7YhHCIxlU/2HyEQ==",
+      "version": "2.24.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
+      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
       "dev": true,
       "dependencies": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.4",
-        "eslint-module-utils": "^2.6.1",
+        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
         "is-core-module": "^2.4.0",
@@ -3788,9 +3776,9 @@
       "dev": true
     },
     "node_modules/lint-staged": {
-      "version": "11.0.1",
-      "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-11.0.1.tgz",
-      "integrity": "sha512-RkTA1ulE6jAGFskxpGAwxfVRXjHp7D9gFg/+KMARUWMPiVFP0t28Em2u0gL8sA0w3/ck3TC57F2v2RNeQ5XPnw==",
+      "version": "11.1.2",
+      "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-11.1.2.tgz",
+      "integrity": "sha512-6lYpNoA9wGqkL6Hew/4n1H6lRqF3qCsujVT0Oq5Z4hiSAM7S6NksPJ3gnr7A7R52xCtiZMcEUNNQ6d6X5Bvh9w==",
       "dev": true,
       "dependencies": {
         "chalk": "^4.1.1",
@@ -3798,7 +3786,6 @@
         "commander": "^7.2.0",
         "cosmiconfig": "^7.0.0",
         "debug": "^4.3.1",
-        "dedent": "^0.7.0",
         "enquirer": "^2.3.6",
         "execa": "^5.0.0",
         "listr2": "^3.8.2",
@@ -6612,9 +6599,9 @@
       }
     },
     "@babel/eslint-parser": {
-      "version": "7.14.7",
-      "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-7.14.7.tgz",
-      "integrity": "sha512-6WPwZqO5priAGIwV6msJcdc9TsEPzYeYdS/Xuoap+/ihkgN6dzHp2bcAAwyWZ5bLzk0vvjDmKvRwkqNaiJ8BiQ==",
+      "version": "7.15.0",
+      "resolved": "https://registry.npmjs.org/@babel/eslint-parser/-/eslint-parser-7.15.0.tgz",
+      "integrity": "sha512-+gSPtjSBxOZz4Uh8Ggqu7HbfpB8cT1LwW0DnVVLZEJvzXauiD0Di3zszcBkRmfGGrLdYeHUwcflG7i3tr9kQlw==",
       "dev": true,
       "requires": {
         "eslint-scope": "^5.1.1",
@@ -6990,9 +6977,9 @@
       }
     },
     "@eslint/eslintrc": {
-      "version": "0.4.2",
-      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.2.tgz",
-      "integrity": "sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg==",
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
+      "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==",
       "dev": true,
       "requires": {
         "ajv": "^6.12.4",
@@ -7650,9 +7637,9 @@
       "dev": true
     },
     "concurrently": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-6.2.0.tgz",
-      "integrity": "sha512-v9I4Y3wFoXCSY2L73yYgwA9ESrQMpRn80jMcqMgHx720Hecz2GZAvTI6bREVST6lkddNypDKRN22qhK0X8Y00g==",
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-6.2.1.tgz",
+      "integrity": "sha512-emgwhH+ezkuYKSHZQ+AkgEpoUZZlbpPVYCVv7YZx0r+T7fny1H03r2nYRebpi2DudHR4n1Rgbo2YTxKOxVJ4+g==",
       "dev": true,
       "requires": {
         "chalk": "^4.1.0",
@@ -7749,9 +7736,9 @@
       "dev": true
     },
     "cypress": {
-      "version": "7.7.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-7.7.0.tgz",
-      "integrity": "sha512-uYBYXNoI5ym0UxROwhQXWTi8JbUEjpC6l/bzoGZNxoKGsLrC1SDPgIDJMgLX/MeEdPL0UInXLDUWN/rSyZUCjQ==",
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-8.2.0.tgz",
+      "integrity": "sha512-jg7S5VxxslwsgEyAkCE9ZCkFADxOUY1bSWScp1cWnga88K0TZgFQ0zdxyG9Mw/4spLGuvkriIZ62am+TR6C04w==",
       "dev": true,
       "requires": {
         "@cypress/request": "^2.88.5",
@@ -7862,12 +7849,6 @@
         }
       }
     },
-    "dedent": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
-      "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=",
-      "dev": true
-    },
     "deep-is": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
@@ -8067,13 +8048,13 @@
       "dev": true
     },
     "eslint": {
-      "version": "7.30.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz",
-      "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==",
+      "version": "7.32.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz",
+      "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==",
       "dev": true,
       "requires": {
         "@babel/code-frame": "7.12.11",
-        "@eslint/eslintrc": "^0.4.2",
+        "@eslint/eslintrc": "^0.4.3",
         "@humanwhocodes/config-array": "^0.5.0",
         "ajv": "^6.10.0",
         "chalk": "^4.0.0",
@@ -8161,36 +8142,30 @@
       "requires": {}
     },
     "eslint-import-resolver-node": {
-      "version": "0.3.4",
-      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz",
-      "integrity": "sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==",
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.5.tgz",
+      "integrity": "sha512-XMoPKjSpXbkeJ7ZZ9icLnJMTY5Mc1kZbCakHquaFsXPpyWOwK0TK6CODO+0ca54UoM9LKOxyUNnoVZRl8TeaAg==",
       "dev": true,
       "requires": {
-        "debug": "^2.6.9",
-        "resolve": "^1.13.1"
+        "debug": "^3.2.7",
+        "resolve": "^1.20.0"
       },
       "dependencies": {
         "debug": {
-          "version": "2.6.9",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "version": "3.2.7",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+          "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
           "dev": true,
           "requires": {
-            "ms": "2.0.0"
+            "ms": "^2.1.1"
           }
-        },
-        "ms": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
-          "dev": true
         }
       }
     },
     "eslint-module-utils": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.1.tgz",
-      "integrity": "sha512-ZXI9B8cxAJIH4nfkhTwcRTEAnrVfobYqwjWy/QMCZ8rHkZHFjf9yO4BzpiF9kCSfNlMG54eKigISHpX0+AaT4A==",
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz",
+      "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==",
       "dev": true,
       "requires": {
         "debug": "^3.2.7",
@@ -8235,17 +8210,17 @@
       }
     },
     "eslint-plugin-import": {
-      "version": "2.23.4",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.23.4.tgz",
-      "integrity": "sha512-6/wP8zZRsnQFiR3iaPFgh5ImVRM1WN5NUWfTIRqwOdeiGJlBcSk82o1FEVq8yXmy4lkIzTo7YhHCIxlU/2HyEQ==",
+      "version": "2.24.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.0.tgz",
+      "integrity": "sha512-Kc6xqT9hiYi2cgybOc0I2vC9OgAYga5o/rAFinam/yF/t5uBqxQbauNPMC6fgb640T/89P0gFoO27FOilJ/Cqg==",
       "dev": true,
       "requires": {
         "array-includes": "^3.1.3",
         "array.prototype.flat": "^1.2.4",
         "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.4",
-        "eslint-module-utils": "^2.6.1",
+        "eslint-import-resolver-node": "^0.3.5",
+        "eslint-module-utils": "^2.6.2",
         "find-up": "^2.0.0",
         "has": "^1.0.3",
         "is-core-module": "^2.4.0",
@@ -9425,9 +9400,9 @@
       "dev": true
     },
     "lint-staged": {
-      "version": "11.0.1",
-      "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-11.0.1.tgz",
-      "integrity": "sha512-RkTA1ulE6jAGFskxpGAwxfVRXjHp7D9gFg/+KMARUWMPiVFP0t28Em2u0gL8sA0w3/ck3TC57F2v2RNeQ5XPnw==",
+      "version": "11.1.2",
+      "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-11.1.2.tgz",
+      "integrity": "sha512-6lYpNoA9wGqkL6Hew/4n1H6lRqF3qCsujVT0Oq5Z4hiSAM7S6NksPJ3gnr7A7R52xCtiZMcEUNNQ6d6X5Bvh9w==",
       "dev": true,
       "requires": {
         "chalk": "^4.1.1",
@@ -9435,7 +9410,6 @@
         "commander": "^7.2.0",
         "cosmiconfig": "^7.0.0",
         "debug": "^4.3.1",
-        "dedent": "^0.7.0",
         "enquirer": "^2.3.6",
         "execa": "^5.0.0",
         "listr2": "^3.8.2",

package.json

@@ -19,7 +19,11 @@
     "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
     "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
     "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "lint:docker-compose": "docker-compose config -q && :",
+    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
+    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
+    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
   },
   "lint-staged": {
     "{services,updater}/**/*.{js,jsx}": "eslint --fix",
@@ -28,10 +32,10 @@
     ".github/workflows/*": "actionlint",
     "Dockerfile": "hadolint",
     "*.sh": "shellcheck -x",
-    "docker-compose.yml": "docker-compose config -q",
-    "docker-compose.real.yml": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q",
-    "docker-compose.ci.yml": "docker-compose -f docker-compose.ci.yml config -q",
-    "docker-compose.test.yml": "docker-compose -f docker-compose.test.yml config -q"
+    "docker-compose.yml": "npm run lint:docker-compose",
+    "docker-compose.real.yml": "npm run lint:docker-compose:real",
+    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
+    "docker-compose.test.yml": "npm run lint:docker-compose:test"
   },
   "repository": {
     "type": "git",
@@ -43,23 +47,23 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/eslint-parser": "^7.14.7",
+    "@babel/eslint-parser": "^7.15.0",
     "@babel/plugin-syntax-jsx": "^7.14.5",
-    "concurrently": "^6.2.0",
-    "cypress": "^7.7.0",
-    "eslint": "^7.30.0",
+    "concurrently": "^6.2.1",
+    "cypress": "^8.2.0",
+    "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-babel": "^5.3.1",
     "eslint-plugin-cypress": "^2.11.3",
-    "eslint-plugin-import": "^2.23.4",
+    "eslint-plugin-import": "^2.24.0",
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^3.4.0",
     "eslint-plugin-react": "^7.24.0",
     "eslint-plugin-react-hooks": "^4.2.0",
     "husky": "^7.0.1",
-    "lint-staged": "^11.0.1",
+    "lint-staged": "^11.1.2",
     "prettier": "^2.3.2",
     "stylelint": "^13.13.1",
     "stylelint-config-prettier": "^8.0.2",

run.sh

@@ -22,7 +22,7 @@ Usage: $(basename "${BASH_SOURCE[0]}") option [version] command --flag
 
 Parameters:
 * Option: oodikone/updater/both
-* Version: anon/real. Not necessary in all cases, such as when running down or logs.
+* Version: anon/real. Not necessary when running down/logs/pull.
 * Command: will be passed to docker-compose.
 EOF
 }
@@ -40,9 +40,9 @@ parse_params() {
 
   [[ ${#args[@]} -eq 1 ]] && usage && die "Wrong number of arguments"
 
-  # Down or logs can be passed without version. Otherwise parse version and then pass
+  # Some arguments can be passed without version. Otherwise parse version and then pass
   # rest to compose
-  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" ]]; then
+  if [[ "${args[1]}" == "down" || "${args[1]}" == "logs" || "${args[1]}" == "pull" ]]; then
     version=""
     compose_command=${args[*]:1}
   else

scripts/functions_for_setup.sh

@@ -18,7 +18,6 @@ SIS_DB_NAME="sis-db"
 SIS_IMPORTER_DB_NAME="sis-importer-db"
 USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
-OODI_DB_NAME="oodi-db" # TODO: Remove when oodi is removed
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
 KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
@@ -26,7 +25,6 @@ SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-si
 SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
 USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
-OODI_DB_REAL_DUMP_URL="svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/latest-pg.sqz" # TODO: Remove when oodi is removed
 
 # Source utility functions
 source "$PROJECT_ROOT"/scripts/utils.sh
@@ -120,14 +118,6 @@ reset_sis_importer_data() {
   reset_databases $database
 }
 
-reset_old_oodi_data() {
-  infomsg "Downloading old oodi-db dump"
-  local database=$OODI_DB_NAME
-  local url=$OODI_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
-}
-
 set_up_oodikone_from_scratch() {
   draw_mopo
   folders_to_set_up=(
@@ -141,9 +131,9 @@ set_up_oodikone_from_scratch() {
 
   infomsg "Cleaning up docker setup and node_modules from possible previous installations"
   local compose_down_args="--remove-orphans --volumes --rmi all"
-  "$PROJECT_ROOT"/run.sh both down "$compose_down_args"
-  docker-compose --file "$PROJECT_ROOT"/docker-compose.test.yml "$compose_down_args"
-  docker-compose --file "$PROJECT_ROOT"/docker-compose.ci.yml "$compose_down_args"
+  "$PROJECT_ROOT"/run.sh both down "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.test.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
+  docker-compose --file "$PROJECT_ROOT"/docker-compose.ci.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
     cd "$folder" || die "Couldn't change directory to folder $folder"
     rm -rf node_modules|| die "Couldn't remove node_modules in folder $folder"

services/backend/Dockerfile

@@ -12,12 +12,12 @@ CMD ["npm", "run", "dev"]
 
 FROM base as production
 ENV NODE_ENV=production
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
-COPY --chown=node:node . /usr/src/app/
-USER node
 ARG SENTRY_ENVIRONMENT
 ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+USER node
 CMD ["npm", "start"]

services/backend/scripts/create_course_groups_kayttis.js

@@ -1,34 +0,0 @@
-const { createCourseGroup } = require('../src/services/courseGroups')
-
-const run = async () => {
-  await Promise.all(groups.map(name => createCourseGroup(name)))
-  process.exit(0)
-}
-
-const groups = [
-  'Yleinen ja aikuiskasvatustiede',
-  'Kasvatustiede, yleinen didaktiikka',
-  'Kasvatuspsykologia',
-  'Varhaiskasvatus (VO+VAKAMO)',
-  'Erityispedagogiikka',
-  'Kotitaloustiede',
-  'Käsityötiede',
-  'Aikuisopetus',
-  'Biologia ja maantiede',
-  'Historia ja yhteiskuntaoppi',
-  'Uskonto',
-  'Elämänkatsomustieto',
-  'Suomen kieli ja kirjallisuus',
-  'Vieraat kielet',
-  'Matematiikka',
-  'Fysiikka ja kemia',
-  'Musiikki',
-  'Kuvataide',
-  'Liikunta',
-  'Draama',
-  'Mediakasvatus',
-  'Tiedekeskuspedagogiikka',
-  'Svenska studier',
-]
-
-run()

services/backend/scripts/create_fmsei_report.js

@@ -1,132 +0,0 @@
-/* eslint-disable no-console */
-
-const { Student, Credit, Studyright } = require('../src/models')
-const Sequelize = require('sequelize')
-const Op = Sequelize.Op
-
-const startDate = new Date(2019, 1, 1, 0, 0, 0, 0)
-
-const wasFMSEIStudent = (student, date) => {
-  const degreeright = student.studyrights.find(sr => sr.extentcode !== 9 && sr.startdate <= date && sr.enddate >= date)
-  return !degreeright
-}
-
-const courseCodes = [
-  'AYTKT200041',
-  'AYTKT21007',
-  'TKT21007',
-  'AYTKT21025en',
-  'AYTKT21009',
-  'TKT21009',
-  'AYTKT21010',
-  'TKT21010',
-  'AYTKT21018',
-  'AYTKT21018fi',
-  'AYTKT21018sv',
-  'AY5823951',
-  'AY5823952',
-  'AY5823953',
-  'AY5823954',
-  'AY5823955',
-  'AY5823956',
-]
-
-const createReport = async () => {
-  console.log('SPLIT')
-  const { fmseiStudentnumbers, otherStudentnumbers } = await courseCodes.reduce(
-    async (accPromise, courseCode) => {
-      const credits = await Credit.findAll({
-        where: {
-          course_code: courseCode,
-          attainment_date: {
-            [Op.gte]: startDate,
-          },
-        },
-      })
-
-      const { fmseiStudentnumbers, otherStudentnumbers } = await credits.reduce(
-        async (accPromise, credit) => {
-          const student = await Student.findOne({
-            where: { studentnumber: credit.student_studentnumber },
-            include: [{ model: Studyright }],
-          })
-          const acc = await accPromise
-
-          if (wasFMSEIStudent(student, credit.attainment_date)) {
-            // FMSEI student fmsei credits
-            console.log(
-              `${credit.student_studentnumber};${credit.course_code};${credit.grade};${credit.credits};${new Date(
-                credit.attainment_date
-              ).toISOString()}`
-            )
-            return {
-              fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(student.studentnumber),
-              otherStudentnumbers: acc.otherStudentnumbers,
-            }
-          }
-          return {
-            fmseiStudentnumbers: acc.fmseiStudentnumbers,
-            otherStudentnumbers: acc.otherStudentnumbers.concat(student.studentnumber),
-          }
-        },
-        { fmseiStudentnumbers: [], otherStudentnumbers: [] }
-      )
-
-      const acc = await accPromise
-
-      return {
-        courseStats: acc.courseStats.concat({
-          courseCode: courseCode,
-          fmsei: fmseiStudentnumbers.length,
-          other: otherStudentnumbers.length,
-        }),
-        fmseiStudentnumbers: acc.fmseiStudentnumbers.concat(fmseiStudentnumbers),
-        otherStudentnumbers: acc.otherStudentnumbers.concat(otherStudentnumbers),
-      }
-    },
-    { courseStats: [], fmseiStudentnumbers: [], otherStudentnumbers: [] }
-  )
-  const fmseiStudents = await Student.findAll({
-    where: { studentnumber: fmseiStudentnumbers },
-    include: [
-      {
-        model: Credit,
-        separate: true,
-      },
-    ],
-  })
-
-  const otherStudents = await Student.findAll({
-    where: { studentnumber: otherStudentnumbers },
-  })
-  console.log('SPLIT')
-  otherStudents.forEach(s => console.log(`${new Date(s.birthdate).getFullYear()};${s.gender_fi}`))
-  console.log('SPLIT')
-
-  const getFirstName = namestring => {
-    const names = namestring.split(' ')
-    const i = names.findIndex(n => n.substring(0, 1) === '*')
-    return i != -1 ? names[i].substring(1) : names[0]
-  }
-
-  fmseiStudents.forEach(s =>
-    console.log(
-      `${s.studentnumber};${s.email || ''};${s.zipcode || ''};${s.city_fi || ''};${new Date(
-        s.birthdate
-      ).getFullYear()};${s.country_fi || ''};${s.home_country_fi || ''};${s.gender_fi};${getFirstName(s.firstnames)}`
-    )
-  )
-  console.log('SPLIT')
-  // FMSEI all credits
-  fmseiStudents.forEach(s => {
-    s.credits.forEach(c =>
-      console.log(
-        `${c.student_studentnumber};${c.course_code};${c.grade};${c.credits};${new Date(
-          c.attainment_date
-        ).toISOString()}`
-      )
-    )
-  })
-}
-
-createReport()

services/backend/scripts/force_sync_database.js

@@ -1,16 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-const { forceSyncDatabase } = require('../src/database/connection')
-
-const sync = async () => {
-  try {
-    await forceSyncDatabase()
-    await seedAllMigrations()
-    console.log('Force sync succeeded. ')
-    process.exit(0)
-  } catch (e) {
-    console.log(`Force sync failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-sync()

services/backend/scripts/refresh_studyright_associations.js

@@ -1,8 +0,0 @@
-const { refreshAssociationsInRedis } = require('../src/services/studyrights')
-
-const run = async () => {
-  await refreshAssociationsInRedis()
-  process.exit(0)
-}
-
-run()

services/backend/scripts/seed_migrations.js

@@ -1,13 +0,0 @@
-const { seedAllMigrations } = require('../src/database/seed_migrations')
-
-const run = async () => {
-  try {
-    await seedAllMigrations()
-    process.exit(0)
-  } catch (e) {
-    console.log(`Seeding migrations failed: ${e}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/scripts/update_top_teachers.js

@@ -1,38 +0,0 @@
-const { findAndSaveTopTeachers } = require('../src/services/topteachers')
-const logger = require('../src/util/logger')
-
-const findAndSaveTeachers = async (startcode, endcode) => {
-  for (let code = startcode; code <= endcode; code++) {
-    logger.info(`Saving top teachers from year ${code}`)
-    await findAndSaveTopTeachers(code)
-  }
-}
-
-const parseargs = (args, SEPARATOR = '=') =>
-  args
-    .filter(arg => arg.includes(SEPARATOR))
-    .reduce((acc, arg) => {
-      const [key, val] = arg.split(SEPARATOR)
-      return {
-        ...acc,
-        [key.trim()]: val.trim(),
-      }
-    }, {})
-
-const run = async () => {
-  try {
-    const { from, to } = parseargs(process.argv)
-    const start = from
-    const end = to || from
-    if (end < start) {
-      throw Error('"to" -argument has to be less than "from" -argument. ')
-    }
-    await findAndSaveTeachers(start, end)
-    process.exit(0)
-  } catch (error) {
-    logger.error(`Failed to update top teachers: ${error.message}`)
-    process.exit(1)
-  }
-}
-
-run()

services/backend/src/app.js

@@ -6,8 +6,7 @@ const conf = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { PORT } = conf
-const { initializeDatabaseConnection } = require('./database/connection')
-const { dbConnections } = require('./databaseV2/connection')
+const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
 
 const SENTRY_ENVIRONMENT = process.env.SENTRY_ENVIRONMENT || ''
 const SENTRY_RELEASE = process.env.SENTRY_ENVIRONMENT || ''

services/backend/src/conf-backend.js

@@ -1,14 +1,5 @@
-const { NODE_ENV } = process.env
 const isTest = process.env.NODE_ENV === 'test'
-let DB_URL = process.env.DB_URL
-let DB_URL_KONE = process.env.DB_URL_KONE
-if (isTest) {
-  DB_URL = process.env.TEST_DB
-  DB_URL_KONE = process.env.TEST_DB_KONE
-} else if (NODE_ENV === 'anon') {
-  DB_URL = process.env.ANON_DB
-  DB_URL_KONE = process.env.ANON_DB_KONE
-}
+const DB_URL_KONE = process.env.DB_URL_KONE
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
 if (isNaN(DB_MAX_CONNECTIONS)) {
@@ -23,15 +14,10 @@ const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const SECRET_TOKEN = process.env.SECRET_TOKEN
-const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const CERT_PATH = process.env.CERT_PATH // production/staging only
-const KEY_PATH = process.env.KEY_PATH // production/staging only
 const USERSERVICE_URL = process.env.USERSERVICE_URL
-const UPDATER_URL = process.env.UPDATER_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
 const PORT = isTest ? 8079 : 8080
-const OODI_SECRET = process.env.OODI_SECRET
 const ANALYTICS_INFLUXDB_URL = process.env.ANALYTICS_INFLUXDB_URL
 const ANALYTICS_INFLUXDB_USER = process.env.ANALYTICS_INFLUXDB_USER
 const ANALYTICS_INFLUXDB_PASSWORD = process.env.ANALYTICS_INFLUXDB_PASSWORD
@@ -39,67 +25,34 @@ const ANALYTICS_INFLUXDB_DB = process.env.ANALYTICS_INFLUXDB_DB
 const MATOMO_SITE_ID = process.env.MATOMO_SITE_ID
 const MATOMO_URL = process.env.MATOMO_URL
 
-const FEATURES = {
-  ERROR_HANDLER: false,
-}
-
 const formatURL = url => {
   return !!url && !url.startsWith('http') ? `http://${url}` : url
 }
 
 const addSlashToEnd = url => (url.endsWith('/') ? url : url + '/')
 
-if (process.env.NODE_ENV === 'development' && process.env.FEATURES) {
-  const toggled = process.env.FEATURES.split(',')
-  toggled
-    .map(toggle => toggle.trim())
-    .forEach(feature => {
-      if (FEATURES[feature] !== undefined) {
-        FEATURES[feature] = true
-      }
-    })
-}
-
-const OODI = {
-  test: 'http://localhost',
-  anon: process.env.OODI_ADDR_ANON,
-}
-
-const OODI_ADDR = OODI[process.env.NODE_ENV] || process.env.OODI_ADDR
 const ACCESS_TOKEN_HEADER_KEY = 'x-access-token'
-const OODI_SECRET_HEADER_KEY = 'x-oodi-secret'
 
-let requiredGroup = ['grp-oodikone-users', 'grp-oodikone-basic-users']
-if (process.env.NODE_ENV === 'staging') {
-  requiredGroup = ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
-}
-if (process.env.NODE_ENV === 'development' || isTest) {
-  requiredGroup = null
-}
+const isStaging = process.env.NODE_ENV === 'staging'
+
+const requiredGroup = isStaging
+  ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
+  : ['grp-oodikone-users', 'grp-oodikone-basic-users']
 
 module.exports = {
   frontend_addr,
-  DB_URL,
   DB_URL_KONE,
   DB_MAX_CONNECTIONS,
   DB_MAX_CRON_CONNECTIONS,
   redis,
   TOKEN_SECRET,
   SECRET_TOKEN,
-  DB_SCHEMA,
   DB_SCHEMA_KONE,
-  OODI_ADDR,
-  CERT_PATH,
-  KEY_PATH,
-  FEATURES,
   USERSERVICE_URL: formatURL(USERSERVICE_URL),
   ACCESS_TOKEN_HEADER_KEY,
   PORT,
   requiredGroup,
-  OODI_SECRET,
-  OODI_SECRET_HEADER_KEY,
   isTest,
-  UPDATER_URL,
   SIS_UPDATER_URL,
   ANALYTICS_INFLUXDB_URL: formatURL(ANALYTICS_INFLUXDB_URL),
   ANALYTICS_INFLUXDB_USER,

services/backend/src/database/connection.js

@@ -1,17 +1,42 @@
 const Sequelize = require('sequelize')
+const EventEmitter = require('events')
 const Umzug = require('umzug')
 const conf = require('../conf-backend')
+const { SIS_DB_URL } = process.env
+class DbConnection extends EventEmitter {
+  constructor() {
+    super()
+    this.RETRY_ATTEMPTS = 15
+    this.established = false
 
-const sequelize = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA,
-  searchPath: conf.DB_SCHEMA,
-  logging: false,
-  pool: {
-    min: 0,
-    max: conf.DB_MAX_CONNECTIONS,
-  },
-})
-sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
+    this.sequelize = new Sequelize(SIS_DB_URL, {
+      dialect: 'postgres',
+      pool: {
+        max: 25,
+        min: 0,
+        acquire: 10000,
+        idle: 300000000,
+      },
+      logging: false,
+    })
+  }
+
+  async connect(attempt = 1) {
+    try {
+      await this.sequelize.authenticate()
+      this.emit('connect')
+      this.established = true
+    } catch (e) {
+      if (attempt > this.RETRY_ATTEMPTS) {
+        this.emit('error', e)
+        return
+      }
+      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
+      console.log(e)
+      setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
+    }
+  }
+}
 
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
@@ -25,7 +50,6 @@ const initializeDatabaseConnection = async () => {
   const waitSeconds = 60
   for (let i = 1; i <= waitSeconds; i++) {
     try {
-      await sequelize.authenticate()
       await sequelizeKone.authenticate()
       break
     } catch (e) {
@@ -65,22 +89,17 @@ const initializeDatabaseConnection = async () => {
 }
 
 const forceSyncDatabase = async () => {
-  try {
-    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
-  } catch (e) {
-    // console.log(e)
-  }
   try {
     await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
   } catch (e) {
-    // console.log(e)
+    //console.log(e)
   }
-  await sequelize.sync({ force: true })
   await sequelizeKone.sync({ force: true })
 }
 
+const dbConnections = new DbConnection()
 module.exports = {
-  sequelize,
+  dbConnections,
   sequelizeKone,
   initializeDatabaseConnection,
   forceSyncDatabase,

services/backend/src/database/migrations/20190710_03_drop_ldapuser.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async queryInterface => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.dropTable('ldapuser', { transaction })
-    })
-  },
-  down: async () => {},
-}