Merge pull request #1317 from UniversityOfHelsinkiCS/trunk

fix studyprogramme column 820ab42

.circleci/config.yml

@@ -0,0 +1,93 @@
+version: 2.1
+
+executors:
+  my-executor:
+    working_directory: ~/app
+    machine:
+      image: ubuntu-1604:201903-01
+
+aliases:
+  - &check_exit_statuses
+    run:
+      name: check_exit_statuses
+      command: |
+        docker ps -a
+        if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+          echo 'Some process had nonzero exit code';
+          npm run docker:ci -- logs
+          exit 1;
+        fi
+
+jobs:
+  build:
+    executor: my-executor
+    steps:
+      - run:
+          name: set_env_variables
+          command: |
+            echo $CIRCLE_BRANCH;
+            echo $CIRCLE_TAG;
+            if [[ -n "$CIRCLE_TAG" ]]; then
+              echo 'export TAG=latest' >> $BASH_ENV;
+              echo 'export DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=staging' >> $BASH_ENV;
+              echo 'USING RELEASE CONFIG';
+            elif [[ "$CIRCLE_BRANCH" = "master" && -z "$CIRCLE_PULL_REQUEST" ]]; then
+              echo 'export TAG=staging' >> $BASH_ENV;
+              echo 'export DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'USING STAGING CONFIG';
+            else
+              echo 'export TAG=trunk' >> $BASH_ENV;
+              echo 'export DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'USING TRUNK CONFIG';
+            fi
+      - checkout
+      - restore_cache:
+          keys:
+            - npmcache-v2-{{ checksum "package-lock.json" }}
+      - run: npm ci
+      - run: npm run cypress -- verify
+      - save_cache:
+          key: npmcache-v2-{{ checksum "package-lock.json" }}
+          paths:
+            - ~/.cache
+      - run: TAG=${CACHE_TAG} npm run docker:ci -- pull --ignore-pull-failures
+      - run: npm run docker:ci -- build
+      - run: npm run docker:ci -- up -d db user_db db_kone
+      - *check_exit_statuses
+      - run: git clone --depth 1 git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
+      - run: source ./scripts/scripts.sh && init_dirs && db_anon_setup_full
+      - *check_exit_statuses
+      - run: npm run docker:ci -- up -d --no-recreate
+      - run:
+          name: log_in_background
+          background: true
+          command: |
+            docker ps -a && npm run docker:ci -- logs -f
+      - *check_exit_statuses
+      - run: npm run test_services
+      - *check_exit_statuses
+      - run: timeout 5m bash -c 'until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf .; sleep 5; done'
+      - run: npm run cypress:ci
+      - *check_exit_statuses
+      - store_artifacts:
+          path: cypress/videos
+      - store_artifacts:
+          path: cypress/screenshots
+      - deploy:
+          command: |
+            if [ "${DEPLOY}" == "true" ]; then
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              npm run docker:ci -- push
+            fi
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/

.gitignore

@@ -1,5 +1,6 @@
 data
 pgdata
+pgdata_kone
 user_pgdata
 analytics_pgdata
 usage_pgdata
@@ -13,17 +14,14 @@ anonyymioodi
 nginx
 kafka
 node_modules
-cypress/support
-cypress/videos
 services/updater_writer/updater/test_assets/testi.bak
 services/updater_writer/updater/test_assets/011120775.json
 services/updater_writer/updater/test_assets/014272112.json
 services/updater_writer/updater/test_assets/014441008.json
 services/updater_writer/updater/test_assets/meta.json
 services/updater_scheduler/all_student_numbers.txt
-services/updater_scheduler/debug.log
 services/updater_scheduler/active_student_numbers.txt
-services/oodikone2-backend/debug.log
 datastore
 services/updater_api/debug.log
-services/backend/updater_writer/dumps
+services/backend/updater_writer/dumps
+debug.log

.travis.yml

@@ -1,67 +0,0 @@
-language: node_js
-
-node_js:
-  # Node 10.3+ includes npm@6 which has good "npm ci" command
-  - 10.8
-
-cache:
-  # cache both npm modules and Cypress binary
-  npm: true
-  directories:
-    - ~/.cache
-
-services:
-  - docker
-
-before_script:
-  - docker --version
-  - docker-compose --version
-  - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")"
-
-jobs:
-  include:
-    - stage: build & test & push & deploy
-      if: NOT tag IS present
-      env:
-        - IMAGE=STAGING
-      script:
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.yml
-        - docker ps
-        - cat nginx/log
-        - docker-compose logs -f &
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
-        - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
-        - docker ps -a
-        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
-        provider: script
-        skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          branch: master
-    - # in parallel
-      if: type = pull_request OR NOT branch = master
-      env:
-        - IMAGE=PRODUCTION
-      script:
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.production.yml
-        - docker ps
-        - cat nginx/log
-        - docker-compose logs -f &
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
-        - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
-        - docker ps -a
-        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
-        provider: script
-        skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          tags: true
-
-after_failure:
-  - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

README.md

@@ -23,7 +23,7 @@ Launch the CLI with the command below and follow the instructions.
 Before running cli [make yourself a deployment key](https://developer.github.com/v3/guides/managing-deploy-keys/#setup-2) for Anonyymioodi private repository and put it in your root folder as private.key. This allows you to download the anonymized dumps from the repository.
 
 ```
-bash run.sh
+bash ./scripts/run.sh
 ```
 
 For local development with anonymized data use `2) Anon setup` (you can run e2e tests with this).
@@ -58,27 +58,27 @@ The mapping of ports and environment variables are also defined in the docker-co
 #### With anonymized data:
 
 ```
-docker-compose up -d
+npm run docker:up
 ```
 
 #### With real data:
 
 ```
-docker-compose -f docker-compose.yml -f docker-compose.real.yml up -d
+docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml up -d
 ```
 
 ### Stop the development environment
 
 ```
-docker-compose down
+npm run docker:down
 ```
 
 ### Restart container(s)
 
 ```
-docker-compose restart          # all
+npm run docker:restart          # all
 
-docker-compose restart backend  # just backend
+npm run docker:restart:backend  # just backend
 ```
 
 ### View the containers in the running environment
@@ -90,9 +90,9 @@ docker-compose ps
 
 
 ```
-docker-compose logs -f          # all
+npm run docker:logs          # all
 
-docker-compose logs -f backend  # just backend
+npm run docker:logs:backend  # just backend
 ```
 
 

adminer-plugins/login-servers.php

@@ -6,6 +6,6 @@ require_once('plugins/login-servers.php');
     * @param string
     */
 return new AdminerLoginServers(
-    $servers = array("db" => "db", "user_db" => "user_db", "analytics_db" => "analytics_db"),
+    $servers = array("db" => "db", "db_kone" => "db_kone", "user_db" => "user_db", "analytics_db" => "analytics_db"),
     $driver = "pgsql"
 );

cypress/.gitignore

@@ -1 +1,2 @@
 screenshots
+videos

cypress/integration/Course_statistics.js

@@ -51,8 +51,8 @@ describe('Course Statistics tests', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
     cy.get("input[placeholder='Search by entering a course code']").type('TKT')
-    cy.contains("tr", "TKT20003").click()
-    cy.contains("tr", "TKT10002").click()
+    cy.contains("td", "TKT20003").click()
+    cy.contains("td", /^TKT10002/).click()
     cy.contains("Fetch statistics").should('be.enabled').click()
     cy.contains("Search for courses").should('not.exist')
 

cypress/integration/Population_statistics.js

@@ -78,6 +78,21 @@ describe('Population Statistics tests', () => {
       cy.contains("Excludes students with cancelled study right")
     })
 
+    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
+      cy.get("tr").its('length').should('be.gte', 10)
+      cy.route('/api/v3/courseyearlystats**').as('coursePage')
+      cy.contains("Opiskelijan digitaidot: orientaatio").siblings().find(".level").click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+    })
+    cy.contains("DIGI-000A")
+    cy.go("back")
+
+    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
+      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
+      cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '9')
+    })
+
     cy.contains("add").click()
     cy.contains("Add filters").siblings().within(() => {
       cy.get(".form").should('have.length', 10)
@@ -88,39 +103,48 @@ describe('Population Statistics tests', () => {
     let filteredStudents = 1328493
     cy.contains("Credits gained during first").parentsUntil(".tab").get("table").within(() => {
       cy.get("tr").eq(1).find("td").eq(1).invoke("text").then(text => filteredStudents = Number(text))
+      cy.route('POST', '/api/v2/populationstatistics/courses**').as('courseData')
       cy.get("tr").eq(1).click()
+      cy.wait('@courseData')
     }).then(() => {
       checkAmountOfStudents(filteredStudents)
     })
 
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.get("tr").its('length').should('be.gte', 10)
-      cy.contains("Opiskelijan digitaidot: orientaatio").siblings().find(".level").click()
-      cy.wait(1000)
-      cy.url().should('include', '/coursestatistics')
-    })
-    cy.contains("DIGI-000A")
-    cy.go("back")
-
-
-
-    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
-      cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '9')
-    })
-
-    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.get(".refresh").click()
-      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
+      cy.contains("number at least").siblings().within(() => cy.get("input").clear())
+      cy.contains("number at least").siblings().within(() => cy.get("input").type("0"))
       cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '1')
     })
 
-    cy.get("button").contains("show").click()
+    cy.contains("button", "show").click()
     cy.contains("Student names hidden").click()
     cy.contains("Oinonen").siblings().eq(2).click()
     cy.contains("Oinonen").invoke('text').then((text) => expect(text).to.equal('Oinonen Heidi Eeva Elisabet, 014473717'))
   })
 
+  it('Student list checking works as intended', () => {
+    cy.contains("Select study programme").click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("See population").click()
+    cy.contains("button", "show").click()
+    cy.contains("010483918")
+    cy.contains("666666666").should('not.exist')
+    cy.contains('button', "Check studentnumbers").click()
+    cy.contains('Check for studentnumbers')
+    cy.get('textarea').type("010483918").type('{enter}').type("666666666")
+    cy.contains('button', 'check students').click()
+    cy.contains('form', 'Results').within(e => {
+      cy.get('div').eq(0).within(e => {
+        cy.contains('student numbers in list but not in oodi')
+        cy.contains('666666666')
+      })
+      cy.get('div').eq(2).within(e => {
+        cy.contains('student numbers in oodi but not in list')
+        cy.contains('014896381')
+      })
+      cy.contains('010483918').should('not.exist')
+    })
+  })
+
   it('All filters working', () => {
     cy.contains("Select study programme", { timeout: 50000 }).click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
     cy.contains("See population").click()
@@ -185,8 +209,7 @@ describe('Population Statistics tests', () => {
     cy.contains("and at least").parentsUntil("form").contains("set filter").click()
 
     cy.contains("Filters").siblings().within(() => {
-      cy.contains('Kehittävä').should('have.text', 'Kehittävä työntutkimus')
-      cy.contains('Kvalitatiiviset').should('have.text', 'Kvalitatiiviset tutkimusmenetelmät I')
+      cy.contains('Kehittävä').should('have.text', 'Avoin yo: Kehittävä työntutkimus')
     })
 
     checkAmountOfStudents(1)
@@ -205,7 +228,6 @@ describe('Population Statistics tests', () => {
 
     cy.contains('Excluded students that graduated')
 
-
     cy.contains("Students that has").parentsUntil("form").within(() => {
       cy.contains("degree").click().siblings().contains("any degree").click()
       cy.contains("programme").click().siblings().contains("Kasvatustieteiden kandiohjelma")
@@ -213,7 +235,7 @@ describe('Population Statistics tests', () => {
       cy.contains("set filter").click()
     })
 
-    checkAmountOfStudents(1)
+    checkAmountOfStudents(3)
 
     cy.contains("Basic filters").parentsUntil("form").contains("set filter").click()
 

cypress/integration/Student_statistics.js

@@ -11,7 +11,6 @@ describe('Student Statistics tests', () => {
                 }
             }
         })
-        console.log(Cypress.config().baseUrl)
         cy.visit(Cypress.config().baseUrl)
         cy.contains("Student statistics").click()
         cy.contains("Student names hidden")
@@ -71,4 +70,12 @@ describe('Student Statistics tests', () => {
         cy.get('.prompt').clear().type('01114')
         cy.contains('Student number').should('not.exist')
     })
-})
+
+    it('Can jump to population page', () => {
+        cy.get('.prompt').type('Oinonen')
+        cy.contains('011143561').click()
+        cy.get('i.level.up.alternate.icon').eq(0).click()
+        cy.contains('Population statistics')
+        cy.contains('Credit accumulation (for 1 students)')
+    })
+})

cypress/integration/Studyprogramme_overview.js

@@ -41,22 +41,66 @@ describe('Studyprogramme overview', () => {
 
   it('renders progress and productivity tables', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.get('table').should('have.length', 2)
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-    cy.contains("2018-2019").siblings().contains('9')
-    cy.contains("2017-2018").siblings().contains('12').siblings().contains('7').siblings().contains('2')
-    cy.get('table').eq(1).contains('2018').siblings().contains('443')
-    cy.get('table').eq(1).contains('2017').siblings().contains('381')
+    cy.get('button:contains("Recalculate")').eq(0).click()
+    cy.get('button:contains("Recalculate")').eq(1).click()
+    cy.contains('Loading...').should('not.be.visible').then(() => {
+      cy.wait(1000)
+      cy.reload()
+      cy.get('table').should('have.length', 2)
+      cy.contains('Population progress')
+      cy.contains('Yearly productivity')
+      cy.contains("2018-2019").siblings().contains('9')
+      cy.contains("2017-2018").siblings().contains('12').siblings().contains('7').siblings().contains('2')
+      cy.get('table').eq(1).contains('2018').siblings().contains('443')
+      cy.get('table').eq(1).contains('2017').siblings().contains('381')
+    })
   })
 
-  it('can search for mandatory courses', () => {
+  it('can add and delete mandatory courses and labels', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
     cy.contains('Mandatory Courses').click()
     cy.get('button').contains('Add Courses').click()
-    cy.get('input').eq(0).type('CSM')
+    cy.route('/api/v2/coursesmulti**').as('searchResponse')
+    cy.get('input').eq(1).type('Code Generation')
+    cy.wait('@searchResponse')
     cy.contains('Searched courses')
-    cy.contains('Code Generation')
+    cy.get('table').eq(0).contains('Code Generation').click()
+    cy.get('table').eq(1).contains('tr', 'Code Generation').contains('select label')
+
+    cy.contains('Group labels').click()
+    const label = 'cypress test label'
+    cy.contains(label).should('not.exist')
+    cy.contains('button', 'Add').should('be.disabled')
+    cy.get('input').eq(0).type(label)
+    cy.contains('button', 'Add').should('be.enabled').click()
+    cy.contains('button', 'Add').should('be.disabled')
+    cy.contains('table', label)
+
+    cy.contains('Mandatory courses').click()
+    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('select label').click()
+    cy.contains(label).click()
+    cy.get('table').eq(0).contains('tr', 'Code Generation').within((el) => {
+      cy.get('div.dropdown>div').eq(0).within((el) => {
+        cy.contains('select label').should('not.exist')
+        cy.contains(label)
+      })
+      cy.get('i.clear').click()
+      cy.get('div.dropdown>div').eq(0).within((el) => {
+        cy.contains(label).should('not.exist')
+        cy.contains('select label')
+      })
+    })
+    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('select label').click()
+    cy.contains(label).click()
+    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('button', 'Delete').click()
+    cy.contains('Code Generation').should('not.exist')
+
+    cy.contains('Group labels').click()
+    cy.contains('tr', label).within((el) => {
+      cy.get('i.remove').click()
+    })
+    cy.contains('button', 'Remove').click()
+    cy.contains(label).should('not.exist')
   })
 
   it('can open Thesis page', () => {

cypress/integration/Teachers.js

@@ -0,0 +1,71 @@
+
+describe('Teachers page tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+          proxy.xhr.setRequestHeader('uid', 'tktl')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    cy.visit(Cypress.config().baseUrl)
+    cy.reload()
+    cy.contains("Teachers").click()
+    cy.contains("Teacher statistics by course providers")
+  })
+
+  it("Check Statistics", () => {
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('Autumn 2017').click()
+    cy.get('.form > :nth-child(2) > .ui').click()
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+    cy.get('.form > .fluid').click()
+    cy.contains('Name')
+    cy.contains('Wu')
+    cy.contains('Jämsä')
+    cy.contains('Kuusinen')
+  })
+
+  it("Teacher search works", () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type('Pekka')
+    cy.contains('Professori Pekka')
+    cy.contains('Saren')
+    cy.contains('Jari')
+    cy.contains('Lopez')
+  })
+
+  it("Can check teacher page", () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type('Pekka')
+    cy.contains('Saren').click()
+    cy.contains('English Academic & Professional Skills: Reading, Writing & Spoken Communication (CEFR B2)*')
+  })
+
+  it("Can check teacher page if teacher doesn't have courses", () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type('Pekka')
+    cy.contains('Professori').click()
+    cy.contains('Name').should('not.exist')
+  })
+
+  it("Check leaderboad works", () => {
+    cy.get('.borderless > :nth-child(2)').click()
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('2017-18').click()
+    cy.contains("Recalculate this year").click()
+    cy.wait(5000)
+    cy.reload()
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('2017-18').click()
+    cy.contains("Passed")
+    cy.contains("Valtonen")
+    cy.contains("Wahlström")
+  })
+})

cypress/integration/Users.js

@@ -15,7 +15,7 @@ describe('Users tests', () => {
     cy.visit(Cypress.config().baseUrl)
     cy.reload()
     cy.contains("Users").click()
-    cy.contains("Enable or disable access to Oodikone")
+    cy.contains("Oodikone users")
   })
 
   it("Admin mocking normal user shows only the mocked user's programmes", () => {
@@ -27,8 +27,11 @@ describe('Users tests', () => {
       cy.get("div[role='listitem']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")
     })
 
+    cy.route('POST', '/api/superlogin/normk').as('superlogin')
     cy.get('i.spy').click()
+    cy.wait('@superlogin')
     cy.contains("mocking as normk")
+    cy.wait(1000)
     cy.contains("Study programme").click().siblings().contains("Search by class").click()
     cy.contains("label", "Study programme")
     cy.contains("label", "Study programme").siblings().within(($row) => {

cypress/support/commands.js

@@ -0,0 +1,25 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add("login", (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This is will overwrite an existing command --
+// Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })

cypress/support/index.js

@@ -0,0 +1,31 @@
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands'
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')
+
+// describe('Before all tests', () => {
+//     it('reset anon database', () => {
+//       cy.exec(
+//         'source ./scripts/scripts.sh && time reset_db_for_cypress',
+//         { timeout: 5*60*1000 }
+//       ).then(result => {
+//         console.log(result)
+//       })
+//     })
+// })

docker-compose.lateste2e.production.yml

@@ -1,236 +0,0 @@
-version: '3'
-
-services:
-  nats:
-    image: nats-streaming
-    command: -cid updaterNATS
-    expose:
-      - "4222"
-    container_name: nats
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
-
-  user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=user_db
-
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
-
-  backend:
-    image: toska/oodikone2-backend:latest
-    build:
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    ports:
-      - "8080:8080"
-      - "9229:9229"
-    environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      FRONT_URL: http://localhost:1337
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
-
-  frontend:
-    image: toska/oodikone2-frontend:latest
-    build:
-      context: services/oodikone2-frontend
-      args:
-        BASE_PATH: /
-    ports:
-      - "5000:5000"
-    environment:
-      WAIT_HOSTS: user_db:5432
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
-
-  userservice:
-    image: toska/oodikone2-userservice:latest
-    build:
-      context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      - WAIT_HOSTS=user_db:5432
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
-
-  usageservice:
-    image: toska/oodikone2-usageservice:latest
-    build:
-      context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      - WAIT_HOSTS=usage_db:5432
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  analytics:
-    image: toska/oodikone2-analytics:latest
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-      - analytics_redis
-    container_name: analytics
-
-  updater_api:
-    image: toska/updater_api:latest
-    build:
-      context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - WAIT_HOSTS=nats:4222
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
-
-  updater_writer:
-    image: toska/updater_writer:latest
-    build:
-      context: services/backend
-      dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      WAIT_HOSTS: nats:4222,db:5432
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo
-    volumes:
-      - ./mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_db
-
-  updater_scheduler:
-    image: toska/updater_scheduler:latest
-    build:
-      context: services/updater_scheduler
-    command: ["echo", "mluukkai on orjuuttaja"]
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_scheduler
-
-  nginx:
-    image: nginx:latest
-    volumes:
-      - ./nginx.production.conf:/etc/nginx/nginx.conf
-      - ./nginx/error.log:/etc/nginx/error_log.log
-      - ./nginx/log:/var/log/nginx/localhost.error_log
-      - ./nginx/cache/:/etc/nginx/cache
-      - /etc/letsencrypt/:/etc/letsencrypt/
-    ports:
-      -  1337:80
-    depends_on:
-      - frontend
-      - backend
-      - userservice
-      - analytics
-
-    container_name: nginx

docker-compose.lateste2e.yml

@@ -1,238 +0,0 @@
-version: '3'
-
-services:
-  nats:
-    image: nats-streaming
-    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
-    expose:
-      - "4222"
-    volumes:
-      - ./datastore:/datastore
-    container_name: nats
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
-
-  user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=user_db
-
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
-
-  backend:
-    image: toska/oodikone2-backend:staging
-    build:
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    ports:
-      - "8080:8080"
-      - "9229:9229"
-    environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      FRONT_URL: http://localhost:1337
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
-
-  frontend:
-    image: toska/oodikone2-frontend:staging
-    build:
-      context: services/oodikone2-frontend
-      args:
-        BASE_PATH: /
-    ports:
-      - "5000:5000"
-    environment:
-      WAIT_HOSTS: user_db:5432
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
-    
-  userservice:
-    image: toska/oodikone2-userservice:staging
-    build:
-      context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      - WAIT_HOSTS=user_db:5432
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
-
-  usageservice:
-    image: toska/oodikone2-usageservice:staging
-    build:
-      context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      - WAIT_HOSTS=usage_db:5432
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  analytics:
-    image: toska/oodikone2-analytics:staging
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-      - analytics_redis
-    container_name: analytics
-
-  updater_api:
-    image: toska/updater_api:staging
-    build:
-      context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - WAIT_HOSTS=nats:4222
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
-
-  updater_writer:
-    image: toska/updater_writer:staging
-    build:
-      context: services/backend
-      dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      WAIT_HOSTS: nats:4222,db:5432
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo
-    volumes:
-      - ./mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_db
-
-  updater_scheduler:
-    image: toska/updater_scheduler:staging
-    build:
-      context: services/updater_scheduler
-    command: ["echo", "mluukkai on orjuuttaja"]
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_scheduler
-
-  nginx:
-    image: nginx:latest
-    volumes:
-      - ./nginx.staging.conf:/etc/nginx/nginx.conf
-      - ./nginx/error.log:/etc/nginx/error_log.log
-      - ./nginx/log:/var/log/nginx/localhost.error_log
-      - ./nginx/cache/:/etc/nginx/cache
-      - /etc/letsencrypt/:/etc/letsencrypt/
-    ports:
-      -  1337:80
-    depends_on:
-      - frontend
-      - backend
-      - userservice
-      - analytics
-
-    container_name: nginx

docker-compose.yml

@@ -1,43 +1,17 @@
-version: '3'
+version: '3.2'
 
 services:
-  # zookeeper:
-  #   image: wurstmeister/zookeeper
-  #   ports:
-  #     - "2181:2181"
-  #   environment:
-  #     - KAFKA_BROKER_ID=1
-  #   container_name: zookeeper
-
-  # kafka:
-  #   image: wurstmeister/kafka
-  #   container_name: kafka
-  #   depends_on:
-  #     - zookeeper
-  #   ports:
-  #     - "9092:9092"
-  #   environment:
-  #     - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}'
-  #     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-  #     - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
-  #     - KAFKA_DELETE_TOPIC_ENABLE=true
-  #     - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1
-  #     - KAFKA_LOG_DIRS=/kafka/kafka-logs-1
-  #   volumes:
-  #     - ./kafka/docker.sock:/var/run/docker.sock
-  #     - ./kafka/logs:/kafka/kafka-logs-1
-
   nats:
-    image: nats-streaming
-    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
+    image: nats-streaming:0.15.1
+    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
     expose:
       - "4222"
-    volumes:
-      - ./datastore:/datastore
     ports:
       - "8222:8222"
       - "4222:4222"
     container_name: nats
+    volumes:
+      - ./datastore:/datastore
 
   analytics_db:
     image: postgres:9.6.3
@@ -59,7 +33,6 @@ services:
     container_name: oodi_user_db
     environment:
       - POSTGRES_USER=postgres
-      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
 
   usage_db:
     image: postgres:9.6.3
@@ -73,7 +46,7 @@ services:
       - POSTGRES_DB=usage_db
 
   db:
-    image: postgres:9.6.3
+    image: postgres:10-alpine
     ports:
       - "5421:5432"
     volumes:
@@ -81,7 +54,18 @@ services:
       - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
     container_name: oodi_db
     environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+      - POSTGRES_USER=postgres
+
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - ./pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
       - POSTGRES_USER=postgres
 
   redis:
@@ -93,32 +77,18 @@ services:
       - ./redis-data:/data
     container_name: redis
 
-  # mongo_db:
-  #   image: mongo
-  #   restart: unless-stopped
-  #   volumes:
-  #     - ./backups/mongo/oodilearn:/dump
-  #     - ./mongodata:/data/db
-  #   ports:
-  #     - "27017:27017"
-  #   container_name: mongo_db
-
   backend:
     build:
       context: services/backend
       dockerfile: oodikone2-backend/Dockerfile
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
     ports:
       - "8080:8080"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      FRONT_URL: http://localhost:8000
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
       USERSERVICE_URL: http://userservice:4567
@@ -127,13 +97,7 @@ services:
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
-    volumes:
-      - ./services/backend/oodikone2-backend/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/src/models
-      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
-
+      UPDATER_URL: http://updater_scheduler:3678
     container_name: backend
     depends_on:
       - db
@@ -143,52 +107,17 @@ services:
   frontend:
     build:
       context: services/oodikone2-frontend
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker']
-    restart: unless-stopped
-    ports:
-      - "8081:8081"
-      - "7777:7777"
     environment:
       BACKEND_ADDR: backend
       ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres
-    volumes:
-      - ./services/oodikone2-frontend/:/usr/src/app
     container_name: frontend
 
-  # oodilearn:
-  #   build:
-  #     context: services/oodilearn/server
-  #     dockerfile: Dockerfile
-  #   restart: unless-stopped
-  #   volumes:
-  #     - ./services/oodilearn/data:/data
-  #     - ./services/oodilearn/server:/src
-  #     - ./services/oodilearn/models:/models
-  #   ports:
-  #     - "5000:5000"
-  #   environment:
-  #     - ENV=development
-  #     - MONGO_URI=mongodb://mongo_db:27017/oodilearn
-  #   container_name: oodilearn
-
   userservice:
     build:
       context: services/oodikone2-userservice
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/oodikone2-userservice/:/usr/src/app
     ports:
       - 4567:4567
     environment:
-      WAIT_HOSTS: user_db:5432
       DB_URL: postgres://postgres@user_db:5432/user_db
       TEST_DB: postgres://postgres@user_db:5432/user_db_test
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -200,19 +129,12 @@ services:
   usageservice:
     build:
       context: services/oodikone2-usageservice
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
     ports:
       - 545:545
     environment:
-      WAIT_HOSTS: usage_db:5432
       PORT: '545'
       DB_URL: postgres://postgres@usage_db:5432/usage_db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    volumes:
-      - ./services/oodikone2-usageservice/:/usr/src/app
     depends_on:
       - usage_db
     container_name: usageservice
@@ -220,121 +142,56 @@ services:
   updater_api:
     build:
       context: services/updater_api
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/updater_api/:/usr/src/app
     depends_on:
       - nats
     environment:
-      - WAIT_HOSTS=nats:4222
       - NATS_URI=nats://nats:4222
       - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
+      - TOKEN=jotaindiibadaabaa
+    #container_name: updater_api
 
   updater_writer:
     build:
       context: services/backend
       dockerfile: updater_writer/Dockerfile
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/backend/updater_writer/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/models
-      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
-
     depends_on:
       - nats
       - db
     environment:
-      WAIT_HOSTS: nats:4222,db:5432
       NATS_URI: nats://nats:4222
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_SCHEMA: updater_test
-      DB_SCHEMA_KONE: kone_data
-
-
+    #container_name: updater_writer
 
   scheduler_mongo:
-    image: mongo
-    restart: unless-stopped
+    image: mongo:4.0.10
     volumes:
       - ./mongodata:/data/db
     expose:
       - "27017"
-    container_name: scheduler_db
+    container_name: scheduler_mongo
 
   updater_scheduler:
     build:
       context: services/updater_scheduler
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/updater_scheduler/:/usr/src/app
     depends_on:
       - nats
       - scheduler_mongo
     environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
       - NATS_URI=nats://nats:4222
       - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-      - LOG_HOST=toska.cs.helsinki.fi
-      - LOG_PORT=9501
-      - LOG_HOSTNAME=updater_scheduler
-
+      - PORT=3678
     container_name: updater_scheduler
 
-
   analytics:
     build:
       context: services/oodikone2-analytics
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/oodikone2-analytics/:/usr/src/app
     ports:
       - 4568:4568
     environment:
-      WAIT_HOSTS: analytics_db:5432,analytics_redis:6379
-      REDIS: analytics_redis
-      DB_URL: postgres://postgres@analytics_db:5432/analytics_db
-      TEST_DB: postgres://postgres@analytics_db:5432/analytics_db_test
+      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
+      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
     depends_on:
       - analytics_db
-      - analytics_redis
     container_name: analytics
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
-
-  adminer:
-    image: adminer:4.6.2
-    environment:
-      - ADMINER_DESIGN=pepa-linha
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-    ports:
-      - 5050:8080
-    restart: always
-    container_name: adminer

docker/docker-compose.dev.real.yml

@@ -1,9 +1,14 @@
-version: '3'
+version: '3.2'
 
 services:
   backend:
     environment:
       DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
   userservice:
     environment:
       DB_URL: postgres://postgres@user_db:5432/user_db_real
+
+  updater_writer:
+    environment:
+      DB_URL: postgres://postgres@db:5432/tkt_oodi_real

docker/docker-compose.dev.yml

@@ -0,0 +1,163 @@
+version: '3.2'
+
+services:
+  user_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+    restart: unless-stopped
+
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+    restart: unless-stopped
+
+  backend:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    environment:
+      FRONT_URL: http://localhost:8000
+    volumes:
+      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/src/models
+      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+    restart: unless-stopped
+
+  frontend:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
+    ports:
+      - "8081:8081"
+      - "7777:7777"
+    environment:
+      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
+      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+      USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres
+      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
+    volumes:
+      - ./services/oodikone2-frontend/:/usr/src/app
+    restart: unless-stopped
+
+  userservice:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/oodikone2-userservice/:/usr/src/app
+    restart: unless-stopped
+
+  usageservice:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/oodikone2-usageservice/:/usr/src/app
+    restart: unless-stopped
+
+  updater_api:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/updater_api/:/usr/src/app
+    restart: unless-stopped
+
+  updater_writer:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/backend/updater_writer/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/models
+      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
+    environment:
+      DB_SCHEMA_KONE: kone_data
+    restart: unless-stopped
+
+  updater_scheduler:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/updater_scheduler/:/usr/src/app
+    restart: unless-stopped
+
+  analytics:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    volumes:
+      - ./services/oodikone2-analytics/:/usr/src/app
+    restart: unless-stopped
+
+  adminer:
+    image: adminer:4.6.2
+    environment:
+      - ADMINER_DESIGN=pepa-linha
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+    ports:
+      - 5050:8080
+    container_name: adminer
+    restart: unless-stopped
+
+  # zookeeper:
+  #   image: wurstmeister/zookeeper
+  #   ports:
+  #     - "2181:2181"
+  #   environment:
+  #     - KAFKA_BROKER_ID=1
+  #   container_name: zookeeper
+
+  # kafka:
+  #   image: wurstmeister/kafka
+  #   container_name: kafka
+  #   depends_on:
+  #     - zookeeper
+  #   ports:
+  #     - "9092:9092"
+  #   environment:
+  #     - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}'
+  #     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+  #     - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+  #     - KAFKA_DELETE_TOPIC_ENABLE=true
+  #     - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1
+  #     - KAFKA_LOG_DIRS=/kafka/kafka-logs-1
+  #   volumes:
+  #     - ./kafka/docker.sock:/var/run/docker.sock
+  #     - ./kafka/logs:/kafka/kafka-logs-1
+
+  # mongo_db:
+  #   image: mongo
+  #   volumes:
+  #     - ./backups/mongo/oodilearn:/dump
+  #     - ./mongodata:/data/db
+  #   ports:
+  #     - "27017:27017"
+  #   container_name: mongo_db
+
+  # oodilearn:
+  #   build:
+  #     context: services/oodilearn/server
+  #     dockerfile: Dockerfile
+  #   volumes:
+  #     - ./services/oodilearn/data:/data
+  #     - ./services/oodilearn/server:/src
+  #     - ./services/oodilearn/models:/models
+  #   ports:
+  #     - "5000:5000"
+  #   environment:
+  #     - ENV=development
+  #     - MONGO_URI=mongodb://mongo_db:27017/oodilearn
+  #   container_name: oodilearn

docker/docker-compose.lateste2e.yml

@@ -0,0 +1,103 @@
+version: '3.2'
+
+services:
+  user_db:
+    environment:
+      - POSTGRES_DB=user_db
+
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
+
+  backend:
+    image: toska/oodikone2-backend:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-backend:trunk
+        - toska/oodikone2-backend:staging
+        - toska/oodikone2-backend:latest
+    ports:
+      - "9229:9229"
+    environment:
+      FRONT_URL: http://localhost:1337
+      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+
+  frontend:
+    image: toska/oodikone2-frontend:${TAG}
+    build:
+      args:
+        BASE_PATH: /
+        SENTRY_URL: ${SENTRY_URL}
+        SENTRY_AUTH_TOKEN: ${SENTRY_AUTH_TOKEN}
+        SENTRY_ORG: sentry
+        SENTRY_PROJECT: oodifront
+      cache_from:
+        - toska/oodikone2-frontend:trunk
+        - toska/oodikone2-frontend:staging
+        - toska/oodikone2-frontend:latest
+    ports:
+      - "5000:5000"
+
+  userservice:
+    image: toska/oodikone2-userservice:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-userservice:trunk
+        - toska/oodikone2-userservice:staging
+        - toska/oodikone2-userservice:latest
+
+  usageservice:
+    image: toska/oodikone2-usageservice:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-usageservice:trunk
+        - toska/oodikone2-usageservice:staging
+        - toska/oodikone2-usageservice:latest
+
+  updater_api:
+    image: toska/updater_api:${TAG}
+    build:
+      cache_from:
+        - toska/updater_api:trunk
+        - toska/updater_api:staging
+        - toska/updater_api:latest
+
+  updater_writer:
+    image: toska/updater_writer:${TAG}
+    build:
+      cache_from:
+        - toska/updater_writer:trunk
+        - toska/updater_writer:staging
+        - toska/updater_writer:latest
+
+  updater_scheduler:
+    image: toska/updater_scheduler:${TAG}
+    build:
+      cache_from:
+        - toska/updater_scheduler:trunk
+        - toska/updater_scheduler:staging
+        - toska/updater_scheduler:latest
+
+  analytics:
+    image: toska/oodikone2-analytics:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-analytics:trunk
+        - toska/oodikone2-analytics:staging
+        - toska/oodikone2-analytics:latest
+
+  nginx:
+    image: nginx:latest
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/error.log:/etc/nginx/error_log.log
+      - ./nginx/log:/var/log/nginx/localhost.error_log
+      - ./nginx/cache/:/etc/nginx/cache
+    ports:
+      -  1337:80
+    depends_on:
+      - frontend
+      - backend
+      - userservice
+      - analytics
+    container_name: nginx

nginx.conf

@@ -4,6 +4,7 @@ http {
 
   rewrite_log on;
   error_log /var/log/nginx/localhost.error_log notice;
+  access_log off;
 
   server {
     listen 80;

nginx.staging.conf

@@ -1,19 +0,0 @@
-events { worker_connections 1024; }
-
-http {
-
-  rewrite_log on;
-  error_log /var/log/nginx/localhost.error_log notice;
-
-  server {
-    listen 80;
-
-    location / {
-      proxy_pass http://frontend:5000/;
-    }
-
-    location /api/ {
-      proxy_pass http://backend:8080/;
-    }
-  }
-}

package-lock.json

@@ -48,9 +48,9 @@
       }
     },
     "ajv": {
-      "version": "6.10.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
-      "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==",
+      "version": "6.10.2",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz",
+      "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==",
       "dev": true,
       "requires": {
         "fast-deep-equal": "^2.0.1",
@@ -77,6 +77,12 @@
       "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
       "dev": true
     },
+    "arch": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/arch/-/arch-2.1.1.tgz",
+      "integrity": "sha512-BLM56aPo9vLLFVa8+/+pJLnrZ7QGGTVHWsCwieAWT9o9K8UeGaQbzZbGoabWLOo2ksBCztoXdqBZBplqLDDCSg==",
+      "dev": true
+    },
     "argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -102,12 +108,12 @@
       "dev": true
     },
     "async": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.4.0.tgz",
-      "integrity": "sha1-SZAgDxjqW4N8LMT4wDGmmFw4VhE=",
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
+      "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
       "dev": true,
       "requires": {
-        "lodash": "^4.14.0"
+        "lodash": "^4.17.10"
       }
     },
     "asynckit": {
@@ -341,9 +347,9 @@
       "dev": true
     },
     "combined-stream": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
-      "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
       "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
@@ -380,9 +386,9 @@
       }
     },
     "concurrently": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-4.1.0.tgz",
-      "integrity": "sha512-pwzXCE7qtOB346LyO9eFWpkFJVO3JQZ/qU/feGeaAHiX1M3Rw3zgXKc5cZ8vSH5DGygkjzLFDzA/pwoQDkRNGg==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-4.1.1.tgz",
+      "integrity": "sha512-48+FE5RJ0qc8azwKv4keVQWlni1hZeSjcWr8shBelOBtBHcKj1aJFM9lHRiSc1x7lq416pkvsqfBMhSRja+Lhw==",
       "dev": true,
       "requires": {
         "chalk": "^2.4.1",
@@ -403,9 +409,9 @@
           "dev": true
         },
         "rxjs": {
-          "version": "6.5.1",
-          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.1.tgz",
-          "integrity": "sha512-y0j31WJc83wPu31vS1VlAFW5JGrnGC+j+TtGAa1fRQphy48+fDYiDmX8tjGloToEsMkxnouOg/1IzXGKkJnZMg==",
+          "version": "6.5.2",
+          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.2.tgz",
+          "integrity": "sha512-HUb7j3kvb7p7eCUHE3FqjoDsC1xfZQ4AHFWfTKSpZ+sAhhz5X1WX0ZuUqWbzB2QhSLp3DoLUG+hMdEDKqWo2Zg==",
           "dev": true,
           "requires": {
             "tslib": "^1.9.0"
@@ -454,41 +460,49 @@
       }
     },
     "cypress": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-3.2.0.tgz",
-      "integrity": "sha512-PN0wz6x634QyNL56/voTzJoeScDfwtecvSfFTHfv5MkHuECVSR4VQcEZTvYtKWln3CMBMUkWbBKPIwwu2+a/kw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-3.4.1.tgz",
+      "integrity": "sha512-1HBS7t9XXzkt6QHbwfirWYty8vzxNMawGj1yI+Fu6C3/VZJ8UtUngMW6layqwYZzLTZV8tiDpdCNBypn78V4Dg==",
       "dev": true,
       "requires": {
         "@cypress/listr-verbose-renderer": "0.4.1",
         "@cypress/xvfb": "1.2.4",
+        "arch": "2.1.1",
         "bluebird": "3.5.0",
         "cachedir": "1.3.0",
         "chalk": "2.4.2",
         "check-more-types": "2.24.0",
         "commander": "2.15.1",
         "common-tags": "1.8.0",
-        "debug": "3.1.0",
+        "debug": "3.2.6",
         "execa": "0.10.0",
         "executable": "4.1.1",
         "extract-zip": "1.6.7",
-        "fs-extra": "4.0.1",
-        "getos": "3.1.0",
-        "glob": "7.1.3",
+        "fs-extra": "5.0.0",
+        "getos": "3.1.1",
         "is-ci": "1.2.1",
         "is-installed-globally": "0.1.0",
         "lazy-ass": "1.6.0",
         "listr": "0.12.0",
-        "lodash": "4.17.11",
+        "lodash": "4.17.15",
         "log-symbols": "2.2.0",
         "minimist": "1.2.0",
         "moment": "2.24.0",
         "ramda": "0.24.1",
         "request": "2.88.0",
-        "request-progress": "0.4.0",
+        "request-progress": "3.0.0",
         "supports-color": "5.5.0",
-        "tmp": "0.0.33",
+        "tmp": "0.1.0",
         "url": "0.11.0",
         "yauzl": "2.10.0"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "4.17.15",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "dev": true
+        }
       }
     },
     "dashdash": {
@@ -507,12 +521,12 @@
       "dev": true
     },
     "debug": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-      "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+      "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
       "dev": true,
       "requires": {
-        "ms": "2.0.0"
+        "ms": "^2.1.1"
       }
     },
     "decamelize": {
@@ -630,6 +644,12 @@
             "ms": "2.0.0"
           }
         },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+          "dev": true
+        },
         "yauzl": {
           "version": "2.4.1",
           "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
@@ -705,13 +725,13 @@
       }
     },
     "fs-extra": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.1.tgz",
-      "integrity": "sha1-f8DGyJV/mD9X8waiTlud3Y0N2IA=",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-5.0.0.tgz",
+      "integrity": "sha512-66Pm4RYbjzdyeuqudYqhFiNBbCIuI9kgRqLPSHIlXHidW8NIQtVdkM1yeZ4lXwuhbTETv3EUGMNHAAw6hiundQ==",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.2",
-        "jsonfile": "^3.0.0",
+        "jsonfile": "^4.0.0",
         "universalify": "^0.1.0"
       }
     },
@@ -740,12 +760,12 @@
       "dev": true
     },
     "getos": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/getos/-/getos-3.1.0.tgz",
-      "integrity": "sha512-i9vrxtDu5DlLVFcrbqUqGWYlZN/zZ4pGMICCAcZoYsX3JA54nYp8r5EThw5K+m2q3wszkx4Th746JstspB0H4Q==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/getos/-/getos-3.1.1.tgz",
+      "integrity": "sha512-oUP1rnEhAr97rkitiszGP9EgDVYnmchgFzfqRzSkgtfv7ai6tEi7Ko8GgjNXts7VLWEqrTWyhsOKLe5C5b/Zkg==",
       "dev": true,
       "requires": {
-        "async": "2.4.0"
+        "async": "2.6.1"
       }
     },
     "getpass": {
@@ -758,9 +778,9 @@
       }
     },
     "glob": {
-      "version": "7.1.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
-      "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
       "dev": true,
       "requires": {
         "fs.realpath": "^1.0.0",
@@ -781,9 +801,9 @@
       }
     },
     "graceful-fs": {
-      "version": "4.1.15",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
-      "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz",
+      "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==",
       "dev": true
     },
     "har-schema": {
@@ -923,9 +943,9 @@
       }
     },
     "inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "dev": true
     },
     "ini": {
@@ -1075,9 +1095,9 @@
       "dev": true
     },
     "jsonfile": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-3.0.1.tgz",
-      "integrity": "sha1-pezG9l9T9mLEQVx2daAzHQmS7GY=",
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
+      "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.6"
@@ -1266,9 +1286,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
+      "version": "4.17.15",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+      "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
       "dev": true
     },
     "lodash.once": {
@@ -1317,18 +1337,18 @@
       }
     },
     "mime-db": {
-      "version": "1.39.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.39.0.tgz",
-      "integrity": "sha512-DTsrw/iWVvwHH+9Otxccdyy0Tgiil6TWK/xhfARJZF/QFhwOgZgOIvA2/VIGpM8U7Q8z5nDmdDWC6tuVMJNibw==",
+      "version": "1.40.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
+      "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.23",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.23.tgz",
-      "integrity": "sha512-ROk/m+gMVSrRxTkMlaQOvFmFmYDc7sZgrjjM76abqmd2Cc5fCV7jAMA5XUccEtJ3cYiYdgixUVI+fApc2LkXlw==",
+      "version": "2.1.24",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
+      "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
       "dev": true,
       "requires": {
-        "mime-db": "~1.39.0"
+        "mime-db": "1.40.0"
       }
     },
     "mimic-fn": {
@@ -1376,9 +1396,9 @@
       "dev": true
     },
     "ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
       "dev": true
     },
     "nice-try": {
@@ -1387,12 +1407,6 @@
       "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
       "dev": true
     },
-    "node-eta": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/node-eta/-/node-eta-0.1.1.tgz",
-      "integrity": "sha1-QGYQmzk3HHYccrfr2pqeoKXeEh8=",
-      "dev": true
-    },
     "normalize-package-data": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@@ -1523,12 +1537,6 @@
         }
       }
     },
-    "os-tmpdir": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
-      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
-      "dev": true
-    },
     "p-defer": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
@@ -1654,15 +1662,15 @@
       }
     },
     "process-nextick-args": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
       "dev": true
     },
     "psl": {
-      "version": "1.1.31",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz",
-      "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.2.0.tgz",
+      "integrity": "sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA==",
       "dev": true
     },
     "pump": {
@@ -1771,13 +1779,12 @@
       }
     },
     "request-progress": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-0.4.0.tgz",
-      "integrity": "sha1-wZVOOQhqqFJpxWYLzuAUKmpw1+c=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-3.0.0.tgz",
+      "integrity": "sha1-TKdUCBx/7GP1BeT6qCWqBs1mnb4=",
       "dev": true,
       "requires": {
-        "node-eta": "^0.1.1",
-        "throttleit": "^0.0.2"
+        "throttleit": "^1.0.0"
       }
     },
     "require-directory": {
@@ -1817,6 +1824,15 @@
         "onetime": "^1.0.0"
       }
     },
+    "rimraf": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
+      "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
+      "dev": true,
+      "requires": {
+        "glob": "^7.1.3"
+      }
+    },
     "run-node": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/run-node/-/run-node-1.0.0.tgz",
@@ -2013,18 +2029,18 @@
       "dev": true
     },
     "throttleit": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-0.0.2.tgz",
-      "integrity": "sha1-z+34jmDADdlpe2H90qg0OptoDq8=",
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz",
+      "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=",
       "dev": true
     },
     "tmp": {
-      "version": "0.0.33",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
-      "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.1.0.tgz",
+      "integrity": "sha512-J7Z2K08jbGcdA1kkQpJSqLF6T0tdQqpR2pnSUXsIchbPdTI9v3e85cLW0d6WDhwuAleOV71j2xWs8qMPfK7nKw==",
       "dev": true,
       "requires": {
-        "os-tmpdir": "~1.0.2"
+        "rimraf": "^2.6.3"
       }
     },
     "tough-cookie": {
@@ -2052,9 +2068,9 @@
       "dev": true
     },
     "tslib": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
-      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==",
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
+      "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==",
       "dev": true
     },
     "tunnel-agent": {

package.json

@@ -2,14 +2,30 @@
   "name": "oodikone",
   "version": "1.0.0",
   "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the docker-compose.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
-  "main": "run.sh",
+  "main": "./scripts/run.sh",
   "scripts": {
-    "cypress:open": "CYPRESS_baseUrl=http://localhost:8081 cypress open",
-    "cypress:run": "CYPRESS_baseUrl=http://localhost:8081 cypress run --config video=false -P ./",
-    "cypress:record": "cypress run --config videoUploadOnPasses=false -P ./",
+    "cli": "./scripts/run.sh",
+    "cypress": "cypress",
+    "cypress:open": "cypress open --config baseUrl=http://localhost:8081/",
+    "cypress:run": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:8081/",
+    "cypress:ci": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:1337/",
     "concurrently": "concurrently",
-    "test": "./run_all_tests.sh",
-    "test_services": "./run_service_tests.sh"
+    "test": "./scripts/run_all_tests.sh",
+    "test_services": "./scripts/run_service_tests.sh",
+    "lint": "./scripts/run_service_lints.sh",
+    "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
+    "docker:real": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml",
+    "docker:ci": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml",
+    "docker:build": "npm run docker -- build",
+    "docker:up": "npm run docker -- up -d",
+    "docker:up:real": "npm run docker:real -- up -d",
+    "docker:down": "npm run docker -- down",
+    "docker:logs": "npm run docker -- logs -f",
+    "docker:logs:backend": "npm run docker -- logs -f backend",
+    "docker:restart": "npm run docker -- restart",
+    "docker:restart:backend": "npm run docker -- restart backend",
+    "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune -a",
+    "start": "npm run docker:up && npm run docker:logs"
   },
   "husky": {
     "hooks": {
@@ -27,8 +43,8 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "concurrently": "^4.1.0",
-    "cypress": "^3.2.0",
+    "concurrently": "^4.1.1",
+    "cypress": "^3.4.1",
     "husky": "^1.3.1"
   },
   "dependencies": {}

scripts/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-source scripts.sh
+source ./scripts/scripts.sh
 
 PS3='Please enter your choice: '
 
@@ -26,7 +26,6 @@ options=(
     "Reset real db"
     "Download latest anon db"
     "Download latest real db"
-    "Full reset"
     "Quit"
 )
 select opt in "${options[@]}"
@@ -34,7 +33,7 @@ do
     case $opt in
         "e2e setup")
             mopo
-            run_e2e_setup docker-compose.lateste2e.yml
+            run_e2e_setup ./docker/docker-compose.lateste2e.yml staging
             ;;
         "Anon setup")
             mopo
@@ -56,9 +55,6 @@ do
         "Download latest real db")
             get_oodikone_server_backup
             ;;
-        "Full reset")
-            purge
-            ;;
         "Quit")
             break
             ;;

scripts/run_service_lints.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# stop on first error
+set -e
+docker-compose exec -T backend npm run lint
+docker-compose exec -T updater_writer npm  run lint
+docker-compose exec -T updater_api npm run lint
+docker-compose exec -T updater_scheduler npm run lint
+docker-compose exec -T userservice npm run lint
+docker-compose exec -T frontend npm run lint
+docker-compose exec -T analytics npm run lint
+docker-compose exec -T userservice npm run lint

scripts/run_service_tests.sh

@@ -3,8 +3,11 @@
 # stop on first error
 set -e
 docker-compose exec -T backend npm test
+docker-compose exec -T updater_writer npm test
+docker-compose exec -T updater_api npm test
+docker-compose exec -T updater_scheduler npm test
 docker-compose exec -T userservice npm test
 docker-compose exec -T frontend npm test
 docker-compose exec -T analytics npm test
+docker-compose exec -T userservice npm test
 docker-compose exec -T usageservice npm test
-# docker-compose exec -T updater_writer npm test

scripts/scripts.sh

@@ -5,9 +5,15 @@ ANONDB_DIR=anonyymioodi
 BACKUP_DIR=backups
 PSQL_DB_BACKUP="$ANONDB_DIR/anon.sqz"
 USER_DB_BACKUP="$ANONDB_DIR/user-dump.sqz"
+KONE_DB_BACKUP="$ANONDB_DIR/anon_kone.sqz"
 PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
+KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz"
 USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 
+docker-compose-dev () {
+    docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@"
+}
+
 retry () {
     for i in {1..60}
     do
@@ -25,20 +31,9 @@ echo_path () {
     echo $(pwd)
 }
 
-purge () {
-    docker-compose down || echo "docker-compose down failed"
-    git clean -f -fdX
-}
-
-megapurge () {
-    git clean -f -fdX
-    docker stop $(docker ps -q)
-    docker container prune
-    docker rmi $(docker images -q)
-}
-
 get_oodikone_server_backup() {
-    scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
+    scp -r -o ProxyCommand="ssh -W %h:%p melkinpaasi.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
+    scp -r -o ProxyCommand="ssh -W %h:%p melkinpaasi.cs.helsinki.fi" svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
 }
 
 get_anon_oodikone() {
@@ -55,19 +50,8 @@ get_anon_oodikone() {
 }
 
 restore_psql_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi -j4 $PSQL_DB_BACKUP
-}
-
-restore_real_psql_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi_real -j4 $PSQL_REAL_DB_BACKUP
-}
-
-restore_userdb_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db -j4 $USER_DB_BACKUP
-}
-
-restore_real_userdb_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db_real -j4 $USER_REAL_DB_BACKUP
+    docker cp $1 $2:/asd.sqz
+    docker exec $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
 }
 
 # oodilearn
@@ -76,19 +60,28 @@ restore_real_userdb_from_backup () {
 # }
 
 ping_psql () {
-    echo "Pinging psql in container $1 with db name $2"
+    drop_psql $1 $2
+    echo "Creating psql in container $1 with db name $2"
     retry docker exec -u postgres $1 pg_isready --dbname=$2
     docker exec -u postgres $1 psql -c "CREATE DATABASE $2" || echo "container $1 DB $2 already exists"
 }
 
+drop_psql () {
+    echo "Dropping psql in container $1 with db name $2"
+    retry docker exec -u postgres $1 pg_isready --dbname=$2
+    docker exec -u postgres $1 psql -c "DROP DATABASE $2" || echo "container $1 DB $2 doesn't exists"
+}
+
 db_setup_full () {
     echo "Restoring PostgreSQL from backup"
-    retry restore_real_psql_from_backup
+    ping_psql "oodi_db" "tkt_oodi_real"
+    restore_psql_from_backup $PSQL_REAL_DB_BACKUP oodi_db tkt_oodi_real
+    ping_psql "db_kone" "db_kone_real"
+    restore_psql_from_backup $KONE_REAL_DB_BACKUP db_kone db_kone_real
+    ping_psql "oodi_user_db" "user_db_real"
+    restore_psql_from_backup $USER_REAL_DB_BACKUP oodi_user_db user_db_real
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
-    echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db_real"
-    retry restore_real_userdb_from_backup
     echo "Database setup finished"
 }
 
@@ -96,47 +89,45 @@ db_anon_setup_full () {
     echo "Restoring PostgreSQL from backup"
     ping_psql "oodi_db" "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
-    retry restore_psql_from_backup
+    restore_psql_from_backup $PSQL_DB_BACKUP oodi_db tkt_oodi
+    ping_psql "db_kone" "db_kone"
+    ping_psql "db_kone" "db_kone_test"
+    restore_psql_from_backup $KONE_DB_BACKUP db_kone db_kone
+    ping_psql "oodi_user_db" "user_db"
+    restore_psql_from_backup $USER_DB_BACKUP oodi_user_db user_db
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
-    echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db"
-    retry restore_userdb_from_backup
     echo "Database setup finished"
 }
 
 reset_real_db () {
-    docker-compose down
-    docker-compose up -d db user_db
-    ping_psql "oodi_db" "tkt_oodi_real"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi_real"
-    ping_psql "oodi_user_db" "user_db_real"
-    docker exec -u postgres oodi_user_db dropdb "user_db_real"
+    docker-compose-dev down
+    docker-compose-dev up -d db user_db db_kone
     db_setup_full
-    docker-compose down
+    docker-compose-dev down
 }
 
 reset_db () {
-    docker-compose down
-    docker-compose up -d db user_db
-    ping_psql "oodi_db" "tkt_oodi"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi"
-    ping_psql "oodi_db" "tkt_oodi_test"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi_test"
-    ping_psql "oodi_user_db" "user_db"
-    docker exec -u postgres oodi_user_db dropdb "user_db"
+    docker-compose-dev down
+    docker-compose-dev up -d db user_db db_kone
     db_anon_setup_full
-    docker-compose down
+    docker-compose-dev down
+}
+
+reset_db_for_cypress () {
+    # stop any service with connections to DB
+    docker-compose stop backend updater_writer
+    db_anon_setup_full
+    # restart services, run migrations
+    docker-compose restart backend updater_writer
+    # wait until backend is up
+    retry curl --silent --fail localhost:8080/ping
 }
 
 install_cli_npm_packages () {
     npm ci
 }
 
-docker_build () {
-    docker-compose up -d --build
-}
-
 show_instructions () {
     cat ./assets/instructions.txt
 }
@@ -151,12 +142,12 @@ run_full_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose build
+    docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose up -d db user_db
+    docker-compose-dev up -d db user_db db_kone
     db_setup_full
     db_anon_setup_full
-    docker-compose down
+    docker-compose-dev down
     show_instructions
 }
 
@@ -168,11 +159,11 @@ run_anon_full_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose build
+    docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose up -d db user_db
+    docker-compose-dev up -d db user_db db_kone
     db_anon_setup_full
-    docker-compose down
+    docker-compose-dev down
     show_instructions
 }
 
@@ -184,10 +175,10 @@ run_e2e_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose -f $1 build
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 build
     echo "Setup oodikone db from dump."
-    docker-compose -f $1 up -d db user_db
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
     db_anon_setup_full
     echo "Starting services."
-    docker-compose -f $1 up -d
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d
 }

services/backend/oodikone2-backend/.eslintrc.json

@@ -23,7 +23,6 @@
             "error",
             "never"
         ],
-        "no-console": 0,
-        "max-len": [2, 120]
+        "no-console": 0
     }
 }

services/backend/oodikone2-backend/Dockerfile

@@ -1,22 +1,21 @@
 FROM node:10.15-alpine
 
+WORKDIR /usr/src/app
+EXPOSE 8080
+CMD npm start
+
+COPY ./oodikone2-backend/package.json /usr/src/app/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/app/package-lock.json
+COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
+RUN npm ci
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
+
 COPY ./oodikone2-backend /usr/src/app
 COPY ./shared/models /usr/src/app/src/models
 COPY ./shared/migrations /usr/src/app/src/database/migrations
 COPY ./shared/migrations_kone /usr/src/app/src/database/migrations_kone
 
-WORKDIR /usr/src/app
-RUN npm ci
-
-COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
-RUN chmod +x /wait
-
-EXPOSE 8080
-CMD /wait && npm start

services/backend/oodikone2-backend/package.json

@@ -15,7 +15,8 @@
     "docker_local": "NODE_ENV=docker_local nodemon index.js",
     "dev": "NODE_ENV=dev nodemon --max_old_space_size=4096 index.js",
     "anon": "NODE_ENV=test nodemon index.js",
-    "test": "NODE_ENV=test jest --runInBand --verbose",
+    "test": "npm run lint && npm run jest",
+    "jest": "NODE_ENV=test jest --runInBand --verbose",
     "test_docker": "docker exec -t backend /bin/bash -c 'npm test'",
     "update_database_dev": "NODE_ENV=dev node src/services/doo_api_database_updater/run_updater.js",
     "update_database": "node src/services/doo_api_database_updater/run_updater.js",
@@ -23,7 +24,7 @@
     "init_redis": "node src/util/init_redis.js",
     "seed_redis": "node src/util/add_json_duplicates_to_redis.js && node src/util/map_course_codes.js",
     "studentnumbers_to_file": "NODE_ENV=dev node src/services/doo_api_database_updater/db_studentnumbers_to_file.js",
-    "lint": "eslint --parser babel-eslint src",
+    "lint": "eslint --parser babel-eslint .",
     "force_sync_database": "node scripts/force_sync_database.js",
     "update_studentnumbers_dev": "NODE_ENV=dev node src/services/doo_api_database_updater/student_list_updater.js",
     "anonymize": "node src/anonymizer/anonymizer.js",
@@ -38,41 +39,42 @@
   "author": "Matti Luukkainen",
   "license": "MIT",
   "dependencies": {
+    "async-lock": "^1.2.2",
     "axios": "^0.19.0",
-    "bluebird": "^3.5.3",
-    "body-parser": "^1.18.3",
+    "bluebird": "^3.5.5",
+    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
-    "dotenv": "^6.1.0",
-    "express": "^4.16.4",
-    "jsonwebtoken": "^8.4.0",
-    "kafka-node": "^4.1.1",
-    "lodash": "^4.17.10",
+    "cron": "^1.7.1",
+    "dotenv": "^6.2.0",
+    "express": "^4.17.1",
+    "jsonwebtoken": "^8.5.1",
+    "kafka-node": "^4.1.3",
+    "lodash": "^4.17.15",
     "mkdirp": "^0.5.1",
-    "moment": "^2.22.2",
+    "moment": "^2.24.0",
     "morgan": "^1.9.1",
-    "node-nats-streaming": "^0.2.2",
     "nodemailer": "^4.7.0",
-    "pg": "^7.7.1",
+    "pg": "^7.11.0",
     "raven": "^2.6.3",
     "redis": "^2.8.0",
-    "sequelize": "^5.3.0",
+    "sequelize": "^5.10.2",
     "umzug": "^2.2.0",
     "uuid": "^3.3.2",
     "winston": "^3.0.0",
     "winston-log2gelf": "^2.0.4",
-    "winston-transport": "^4.2.0"
+    "winston-transport": "^4.3.0"
   },
   "devDependencies": {
-    "ava": "^0.25.0",
+    "ava": "^2.2.0",
     "babel-eslint": "^8.2.6",
-    "eslint": "^5.9.0",
+    "eslint": "^5.16.0",
     "faker": "^4.1.0",
-    "jest": "^24.1.0",
-    "nock": "^10.0.2",
+    "jest": "^24.8.0",
+    "nock": "^10.0.6",
     "node-status": "^1.0.0",
-    "nodemon": "^1.18.7",
-    "nyc": "^13.1.0",
-    "supertest": "^3.1.0"
+    "nodemon": "^1.19.1",
+    "nyc": "^13.3.0",
+    "supertest": "^3.4.2"
   },
   "ava": {
     "files": [

services/backend/oodikone2-backend/src/anonymizer/anon_oodi_interface.js

@@ -13,9 +13,10 @@ axios.defaults.auth = {
   username: 'tktl',
 }
 
-axios.defaults.params = {
-  token: process.env.TOKEN
-}
+instance.interceptors.request.use((config) => {
+  config.params = {token: process.env.TOKEN}
+  return config
+})
 
 const getStudent = (studentNumber) => {
   return instance.get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)

services/backend/oodikone2-backend/src/app.js

@@ -4,33 +4,43 @@ const bodyParser = require('body-parser')
 const Raven = require('raven')
 const conf = require('./conf-backend')
 const routes = require('./routes')
-const { StartNats } = require('./events')
+const { startCron } = require('./events')
 const { PORT } = conf
-const app = express()
+const { initializeDatabaseConnection } = require('./database/connection')
 
-StartNats()
+initializeDatabaseConnection().then(() => {
+  const app = express()
 
-Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
+  startCron()
 
-app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-app.use(bodyParser.json())
+  Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
 
-app.get('/ping', async (req, res) => {
-  res.json({ data: 'pong' })
-})
+  app.use(cors({ credentials: true, origin: conf.frontend_addr }))
+  app.use(bodyParser.json())
 
-let BASE_URL = ''
-if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
-  BASE_URL = '/api'
-}
+  app.get('/ping', async (req, res) => {
+    res.json({ data: 'pong' })
+  })
 
-routes(app, BASE_URL)
+  let BASE_URL = ''
+  if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
+    BASE_URL = '/api'
+  }
 
-app.get('*', async (req, res) => {
-  const results = { error: 'unknown endpoint' }
-  res.status(404).json(results)
-})
+  routes(app, BASE_URL)
 
-module.exports = app.listen(PORT, () => {
-  console.log('Example app listening on port ' + PORT + '!')
+  app.get('*', async (req, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+
+  const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+  process.on('SIGTERM', () => {
+    server.close(() => {
+      console.log('Process terminated')
+    })
+  })
+}).catch(e => {
+  process.exitCode = 1
+  console.log(e)
 })

services/backend/oodikone2-backend/src/conf-backend.js

@@ -1,23 +1,27 @@
 require('dotenv').config()
 
 const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
 const isTest = process.env.NODE_ENV === 'test'
+let DB_URL = process.env.DB_URL
+let DB_URL_KONE = process.env.DB_URL_KONE
 if (isTest) {
   DB_URL = process.env.TEST_DB
+  DB_URL_KONE = process.env.TEST_DB_KONE
 }
 else if (NODE_ENV === 'anon') {
   DB_URL = process.env.ANON_DB
+  DB_URL_KONE = process.env.ANON_DB_KONE
 }
 const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'kone_data'
+const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
 const OODILEARN_URL = process.env.OODILEARN_URL
 const USERSERVICE_URL = process.env.USERSERVICE_URL
+const UPDATER_URL = process.env.UPDATER_URL
 const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
 const PORT = isTest ? 8079 : 8080
@@ -58,10 +62,26 @@ if (process.env.NODE_ENV === 'dev' || isTest) {
 }
 
 module.exports = {
-  frontend_addr, DB_URL, redis, TOKEN_SECRET, DB_SCHEMA, DB_SCHEMA_KONE, OODI_ADDR, CERT_PATH, KEY_PATH, FEATURES, OODILEARN_URL,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL), ACCESS_TOKEN_HEADER_KEY, PORT,
+  frontend_addr,
+  DB_URL,
+  DB_URL_KONE,
+  redis,
+  TOKEN_SECRET,
+  DB_SCHEMA,
+  DB_SCHEMA_KONE,
+  OODI_ADDR,
+  CERT_PATH,
+  KEY_PATH,
+  FEATURES,
+  OODILEARN_URL,
+  USERSERVICE_URL: formatURL(USERSERVICE_URL),
+  ACCESS_TOKEN_HEADER_KEY,
+  PORT,
   ANALYTICS_URL: formatURL(ANALYTICS_URL),
   USAGESERVICE_URL,
-  requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY,
-  isTest
+  requiredGroup,
+  OODI_SECRET,
+  OODI_SECRET_HEADER_KEY,
+  isTest,
+  UPDATER_URL
 }

services/backend/oodikone2-backend/src/database/connection.js

@@ -5,77 +5,65 @@ const conf = require('../conf-backend')
 const sequelize = new Sequelize(conf.DB_URL, {
   schema: conf.DB_SCHEMA,
   searchPath: conf.DB_SCHEMA,
-  logging: false,
-  operatorsAliases: false
+  logging: false
 })
 sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
 
-const sequelizeKone = new Sequelize(conf.DB_URL, {
+const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
-  logging: false,
-  operatorsAliases: false
+  logging: false
 })
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
-// See https://github.com/sequelize/sequelize/issues/10875
 
-const runMigrations = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelize,
-        tableName: 'migrations'
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelize.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/src/database/migrations`,
-        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA
+
+const initializeDatabaseConnection = async () => {
+  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+  const waitSeconds = 60
+  for (let i = 1; i <= waitSeconds; i++) {
+    try {
+      await sequelize.authenticate()
+      await sequelizeKone.authenticate()
+      break
+    } catch (e) {
+      if (i === waitSeconds) {
+        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        throw e
       }
-    })
-    const migrations = await migrator.up()
+      console.log('.')
+      await sleep(1000)
+    }
+  }
+  if (!conf.isTest) {
+    try {
+      const migrator = new Umzug({
+        storage: 'sequelize',
+        storageOptions: {
+          sequelize: sequelizeKone,
+          tableName: 'migrations',
+          schema: conf.DB_SCHEMA_KONE
+        },
+        logging: console.log,
+        migrations: {
+          params: [
+            sequelizeKone.getQueryInterface(),
+            Sequelize
+          ],
+          path: `${process.cwd()}/src/database/migrations_kone`,
+          pattern: /\.js$/,
+          schema: conf.DB_SCHEMA_KONE
+        },
+      })
+      const migrations = await migrator.up()
 
-    console.log('Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Migration error, message:', e)
+      console.log('Kone Migrations up to date', migrations)
+    } catch (e) {
+      console.log('Kone Migration error')
+      throw e
+    }
   }
 }
 
-const runMigrationsKone = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelizeKone,
-        tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelizeKone.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/src/database/migrations_kone`,
-        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE
-      },
-    })
-    const migrations = await migrator.up()
-
-    console.log('Kone Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Kone Migration error, message:', e)
-  }
-}
-
-const migrationPromise = !conf.isTest ? runMigrations().then(() => runMigrationsKone()) : Promise.resolve()
-
 const forceSyncDatabase = async () => {
   try {
     await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
@@ -94,6 +82,6 @@ const forceSyncDatabase = async () => {
 module.exports = {
   sequelize,
   sequelizeKone,
-  migrationPromise,
+  initializeDatabaseConnection,
   forceSyncDatabase
 }

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -1,5 +1,6 @@
 const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
+const { Migration } = require('../models/index')
+const { MigrationKone } = require('../models/models_kone')
 
 const DEFAULT_PATH = `${process.cwd()}/src/database/migrations`
 const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`

services/backend/oodikone2-backend/src/events.js

@@ -1,117 +1,101 @@
-var stan = require('node-nats-streaming').connect(
-  'updaterNATS',
-  process.env.HOSTNAME,
-  process.env.NATS_URI
-)
-
+const { CronJob } = require('cron')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studytrack')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
+const { calculateFacultyYearlyStats } = require('./services/faculties')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput, patchFacultyYearlyStats } = require('./services/analyticsService')
 
-const taskStatuses = { }
-const handleMessage = (asyncHandlerFunction) => async (msg) => {
-  const key = msg.getSubject()+'_'+msg.getData()
-  if (taskStatuses[key])
-    return
-  stan.publish('status', key+':RECEIVED', (err) => {
-    if(err) {
-      console.log('publish failed', err)
-    }
-  })
-  taskStatuses[key] = true
-  const success = await asyncHandlerFunction(msg)
-  taskStatuses[key] = false
-  stan.publish('status', key+(success ? ':DONE' : ':ERRORED'), (err) => {
-    if(err) {
-      console.log('publish failed', err)
-    }
-  })
-  msg.ack()
+const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
+
+const refreshFacultyYearlyStats = async () => {
+  try {
+    console.log('Refreshing faculty yearly stats...')
+    const data = await calculateFacultyYearlyStats()
+    await patchFacultyYearlyStats(data)
+  } catch (e) {
+    console.error(e)
+  }
 }
 
-const StartNats = () => {
-  console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-  var opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  // opts.setAckWait(6*60*60*1000)
-  // opts.setDeliverAllAvailable()
-  // opts.setDurableName('durable')
-  opts.setMaxInFlight(1)
-  stan.on('connect', async function() {
-    stan.subscribe('RefreshStudyrightAssociations', opts).on('message', handleMessage(async () => {
+const refreshStudyrightAssociations = async () => {
+  try {
+    console.log('Refreshing studyright associations...')
+    await refreshAssociationsInRedis()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+const refreshOverview = async () => {
+  try {
+    console.log('Refreshing overview...')
+    const codes = (await getAllProgrammes()).map(p => p.code)
+    let ready = 0
+    for (const code of codes) {
       try {
-        await refreshAssociationsInRedis()
+        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+        const since = new Date().getFullYear() - 5
+        const data = await throughputStatsForStudytrack(
+          code,
+          since
+        )
+        await setThroughput(
+          data
+        )
       } catch (e) {
-        console.error(e)
-        return false
-      }
-      return true
-    }))
-    stan.subscribe('RefreshOverview', opts).on('message', handleMessage(async () => {
-      try {
-        console.log('RefreshOverview starting')
-        const codes = (await getAllProgrammes()).map(p => p.code)
-
-        let ready = 0
-        for(const code of codes) {
-          try {
-            await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-            const since = new Date().getFullYear() - 5
-            const data = await throughputStatsForStudytrack(
-              code,
-              since
-            )
-            await setThroughput(
-              data
-            )
-          } catch (e) {
-            try {
-              await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-            } catch (e) {
-              console.error(
-                e
-              )
-            }
-            console.log(
-              `Failed to update throughput stats for code: ${code}, reason: ${
-                e.message
-              }`
-            )
-          }
-          try {
-            await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-            const since = '2017-08-01'
-            const data = await productivityStatsForStudytrack(code, since)
-            await setProductivity(data)
-
-          } catch (e) {
-            try {
-              await patchProductivity({
-                [code]: { status: 'RECALCULATION ERRORED' }
-              })
-            } catch (e) {
-              console.error(e)
-            }
-            console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-          }
-          ready += 1
-          console.log(
-            `RefreshOverview ${ready}/${
-              codes.length
-            } done`
+        try {
+          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+        } catch (e) {
+          console.error(
+            e
           )
         }
-      } catch (e) {
         console.error(e)
-        return false
+        console.log(
+          `Failed to update throughput stats for code: ${code}, reason: ${
+            e.message
+          }`
+        )
       }
-      return true
-    }))
-  })
-  return stan
+      try {
+        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+        const since = '2017-08-01'
+        const data = await productivityStatsForStudytrack(code, since)
+        await setProductivity(data)
+
+      } catch (e) {
+        try {
+          await patchProductivity({
+            [code]: { status: 'RECALCULATION ERRORED' }
+          })
+        } catch (e) {
+          console.error(e)
+        }
+        console.error(e)
+        console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+      }
+      ready += 1
+      console.log(
+        `RefreshOverview ${ready}/${
+          codes.length
+        } done`
+      )
+    }
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+const startCron = () => {
+  if (process.env.NODE_ENV === 'production') {
+    schedule('0 6 * * *', async () => {
+      await refreshFacultyYearlyStats()
+      await refreshStudyrightAssociations()
+      await refreshOverview()
+    })
+  }
 }
 
 module.exports = {
-  StartNats
+  startCron
 }

services/backend/oodikone2-backend/src/middleware/accesslogger.js

@@ -1,20 +1,9 @@
 const morgan = require('morgan')
-const UsageService = require('../services/usageService')
 
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const message = [
-    req.decodedToken.name, ':',
-    tokens['method'](req, res),
-    tokens['url'](req, res),
-    tokens['status'](req, res),
-    '-',
-    tokens['response-time'](req, res), 'ms'
-  ].join(' ')
   const meta = req.decodedToken
   fields.forEach(field => meta[field] = tokens[field](req, res))
-  UsageService.log(message, meta).catch(console.log)
- 
 })
 
 module.exports = accessLogger

services/backend/oodikone2-backend/src/middleware/auth.js

@@ -1,6 +1,7 @@
 const jwt = require('jsonwebtoken')
 const conf = require('../conf-backend')
 const blacklist = require('../services/blacklist')
+const { getAccessGroupCodesFor, getCodesFromElementDetails } = require('../services/userService')
 const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
 const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
 
@@ -15,9 +16,11 @@ const checkAuth = async (req, res, next) => {
       if (err) {
         res.status(403).json(err)
       } else if (decoded.version !== TOKEN_VERSION) {
-        res.status(401).json({ error: 'Token needs to be refreshed - invalid version', reloadPage: true })
+        res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
       } else if (decoded.mockedBy ? isShibboUser(decoded.mockedBy, uid) : isShibboUser(decoded.userId, uid)) {
         req.decodedToken = decoded
+        req.roles = await getAccessGroupCodesFor(decoded.userId)
+        req.rights = await getCodesFromElementDetails(decoded.userId)
         next()
       } else {
         res.status(403).json({ error: 'User shibboleth id and token id did not match' })
@@ -28,17 +31,15 @@ const checkAuth = async (req, res, next) => {
   }
 }
 
-const roles = requiredRoles => (req, res, next) => {
-  if (req.decodedToken && req.decodedToken.roles != null) {
-    const roles = req.decodedToken.roles.map(r => r.group_code)
-    console.log(`Request has roles: ${roles}`)
+const roles = requiredRoles => async (req, res, next) => {
+  if (req.decodedToken) {
+    const { roles } = req
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
-      console.log(`authorized for ${requiredRoles}`)
       next()
       return
     }
   }
-  console.log(`missing required roles ${requiredRoles}`)
+  console.log(`Missing required roles ${ requiredRoles }`)
   res.status(403).json({ error: 'missing required roles' })
 }
 
@@ -48,8 +49,7 @@ const checkRequiredGroup = async (req, res, next) => {
   const tokenOutdated = req.decodedToken.enabled !== enabled
   if (tokenOutdated) {
     res.status(401).json({
-      error: 'Token needs to be refreshed - enabled doesnt match hy-group requirement',
-      reloadPage: true
+      error: 'Token needs to be refreshed - enabled doesnt match hy-group requirement'
     })
   } else if (!enabled) {
     res.status(403).json({ error: 'User is not enabled' })
@@ -62,7 +62,7 @@ const checkUserBlacklisting = async (req, res, next) => {
   const { userId, createdAt } = req.decodedToken
   const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
   if (isBlacklisted) {
-    res.status(401).json({ error: 'Token needs to be refreshed - blacklisted', reloadPage: true })
+    res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
   } else {
     next()
   }

services/backend/oodikone2-backend/src/routes.js

@@ -23,6 +23,7 @@ const oodi = require('./routes/oodi')
 const task = require('./routes/tasks')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
+const updater = require('./routes/updater')
 
 module.exports = (app, url) => {
   app.use(url, log)
@@ -38,15 +39,16 @@ module.exports = (app, url) => {
   app.use(url, providers)
   app.use(url, faculties)
   app.use(url, semesters)
+  app.use(url, tags)
+  app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['users']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/usage`, auth.roles(['usage']), usage)
   app.use(`${url}/oodilearn`, auth.roles(['oodilearn']), oodilearn)
   app.use(`${url}/course-groups`, auth.roles(['coursegroups']), courseGroups)
-  app.use(`${url}/mandatory_courses`, mandatoryCourses),
-  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels),
+  app.use(`${url}/mandatory_courses`, mandatoryCourses)
+  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
   app.use(`${url}/oodi`, auth.roles(['dev']), oodi)
   app.use(url, auth.roles(['dev', 'admin']), task)
-  app.use(url, tags)
 }

services/backend/oodikone2-backend/src/routes/courseGroups.js

@@ -1,12 +1,11 @@
 const router = require('express').Router()
-
 const CourseGroupService = require('../services/courseGroups')
 
 router.get('/programme/:programmeId/:force?', async (req, res) => {
   const { programmeId, force } = req.params
-  const { rights, roles } = req.decodedToken
+  const { rights, roles } = req
   const semesterCode = req.query.semester
-  if (rights.includes(programmeId) || (roles && roles.map(r => r.group_code).includes('admin'))) {
+  if (rights.includes(programmeId) || (roles && roles.includes('admin'))) {
     const courseGroups = await CourseGroupService.getCourseGroupsWithTotals(
       programmeId, semesterCode, force === 'force'
     )

services/backend/oodikone2-backend/src/routes/courses.js

@@ -39,8 +39,8 @@ router.get('/coursedisciplines', async (req, res) => {
 
 router.get('/v2/courseyearlystats', async (req, res) => {
   let results = []
-  const { rights, roles } = req.decodedToken
-  const admin = roles.map(r => r.group_code).includes('admin')
+  const { rights, roles } = req
+  const admin = roles.includes('admin')
   if (!admin) {
     if (rights.length <= 0) {
       return res.status(403).json({ error: 'No programmes so no access to course stats' })
@@ -57,19 +57,21 @@ router.get('/v2/courseyearlystats', async (req, res) => {
 
 router.get('/v3/courseyearlystats', async (req, res) => {
   try {
-    const { rights, roles } = req.decodedToken
-    const admin = roles.map(r => r.group_code).includes('admin')
+    const { rights, roles } = req
+    const admin = roles.includes('admin')
     if (!admin) {
+      // If user has rights to see at least one programme, then they are allowed
+      // to see all of them
       if (rights.length <= 0) {
         return res.status(403).json({ error: 'No programmes so no access to course stats' })
       }
     }
-    const { codes, startyearcode, endyearcode, separate: sep } = req.query
+    const { codes, separate: sep } = req.query
     const separate = !sep ? false : JSON.parse(sep)
-    if (!codes || !startyearcode) {
+    if (!codes) {
       res.status(422).send('Missing required query parameters')
     } else {
-      const results = await Course.courseYearlyStats(codes, separate, startyearcode, endyearcode)
+      const results = await Course.courseYearlyStats(codes, separate)
       res.json(results)
     }
   } catch (e) {

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -1,11 +1,20 @@
 const router = require('express').Router()
-const { getAllDegreesAndProgrammes, getAllProgrammes } = require('../services/studyrights')
+const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
 const MandatoryCourses = require('../services/mandatoryCourses')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('../services/studytrack')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
 const { getProductivity, setProductivity, getThroughput, setThroughput,
   patchProductivity, patchThroughput, ping } = require('../services/analyticsService')
 
+router.get('/elementdetails/all', async (req, res) => {
+  try {
+    const elementdetails = await getAllElementDetails()
+    res.json(elementdetails)
+  } catch (e) {
+    res.status(500).json(e)
+  }
+})
+
 router.get('/studyprogrammes', async (req, res) => {
   try {
     const studyrights = await getAllDegreesAndProgrammes()
@@ -89,6 +98,7 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
         console.error(e)
         return
       }
+      console.error(e)
       console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
     }
     ready += 1
@@ -149,6 +159,7 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
         console.error(e)
         return
       }
+      console.error(e)
       console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
     }
     ready += 1

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -1,9 +1,29 @@
 const router = require('express').Router()
 const { all } = require('../services/organisations')
+const { byType: getElementDetailsByType } = require('../services/elementdetails')
+const { getFacultyYearlyStats } = require('../services/analyticsService')
 
 router.get('/faculties', async (req, res) => {
   const faculties = await all()
   res.json(faculties)
 })
 
+router.get('/faculties/yearlystats', async (req, res) => {
+  try {
+    const facultyYearlyStats = await getFacultyYearlyStats()
+    res.status(200).json(facultyYearlyStats)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
+router.get('/faculties/programmes', async (req, res) => {
+  try {
+    const facultyProgrammes = await getElementDetailsByType(20)
+    res.json(facultyProgrammes)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/mandatorycourses.js

@@ -4,56 +4,49 @@ const MandatoryCourses = require('../services/mandatoryCourses')
 router.delete('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.remove(programme, course)
-      res.status(200).json(course)
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.remove(programme, course)
+    res.status(200).json(course)
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 
 router.post('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.create(programme, course)
-      const newCourse = await MandatoryCourses.find(programme, course)
-      res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.create(programme, course)
+    const newCourse = await MandatoryCourses.find(programme, course)
+    if (!newCourse) return res.status(400).end()
+    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 
 router.post('/:programme/label/:course', async (req, res) => {
   const { programme, course } = req.params
   const { label } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.updateLabel(programme, course, label)
-      const updatedCourse = await MandatoryCourses.find(programme, course)
-      res
-        .status(200)
-        .json({
-          name: updatedCourse.course.name,
-          code: updatedCourse.course_code,
-          label: updatedCourse.mandatory_course_label
-        })
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.updateLabel(programme, course, label)
+    const updatedCourse = await MandatoryCourses.find(programme, course)
+    if (!updatedCourse) return res.status(400).end()
+    res
+      .status(200)
+      .json({
+        name: updatedCourse.course.name,
+        code: updatedCourse.course_code,
+        label: updatedCourse.mandatory_course_label
+      })
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 

services/backend/oodikone2-backend/src/routes/population.js

@@ -1,13 +1,17 @@
 const router = require('express').Router()
 const Population = require('../services/populations')
 const Filters = require('../services/filters')
-const { updateStudents } = require('../services/doo_api_database_updater/database_updater')
+const { updateStudents } = require('../services/updaterService')
+const { isValidStudentId } = require('../util/index')
+
+const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
+const UserService = require('../services/userService')
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
   try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
+    if (!req.body.startYear || !req.body.semesters || !req.body.studyRights) {
       res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
       return
     }
@@ -27,28 +31,106 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
 
     res.json(result)
   } catch (e) {
-    res.status(400).json({ error: e })
+    console.log(e)
+    res.status(500).json({ error: e })
+  }
+})
+
+router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
+  try {
+    if (!req.body.yearcode || !req.body.coursecodes) {
+      res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+      return
+    }
+    const { coursecodes, yearcode } = req.body
+    let studentnumberlist
+    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, yearcode)
+    const { decodedToken: { userId }, roles } = req
+
+    if (roles && roles.includes('admin')) {
+      studentnumberlist = studentnumbers
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+    }
+
+    const result = await Population.bottlenecksOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000
+    }, studentnumberlist)
+
+    if (result.error) {
+      res.status(400).json(result)
+      return
+    }
+
+    res.json(result)
+  } catch (e) {
+    console.log(e)
+    res.status(500).json({ error: e })
+  }
+})
+
+router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
+  try {
+    if (!req.body.studentnumberlist) {
+      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
+      return
+    }
+    let studentnumberlist
+    const { decodedToken: { userId }, roles } = req
+
+    if (roles && roles.includes('admin')) {
+      studentnumberlist = req.body.studentnumberlist
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+    }
+
+    const result = await Population.bottlenecksOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000
+    }, studentnumberlist)
+    if (result.error) {
+      res.status(400).json(result)
+      return
+    }
+
+    res.json(result)
+
+  } catch (err) {
+    console.log(err)
+    res.status(500).json({ error: err })
   }
 })
 
 router.get('/v3/populationstatistics', async (req, res) => {
-  const { year, semesters, studyRights: studyRightsJSON } = req.query
+  const { startYear, semesters, studyRights: studyRightsJSON } = req.query
   try {
-    if (!year || !semesters || !studyRightsJSON) {
+    if (!startYear || !semesters || !studyRightsJSON) {
       res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
       return
     }
     let studyRights = null
     try {
       studyRights = JSON.parse(studyRightsJSON)
-      const { roles, rights } = req.decodedToken
-      if (!roles || !roles.map(r => r.group_code).includes('admin')) {
+      const { rights, roles } = req
+
+      if (!roles || !roles.includes('admin')) {
         if (!rights.includes(studyRights.programme)) {
           res.status(403).json([])
           return
         }
       }
-    } catch(e) {
+    } catch (e) {
       console.error(e)
       res.status(400).json({ error: 'The query had invalid studyRights' })
       return
@@ -57,8 +139,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
     if (req.query.months == null) {
       req.query.months = 12
     }
-
-    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights})
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
 
     if (result.error) {
       console.log(result.error)
@@ -70,7 +151,78 @@ router.get('/v3/populationstatistics', async (req, res) => {
     res.json(result)
   } catch (e) {
     console.log(e)
-    res.status(400).json({ error: e })
+    res.status(500).json({ error: e })
+  }
+})
+
+router.get('/v3/populationstatisticsbycourse', async (req, res) => {
+  const { coursecodes, yearcode } = req.query
+  if (!coursecodes || !yearcode) {
+    res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+    return
+  }
+  const semesters = ['FALL', 'SPRING']
+  let studentnumberlist
+  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), yearcode)
+  const { decodedToken: { userId }, roles } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+  try {
+    const result = await Population.optimizedStatisticsOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters,
+      months: 10000
+    }, studentnumberlist)
+
+    if (result.error) {
+      console.log(result.error)
+      res.status(400).end()
+      return
+    }
+
+    console.log(`request completed ${new Date()}`)
+    res.json(result)
+  } catch (e) {
+    console.log(e)
+    res.status(500).json({ error: e })
+  }
+})
+
+router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
+  const { studentnumberlist } = req.body
+  const { roles } = req
+
+  if (!(roles && roles.includes('admin'))) {
+    console.log('no rights')
+    res.status(400).end()
+    return
+  } else {
+    try {
+      const result = await Population.optimizedStatisticsOf({
+        startYear: 1900,
+        endYear: 2200,
+        studyRights: [],
+        semesters: ['FALL', 'SPRING'],
+        months: 10000
+      }, studentnumberlist)
+      if (result.error) {
+        console.log(result.error)
+        res.status(400).end()
+        return
+      }
+      res.status(200).json(result)
+    } catch (err) {
+      console.log(err)
+      res.status(400).end()
+    }
   }
 })
 
@@ -120,19 +272,24 @@ router.delete('/v2/populationstatistics/filters', async (req, res) => {
 
 router.post('/updatedatabase', async (req, res) => {
   const studentnumbers = req.body
-  console.log(studentnumbers)
-  if (studentnumbers) {
-    await updateStudents(studentnumbers, 128)
-    res.status(200).json('Updated')
-  } else {
+  if (!(studentnumbers && studentnumbers.every(sn => isValidStudentId(sn)))) {
     res.status(400).end()
+    return
+  }
+  try {
+    const response = await updateStudents(studentnumbers)
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
   }
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
   try {
-    const { rights, roles } = req.decodedToken
-    if (roles && roles.map(r => r.group_code).includes('admin')) {
+    const { rights, roles } = req
+    if (roles && roles.includes('admin')) {
       const studyrights = await StudyrightService.getAssociations()
       res.json(studyrights)
     } else {

services/backend/oodikone2-backend/src/routes/semesters.js

@@ -2,8 +2,7 @@ const router = require('express').Router()
 const { getSemestersAndYears } = require('../services/semesters')
 
 router.get('/semesters/codes', async(req, res) => {
-  const before = new Date()
-  const providers = await getSemestersAndYears(before)
+  const providers = await getSemestersAndYears()
   res.json(providers)
 })
 

services/backend/oodikone2-backend/src/routes/students.js

@@ -4,8 +4,9 @@ const userService = require('../services/userService')
 const Unit = require('../services/units')
 
 router.get('/students', async (req, res) => {
-  const { roles, userId } = req.decodedToken
-  if (roles && roles.map(r => r.group_code).includes('admin')) {
+  const { roles, decodedToken: { userId } } = req
+
+  if (roles && roles.includes('admin')) {
     let results = []
     if (req.query.searchTerm) {
       results = await Student.bySearchTerm(req.query.searchTerm)
@@ -20,15 +21,21 @@ router.get('/students', async (req, res) => {
 })
 
 router.get('/students/:id', async (req, res) => {
-  const studentId = req.params.id
-  const { roles } = req.decodedToken
-  if (roles && roles.map(r => r.group_code).includes('admin')) {
+  const { id: studentId } = req.params
+  const { roles } = req
+
+  if (roles && roles.includes('admin')) {
     const results = await Student.withId(studentId)
-    return res.json(results)
+    return results.error ?
+      res.status(400).json({ error: 'error finding student' }) :
+      res.json(results)
   }
 
   const uid = req.decodedToken.userId
   const student = await Student.withId(studentId)
+  if (student.error) {
+    return res.status(400).json({ error: 'error finding student' })
+  }
   const units = await userService.getUnitsFromElementDetails(uid)
 
   const rights = await Promise.all(units.map(async unit => {
@@ -39,9 +46,8 @@ router.get('/students/:id', async (req, res) => {
   if (rights.some(right => right !== null)) {
     res.json(student).end()
   } else {
-    res.json([]).end()
+    res.status(400).json({ error: 'error finding student' })
   }
-
 })
 
 module.exports = router

services/backend/oodikone2-backend/src/routes/tags.js

@@ -1,6 +1,8 @@
 const router = require('express').Router()
 const Tags = require('../services/tags')
 const TagStudent = require('../services/tagstudent')
+const Students = require('../services/students')
+const { difference } = require('lodash')
 
 router.get('/tags', async (req, res) => {
   try {
@@ -8,40 +10,54 @@ router.get('/tags', async (req, res) => {
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
-  const { studytrack } = req.params
   try {
+    const { studytrack } = req.params
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     const tags = await Tags.findTagsByStudytrack(studytrack)
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.post('/tags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await Tags.createNewTag(tag)
-    res.status(200).json(result)
+    const { tag } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await Tags.createNewTag(tag)
+    const tags = await Tags.findTagsByStudytrack(tag.studytrack)
+    res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err.message)
+    res.status(500).json(err.message)
   }
 })
 
 router.delete('/tags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await Tags.deleteTag(tag)
-    res.status(200).json(result)
+    const { tag } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await Tags.deleteTag(tag)
+    const t = await Tags.findTagsByStudytrack(tag.studytrack)
+    res.status(200).json(t)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 
 })
@@ -52,52 +68,94 @@ router.get('/studenttags', async (req, res) => {
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/studenttags/:studytrack', async (req, res) => {
-  const { studytrack } = req.params
   try {
+    const { studytrack } = req.params
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
-  const { studentnumber } = req.params
   try {
+    const { studentnumber } = req.params
     const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    console.log(result)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
+  }
+})
+
+router.post('/studenttags/:studentnumber', async (req, res) => {
+  try {
+    const { tag, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await TagStudent.createStudentTag(tag)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(500).json(err)
   }
 })
 
 router.post('/studenttags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await TagStudent.createStudentTag(tag)
+    const { tags, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const existingtags = await Tags.findTagsByStudytrack(studytrack)
+    const existingTagids = existingtags.map(t => t.tag_id)
+    const tagids = [...new Set(tags.map(t => t.tag_id))]
+    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
+
+    const studentnumbers = tags.map(t => t.studentnumber)
+    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+    const missingStudents = difference(studentnumbers, students)
+    if (missingStudents.length !== 0) return res.status(400).json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+
+    await TagStudent.createMultipleStudentTags(tags)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.delete('/studenttags', async (req, res) => {
-  const { id } = req.body
   try {
-    const result = await TagStudent.deleteStudentTag(id)
+    const { tag_id, studentnumber, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+
+    await TagStudent.deleteStudentTag(studentnumber, tag_id)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -45,14 +45,15 @@ const mapToProviders = rights => rights.map((r) => {
 })
 
 router.get('/stats', async (req, res) => {
-  const { rights, roles } = req.decodedToken
+  const { rights, roles } = req
+
   const { providers, semesterStart, semesterEnd } = req.query
   if (!providers || !semesterStart) {
     return res.status(422).send('Missing required query parameters.')
   }
   const providerRights = mapToProviders(rights)
 
-  if (!(providers.every(p => providerRights.includes(p)) || roles.map(r => r.group_code).includes('admin'))) {
+  if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')
   }
   const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd||semesterStart + 1)

services/backend/oodikone2-backend/src/routes/updater.js

@@ -0,0 +1,129 @@
+const router = require('express').Router()
+const {
+  updateOldestStudents,
+  getStatuses,
+  updateActiveStudents,
+  updateAllStudents,
+  updateAttainments,
+  updateMetadata,
+  updateStudentlist,
+  rescheduleScheduled,
+  rescheduleFetched,
+  updateNoStudents
+} = require('../services/updaterService')
+
+router.post('/update/oldest', async (req, res) => {
+  const { amount } = req.body
+  if (!Number.isInteger(Number(amount))) {
+    res.status(400).end()
+    return
+  }
+  try {
+    const response = await updateOldestStudents(amount)
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/all', async (req, res) => {
+  try {
+    const response = await updateAllStudents()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/active', async (req, res) => {
+  try {
+    const response = await updateActiveStudents()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/no_student', async (req, res) => {
+  try {
+    const response = await updateNoStudents()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/attainment', async (req, res) => {
+  try {
+    const response = await updateAttainments()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/meta', async (req, res) => {
+  try {
+    const response = await updateMetadata()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/studentlist', async (req, res) => {
+  try {
+    const response = await updateStudentlist()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.get('/status', async (req, res) => {
+  try {
+    const statuses = await getStatuses()
+    res.status(200).json(statuses)
+  } catch (err) {
+    console.log(err)
+    res.status(500).json([{ label: 'request status', value: 'request failed' }])
+  }
+})
+
+router.post('/reschedule/scheduled', async (req, res) => {
+  try {
+    const response = await rescheduleScheduled()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/reschedule/fetched', async (req, res) => {
+  try {
+    const response = await rescheduleFetched()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/users.js

@@ -8,11 +8,6 @@ router.get('/', async (req, res) => {
   res.json(results)
 })
 
-router.get('/enabled', async (req, res) => {
-  const results = await userService.findAllEnabled()
-  res.json(results)
-})
-
 router.get('/access_groups', async (req, res) => {
   const result = await userService.getAccessGroups()
   res.json(result)

services/backend/oodikone2-backend/src/services/analyticsService.js

@@ -38,6 +38,16 @@ const patchThroughput = async (data) => {
   return response.data
 }
 
+const patchFacultyYearlyStats = async (data) => {
+  const response = await client.patch('/facultystats', { data })
+  return response.data
+}
+
+const getFacultyYearlyStats = async (data) => {
+  const response = await client.get('/facultystats', { data })
+  return response.data
+}
+
 module.exports = {
   ping,
   getProductivity,
@@ -45,5 +55,7 @@ module.exports = {
   patchProductivity,
   getThroughput,
   setThroughput,
-  patchThroughput
+  patchThroughput,
+  patchFacultyYearlyStats,
+  getFacultyYearlyStats
 }

services/backend/oodikone2-backend/src/services/blacklist.js

@@ -12,7 +12,6 @@ const isUserBlacklisted = async (userId, tokenCreatedAt) => {
   const isBlacklisted = async () => {
     const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
     const status = !!(tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp))
-    console.log('BLACKLISTED? ', {isBlacklisted: status, userId, tokenCreatedAt, blacklistTimestamp})
     return status
   }
   const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))

services/backend/oodikone2-backend/src/services/courseGroups.js

@@ -1,5 +1,8 @@
 const { redisClient } = require('../services/redis')
-const { Teacher, CourseGroup } = require('../models')
+const { Teacher } = require('../models')
+const { CourseGroup, TeacherCourseGroup } = require('../models/models_kone')
+const { Op } = require('sequelize')
+const { flatMap } = require('lodash')
 
 const {
   getCurrentAcademicYear,
@@ -26,18 +29,27 @@ const getCurrentAcademicYearSemesterCode = async () => {
   return academicYear[0].semestercode
 }
 
-const getTeachersForCourseGroup = (courseGroupId) => {
-  return Teacher.findAll({
-    attributes: ['name', 'code', 'id'],
+const getTeachersForCourseGroup = async (courseGroupId) => {
+  const courseGroups = await CourseGroup.findAll({
+    model: CourseGroup,
     include: {
-      model: CourseGroup,
-      attributes: [],
+      model: TeacherCourseGroup,
       required: true,
-      where: {
-        id: courseGroupId
-      },
+    },
+    where: {
+      id: courseGroupId
     },
   })
+  const teacherIds = flatMap(courseGroups, e => e.teacher_course_groups.map(tcg => tcg.teacher_id))
+  const teachers = await Teacher.findAll({
+    attributes: ['name', 'code', 'id'],
+    where: {
+      id: {
+        [Op.in]: teacherIds
+      }
+    },
+  })
+  return teachers
 }
 
 const createCourseGroup = (name) => CourseGroup.create({ name })
@@ -141,17 +153,13 @@ const getCoursesByTeachers = async (teacherIds, semesterCode) => {
 }
 
 const addTeacher = async (courseGroupId, teacherid) => {
-  const group = await CourseGroup.findByPk(courseGroupId)
   const teacher = await Teacher.findByPk(teacherid)
-  if (!group || !teacher) return
-  return await group.addTeacher(teacher)
+  if (!teacher) return
+  return TeacherCourseGroup.create({ teacher_id: teacherid, course_group_id: courseGroupId })
 }
 
-const removeTeacher = async (courseGroupId, teacherid) => {
-  const group = await CourseGroup.findByPk(courseGroupId)
-  const teacher = await Teacher.findByPk(teacherid)
-  if (!group || !teacher) return
-  return await group.removeTeacher(teacher)
+const removeTeacher = (courseGroupId, teacherid) => {
+  return TeacherCourseGroup.destroy({ where: { teacher_id: teacherid, course_group_id: courseGroupId } })
 }
 
 module.exports = {

services/backend/oodikone2-backend/src/services/course_stats_counter.js

@@ -19,7 +19,8 @@ class CourseStatsCounter {
       failedMany: {},
       improvedPassedGrade: {},
       notParticipated: { ...allstudents },
-      notParticipatedOrFailed: { ...allstudents }
+      notParticipatedOrFailed: { ...allstudents },
+      markedToSemester: {}
     }
     this.stats = {
       students: 0,
@@ -67,6 +68,10 @@ class CourseStatsCounter {
     this.stats.passingSemesters[semester]++
   }
 
+  markToSemester(studentnumber) {
+    this.students.markedToSemester[studentnumber] = true
+  }
+
   markCredit(studentnumber, grade, passed, failed, improved, semester) {
     // studentnumber = `${shajs('sha256').update(process.env.key + studentnumber).digest('hex')}`
     this.markAttempt()
@@ -74,10 +79,11 @@ class CourseStatsCounter {
     this.markGrade(grade, passed, failed, improved)
     this.markToAll(studentnumber)
     if (passed) {
-      this.markPassingGrade(studentnumber)
-      if (!improved) {
+      if (!this.students.markedToSemester[studentnumber]) {
         this.markPassedSemester(semester)
+        this.markToSemester(studentnumber)
       }
+      this.markPassingGrade(studentnumber)
     } else if (improved) {
       this.markImprovedGrade(studentnumber)
     } else if (failed) {
@@ -93,11 +99,15 @@ class CourseStatsCounter {
     return (this.students.passed[studentnumber] !== undefined)
   }
 
-  removeFromFailed (studentnumber) {
+  removeFromFailed(studentnumber) {
     delete this.students.failed[studentnumber]
     delete this.students.failedMany[studentnumber]
   }
 
+  removeFromPassed(studentnumber) {
+    delete this.students.passed[studentnumber]
+  }
+
   markPassingGrade(studentnumber) {
     delete this.students.notParticipatedOrFailed[studentnumber]
     this.students.passed[studentnumber] = true
@@ -148,7 +158,7 @@ class CourseStatsCounter {
     cumulativeStats['0-SPRING'] = cumulativeStats['0-FALL'] + passingSemesters['0-SPRING']
 
     for (let i = 1; i < 7; i++) {
-      cumulativeStats[`${i}-FALL`] = cumulativeStats[`${i-1}-SPRING`] + passingSemesters[`${i}-FALL`]
+      cumulativeStats[`${i}-FALL`] = cumulativeStats[`${i - 1}-SPRING`] + passingSemesters[`${i}-FALL`]
       cumulativeStats[`${i}-SPRING`] = cumulativeStats[`${i}-FALL`] + passingSemesters[`${i}-SPRING`]
     }
 

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js

@@ -21,10 +21,11 @@ class CourseYearlyStatsCounter {
     this.programmes[code] = { name, students: [] }
   }
 
-  initGroup(groupcode, name) {
+  initGroup(groupcode, name, coursecode, yearcode) {
     this.groups[groupcode] = {
       code: groupcode,
       name,
+      coursecode,
       attempts: {
         grades: {},
         classes: {
@@ -32,7 +33,8 @@ class CourseYearlyStatsCounter {
           failed: []
         }
       },
-      students: {}
+      students: {},
+      yearcode
     }
   }
 
@@ -47,8 +49,9 @@ class CourseYearlyStatsCounter {
     if (!this.programmes[code]) {
       this.initProgramme(code, name)
     }
-    const prog = this.programmes[code]
-    prog.students.push(studentnumber)
+    if (!this.programmes[code].students.includes(studentnumber)) {
+      this.programmes[code].students.push(studentnumber)
+    }
   }
 
   markStudyProgrammes(studentnumber, programmes) {
@@ -88,9 +91,9 @@ class CourseYearlyStatsCounter {
     }
   }
 
-  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname) {
+  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode) {
     if (!this.groups[groupcode]) {
-      this.initGroup(groupcode, groupname)
+      this.initGroup(groupcode, groupname, coursecode, yearcode)
     }
     this.markCreditToStudents(studentnumber, passed, grade, groupcode)
     this.markCreditToAttempts(studentnumber, passed, grade, groupcode)
@@ -114,12 +117,14 @@ class CourseYearlyStatsCounter {
   formatStudentStatistics(students) {
     const grades = {}
     const classes = {}
+    const studentnumbers = []
     Object.entries(students).forEach(([studentnumber, stat]) => {
       const { grade, category } = stat
       grades[grade] = grades[grade] ? grades[grade].concat(studentnumber) : [studentnumber]
       classes[category] = classes[category] ? classes[category].concat(studentnumber) : [studentnumber]
+      studentnumbers.includes(studentnumber) ? null : studentnumbers.push(studentnumber)
     })
-    return { grades, classes }
+    return { grades, classes, studentnumbers }
   }
 
   formatGroupStatistics() {

services/backend/oodikone2-backend/src/services/courses.js

@@ -4,8 +4,11 @@ const {
   sequelize, Student, Credit,
   Course, CourseType, Discipline,
   ElementDetails, StudyrightElement,
-  Studyright, Semester, CourseDuplicates,
+  Studyright, Semester,
 } = require('../models')
+const {
+  sequelizeKone, CourseDuplicates,
+} = require('../models/models_kone')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('../services/course_yearly_stats_counter')
 const _ = require('lodash')
@@ -36,7 +39,11 @@ const byName = (name, language) => Course.findAll({
         [Op.eq]: name
       }
     }
-  }
+  },
+  order: [
+    ['latest_instance_date', 'DESC']
+  ],
+  limit: 1
 })
 
 const byNameOrCodeTypeAndDiscipline = (searchTerm, type, discipline, language) => {
@@ -97,7 +104,7 @@ const creditsForCourses = (codes) => Credit.findAll({
       attributes: ['studentnumber'],
       include: {
         model: StudyrightElement,
-        attributes: ['code'],
+        attributes: ['code', 'startdate'],
         include: [
           {
             model: ElementDetails,
@@ -193,6 +200,7 @@ const creditsOf = async (codes) => {
       fail: credits.filter(Credit.failed).length,
       pass: credits.filter(Credit.passed).length,
       students: credits.length,
+      studentnumbers: credits.map(cr => cr.student_studentnumber)
     }
   }
 
@@ -235,7 +243,7 @@ const oneYearStats = (instances, year, separate, allInstancesUntilYear) => {
     return {
       studentsThatPassedThisYear, studentsThatFailedThisYear, allStudentsThatFailedEver,
       passedStudentsThatFailedBefore, passedStudentsOnFirstTry, failedStudentsThatFailedBefore,
-      failedStudentsOnFirstTry, gradeDistribution
+      failedStudentsOnFirstTry, gradeDistribution, studentnumbers: thisSemester.studentnumbers
     }
   }
   const stats = []
@@ -326,8 +334,7 @@ const yearlyStatsOf = async (code, year, separate, language) => {
       return b
     }, resultProgrammes)
 
-  const alternatives = await getDuplicateCodes(code)
-  const codes = alternatives ? alternatives : [code]
+  const codes = await alternativeCodes(code)
   const allInstances = await creditsOf(codes)
   const yearInst = allInstances
     .filter(inst => moment(new Date(inst.date)).isBetween(year.start + '-09-01', year.end + '-08-01'))
@@ -409,7 +416,7 @@ const deleteDuplicateCode = async (code) => {
         coursecode: code
       }
     })
-    await sequelize.query(
+    await sequelizeKone.query(
       `DELETE FROM course_duplicates
       WHERE groupid in ( SELECT groupid FROM course_duplicates
       group by groupid
@@ -421,12 +428,17 @@ const deleteDuplicateCode = async (code) => {
   }
 }
 
-const getDuplicateCodesWithCourses = () => {
-  return CourseDuplicates.findAll({
-    include: {
-      model: Course,
+const getDuplicateCodesWithCourses = async () => {
+  const courseDuplicates = await CourseDuplicates.findAll()
+  const courses = await Course.findAll({
+    where: {
+      code: {
+        [Op.in]: courseDuplicates.map(e => e.coursecode)
+      }
     }
   })
+  const codeToCourse = courses.reduce((acc, c) => { acc[c.code] = c; return acc }, {})
+  return courseDuplicates.map(cd => ({ ...cd.get(), course: codeToCourse[cd.coursecode] }))
 }
 
 const getDuplicatesToIdMap = () => {
@@ -463,20 +475,19 @@ const getMainCodeToDuplicates = async () => {
     )[0]
     acc[main.code] = {
       maincourse: { code: main.code, name: main.name },
-      duplicates: courses.map( c => ({ code: c.code, name: c.name }))
+      duplicates: courses.map(c => ({ code: c.code, name: c.name }))
     }
     return acc
   }, {})
   return maincodeToDuplicates
 }
 
-const getCodeToMainCodeMap = async () => {
+const getCodeToMainCourseMap = async () => {
   try {
     const maincodeToDuplicates = await getMainCodeToDuplicates()
     const codeToMainCode = Object.values(maincodeToDuplicates).reduce((acc, d) => {
-      const maincode = d.maincourse.code
       d.duplicates.forEach((c) => {
-        acc[c.code] = maincode
+        acc[c.code] = d.maincourse
       })
       return acc
     }, {})
@@ -487,37 +498,19 @@ const getCodeToMainCodeMap = async () => {
   return {}
 }
 
-const getMainCodesMap = async () => {
-  return await getCodeToMainCodeMap()
-}
-
-const getMaincodeToDuplicateCodesMap = async () => {
-  try {
-    const codeToMainCodeMap = await getCodeToMainCodeMap()
-    const codes = Object.keys(codeToMainCodeMap)
-    const mainCodeToCoursesMap = codes.reduce((acc, code) => {
-      const maincode = codeToMainCodeMap[code]
-      const duplicates = acc[maincode] || []
-      duplicates.push(code)
-      acc[maincode] = duplicates
-      return acc
-    }, {})
-    return mainCodeToCoursesMap
-  } catch (e) {
-    console.error(e)
-  }
-  return {}
+const getMainCodeToDuplicatesAndCodeToMainCode = async () => {
+  return [await getMainCodeToDuplicates(), await getCodeToMainCourseMap()]
 }
 
 const getMainCourseToCourseMap = async (/*programme*/) => {
   try {
-    const codeToMainCodeMap = await getCodeToMainCodeMap()
+    const codeToMainCodeMap = await getCodeToMainCourseMap()
     const courses = await byCodes(Object.keys(codeToMainCodeMap))
     const mainCodeToCoursesMap = courses.reduce((acc, course) => {
-      const maincode = codeToMainCodeMap[course.code]
-      const duplicates = acc[maincode] || []
+      const maincourse = codeToMainCodeMap[course.code]
+      const duplicates = acc[maincourse.code] || []
       duplicates.push(course)
-      acc[maincode] = duplicates
+      acc[maincourse.code] = duplicates
       return acc
     }, {})
     return mainCodeToCoursesMap
@@ -528,9 +521,10 @@ const getMainCourseToCourseMap = async (/*programme*/) => {
 }
 
 const getDuplicateCodes = async (code) => {
-  const maincodeToDuplicatesMap = await getMaincodeToDuplicateCodesMap()
-  const duplicates = maincodeToDuplicatesMap[code]
-  return duplicates
+  const [mainCodeToDuplicates, codeToMainCourseMap] = await getMainCodeToDuplicatesAndCodeToMainCode()
+  const maincourse = codeToMainCourseMap[code]
+  if (!maincourse) return null
+  return mainCodeToDuplicates[maincourse.code].duplicates.map(e => e.code)
 }
 
 const setDuplicateCode = async (code1, code2) => {
@@ -541,12 +535,12 @@ const setDuplicateCode = async (code1, code2) => {
       if (course1 && course2) {
         const all = await getDuplicatesToIdMap()
         // make sure both dont have a group
-        if ([all[code1], all[code2]].filter(e=>e).length <= 1) {
+        if ([all[code1], all[code2]].filter(e => e).length <= 1) {
           let groupid = all[code1] || all[code2]
           if (!groupid) {
             // neither has a group, make one
-            groupid = Math.max(0, ...Object.values(all).filter(e=>e))
-            groupid = groupid && !isNaN(groupid) ? groupid+1 : 1
+            groupid = Math.max(0, ...Object.values(all).filter(e => e))
+            groupid = groupid && !isNaN(groupid) ? groupid + 1 : 1
           }
           await CourseDuplicates.bulkCreate([
             { groupid, coursecode: code1 },
@@ -572,13 +566,14 @@ const alternativeCodes = async code => {
   return alternatives ? alternatives : [code]
 }
 
-const formatStudyrightElement = ({ code, element_detail }) => ({
+const formatStudyrightElement = ({ code, element_detail, startdate }) => ({
   code,
-  name: element_detail.name
+  name: element_detail.name,
+  startdate
 })
 
 const parseCredit = credit => {
-  const { student, semester, grade } = credit
+  const { student, semester, grade, course_code } = credit
   const { studentnumber, studyright_elements: elements } = student
   const { yearcode, yearname, semestercode, name: semestername } = semester
   return {
@@ -587,6 +582,7 @@ const parseCredit = credit => {
     yearname,
     semestercode,
     semestername,
+    coursecode: course_code,
     grade,
     passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
     studentnumber,
@@ -594,7 +590,7 @@ const parseCredit = credit => {
   }
 }
 
-const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode) => {
+const yearlyStatsOfNew = async (coursecode, separate) => {
   const codes = await alternativeCodes(coursecode)
   const [credits, course] = await Promise.all([creditsForCourses(codes), Course.findByPk(coursecode)])
   const counter = new CourseYearlyStatsCounter()
@@ -602,25 +598,22 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
     const {
       studentnumber, grade, passed,
       semestercode, semestername,
-      yearcode, yearname, programmes
+      yearcode, yearname, programmes,
+      coursecode
     } = parseCredit(credit)
-
-    if (startyearcode <= yearcode && yearcode <= endyearcode) {
-      const groupcode = separate ? semestercode : yearcode
-      const groupname = separate ? semestername : yearname
-      const unknownProgramme = [{
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra'
-        }
-      }]
-      counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
-      counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname)
-    } else {
-      counter.markCreditToHistory(studentnumber, passed)
-    }
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+    const unknownProgramme = [{
+      code: 'OTHER',
+      name: {
+        en: 'Other',
+        fi: 'Muu',
+        sv: 'Andra'
+      }
+    }]
+    counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
+    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
+    counter.markCreditToHistory(studentnumber, passed)
   }
   const statistics = counter.getFinalStatistics()
   return {
@@ -631,10 +624,9 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
   }
 }
 
-const courseYearlyStats = async (coursecodes, separate, startyearcode, endyearcode) => {
+const courseYearlyStats = async (coursecodes, separate) => {
   const stats = await Promise.all(coursecodes
-    .map(code => yearlyStatsOfNew(code, separate, startyearcode, endyearcode || startyearcode)))
-
+    .map(code => yearlyStatsOfNew(code, separate)))
   return stats
 }
 
@@ -662,7 +654,7 @@ const nameLikeTerm = (name) => {
 
 const codeLikeTerm = (code) => !code ? undefined : {
   code: {
-    [Op.iLike]: `${code.trim()}%`
+    [Op.iLike]: `%${code.trim()}%`
   }
 }
 
@@ -701,12 +693,13 @@ module.exports = {
   findDuplicates,
   setDuplicateCode,
   deleteDuplicateCode,
-  getMainCodesMap,
+  getCodeToMainCourseMap,
   getMainCourseToCourseMap,
   getAllCourseTypes,
   getAllDisciplines,
   yearlyStatsOfNew,
   courseYearlyStats,
   byNameAndOrCodeLike,
-  byCodes
+  byCodes,
+  getMainCodeToDuplicatesAndCodeToMainCode
 }

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_data_mapper.js

@@ -250,12 +250,7 @@ const getTransfersFromData = (data, studentnumber) => {
   const studytracks = data.elements.filter(element => element.element_id === 20)
   const sorted = _.sortBy(studytracks, 'end_date')
   let transfers = []
-  let i = 0
-  while (i < sorted.length) {
-    if (i === 0) {
-      i++
-      continue
-    }
+  for (let i = 1; i < sorted.length; ++i) {
     let target = sorted[i].code
     let source = sorted[i - 1].code
     let transferdate = sorted[i - 1].end_date
@@ -266,7 +261,6 @@ const getTransfersFromData = (data, studentnumber) => {
       transferdate,
       studentnumber,
       studyrightid: `${data.studyright_id}`})
-    i++
   }
   return(transfers)
 }

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_interface.js

@@ -5,10 +5,10 @@ const https = require('https')
 const fs = require('fs')
 const base_url = OODI_ADDR
 
-const agent = KEY_PATH && CERT_PATH ? 
+const agent = KEY_PATH && CERT_PATH ?
   new https.Agent({
     cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-    key: fs.readFileSync(process.env.KEY_PATH, 'utf8'), 
+    key: fs.readFileSync(process.env.KEY_PATH, 'utf8'),
   }) :
   new https.Agent({
     rejectUnauthorized: false
@@ -48,9 +48,10 @@ if (process.env.NODE_ENV === 'dev') {
     username: 'tktl',
     password: process.env.OODI_PW
   }
-  axios.defaults.params = {
-    token: process.env.TOKEN
-  }
+  instance.interceptors.request.use((config) => {
+    config.params = {token: process.env.TOKEN}
+    return config
+  })
 
 }
 

services/backend/oodikone2-backend/src/services/elementdetails.js

@@ -1,7 +1,13 @@
 const { ElementDetails } = require('../models/index')
 const { Op } = require('sequelize')
 
-const byId = id =>  ElementDetails.findByPk(id)
+const byId = id => ElementDetails.findByPk(id)
+
+const byType = type => ElementDetails.findAll({
+  where: {
+    type
+  }
+})
 
 const byCodes = codes => ElementDetails.findAll({
   where: {
@@ -11,4 +17,4 @@ const byCodes = codes => ElementDetails.findAll({
   }
 })
 
-module.exports = { byId, byCodes }
+module.exports = { byId, byType, byCodes }

services/backend/oodikone2-backend/src/services/faculties.js

@@ -0,0 +1,72 @@
+const axios = require('axios')
+const moment = require('moment')
+const AsyncLock = require('async-lock')
+const { USERSERVICE_URL } = require('../conf-backend')
+const { Credit, StudyrightElement } = require('../models')
+
+const client = axios.create({ baseURL: USERSERVICE_URL, headers: { 'secret': process.env.USERSERVICE_SECRET } })
+
+const calculateFacultyYearlyStats = async () => {
+  const { data: facultyProgrammes } = await client.get('/faculty_programmes')
+  const res = {}
+  const lock = new AsyncLock()
+
+  let amountDone = 0
+  await Promise.all(facultyProgrammes.map(({ faculty_code, programme_code }) => (
+    new Promise(async (facultyRes) => {
+      if (!res[faculty_code]) res[faculty_code] = {}
+      if (!res[faculty_code][programme_code]) res[faculty_code][programme_code] = {}
+
+      const facultyStudents = await StudyrightElement.findAll({
+        where: {
+          code: programme_code
+        },
+        attributes: ['studentnumber']
+      })
+
+      await Promise.all(facultyStudents.map((student) => (
+        new Promise(async (studentRes) => {
+          const filteredStudentCredits = await Credit.findAll({
+            where: {
+              student_studentnumber: student.studentnumber,
+              credittypecode: [4, 10],
+              isStudyModule: false
+            }
+          })
+
+          filteredStudentCredits.forEach((c) => {
+            const attainmentYear = moment(c.attainment_date).year()
+            if (!res[faculty_code][attainmentYear]) {
+              lock.acquire(faculty_code, (done) => {
+                if (!res[faculty_code][programme_code][attainmentYear]) {
+                  const facultyYearStats = {}
+                  facultyYearStats.studentCredits = 0
+                  facultyYearStats.coursesPassed = 0
+                  facultyYearStats.coursesFailed = 0
+                  res[faculty_code][programme_code][attainmentYear] = facultyYearStats
+                }
+                done()
+              })
+            }
+            const facultyYearStats = res[faculty_code][programme_code][attainmentYear]
+            if (c.credittypecode === 4) {
+              facultyYearStats.studentCredits += c.credits
+              facultyYearStats.coursesPassed += 1
+            } else {
+              facultyYearStats.coursesFailed += 1
+            }
+          })
+          studentRes()
+        })
+      )))
+      amountDone += 1
+      console.log(`Faculty programmes done ${amountDone}/${facultyProgrammes.length}`)
+      facultyRes()
+    })
+  )))
+  return res
+}
+
+module.exports = {
+  calculateFacultyYearlyStats
+}

services/backend/oodikone2-backend/src/services/filters.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { Filters } = require('../models')
+const { Filters } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 

services/backend/oodikone2-backend/src/services/mailservice.js

@@ -66,7 +66,7 @@ const feedback = (content, user, email, full_name) => {
   return ({
     to: 'Toska <grp-toska@helsinki.fi>',
     replyTo: email,
-    subject: `New message from Oodikone feedback`,
+    subject: 'New message from Oodikone feedback',
     text: `New message from user ${user}`,
     html: `${content}
     <p>sent by ${full_name}, userid: ${user}, email: ${email}</p>

services/backend/oodikone2-backend/src/services/mandatoryCourseLabels.js

@@ -1,10 +1,7 @@
 const Sequelize = require('sequelize')
 const _ = require('lodash')
-const { MandatoryCourseLabels } = require('../models')
+const { MandatoryCourseLabels, sequelizeKone } = require('../models/models_kone')
 const { Op } = Sequelize
-const {
-  sequelize
-} = require('../models/index')
 
 const create = async (studyprogramme_id, { label }) => {
   const labels = await labelsByStudyprogramme(studyprogramme_id)
@@ -23,7 +20,7 @@ const move = async (studyprogramme_id, { id }, direction) => {
   if (indexToSwap < 0 || indexToSwap >= orderedlabels.length) return null;
   [orderedlabels[indexToMove], orderedlabels[indexToSwap]] = [orderedlabels[indexToSwap], orderedlabels[indexToMove]]
   const newOrder = orderedlabels.map((e, index) => ({ ...e, orderNumber: index }))
-  await sequelize.transaction(async transaction => {
+  await sequelizeKone.transaction(async transaction => {
     for (let e of newOrder) {
       await MandatoryCourseLabels.update(
         {orderNumber: e.orderNumber}, { where: { studyprogramme_id, id: e.id }, transaction }

services/backend/oodikone2-backend/src/services/mandatoryCourses.js

@@ -1,15 +1,12 @@
 const Sequelize = require('sequelize')
-const { Course, MandatoryCourse, MandatoryCourseLabels } = require('../models')
+const { Course } = require('../models')
+const { MandatoryCourse, MandatoryCourseLabels } = require('../models/models_kone')
 const { Op } = Sequelize
 
-const byStudyprogramme = (studyProgrammeId) => {
-  const courses = MandatoryCourse.findAll({
+const byStudyprogramme = async (studyProgrammeId) => {
+  const mandatoryCourses = await MandatoryCourse.findAll({
     attributes: ['course_code'],
     include: [
-      {
-        model: Course,
-        attributes: ['name'],
-      },
       {
         model: MandatoryCourseLabels,
         attributes: ['id', 'label', 'orderNumber'],
@@ -22,10 +19,20 @@ const byStudyprogramme = (studyProgrammeId) => {
     }
   })
 
-  return courses.map(course => ({
-    name: course.course.name,
-    code: course.course_code,
-    label: course.mandatory_course_label
+  const courses = await Course.findAll({
+    model: Course,
+    where: {
+      code: {
+        [Op.in]: mandatoryCourses.map(c => c.course_code)
+      }
+    }
+  })
+  const courseCodeToCourse = courses.reduce((acc, c) => { acc[c.code] = c; return acc }, {})
+
+  return mandatoryCourses.map(mc => ({
+    name: courseCodeToCourse[mc.course_code].name,
+    code: mc.course_code,
+    label: mc.mandatory_course_label
   }))
 }
 
@@ -36,14 +43,10 @@ const create = (studyProgrammeId, code) => {
   })
 }
 
-const find = (studyProgrammeId, code) => {
-  return MandatoryCourse.findOne({
+const find = async (studyProgrammeId, code) => {
+  const mandatoryCourse = await MandatoryCourse.findOne({
     attributes: ['course_code', 'label'],
     include: [
-      {
-        model: Course,
-        attributes: ['name'],
-      },
       {
         model: MandatoryCourseLabels,
         attributes: ['id', 'label', 'orderNumber'],
@@ -58,6 +61,16 @@ const find = (studyProgrammeId, code) => {
       }
     }
   })
+  if (!mandatoryCourse) return null
+
+  const course = await Course.findOne({
+    where: {
+      code: mandatoryCourse.course_code
+    }
+  })
+  if (!course) return null
+
+  return { ...mandatoryCourse.get(), course: course.get() }
 }
 
 const remove = (studyProgrammeId, code) => {

services/backend/oodikone2-backend/src/services/populations.js

@@ -1,12 +1,14 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
-const { orderBy } = require('lodash')
+
 const {
   Student, Credit, Course, sequelize, Studyright, StudyrightExtent, ElementDetails,
-  Discipline, CourseType, SemesterEnrollment, Semester, Transfers, StudyrightElement,
-  Tag, TagStudent
+  Discipline, CourseType, SemesterEnrollment, Semester, Transfers, StudyrightElement
 } = require('../models')
-const { getMainCodesMap, byName } = require('./courses')
+const {
+  Tag, TagStudent
+} = require('../models/models_kone')
+const { getCodeToMainCourseMap } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 
@@ -25,7 +27,7 @@ const formatStudentForPopulationStatistics = ({
   firstnames, lastname, studentnumber, dateofuniversityenrollment, creditcount,
   matriculationexamination, credits, abbreviatedname, email, studyrights,
   semester_enrollments, transfers, updatedAt, createdAt, gender_code,
-  gender_fi, gender_sv, gender_en, tag_students
+  gender_fi, gender_sv, gender_en, tags
 }, startDate, endDate) => {
 
   const toCourse = ({ grade, attainment_date, credits, course, credittypecode, isStudyModule }) => {
@@ -99,7 +101,7 @@ const formatStudentForPopulationStatistics = ({
     email,
     semesterenrollments,
     updatedAt: updatedAt || createdAt,
-    tags: tag_students || [],
+    tags: tags || [],
     studyrightStart: startDate,
     starting: moment(started).isBetween(startDate, endDate, null, '[]')
   }
@@ -107,7 +109,8 @@ const formatStudentForPopulationStatistics = ({
 
 const dateMonthsFromNow = (date, months) => moment(date).add(months, 'months').format('YYYY-MM-DD')
 
-const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDate, studyright) => {
+const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDate, studyright, tag) => {
+
   const creditsOfStudentOther = {
     student_studentnumber: {
       [Op.in]: studentnumbers
@@ -138,9 +141,9 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
 
   const creditsOfStudent = ['320001', 'MH30_001'].includes(studyright[0]) ?
     creditsOfStudentLaakis : creditsOfStudentOther
-  
+
   const students = await Student.findAll({
-    attributes: ['firstnames', 'lastname', 'studentnumber',
+    attributes: ['firstnames', 'lastname', 'studentnumber', 'home_country_en',
       'dateofuniversityenrollment', 'creditcount', 'matriculationexamination',
       'abbreviatedname', 'email', 'updatedAt', 'gender_code', 'gender_fi', 'gender_sv', 'gender_en'],
     include: [
@@ -213,17 +216,21 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
           }
         }
       },
-      {
-        model: TagStudent,
-        attributes: ['id'],
-        include: [
-          {
-            model: Tag,
-            attributes: ['tag_id','tagname']
-          }
-        ],
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers
       }
+    },
+  })
 
+  const studentTags = await TagStudent.findAll({
+    attributes: ['tag_id', 'studentnumber'],
+    include: [
+      {
+        model: Tag,
+        attributes: ['tag_id', 'tagname'],
+      }
     ],
     where: {
       studentnumber: {
@@ -231,6 +238,17 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       }
     }
   })
+  const studentNumberToTags = studentTags.reduce((acc, t) => {
+    acc[t.studentnumber] = acc[t.studentnumber] || []
+    acc[t.studentnumber].push(t)
+    return acc
+  }, {})
+
+  students.forEach(student => {
+    student.tags = studentNumberToTags[student.studentnumber] || []
+  })
+
+  if (tag) return students.filter(student => student.tags.some(t => t.tag_id === tag))
   return students
 }
 
@@ -243,7 +261,7 @@ const count = (column, count, distinct = false) => {
   )
 }
 
-const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents) => { // eslint-disable-line
+const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag) => { // eslint-disable-line
   const filteredExtents = []
   let studyrightWhere = {
     extentcode: {
@@ -260,6 +278,21 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
     studyrightWhere.canceldate = null
   }
 
+  let studentWhere = {}
+  if (tag) {
+    const taggedStudentnumbers = await TagStudent.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        tag_id: tag
+      }
+    })
+    studentWhere.where = {
+      student_studentnumber: {
+        [Op.in]: taggedStudentnumbers.map(sn => sn.studentnumber)
+      }
+    }
+  }
+
   const students = await Studyright.findAll({
     attributes: ['student_studentnumber'],
     include: {
@@ -293,21 +326,22 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
         }
       ],
       ...studyrightWhere
+
     },
+    ...studentWhere,
     having: count('studyright_elements.code', studyRights.length, true),
   })
-
   return [...new Set(students.map(s => s.student_studentnumber))]
 }
 
 const parseQueryParams = query => {
-  const { semesters, studentStatuses, year, studyRights, months } = query
+  const { semesters, studentStatuses, studyRights, months, endYear, startYear, tag } = query
   const startDate = semesters.includes('FALL') ?
-    `${year}-${semesterStart[semesters.find(s => s === 'FALL')]}` :
-    `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
+    `${startYear}-${semesterStart[semesters.find(s => s === 'FALL')]}` :
+    `${moment(startYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
   const endDate = semesters.includes('SPRING') ?
-    `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}` :
-    `${year}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
+    `${moment(endYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}` :
+    `${endYear}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
   const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
   const cancelledStudents = studentStatuses && studentStatuses.includes('CANCELLED')
   const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
@@ -318,7 +352,8 @@ const parseQueryParams = query => {
     studyRights: Array.isArray(studyRights) ? studyRights : Object.values(studyRights),
     months,
     startDate,
-    endDate
+    endDate,
+    tag
   }
 }
 
@@ -365,32 +400,28 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights
     stats.students.push(formatStudentForPopulationStatistics(student, startDate, endDate))
     return stats
   }, {
-      students: [],
-      extents: {},
-      semesters: {},
-      transfers: {
-        targets: {},
-        sources: {}
-      },
-      studyrights: {
-        degrees: [],
-        programmes: []
-      }
-    })
+    students: [],
+    extents: {},
+    semesters: {},
+    transfers: {
+      targets: {},
+      sources: {}
+    },
+    studyrights: {
+      degrees: [],
+      programmes: []
+    }
+  })
 
+  const [momentstart, momentend] = [moment(startDate), moment(endDate)]
   const transferredStudyright = (s) => {
-    const studyright = s.studyrights.find(s => s.studyrightElements
-      .map(d => d.element_detail.code)
-      .includes(studyRights.programme))
-
-    if (studyright) {
-      s.transferredStudyright = moment(startDate).isAfter(moment(studyright.startdate))
-      if (s.transferredStudyright) {
-        const previousRights = studyright.studyrightElements
-          .filter(e => e.element_detail.type === 20 && e.element_detail.code !== studyRights.programme)
-        s.previousRights = previousRights
-      }
-
+    const transferred_from = s.transfers.find(
+      t => t.target.code === studyRights.programme &&
+        moment(t.transferdate).isBetween(momentstart, momentend)
+    )
+    if (transferred_from) {
+      s.transferredStudyright = true
+      s.transferSource = transferred_from.source
     }
     return s
   }
@@ -404,44 +435,60 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights
   }
 }
 
-const optimizedStatisticsOf = async (query) => {
-  if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
+const formatQueryParamArrays = (query, params) => {
+  let res = { ...query }
+  params.forEach(p => {
+    if (!res[p]) return
+    res[p] = Array.isArray(res[p]) ? res[p] : [res[p]]
+  })
+  return res
+}
+
+const optimizedStatisticsOf = async (query, studentnumberlist) => {
+  const formattedQueryParams = formatQueryParamArrays(query, ['semesters', 'studentStatuses'])
+
+  if (!formattedQueryParams.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
     return { error: 'Semester should be either SPRING OR FALL' }
   }
-  if (query.studentStatuses &&
-    !query.studentStatuses.map(
+
+  if (formattedQueryParams.studentStatuses &&
+    !formattedQueryParams.studentStatuses.map(
       status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
     ).every(e => e === true)
   ) {
     return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
   }
-
   const {
-    studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents, nondegreeStudents
-  } = parseQueryParams(query)
+    studyRights, startDate, months, endDate, exchangeStudents, cancelledStudents, nondegreeStudents
+  } = parseQueryParams(formattedQueryParams)
 
-  const studentnumbers =
+  const studentnumbers = studentnumberlist ?
+    studentnumberlist :
     await studentnumbersWithAllStudyrightElements(
       studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents
     )
-  const students =
-    await getStudentsIncludeCoursesBetween(studentnumbers, startDate, dateMonthsFromNow(startDate, months), studyRights)
+  const students = await getStudentsIncludeCoursesBetween(
+    studentnumbers,
+    startDate,
+    dateMonthsFromNow(startDate, months),
+    studyRights,
+    formattedQueryParams.tag
+  )
 
-  const formattedStudents = await formatStudentsForApi(students, startDate, endDate, query)
+  const formattedStudents = await formatStudentsForApi(students, startDate, endDate, formattedQueryParams)
   return formattedStudents
 }
 
 const unifyOpenUniversity = (code) => {
-  if (code[0] === 'A') {
-    return code.substring(code[1] === 'Y' ? 2 : 1)
-  }
-  return code
+  const regexresult = code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+  if (!regexresult) return code
+  return regexresult[1]
 }
 
-const getUnifiedCode = (code, codeduplicates) => {
-  const formattedcode = unifyOpenUniversity(code)
-  const unifiedcode = codeduplicates[formattedcode]
-  return !unifiedcode ? formattedcode : unifiedcode
+const getMainCourse = (course, codeduplicates) => {
+  const formattedcode = unifyOpenUniversity(course.code)
+  const maincourse = codeduplicates[formattedcode]
+  return maincourse || course
 }
 
 const findCourses = (studentnumbers, beforeDate) => {
@@ -459,8 +506,7 @@ const findCourses = (studentnumbers, beforeDate) => {
           attainment_date: {
             [Op.lt]: beforeDate
           }
-        },
-        order: 'attainment_date'
+        }
       },
       {
         model: Discipline
@@ -486,53 +532,71 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date
 })
 
-const bottlenecksOf = async (query) => {
-  if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
-    return { error: 'Semester should be either SPRING OR FALL' }
+const bottlenecksOf = async (query, studentnumberlist) => {
+  const isValidRequest = async (query, params) => {
+    const { studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents } = params
+    if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
+      return { error: 'Semester should be either SPRING OR FALL' }
+    }
+    if (query.studentStatuses &&
+      !query.studentStatuses.every(
+        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
+      )) {
+      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
+    }
+    if (query.selectedStudents) {
+      const allStudents =
+        await studentnumbersWithAllStudyrightElements(
+          studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, query.tag
+        )
+      const disallowedRequest =
+        checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
+      if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
+    }
+    return null
   }
-  if (query.studentStatuses &&
-    !query.studentStatuses.map(
-      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
-    ).every(e => e === true)) {
-    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
+    if (!studentnumberlist) {
+      const { months, studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag } = params
+      const studentnumbers = selectedStudents ||
+        await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag)
+      const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
+      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+      return [allstudents, courses]
+    } else {
+      const allstudents = studentnumberlist
+      const courses = await findCourses(allstudents, new Date())
+      return [allstudents, courses]
+    }
   }
-  const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query)
-
-  if (query.selectedStudents) {
-    const allStudents =
-      await studentnumbersWithAllStudyrightElements(
-        studyRights, startDate, endDate, exchangeStudents, cancelledStudents
+  const params = parseQueryParams(query)
+  const [[allstudents, courses], codeToMainCourse, error] = await Promise.all(
+    [
+      getStudentsAndCourses(query.selectedStudents, studentnumberlist),
+      getCodeToMainCourseMap(),
+      isValidRequest(
+        query,
+        params
       )
-    const disallowedRequest =
-      checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
-    if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
-  }
+    ]
+  )
+  if (error) return error
 
   const bottlenecks = {
     disciplines: {},
     coursetypes: {}
   }
 
-  const codeduplicates = await getMainCodesMap()
+  const stats = {}
 
-  const studentnumbers = query.selectedStudents ||
-    await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents)
-  const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
-  const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
-
-  const allcoursestatistics = await courses.reduce(async (coursestatistics, course) => {
-    const stats = await coursestatistics
-    let { code, name, disciplines, course_type } = course
-
-    if (name.fi && name.fi.includes('Avoin yo:')) {
-      const courses = await byName(name.fi.split(': ')[1], 'fi')
-      const theOne = courses.length > 0 ? orderBy(courses, ['date'], ['desc'])[0] : { name, code }
-      code = theOne.code
-      name = theOne.name
+  const startYear = parseInt(query.endYear, 10)
+  courses.forEach(course => {
+    let { disciplines, course_type } = course
+    const maincourse = getMainCourse(course, codeToMainCourse)
+    if (!stats[maincourse.code]) {
+      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudents)
     }
-
-    const unifiedcode = getUnifiedCode(code, codeduplicates)
-    const coursestats = stats[unifiedcode] || new CourseStatsCounter(unifiedcode, name, allstudents)
+    const coursestats = stats[maincourse.code]
 
     coursestats.addCourseType(course_type.coursetypecode, course_type.name)
     bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
@@ -543,18 +607,13 @@ const bottlenecksOf = async (query) => {
 
     course.credits.forEach(credit => {
       const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
-      const semester = getPassingSemester(parseInt(query.year, 10), date)
+      const semester = getPassingSemester(startYear, date)
       coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
     })
-
-    stats[unifiedcode] = coursestats
-
-    return stats
-  }, Promise.resolve({}))
-
-  bottlenecks.coursestatistics = Object.values(allcoursestatistics)
+    stats[maincourse.code] = coursestats
+  })
+  bottlenecks.coursestatistics = Object.values(stats)
     .map(coursestatistics => coursestatistics.getFinalStats())
-
   return bottlenecks
 }
 

services/backend/oodikone2-backend/src/services/semesters.js

@@ -1,15 +1,8 @@
 const sequelize = require('sequelize')
 const { Semester } = require('../models/index')
-const { Op } = sequelize
 
-const getSemestersAndYears = async before => {
-  const semesters = await Semester.findAll({
-    where: {
-      startdate: {
-        [Op.lt]: before
-      }
-    }
-  })
+const getSemestersAndYears = async () => {
+  const semesters = await Semester.findAll({})
   const result = semesters.reduce((acc, semester) => {
     const { semestercode, name, yearcode, yearname }  = semester
     const semesters = { ...acc.semesters, [semestercode]: { semestercode, name, yearcode }}

services/backend/oodikone2-backend/src/services/students.js

@@ -1,4 +1,5 @@
 const Sequelize = require('sequelize')
+const { sequelize } = require('../database/connection')
 const moment = require('moment')
 const { Student, Credit, Course, Studyright, StudyrightElement, ElementDetails } = require('../models')
 const Op = Sequelize.Op
@@ -41,6 +42,23 @@ const byId = async (id) => Student.findByPk(id, {
   ]
 })
 
+const findByCourseAndSemesters = async (coursecodes, yearcode) =>
+  sequelize.query(`
+    SELECT
+      studentnumber, credit.course_code, attainment_date
+    FROM student
+    INNER JOIN credit ON
+      student.studentnumber=credit.student_studentnumber
+    WHERE 
+      course_code IN (:coursecodes) AND
+      attainment_date
+    BETWEEN
+      (select startdate FROM semesters where yearcode=:yearcode ORDER BY semestercode LIMIT 1) AND
+      (select enddate FROM semesters where yearcode=:yearcode ORDER BY semestercode DESC LIMIT 1);
+  `,
+  { replacements: { coursecodes, yearcode }, type: sequelize.QueryTypes.SELECT })
+    .map(st => st.studentnumber)
+
 const byAbreviatedNameOrStudentNumber = (searchTerm) => {
   return Student.findAll({
     where: {
@@ -280,10 +298,36 @@ const bySearchTermAndElementsNew = async (searchterm, codes) => {
   return matches.map(formatStudent)
 }
 
+const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
+  const students = await Student.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      where: {
+        code: {
+          [Op.in]: codes
+        }
+      }
+    },
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers
+      }
+    }
+  })
+  return students.map(student => student.studentnumber)
+}
+
 const NEW_SEARCH = true
 const bySearchTerm = NEW_SEARCH ? bySearchTermNew : bySearchTermOld
 const bySearchTermAndElements = NEW_SEARCH ? bySearchTermAndElementsNew : bySearchTermAndElementsOld
 
 module.exports = {
-  withId, bySearchTerm, createStudent, updateStudent, bySearchTermAndElements
+  withId,
+  bySearchTerm,
+  createStudent,
+  updateStudent,
+  bySearchTermAndElements,
+  filterStudentnumbersByAccessrights,
+  findByCourseAndSemesters
 }

services/backend/oodikone2-backend/src/services/studyrights.js

@@ -156,12 +156,12 @@ const getAllStudyrightElementsAndAssociations = async () => {
 }
 
 const getStudyrightElementsAndAssociationsForUser = async username => {
-  const studyrightelements = await getUserElementDetails(username)
-  if (studyrightelements.length === 0) {
+  const studyrightelementcodes = await getUserElementDetails(username)
+  if (studyrightelementcodes.length === 0) {
     return []
   }
-  const associations = await getAssociatedStudyrights(studyrightelements.map(element => element.code))
-  return formatStudyrightElements(studyrightelements, associations)
+  const associations = await getAssociatedStudyrights(studyrightelementcodes)
+  return formatStudyrightElements(studyrightelementcodes, associations)
 }
 
 const getAllDegreesAndProgrammes = async () => {
@@ -187,6 +187,11 @@ const getAllProgrammes = async () => {
   return elementDetails
 }
 
+const getAllElementDetails = async () => {
+  const elementDetails = ElementDetails.findAll()
+  return elementDetails
+}
+
 const associatedStudyrightElements = async (offset, limit) => {
   const studyrights = await Studyright.findAll({
     attributes: [],
@@ -355,8 +360,7 @@ const getFilteredAssociations = async (codes) => {
 }
 
 const getUserAssociations = async (userid) => {
-  const elements = await getUserElementDetails(userid)
-  const codes = elements.map(e => e.code)
+  const codes = await getUserElementDetails(userid)
   const associations = await getFilteredAssociations(codes)
   return associations
 }
@@ -375,5 +379,6 @@ module.exports = {
   getFilteredAssociations,
   getUserAssociations,
   refreshAssociationsInRedis,
-  getAllProgrammes
+  getAllProgrammes,
+  getAllElementDetails
 }

services/backend/oodikone2-backend/src/services/studytrack.js

@@ -1,9 +1,13 @@
 const sequelize = require('sequelize')
 const { Op } = sequelize
 const moment = require('moment')
+const { flatMap } = require('lodash')
 const { Credit, Student, Course, Provider, Studyright, StudyrightElement,
-  ElementDetails, ThesisCourse, ThesisTypeEnums
+  ElementDetails, Transfers
 } = require('../models')
+const {
+  ThesisCourse, ThesisTypeEnums
+} = require('../models/models_kone')
 const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const { semesterStart, semesterEnd } = require('../util/semester')
 const isNumber = str => !Number.isNaN(Number(str))
@@ -169,30 +173,39 @@ const graduatedStatsForStudytrack = async (studytrack, since) => {
   return graduatedStatsFromStudyrights(studyrights)
 }
 
-const findProgrammeThesisCredits = code => Credit.findAll({
-  include: {
-    model: Course,
-    required: true,
+const findProgrammeThesisCredits = async code => {
+  const thesiscourses = await ThesisCourse.findAll({
+    model: ThesisCourse,
+    where: {
+      programmeCode: code
+    }
+  })
+  const courseCodeToThesisCourse = thesiscourses.reduce((acc, tc) => { acc[tc.courseCode] = tc; return acc }, {})
+
+  const credits = await Credit.findAll({
     include: {
-      model: ThesisCourse,
+      model: Course,
       required: true,
       where: {
-        programmeCode: code
+        code: {
+          [Op.in]: thesiscourses.map(tc => tc.courseCode)
+        }
+      }
+    },
+    where: {
+      credittypecode: {
+        [Op.ne]: 10
       }
     }
-  },
-  where: {
-    credittypecode: {
-      [Op.ne]: 10
-    }
-  }
-}).map(credit => {
-  const { id, course, attainment_date } = credit
-  const { code, thesis_courses } = course
-  const { thesisType: type } = thesis_courses[0]
-  const year = attainment_date && attainment_date.getFullYear()
-  return { id, code, type, year }
-})
+  })
+
+  return credits.map(credit => {
+    const { id, course, attainment_date } = credit
+    const { code } = course
+    const year = attainment_date && attainment_date.getFullYear()
+    return { id, code, type: courseCodeToThesisCourse[code].thesisType, year }
+  })
+}
 
 const thesisProductivityFromCredits = credits => {
   const stats = {}
@@ -215,9 +228,25 @@ const thesisProductivityForStudytrack = async code => {
 }
 
 const combineStatistics = (creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits) => {
-  const stats = { ...creditStats }
-  Object.keys(stats).forEach(year => {
+  const allYears = [
+    ...new Set(
+      flatMap(
+        [
+          creditStats,
+          studyrightStats,
+          thesisStats,
+          creditsForMajors,
+          transferredCredits
+        ],
+        Object.keys
+      )
+    )
+  ]
+  const stats = {}
+  allYears.forEach(year => {
     const thesis = thesisStats[year] || {}
+    stats[year] = {}
+    stats[year] = creditStats[year] || { credits: 0, year }
     stats[year].graduated = studyrightStats[year] ? studyrightStats[year].graduated : 0
     // stats[year].medianGraduationTime = studyrightStats[year] ? studyrightStats[year].medianGraduationTime : 0
     stats[year].bThesis = thesis.bThesis || 0
@@ -273,16 +302,18 @@ const creditsAfter = (studentnumbers, startDate) => {
 }
 
 const thesesFromClass = async (studentnumbers, startDate, code) => {
-  return Credit.findAll({
+  const thesiscourses = await ThesisCourse.findAll({
+    where: {
+      programmeCode: code
+    }
+  })
+  const credits = await Credit.findAll({
     include: {
       model: Course,
       required: true,
-      include: {
-        attributes: ['thesisType'],
-        model: ThesisCourse,
-        required: true,
-        where: {
-          programmeCode: code
+      where: {
+        code: {
+          [Op.in]: thesiscourses.map(tc => tc.courseCode)
         }
       }
     },
@@ -297,12 +328,16 @@ const thesesFromClass = async (studentnumbers, startDate, code) => {
         [Op.in]: studentnumbers
       }
     }
-  }).reduce((acc, curr) => {
-    curr.course.thesis_courses.map(th => {
-      acc[th.thesisType] = acc[th.thesisType] ? acc[th.thesisType] + 1 : 1
-    })
+  })
+
+  const courseCodeToThesisCourse = thesiscourses.reduce((acc, tc) => { acc[tc.courseCode] = tc; return acc }, {})
+  const courseCodes = credits.map(c => c.course.code)
+  const theses = courseCodes.reduce((acc, code) => {
+    const thesisType = courseCodeToThesisCourse[code].thesisType
+    acc[thesisType] = acc[thesisType] ? acc[thesisType] + 1 : 1
     return acc
   }, {})
+  return theses
 }
 
 const graduationsFromClass = async (studentnumbers, studytrack) => {
@@ -345,49 +380,32 @@ const gendersFromClass = async (studentnumbers) => {
   }, {})
 }
 
-const countriesFromClass = async (studentnumbers) => {
+const nationalitiesFromClass = async (studentnumbers) => {
   return Student.findAll({
-    attributes: [[sequelize.fn('count', sequelize.col('country_en')), 'count'], 'country_en'],
     where: {
       studentnumber: {
         [Op.in]: studentnumbers
-      },
+      }
     },
-    group: ['country_en'],
-    raw: true
-  }).reduce((acc, curr) => {
-    acc[curr.country_en] = curr.count
+    attributes: ['home_country_en']
+  }).reduce((acc, { home_country_en }) => {
+    const country = home_country_en || 'Unknown'
+    if (!acc[country]) acc[country] = 0
+    acc[country] += 1
     return acc
   }, {})
 }
 
 const tranferredToStudyprogram = async (studentnumbers, startDate, studytrack, endDate) => {
-  return Studyright.findAndCountAll({
-    include: {
-      include: {
-        model: ElementDetails,
-        where: {
-          type: {
-            [Op.eq]: 20
-          }
-        }
-      },
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack
-        },
-        startdate: {
-          [Op.gt]: moment(startDate).add(1, 'days'), // because somehow startdates have a time that is not 00:00
-          [Op.lt]: new Date(endDate)
-        }
-      }
-    },
+  return Transfers.count({
     where: {
-      student_studentnumber: {
+      studentnumber: {
         [Op.in]: studentnumbers
-      }
+      },
+      transferdate: {
+        [Op.between]: [startDate, endDate]
+      },
+      targetcode: studytrack
     }
   })
 }
@@ -461,9 +479,9 @@ const statsForClass = async (studentnumbers, startDate, studytrack, endDate) =>
     graduationsFromClass(studentnumbers, studytrack),
     thesesFromClass(studentnumbers, startDate, studytrack),
     gendersFromClass(studentnumbers),
-    countriesFromClass(studentnumbers),
     tranferredToStudyprogram(studentnumbers, startDate, studytrack, endDate),
-    endedStudyright(studentnumbers, startDate, studytrack, endDate)
+    endedStudyright(studentnumbers, startDate, studytrack, endDate),
+    nationalitiesFromClass(studentnumbers)
   ])
 }
 
@@ -485,13 +503,13 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       mte150: 0,
     },
     genders: {},
-    countries: {},
     thesisM: 0,
     thesisB: 0,
     students: 0,
     graduated: 0,
     inTargetTime: 0,
-    transferred: 0
+    transferred: 0,
+    nationalities: {}
   }
   const years = getYears(since)
   // studyprogramme starts with K if bachelors and M if masters
@@ -515,14 +533,12 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     const studentnumbers = await studentnumbersWithAllStudyrightElements([studytrack], startDate, endDate, false, false)
     const creditsForStudyprogramme =
       await productivityCreditsFromStudyprogrammeStudents(studytrack, startDate, studentnumbers)
-
-    const [credits, graduated, theses, genders, countries, transferredTo, endedStudyright] =
+    const [credits, graduated, theses, genders, transferredTo, endedStudyright, nationalities] =
       await statsForClass(studentnumbers, startDate, studytrack, endDate)
     //console.log(year)
     //console.log(transferredFrom.rows.map(r => r.get({ plain: true })))
     // theres so much shit in the data that transefferFrom doesnt rly mean anything
     delete genders[null]
-    delete countries[null]
     const creditValues = credits.reduce((acc, curr) => {
       acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
       acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60
@@ -539,10 +555,9 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
         totals.genders[genderKey] + Number(genders[genderKey]) :
         Number(genders[genderKey])
     })
-    Object.keys(countries).forEach(countryKey => {
-      totals.countries[countryKey] = totals.countries[countryKey] ?
-        totals.countries[countryKey] + Number(countries[countryKey]) :
-        Number(countries[countryKey])
+    Object.keys(nationalities).forEach(nationality => {
+      if (!totals.nationalities[nationality]) totals.nationalities[nationality] = 0
+      totals.nationalities[nationality] += nationalities[nationality]
     })
     const graduationTimes = graduated.map(g => moment(g.enddate).diff(g.studystartdate, 'months'))
     const inTargetTime = graduationTimes.filter(time =>
@@ -556,7 +571,7 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     totals.ended = totals.ended + endedStudyright.count,
     totals.medianGraduationTime = median(allGraduationTimes)
     totals.inTargetTime = totals.inTargetTime + inTargetTime
-    totals.transferred = totals.transferred + transferredTo.count
+    totals.transferred = totals.transferred + transferredTo
     return {
       year: `${year}-${year + 1}`,
       credits: credits.map(cr => cr === null ? 0 : cr),
@@ -567,13 +582,12 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       thesisM: theses.MASTER || 0,
       thesisB: theses.BACHELOR || 0,
       genders,
-      countries,
       creditValues,
-      transferred: transferredTo.count,
-      ended: endedStudyright.count
+      transferred: transferredTo,
+      ended: endedStudyright.count,
+      nationalities
     }
   }))
-
   return { id: studytrack, status: null, data: { years: arr, totals } }
 }
 

services/backend/oodikone2-backend/src/services/studytrack.test.js

@@ -11,7 +11,8 @@ const {
   thesisProductivityFromCredits,
   thesisProductivityForStudytrack
 } = require('./studytrack')
-const { sequelize, ThesisTypeEnums } = require('../models')
+const { sequelize } = require('../models')
+const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
 const { readFileSync } = require('fs')
 
 const { MASTER, BACHELOR } = ThesisTypeEnums
@@ -19,9 +20,15 @@ const { MASTER, BACHELOR } = ThesisTypeEnums
 const provider = '500-M010'
 const studytrack = 'MH50_010'
 
+const thesisCourse = {
+  // from studytrack.test.sql
+  programmeCode: 'MH50_010', courseCode: 'THESIS_01', thesisType: MASTER
+}
+
 beforeAll(async () => {
   const query = readFileSync('./src/services/studytrack.test.sql', 'utf8')
   await sequelize.query(query)
+  await ThesisCourse.create(thesisCourse)
 })
 
 test('Credits for provider should contain passed course credit', async () => {

services/backend/oodikone2-backend/src/services/studytrack.test.sql

@@ -16,7 +16,7 @@ BEGIN
   VALUES
   (5, ts, ts)
   ;
-  
+
   INSERT INTO course
   ("code", "name", "is_study_module")
   VALUES
@@ -25,14 +25,14 @@ BEGIN
   (course_02, '{ "en": "COURSE_EN" }', 'false'),
   (thesis, '{ "en": "Bachelor thesis" }', 'false')
   ;
-  
+
   INSERT INTO credit_types
   ("credittypecode", "createdAt", "updatedAt")
   VALUES
   (10, ts, ts),
   (4, ts, ts)
   ;
-  
+
   INSERT INTO credit
   ("id", "createddate", "lastmodifieddate", "semestercode", "course_code", "credittypecode", "attainment_date", "credits")
   VALUES
@@ -44,14 +44,14 @@ BEGIN
   ('CREDIT_06', ts, ts, 5, course_01, 4, date_2015, 5),
   ('CREDIT_07', ts, ts, 5, thesis, 4, date_2015, 30)
   ;
-  
+
   INSERT INTO providers
   ("providercode", "createdAt", "updatedAt")
   VALUES
   (provider_01, ts, ts),
   (provider_02, ts, ts)
   ;
-  
+
   INSERT INTO course_providers
   ("coursecode", "providercode", "createdAt", "updatedAt")
   VALUES
@@ -84,8 +84,4 @@ BEGIN
   (12, studytrk_02, ts, ts)
   ;
 
-  INSERT INTO thesis_courses
-  ("programmeCode", "courseCode", "thesisType", "createdAt", "updatedAt")
-  VALUES
-  (studytrk_01, thesis, 'MASTER', ts, ts);
-END $$;
+END $$;

services/backend/oodikone2-backend/src/services/tags.js

@@ -1,11 +1,11 @@
 const Sequelize = require('sequelize')
-const { Tag } = require('../models')
+const { Tag } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 
 const findTags = () => Tag.findAll({})
 
-const findTagsByStudytrack = (studytrack) => {
+const findTagsByStudytrack = async (studytrack) => {
   return Tag.findAll({
     where: {
       studytrack: {
@@ -15,6 +15,17 @@ const findTagsByStudytrack = (studytrack) => {
   })
 }
 
+const findTagsFromStudytrackById = async (studytrack, tag_ids) => {
+  return Tag.findAll({
+    where: {
+      studytrack,
+      tag_id: {
+        [Op.in]: tag_ids
+      }
+    }
+  })
+}
+
 const createNewTag = async (tag) => {
   return Tag.create(tag)
 }
@@ -33,5 +44,6 @@ module.exports = {
   createNewTag,
   findTags,
   findTagsByStudytrack,
+  findTagsFromStudytrackById,
   deleteTag
 }

services/backend/oodikone2-backend/src/services/tagstudent.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { TagStudent, Tag } = require('../models')
+const { TagStudent, Tag } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 
@@ -33,12 +33,15 @@ const createStudentTag = async (tag) => {
   return TagStudent.create(tag)
 }
 
-const deleteStudentTag = async (id) => {
+const createMultipleStudentTags = async (tags) => {
+  return TagStudent.bulkCreate(tags, { ignoreDuplicates: true })
+}
+
+const deleteStudentTag = async (studentnumber, tag_id) => {
   return TagStudent.destroy({
     where: {
-      id: {
-        [Op.eq]: id
-      }
+      tag_id,
+      studentnumber
     }
   })
 }
@@ -48,5 +51,6 @@ module.exports = {
   getStudentTagsByStudentnumber,
   getStudentTagsByStudytrack,
   createStudentTag,
+  createMultipleStudentTags,
   deleteStudentTag
 }

services/backend/oodikone2-backend/src/services/teachers.js

@@ -1,5 +1,5 @@
 const { Op } = require('sequelize')
-const { Teacher, Credit, Course, Semester, Provider, CourseGroup } = require('../models/index')
+const { Teacher, Credit, Course, Semester, Provider } = require('../models/index')
 
 const splitByEmptySpace = str => str.replace(/\s\s+/g, ' ').split(' ')
 
@@ -40,11 +40,6 @@ const bySearchTerm = async (rawTerm) => {
         codeLike(terms),
         matchesId(searchTerm)
       ]
-    },
-    include: {
-      model: CourseGroup,
-      attributes: ['id', 'name'],
-      required: false
     }
   })
 }
@@ -93,7 +88,7 @@ const markCredit = (stats, passed, failed, credits, transferred) => {
       ...stats,
       credits: transferred ? stats.credits : stats.credits + credits,
       passed: stats.passed + 1,
-      transferred: transferred ? stats.transferred + credits : stats.transferred 
+      transferred: transferred ? stats.transferred + credits : stats.transferred
     }
   } else if (failed) {
     return {

services/backend/oodikone2-backend/src/services/thesis.js

@@ -1,4 +1,4 @@
-const { ThesisCourse } = require('../models/index')
+const { ThesisCourse } = require('../models/models_kone')
 
 const createThesisCourse = (programme, course, type) => ThesisCourse.create({
   programmeCode: programme,

services/backend/oodikone2-backend/src/services/thesis.test.js

@@ -1,5 +1,6 @@
 const { forceSyncDatabase } = require('../database/connection')
-const { Course, ElementDetails, ThesisTypeEnums, ThesisCourse } = require('../models')
+const { Course, ElementDetails } = require('../models')
+const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
 const { createThesisCourse, deleteThesisCourse, findProgrammeTheses }  = require('./thesis')
 
 const courseOne = {

services/backend/oodikone2-backend/src/services/updaterService.js

@@ -0,0 +1,80 @@
+const axios = require('axios')
+const { UPDATER_URL } = require('../conf-backend')
+
+const client = axios.create({ baseURL: UPDATER_URL })
+
+const ping = async () => {
+  const url = '/ping'
+  const response = await axios.get(url)
+  return response.data
+}
+
+const updateStudents = async (studentNumbers) => {
+  const response = await client.post('/update', studentNumbers)
+  return response.data
+}
+
+const updateOldestStudents = async (amount) => {
+  const response = await client.post('/update/oldest', { amount })
+  return response.data
+}
+
+const updateAllStudents = async () => {
+  const response = await client.post('/update/all')
+  return response.data
+}
+
+const updateActiveStudents = async () => {
+  const response = await client.post('/update/active')
+  return response.data
+}
+
+const updateNoStudents = async () => {
+  const response = await client.post('/update/no_student')
+  return response.data
+}
+
+const updateMetadata = async () => {
+  const response = await client.post('/update/meta')
+  return response.data
+}
+
+const updateAttainments = async () => {
+  const response = await client.post('/update/attainment')
+  return response.data
+}
+
+const updateStudentlist = async () => {
+  const response = await client.post('/update/studentlist')
+  return response.data
+}
+
+const getStatuses = async () => {
+  const response = await client.get('/statuses')
+  return response.data
+}
+
+const rescheduleScheduled = async () => {
+  const response = await client.post('/reschedule/scheduled')
+  return response.data
+}
+
+const rescheduleFetched = async () => {
+  const response = await client.post('/reschedule/fetched')
+  return response.data
+}
+
+module.exports = {
+  ping,
+  updateStudents,
+  updateOldestStudents,
+  updateAllStudents,
+  updateActiveStudents,
+  updateMetadata,
+  updateAttainments,
+  updateStudentlist,
+  getStatuses,
+  rescheduleScheduled,
+  rescheduleFetched,
+  updateNoStudents
+}

services/backend/oodikone2-backend/src/services/usageStatistics.js

@@ -1,6 +1,6 @@
 const _ = require('lodash')
 const moment = require('moment')
-const { UsageStatistic } = require('../models')
+const { UsageStatistic } = require('../models/models_kone')
 const Sequelize = require('sequelize')
 const { Op } = Sequelize
 

services/backend/oodikone2-backend/src/services/userService.js

@@ -1,6 +1,7 @@
 const axios = require('axios')
 const { USERSERVICE_URL } = require('../conf-backend')
 const UnitService = require('./units')
+const elementDetailService = require('./elementdetails')
 
 const client = axios.create({ baseURL: USERSERVICE_URL, headers: { 'secret': process.env.USERSERVICE_SECRET } })
 
@@ -15,11 +16,6 @@ const findAll = async () => {
   return response.data
 }
 
-const findAllEnabled = async () => {
-  const response = await client.get('/findallenabled')
-  return response.data
-}
-
 const login = async (uid, full_name, hyGroups, affiliations, email) => {
   const response = await client.post('/login', {
     uid, full_name, hyGroups, affiliations, email,
@@ -43,7 +39,8 @@ const byUsername = async (uid) => {
 const getUserElementDetails = async (username) => {
   const url = `/user/elementdetails/${username}`
   const response = await client.get(url)
-  return response.data
+  const elementdetailcodes = response.data
+  return elementDetailService.byCodes(elementdetailcodes)
 }
 
 const byId = async (id) => {
@@ -89,12 +86,15 @@ const setFaculties = async (uid, faculties) => {
 }
 
 const getUnitsFromElementDetails = async username => {
-  const url = `/user/elementdetails/${username}`
-  const response = await client.get(url)
-  const elementDetails = response.data
+  const elementDetails = await getUserElementDetails(username)
   return elementDetails.map(element => UnitService.parseUnitFromElement(element))
 }
 
+const getCodesFromElementDetails = async uid => {
+  const elementDetails = await getUserElementDetails(uid)
+  return elementDetails.map(e => e.code)
+}
+
 const modifyAccess = async (body) => {
   const response = await client.post('/modifyaccess', body)
   return response.data
@@ -105,6 +105,11 @@ const getAccessGroups = async () => {
   return response.data
 }
 
+const getAccessGroupCodesFor = async (user) => {
+  const response = await client.get(`/get_accessgroupcodes/${user}`)
+  return response.data
+}
+
 module.exports = {
   ping,
   byUsername,
@@ -117,10 +122,11 @@ module.exports = {
   enableElementDetails,
   removeElementDetails,
   findAll,
-  findAllEnabled,
   modifyAccess,
   getAccessGroups,
+  getAccessGroupCodesFor,
   getUnitsFromElementDetails,
+  getCodesFromElementDetails,
   getRolesFor,
   setFaculties
 }

services/backend/oodikone2-backend/src/util/index.js

@@ -17,6 +17,21 @@ const getStudentNumberChecksum = studentNumber => {
   return (10 - (checksum % 10)) % 10
 }
 
+const isValidStudentId = (id) => {
+  if (/^0\d{8}$/.test(id)) {
+    // is a 9 digit number
+    const multipliers = [7, 1, 3, 7, 1, 3, 7]
+    const checksum = id
+      .substring(1, 8)
+      .split('')
+      .reduce((sum, curr, index) => {
+        return (sum + curr * multipliers[index]) % 10
+      }, 0)
+    return (10 - checksum) % 10 == id[8]
+  }
+  return false
+}
+
 // helpers to print complex sequelizedata
 const plainPrint = (sequelizeData) => console.log(JSON.parse(JSON.stringify(sequelizeData)))
 
@@ -33,5 +48,6 @@ module.exports = {
   getStudentNumberChecksum,
   plainPrint,
   plainify,
+  isValidStudentId,
   oldToNew
 }

services/backend/oodikone2-backend/src/util/logSaver.js

@@ -1,6 +1,6 @@
 const Transport = require('winston-transport')
 const uuid = require('uuid')
-const { UsageStatistic } = require('../models')
+const { UsageStatistic } = require('../models/models_kone')
 
 module.exports = class LogSaverTransport extends Transport {
   log(payload, callback) {
@@ -16,7 +16,7 @@ module.exports = class LogSaverTransport extends Transport {
         status: payload.status,
         data: payload
       }
-      
+
       UsageStatistic.create(object)
     }
     callback()

services/backend/oodikone2-backend/src/util/utils.js

@@ -2,7 +2,6 @@ const { requiredGroup } = require('../conf-backend')
 
 const hasRequiredGroup = (hyGroups) => {
   const hasGroup = requiredGroup === null || hyGroups.some(e => e === requiredGroup)
-  console.log('has required group?:', hyGroups, requiredGroup, hasGroup)
   return hasGroup
 }
 

services/backend/oodikone2-backend/test/endpoint/teacher_test.js

@@ -3,19 +3,18 @@ process.env.DB_SCHEMA = schema
 
 const test = require('ava')
 const supertest = require('supertest')
-const jwt = require('jsonwebtoken')
 
 const { sequelize } = require('../../src/models')
 const app = require('../../src/app')
-const conf = require('../../src/conf-backend')
 const api = supertest(app)
 
-const uid = 'tktl', fullname = ''
-const payload = { userId: uid, name: fullname, enabled: true }
-
-const token = jwt.sign(payload, conf.TOKEN_SECRET, {
-  expiresIn: '24h'
-})
+// const jwt = require('jsonwebtoken')
+// const conf = require('../../src/conf-backend')
+// const uid = 'tktl', fullname = ''
+// const payload = { userId: uid, name: fullname, enabled: true }
+// const token = jwt.sign(payload, conf.TOKEN_SECRET, {
+//   expiresIn: '24h'
+// })
 
 test.before(async () => {
   await sequelize.createSchema(schema)

services/backend/oodikone2-backend/test/jest/globals.js

@@ -1,4 +1,5 @@
 const { sequelize } = require('../../src/models/index')
+const { sequelizeKone } = require('../../src/models/models_kone')
 const { redisClient } = require('../../src/services/redis')
 const { forceSyncDatabase } = require('../../src/database/connection')
 const { seedAllMigrations } = require('../../src/database/seed_migrations')
@@ -10,5 +11,7 @@ beforeAll(async () => {
 
 afterAll(async () => {
   await sequelize.close()
-  await redisClient.quitAsync()
+  await sequelizeKone.close()
+  // https://stackoverflow.com/a/54560610
+  await new Promise(res => redisClient.quit(() => setImmediate(res)))
 })

services/backend/oodikone2-backend/test/jest/services/population.test.js

@@ -3,6 +3,7 @@ const {
   Studyright, Credit, StudyrightExtent, Semester
 } = require('../../../src/models/index')
 const { optimizedStatisticsOf } = require('../../../src/services/populations')
+jest.setTimeout(10000)
 
 const langify = name => ({
   en: `${name}_en`,
@@ -58,7 +59,8 @@ const semesters = {
 
 const createQueryObject = (year, semester, codes, months) => ({
   studyRights: codes,
-  year,
+  endYear: year,
+  startYear: year,
   semesters: [semester],
   months
 })
@@ -211,7 +213,7 @@ describe('optimizedStatisticsOf tests', () => {
       expect(
         courseinstances.some(instance =>
           (instance.date.getTime() === courseinstanceFall.coursedate.getTime()) &&
-          (instance.course.code === courseinstanceFall.course_code))
+            (instance.course.code === courseinstanceFall.course_code))
       ).toBe(true)
     })
 

services/backend/oodikone2-backend/test/supertests/api_spec.js

@@ -2,7 +2,7 @@ const test = require('ava')
 const supertest = require('supertest')
 const app = require('../../src/app')
 const api = supertest(app)
-const { sequelize } = require('../../src/models')
+const { sequelizeKone } = require('../../src/models/models_kone')
 const conf = require('../../src/conf-backend')
 const jwt = require('jsonwebtoken')
 
@@ -22,9 +22,9 @@ test.before(async () => {
     expiresIn: '24h'
   })
   //console.log(token)
-  sequelize.query(
-    `DELETE 
-      FROM tag_student 
+  sequelizeKone.query(
+    `DELETE
+      FROM tag_student
       WHERE tags_tagname = 'test3'`
   )
 })
@@ -33,7 +33,7 @@ test.after.always(async () => {
   const tagToAdd = { text: 'mooc-2013' }
   console.log('ennd  bfefore')
 
-  let res = await api
+  await api
     .delete('/api/students/014424850/tags')
     .send(tagToAdd)
     .set('x-access-token', token)
@@ -186,7 +186,6 @@ test.only('new api populations can be fetched', async t => {
     .expect('Content-Type', /application\/json/)
 
   const stats = res.body
-  console.log(stats)
   t.is(stats.length, 6)
 
 })

services/backend/shared/migrations/20180803_add_language_to_user.js

@@ -1,12 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('users', 'language',
-      {
-        type: Sequelize.STRING,
-        defaultValue: 'fi'
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('users', 'language')
-  }
-}

services/backend/shared/migrations/20180807_add_attainment_date_column.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('credit', 'attainment_date', { type: Sequelize.DATE })
-  },
-  down: async (queryInterface) => {
-    queryInterface.removeColumn('credit', 'attainment_date')
-  }
-}

services/backend/shared/migrations/20180808_01_add_credit_ courseinstace_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    queryInterface.addIndex('credit', {
-      fields: ['courseinstance_id'],
-      name: 'credit_courseinstance_id'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180808_02_add_credit_course_column.js

@@ -1,19 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn(
-      'credit', 
-      'course_code', 
-      { 
-        type: Sequelize.STRING,
-        references: {
-          model: 'course',
-          key: 'code',
-          foreignKey: 'credit_course_fkey'
-        },
-      }
-    )
-  },
-  down: async (queryInterface) => {
-    await queryInterface.removeColumn('credit', 'course_code')
-  }
-}

services/backend/shared/migrations/20180808_03_update_credit_attainment_and_course_values.js

@@ -1,15 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.sequelize.query(`
-        UPDATE credit
-        SET 
-          attainment_date = ci.coursedate,
-          course_code = ci.course_code
-        FROM courseinstance ci
-        WHERE ci.id = credit.courseinstance_id
-        ;
-    `)
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180808_04_add_credit_course_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.addIndex('credit', {
-      fields: ['course_code'],
-      name: 'credit_course_code'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180903_add_description_to_filters.js

@@ -1,12 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('filters', 'description',
-      {
-        type: Sequelize.STRING,
-        defaultValue: ''
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('filters', 'description')
-  }
-}

services/backend/shared/migrations/20180910_01_add_credit_attainment_date_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.addIndex('credit', {
-      fields: ['attainment_date'],
-      name: 'credit_attainment_date'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_02_add_enrollment_studentnumber_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.addIndex('semester_enrollments', {
-      fields: ['studentnumber'],
-      name: 'semester_enrollment_studentnumber'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_03_add_transfer_studentnumber_index.js

@@ -1,14 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    try {
-      await queryInterface.addIndex('transfers', {
-        fields: ['studentnumber'],
-        name: 'transfers_studentnumber'
-      })
-    } catch (e) {
-      console.log('transfers_studentnumber index in table transfers already existed')
-    }
-  },
-  down: async () => {
-  }
-}