Merge pull request #1353 from UniversityOfHelsinkiCS/trunk

fast bug fix to production

.circleci/config.yml

@@ -0,0 +1,94 @@
+version: 2.1
+
+executors:
+  my-executor:
+    working_directory: ~/app
+    machine:
+      image: ubuntu-1604:201903-01
+
+aliases:
+  - &check_exit_statuses
+    run:
+      name: check_exit_statuses
+      command: |
+        docker ps -a
+        if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+          echo 'Some process had nonzero exit code';
+          npm run docker:ci -- logs
+          exit 1;
+        fi
+
+jobs:
+  build:
+    executor: my-executor
+    steps:
+      - run:
+          name: set_env_variables
+          command: |
+            echo $CIRCLE_BRANCH;
+            echo $CIRCLE_TAG;
+            if [[ -n "$CIRCLE_TAG" ]]; then
+              echo 'export TAG=latest' >> $BASH_ENV;
+              echo 'export NOTIFY_DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=staging' >> $BASH_ENV;
+              echo 'USING RELEASE CONFIG';
+            elif [[ "$CIRCLE_BRANCH" = "master" && -z "$CIRCLE_PULL_REQUEST" ]]; then
+              echo 'export TAG=staging' >> $BASH_ENV;
+              echo 'export NOTIFY_DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'USING STAGING CONFIG';
+            else
+              echo 'export TAG=trunk' >> $BASH_ENV;
+              echo 'export NOTIFY_DEPLOY=false' >> $BASH_ENV;
+              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'USING TRUNK CONFIG';
+            fi
+      - checkout
+      - restore_cache:
+          keys:
+            - npmcache-v2-{{ checksum "package-lock.json" }}
+      - run: npm ci
+      - run: npm run cypress -- verify
+      - save_cache:
+          key: npmcache-v2-{{ checksum "package-lock.json" }}
+          paths:
+            - ~/.cache
+      - run: TAG=${CACHE_TAG} npm run docker:ci -- pull --ignore-pull-failures
+      - run: npm run docker:ci -- build
+      - run: npm run docker:ci -- up -d db user_db db_kone
+      - *check_exit_statuses
+      - run: git clone --depth 1 git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
+      - run: source ./scripts/scripts.sh && init_dirs && db_anon_setup_full
+      - *check_exit_statuses
+      - run: npm run docker:ci -- up -d --no-recreate
+      - run:
+          name: log_in_background
+          background: true
+          command: |
+            docker ps -a && npm run docker:ci -- logs -f
+      - *check_exit_statuses
+      - run: npm run test_services
+      - *check_exit_statuses
+      - run: timeout 5m bash -c 'until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf .; sleep 5; done'
+      - run: npm run cypress:ci
+      - *check_exit_statuses
+      - store_artifacts:
+          path: cypress/videos
+      - store_artifacts:
+          path: cypress/screenshots
+      - deploy:
+          command: |
+            docker login -u $DOCKER_USER -p $DOCKER_PASS
+            npm run docker:ci -- push
+            if [ "${NOTIFY_DEPLOY}" == "true" ]; then
+              curl --silent -X POST -H 'Content-type: application/json' --data "{\"text\":\"${CIRCLE_BRANCH}${CIRCLE_TAG} started auto deployment!\"}" ${SLACKBOT_URL}
+            fi
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/

.gitignore

@@ -14,17 +14,14 @@ anonyymioodi
 nginx
 kafka
 node_modules
-cypress/support
-cypress/videos
 services/updater_writer/updater/test_assets/testi.bak
 services/updater_writer/updater/test_assets/011120775.json
 services/updater_writer/updater/test_assets/014272112.json
 services/updater_writer/updater/test_assets/014441008.json
 services/updater_writer/updater/test_assets/meta.json
 services/updater_scheduler/all_student_numbers.txt
-services/updater_scheduler/debug.log
 services/updater_scheduler/active_student_numbers.txt
-services/oodikone2-backend/debug.log
 datastore
 services/updater_api/debug.log
 services/backend/updater_writer/dumps
+debug.log

.travis.yml

@@ -1,72 +0,0 @@
-language: node_js
-
-node_js:
-  # Node 10.3+ includes npm@6 which has good "npm ci" command
-  - 10.8
-
-cache:
-  # cache both npm modules and Cypress binary
-  npm: true
-  directories:
-    - ~/.cache
-
-services:
-  - docker
-
-before_script:
-  - docker --version
-  - docker-compose --version
-  - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")"
-
-jobs:
-  include:
-    - stage: build & test & push & deploy staging
-      env:
-        - IMAGE=STAGING
-        - TAG=staging
-      script:
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG}
-        - docker ps
-        - cat nginx/log
-        - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
-        - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
-        - docker ps -a
-        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy:
-        provider: script
-        skip_cleanup: true
-        script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          branch: master
-    - stage: build & test & push & deploy latest
-      env:
-        - IMAGE=PRODUCTION
-        - TAG=latest
-      script:
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG}
-        - docker ps
-        - cat nginx/log
-        - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
-        - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
-        - docker ps -a
-        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy:
-        provider: script
-        skip_cleanup: true
-        script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          tags: true
-stages:
-  - name: build & test & push & deploy staging
-    if: NOT tag IS present
-  - name: build & test & push & deploy latest
-    if: tag IS present
-
-after_failure:
-  - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

README.md

@@ -23,7 +23,7 @@ Launch the CLI with the command below and follow the instructions.
 Before running cli [make yourself a deployment key](https://developer.github.com/v3/guides/managing-deploy-keys/#setup-2) for Anonyymioodi private repository and put it in your root folder as private.key. This allows you to download the anonymized dumps from the repository.
 
 ```
-bash run.sh
+bash ./scripts/run.sh
 ```
 
 For local development with anonymized data use `2) Anon setup` (you can run e2e tests with this).

cypress/.gitignore

@@ -1 +1,2 @@
 screenshots
+videos

cypress/integration/Course_statistics.js

@@ -51,8 +51,8 @@ describe('Course Statistics tests', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
     cy.get("input[placeholder='Search by entering a course code']").type('TKT')
-    cy.contains("tr", "TKT20003").click()
-    cy.contains("tr", "TKT10002").click()
+    cy.contains("td", "TKT20003").click()
+    cy.contains("td", /^TKT10002/).click()
     cy.contains("Fetch statistics").should('be.enabled').click()
     cy.contains("Search for courses").should('not.exist')
 

cypress/integration/Population_statistics.js

@@ -78,6 +78,21 @@ describe('Population Statistics tests', () => {
       cy.contains("Excludes students with cancelled study right")
     })
 
+    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
+      cy.get("tr").its('length').should('be.gte', 10)
+      cy.route('/api/v3/courseyearlystats**').as('coursePage')
+      cy.contains("Opiskelijan digitaidot: orientaatio").siblings().find(".level").click()
+      cy.wait('@coursePage')
+      cy.url().should('include', '/coursestatistics')
+    })
+    cy.contains("DIGI-000A")
+    cy.go("back")
+
+    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
+      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
+      cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '9')
+    })
+
     cy.contains("add").click()
     cy.contains("Add filters").siblings().within(() => {
       cy.get(".form").should('have.length', 10)
@@ -88,39 +103,48 @@ describe('Population Statistics tests', () => {
     let filteredStudents = 1328493
     cy.contains("Credits gained during first").parentsUntil(".tab").get("table").within(() => {
       cy.get("tr").eq(1).find("td").eq(1).invoke("text").then(text => filteredStudents = Number(text))
+      cy.route('POST', '/api/v2/populationstatistics/courses**').as('courseData')
       cy.get("tr").eq(1).click()
+      cy.wait('@courseData')
     }).then(() => {
       checkAmountOfStudents(filteredStudents)
     })
 
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.get("tr").its('length').should('be.gte', 10)
-      cy.contains("Opiskelijan digitaidot: orientaatio").siblings().find(".level").click()
-      cy.wait(1000)
-      cy.url().should('include', '/coursestatistics')
-    })
-    cy.contains("DIGI-000A")
-    cy.go("back")
-
-
-
-    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
-      cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '9')
-    })
-
-    cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
-      cy.get(".refresh").click()
-      cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
+      cy.contains("number at least").siblings().within(() => cy.get("input").clear())
+      cy.contains("number at least").siblings().within(() => cy.get("input").type("0"))
       cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '1')
     })
 
-    cy.get("button").contains("show").click()
+    cy.contains("button", "show").click()
     cy.contains("Student names hidden").click()
     cy.contains("Oinonen").siblings().eq(2).click()
     cy.contains("Oinonen").invoke('text').then((text) => expect(text).to.equal('Oinonen Heidi Eeva Elisabet, 014473717'))
   })
 
+  it('Student list checking works as intended', () => {
+    cy.contains("Select study programme").click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains("See population").click()
+    cy.contains("button", "show").click()
+    cy.contains("010483918")
+    cy.contains("666666666").should('not.exist')
+    cy.contains('button', "Check studentnumbers").click()
+    cy.contains('Check for studentnumbers')
+    cy.get('textarea').type("010483918").type('{enter}').type("666666666")
+    cy.contains('button', 'check students').click()
+    cy.contains('form', 'Results').within(e => {
+      cy.get('div').eq(0).within(e => {
+        cy.contains('student numbers in list but not in oodi')
+        cy.contains('666666666')
+      })
+      cy.get('div').eq(2).within(e => {
+        cy.contains('student numbers in oodi but not in list')
+        cy.contains('014896381')
+      })
+      cy.contains('010483918').should('not.exist')
+    })
+  })
+
   it('All filters working', () => {
     cy.contains("Select study programme", { timeout: 50000 }).click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
     cy.contains("See population").click()
@@ -185,8 +209,7 @@ describe('Population Statistics tests', () => {
     cy.contains("and at least").parentsUntil("form").contains("set filter").click()
 
     cy.contains("Filters").siblings().within(() => {
-      cy.contains('Kehittävä').should('have.text', 'Kehittävä työntutkimus')
-      cy.contains('Kvalitatiiviset').should('have.text', 'Kvalitatiiviset tutkimusmenetelmät I')
+      cy.contains('Kehittävä').should('have.text', 'Avoin yo: Kehittävä työntutkimus')
     })
 
     checkAmountOfStudents(1)
@@ -205,7 +228,6 @@ describe('Population Statistics tests', () => {
 
     cy.contains('Excluded students that graduated')
 
-
     cy.contains("Students that has").parentsUntil("form").within(() => {
       cy.contains("degree").click().siblings().contains("any degree").click()
       cy.contains("programme").click().siblings().contains("Kasvatustieteiden kandiohjelma")
@@ -213,7 +235,7 @@ describe('Population Statistics tests', () => {
       cy.contains("set filter").click()
     })
 
-    checkAmountOfStudents(1)
+    checkAmountOfStudents(3)
 
     cy.contains("Basic filters").parentsUntil("form").contains("set filter").click()
 

cypress/integration/Student_statistics.js

@@ -11,7 +11,6 @@ describe('Student Statistics tests', () => {
                 }
             }
         })
-        console.log(Cypress.config().baseUrl)
         cy.visit(Cypress.config().baseUrl)
         cy.contains("Student statistics").click()
         cy.contains("Student names hidden")
@@ -71,4 +70,12 @@ describe('Student Statistics tests', () => {
         cy.get('.prompt').clear().type('01114')
         cy.contains('Student number').should('not.exist')
     })
-})
+
+    it('Can jump to population page', () => {
+        cy.get('.prompt').type('Oinonen')
+        cy.contains('011143561').click()
+        cy.get('i.level.up.alternate.icon').eq(0).click()
+        cy.contains('Population statistics')
+        cy.contains('Credit accumulation (for 1 students)')
+    })
+})

cypress/integration/Studyprogramme_overview.js

@@ -41,22 +41,66 @@ describe('Studyprogramme overview', () => {
 
   it('renders progress and productivity tables', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.get('table').should('have.length', 2)
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-    cy.contains("2018-2019").siblings().contains('9')
-    cy.contains("2017-2018").siblings().contains('12').siblings().contains('7').siblings().contains('2')
-    cy.get('table').eq(1).contains('2018').siblings().contains('443')
-    cy.get('table').eq(1).contains('2017').siblings().contains('381')
+    cy.get('button:contains("Recalculate")').eq(0).click()
+    cy.get('button:contains("Recalculate")').eq(1).click()
+    cy.contains('Loading...').should('not.be.visible').then(() => {
+      cy.wait(1000)
+      cy.reload()
+      cy.get('table').should('have.length', 2)
+      cy.contains('Population progress')
+      cy.contains('Yearly productivity')
+      cy.contains("2018-2019").siblings().contains('9')
+      cy.contains("2017-2018").siblings().contains('12').siblings().contains('7').siblings().contains('2')
+      cy.get('table').eq(1).contains('2018').siblings().contains('443')
+      cy.get('table').eq(1).contains('2017').siblings().contains('381')
+    })
   })
 
-  it('can search for mandatory courses', () => {
+  it('can add and delete mandatory courses and labels', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
     cy.contains('Mandatory Courses').click()
     cy.get('button').contains('Add Courses').click()
-    cy.get('input').eq(0).type('CSM')
+    cy.route('/api/v2/coursesmulti**').as('searchResponse')
+    cy.get('input').eq(1).type('Code Generation')
+    cy.wait('@searchResponse')
     cy.contains('Searched courses')
-    cy.contains('Code Generation')
+    cy.get('table').eq(0).contains('Code Generation').click()
+    cy.get('table').eq(1).contains('tr', 'Code Generation').contains('select label')
+
+    cy.contains('Group labels').click()
+    const label = 'cypress test label'
+    cy.contains(label).should('not.exist')
+    cy.contains('button', 'Add').should('be.disabled')
+    cy.get('input').eq(0).type(label)
+    cy.contains('button', 'Add').should('be.enabled').click()
+    cy.contains('button', 'Add').should('be.disabled')
+    cy.contains('table', label)
+
+    cy.contains('Mandatory courses').click()
+    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('select label').click()
+    cy.contains(label).click()
+    cy.get('table').eq(0).contains('tr', 'Code Generation').within((el) => {
+      cy.get('div.dropdown>div').eq(0).within((el) => {
+        cy.contains('select label').should('not.exist')
+        cy.contains(label)
+      })
+      cy.get('i.clear').click()
+      cy.get('div.dropdown>div').eq(0).within((el) => {
+        cy.contains(label).should('not.exist')
+        cy.contains('select label')
+      })
+    })
+    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('select label').click()
+    cy.contains(label).click()
+    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('button', 'Delete').click()
+    cy.contains('Code Generation').should('not.exist')
+
+    cy.contains('Group labels').click()
+    cy.contains('tr', label).within((el) => {
+      cy.get('i.remove').click()
+    })
+    cy.contains('button', 'Remove').click()
+    cy.contains(label).should('not.exist')
   })
 
   it('can open Thesis page', () => {

cypress/integration/Teachers.js

@@ -0,0 +1,71 @@
+
+describe('Teachers page tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+          proxy.xhr.setRequestHeader('uid', 'tktl')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    cy.visit(Cypress.config().baseUrl)
+    cy.reload()
+    cy.contains("Teachers").click()
+    cy.contains("Teacher statistics by course providers")
+  })
+
+  it("Check Statistics", () => {
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('Autumn 2017').click()
+    cy.get('.form > :nth-child(2) > .ui').click()
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+    cy.get('.form > .fluid').click()
+    cy.contains('Name')
+    cy.contains('Wu')
+    cy.contains('Jämsä')
+    cy.contains('Kuusinen')
+  })
+
+  it("Teacher search works", () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type('Pekka')
+    cy.contains('Professori Pekka')
+    cy.contains('Saren')
+    cy.contains('Jari')
+    cy.contains('Lopez')
+  })
+
+  it("Can check teacher page", () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type('Pekka')
+    cy.contains('Saren').click()
+    cy.contains('English Academic & Professional Skills: Reading, Writing & Spoken Communication (CEFR B2)*')
+  })
+
+  it("Can check teacher page if teacher doesn't have courses", () => {
+    cy.url().should('include', '/teachers')
+    cy.get('.borderless > :nth-child(3)').click()
+    cy.get('.prompt').type('Pekka')
+    cy.contains('Professori').click()
+    cy.contains('Name').should('not.exist')
+  })
+
+  it("Check leaderboad works", () => {
+    cy.get('.borderless > :nth-child(2)').click()
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('2017-18').click()
+    cy.contains("Recalculate this year").click()
+    cy.wait(5000)
+    cy.reload()
+    cy.get(':nth-child(1) > .ui > .search').click()
+    cy.contains('2017-18').click()
+    cy.contains("Passed")
+    cy.contains("Valtonen")
+    cy.contains("Wahlström")
+  })
+})

cypress/integration/Users.js

@@ -15,7 +15,7 @@ describe('Users tests', () => {
     cy.visit(Cypress.config().baseUrl)
     cy.reload()
     cy.contains("Users").click()
-    cy.contains("Enable or disable access to Oodikone")
+    cy.contains("Oodikone users")
   })
 
   it("Admin mocking normal user shows only the mocked user's programmes", () => {
@@ -27,7 +27,9 @@ describe('Users tests', () => {
       cy.get("div[role='listitem']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")
     })
 
+    cy.route('POST', '/api/superlogin/normk').as('superlogin')
     cy.get('i.spy').click()
+    cy.wait('@superlogin')
     cy.contains("mocking as normk")
     cy.wait(1000)
     cy.contains("Study programme").click().siblings().contains("Search by class").click()

cypress/support/commands.js

@@ -0,0 +1,25 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add("login", (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This is will overwrite an existing command --
+// Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })

cypress/support/index.js

@@ -0,0 +1,31 @@
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands'
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')
+
+// describe('Before all tests', () => {
+//     it('reset anon database', () => {
+//       cy.exec(
+//         'source ./scripts/scripts.sh && time reset_db_for_cypress',
+//         { timeout: 5*60*1000 }
+//       ).then(result => {
+//         console.log(result)
+//       })
+//     })
+// })

docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3'
+version: '3.2'
 
 services:
   nats:
-    image: nats-streaming
+    image: nats-streaming:0.15.1
     command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
     expose:
       - "4222"
@@ -46,7 +46,7 @@ services:
       - POSTGRES_DB=usage_db
 
   db:
-    image: postgres:9.6.3
+    image: postgres:10-alpine
     ports:
       - "5421:5432"
     volumes:
@@ -84,7 +84,6 @@ services:
     ports:
       - "8080:8080"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
@@ -99,7 +98,6 @@ services:
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
       UPDATER_URL: http://updater_scheduler:3678
-
     container_name: backend
     depends_on:
       - db
@@ -120,7 +118,6 @@ services:
     ports:
       - 4567:4567
     environment:
-      WAIT_HOSTS: user_db:5432
       DB_URL: postgres://postgres@user_db:5432/user_db
       TEST_DB: postgres://postgres@user_db:5432/user_db_test
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -135,7 +132,6 @@ services:
     ports:
       - 545:545
     environment:
-      WAIT_HOSTS: usage_db:5432
       PORT: '545'
       DB_URL: postgres://postgres@usage_db:5432/usage_db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -149,11 +145,10 @@ services:
     depends_on:
       - nats
     environment:
-      - WAIT_HOSTS=nats:4222
       - NATS_URI=nats://nats:4222
       - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
+      - TOKEN=jotaindiibadaabaa
+    #container_name: updater_api
 
   updater_writer:
     build:
@@ -163,15 +158,14 @@ services:
       - nats
       - db
     environment:
-      WAIT_HOSTS: nats:4222,db:5432
       NATS_URI: nats://nats:4222
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
+    #container_name: updater_writer
 
   scheduler_mongo:
-    image: mongo
+    image: mongo:4.0.10
     volumes:
       - ./mongodata:/data/db
     expose:
@@ -185,16 +179,8 @@ services:
       - nats
       - scheduler_mongo
     environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
       - NATS_URI=nats://nats:4222
       - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-      - LOG_HOST=toska.cs.helsinki.fi
-      - LOG_PORT=9501
-      - LOG_HOSTNAME=updater_scheduler
       - PORT=3678
     container_name: updater_scheduler
 
@@ -204,20 +190,8 @@ services:
     ports:
       - 4568:4568
     environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
       - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
       - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
     depends_on:
       - analytics_db
-      - analytics_redis
     container_name: analytics
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis

docker/docker-compose.dev.real.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.2'
 
 services:
   backend:
@@ -8,3 +8,7 @@ services:
   userservice:
     environment:
       DB_URL: postgres://postgres@user_db:5432/user_db_real
+
+  updater_writer:
+    environment:
+      DB_URL: postgres://postgres@db:5432/tkt_oodi_real

docker/docker-compose.dev.yml

@@ -1,20 +1,21 @@
-version: '3'
+version: '3.2'
 
 services:
   user_db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+    restart: unless-stopped
 
   db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+    restart: unless-stopped
 
   backend:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     environment:
       FRONT_URL: http://localhost:8000
     volumes:
@@ -22,13 +23,13 @@ services:
       - ./services/backend/shared/models:/usr/src/app/src/models
       - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
       - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+    restart: unless-stopped
 
   frontend:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
     ports:
       - "8081:8081"
       - "7777:7777"
@@ -39,40 +40,40 @@ services:
       KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
     volumes:
       - ./services/oodikone2-frontend/:/usr/src/app
+    restart: unless-stopped
 
   userservice:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/oodikone2-userservice/:/usr/src/app
+    restart: unless-stopped
 
   usageservice:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/oodikone2-usageservice/:/usr/src/app
+    restart: unless-stopped
 
   updater_api:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/updater_api/:/usr/src/app
+    restart: unless-stopped
 
   updater_writer:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/backend/updater_writer/:/usr/src/app
       - ./services/backend/shared/models:/usr/src/app/models
@@ -80,31 +81,25 @@ services:
       - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
     environment:
       DB_SCHEMA_KONE: kone_data
-
-  scheduler_mongo:
     restart: unless-stopped
 
   updater_scheduler:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/updater_scheduler/:/usr/src/app
-    environment:
-      - LOG_HOST=toska.cs.helsinki.fi
-      - LOG_PORT=9501
-      - LOG_HOSTNAME=updater_scheduler
+    restart: unless-stopped
 
   analytics:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/oodikone2-analytics/:/usr/src/app
+    restart: unless-stopped
 
   adminer:
     image: adminer:4.6.2
@@ -114,8 +109,8 @@ services:
       - ./adminer-plugins:/var/www/html/plugins-enabled
     ports:
       - 5050:8080
-    restart: always
     container_name: adminer
+    restart: unless-stopped
 
   # zookeeper:
   #   image: wurstmeister/zookeeper
@@ -145,7 +140,6 @@ services:
 
   # mongo_db:
   #   image: mongo
-  #   restart: unless-stopped
   #   volumes:
   #     - ./backups/mongo/oodilearn:/dump
   #     - ./mongodata:/data/db
@@ -157,7 +151,6 @@ services:
   #   build:
   #     context: services/oodilearn/server
   #     dockerfile: Dockerfile
-  #   restart: unless-stopped
   #   volumes:
   #     - ./services/oodilearn/data:/data
   #     - ./services/oodilearn/server:/src

docker/docker-compose.lateste2e.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.2'
 
 services:
   user_db:
@@ -11,6 +11,11 @@ services:
 
   backend:
     image: toska/oodikone2-backend:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-backend:trunk
+        - toska/oodikone2-backend:staging
+        - toska/oodikone2-backend:latest
     ports:
       - "9229:9229"
     environment:
@@ -22,27 +27,64 @@ services:
     build:
       args:
         BASE_PATH: /
+        SENTRY_URL: ${SENTRY_URL}
+        SENTRY_AUTH_TOKEN: ${SENTRY_AUTH_TOKEN}
+        SENTRY_ORG: sentry
+        SENTRY_PROJECT: oodifront
+      cache_from:
+        - toska/oodikone2-frontend:trunk
+        - toska/oodikone2-frontend:staging
+        - toska/oodikone2-frontend:latest
     ports:
       - "5000:5000"
 
   userservice:
     image: toska/oodikone2-userservice:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-userservice:trunk
+        - toska/oodikone2-userservice:staging
+        - toska/oodikone2-userservice:latest
 
   usageservice:
     image: toska/oodikone2-usageservice:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-usageservice:trunk
+        - toska/oodikone2-usageservice:staging
+        - toska/oodikone2-usageservice:latest
 
   updater_api:
     image: toska/updater_api:${TAG}
+    build:
+      cache_from:
+        - toska/updater_api:trunk
+        - toska/updater_api:staging
+        - toska/updater_api:latest
 
   updater_writer:
     image: toska/updater_writer:${TAG}
+    build:
+      cache_from:
+        - toska/updater_writer:trunk
+        - toska/updater_writer:staging
+        - toska/updater_writer:latest
 
   updater_scheduler:
     image: toska/updater_scheduler:${TAG}
-    command: ["echo", "mluukkai on orjuuttaja"]
+    build:
+      cache_from:
+        - toska/updater_scheduler:trunk
+        - toska/updater_scheduler:staging
+        - toska/updater_scheduler:latest
 
   analytics:
     image: toska/oodikone2-analytics:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-analytics:trunk
+        - toska/oodikone2-analytics:staging
+        - toska/oodikone2-analytics:latest
 
   nginx:
     image: nginx:latest
@@ -51,7 +93,6 @@ services:
       - ./nginx/error.log:/etc/nginx/error_log.log
       - ./nginx/log:/var/log/nginx/localhost.error_log
       - ./nginx/cache/:/etc/nginx/cache
-      - /etc/letsencrypt/:/etc/letsencrypt/
     ports:
       -  1337:80
     depends_on:

nginx.conf

@@ -4,6 +4,7 @@ http {
 
   rewrite_log on;
   error_log /var/log/nginx/localhost.error_log notice;
+  access_log off;
 
   server {
     listen 80;

package-lock.json

@@ -48,9 +48,9 @@
       }
     },
     "ajv": {
-      "version": "6.10.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
-      "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==",
+      "version": "6.10.2",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz",
+      "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==",
       "dev": true,
       "requires": {
         "fast-deep-equal": "^2.0.1",
@@ -77,6 +77,12 @@
       "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
       "dev": true
     },
+    "arch": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/arch/-/arch-2.1.1.tgz",
+      "integrity": "sha512-BLM56aPo9vLLFVa8+/+pJLnrZ7QGGTVHWsCwieAWT9o9K8UeGaQbzZbGoabWLOo2ksBCztoXdqBZBplqLDDCSg==",
+      "dev": true
+    },
     "argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -102,12 +108,12 @@
       "dev": true
     },
     "async": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.4.0.tgz",
-      "integrity": "sha1-SZAgDxjqW4N8LMT4wDGmmFw4VhE=",
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
+      "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
       "dev": true,
       "requires": {
-        "lodash": "^4.14.0"
+        "lodash": "^4.17.10"
       }
     },
     "asynckit": {
@@ -341,9 +347,9 @@
       "dev": true
     },
     "combined-stream": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
-      "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
       "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
@@ -380,9 +386,9 @@
       }
     },
     "concurrently": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-4.1.0.tgz",
-      "integrity": "sha512-pwzXCE7qtOB346LyO9eFWpkFJVO3JQZ/qU/feGeaAHiX1M3Rw3zgXKc5cZ8vSH5DGygkjzLFDzA/pwoQDkRNGg==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-4.1.1.tgz",
+      "integrity": "sha512-48+FE5RJ0qc8azwKv4keVQWlni1hZeSjcWr8shBelOBtBHcKj1aJFM9lHRiSc1x7lq416pkvsqfBMhSRja+Lhw==",
       "dev": true,
       "requires": {
         "chalk": "^2.4.1",
@@ -403,9 +409,9 @@
           "dev": true
         },
         "rxjs": {
-          "version": "6.5.1",
-          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.1.tgz",
-          "integrity": "sha512-y0j31WJc83wPu31vS1VlAFW5JGrnGC+j+TtGAa1fRQphy48+fDYiDmX8tjGloToEsMkxnouOg/1IzXGKkJnZMg==",
+          "version": "6.5.2",
+          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.2.tgz",
+          "integrity": "sha512-HUb7j3kvb7p7eCUHE3FqjoDsC1xfZQ4AHFWfTKSpZ+sAhhz5X1WX0ZuUqWbzB2QhSLp3DoLUG+hMdEDKqWo2Zg==",
           "dev": true,
           "requires": {
             "tslib": "^1.9.0"
@@ -454,41 +460,49 @@
       }
     },
     "cypress": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-3.2.0.tgz",
-      "integrity": "sha512-PN0wz6x634QyNL56/voTzJoeScDfwtecvSfFTHfv5MkHuECVSR4VQcEZTvYtKWln3CMBMUkWbBKPIwwu2+a/kw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-3.4.1.tgz",
+      "integrity": "sha512-1HBS7t9XXzkt6QHbwfirWYty8vzxNMawGj1yI+Fu6C3/VZJ8UtUngMW6layqwYZzLTZV8tiDpdCNBypn78V4Dg==",
       "dev": true,
       "requires": {
         "@cypress/listr-verbose-renderer": "0.4.1",
         "@cypress/xvfb": "1.2.4",
+        "arch": "2.1.1",
         "bluebird": "3.5.0",
         "cachedir": "1.3.0",
         "chalk": "2.4.2",
         "check-more-types": "2.24.0",
         "commander": "2.15.1",
         "common-tags": "1.8.0",
-        "debug": "3.1.0",
+        "debug": "3.2.6",
         "execa": "0.10.0",
         "executable": "4.1.1",
         "extract-zip": "1.6.7",
-        "fs-extra": "4.0.1",
-        "getos": "3.1.0",
-        "glob": "7.1.3",
+        "fs-extra": "5.0.0",
+        "getos": "3.1.1",
         "is-ci": "1.2.1",
         "is-installed-globally": "0.1.0",
         "lazy-ass": "1.6.0",
         "listr": "0.12.0",
-        "lodash": "4.17.11",
+        "lodash": "4.17.15",
         "log-symbols": "2.2.0",
         "minimist": "1.2.0",
         "moment": "2.24.0",
         "ramda": "0.24.1",
         "request": "2.88.0",
-        "request-progress": "0.4.0",
+        "request-progress": "3.0.0",
         "supports-color": "5.5.0",
-        "tmp": "0.0.33",
+        "tmp": "0.1.0",
         "url": "0.11.0",
         "yauzl": "2.10.0"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "4.17.15",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "dev": true
+        }
       }
     },
     "dashdash": {
@@ -507,12 +521,12 @@
       "dev": true
     },
     "debug": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-      "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+      "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
       "dev": true,
       "requires": {
-        "ms": "2.0.0"
+        "ms": "^2.1.1"
       }
     },
     "decamelize": {
@@ -630,6 +644,12 @@
             "ms": "2.0.0"
           }
         },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+          "dev": true
+        },
         "yauzl": {
           "version": "2.4.1",
           "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
@@ -705,13 +725,13 @@
       }
     },
     "fs-extra": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.1.tgz",
-      "integrity": "sha1-f8DGyJV/mD9X8waiTlud3Y0N2IA=",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-5.0.0.tgz",
+      "integrity": "sha512-66Pm4RYbjzdyeuqudYqhFiNBbCIuI9kgRqLPSHIlXHidW8NIQtVdkM1yeZ4lXwuhbTETv3EUGMNHAAw6hiundQ==",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.2",
-        "jsonfile": "^3.0.0",
+        "jsonfile": "^4.0.0",
         "universalify": "^0.1.0"
       }
     },
@@ -740,12 +760,12 @@
       "dev": true
     },
     "getos": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/getos/-/getos-3.1.0.tgz",
-      "integrity": "sha512-i9vrxtDu5DlLVFcrbqUqGWYlZN/zZ4pGMICCAcZoYsX3JA54nYp8r5EThw5K+m2q3wszkx4Th746JstspB0H4Q==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/getos/-/getos-3.1.1.tgz",
+      "integrity": "sha512-oUP1rnEhAr97rkitiszGP9EgDVYnmchgFzfqRzSkgtfv7ai6tEi7Ko8GgjNXts7VLWEqrTWyhsOKLe5C5b/Zkg==",
       "dev": true,
       "requires": {
-        "async": "2.4.0"
+        "async": "2.6.1"
       }
     },
     "getpass": {
@@ -758,9 +778,9 @@
       }
     },
     "glob": {
-      "version": "7.1.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
-      "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
       "dev": true,
       "requires": {
         "fs.realpath": "^1.0.0",
@@ -781,9 +801,9 @@
       }
     },
     "graceful-fs": {
-      "version": "4.1.15",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
-      "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz",
+      "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==",
       "dev": true
     },
     "har-schema": {
@@ -923,9 +943,9 @@
       }
     },
     "inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "dev": true
     },
     "ini": {
@@ -1075,9 +1095,9 @@
       "dev": true
     },
     "jsonfile": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-3.0.1.tgz",
-      "integrity": "sha1-pezG9l9T9mLEQVx2daAzHQmS7GY=",
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
+      "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.6"
@@ -1266,9 +1286,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
+      "version": "4.17.15",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+      "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
       "dev": true
     },
     "lodash.once": {
@@ -1317,18 +1337,18 @@
       }
     },
     "mime-db": {
-      "version": "1.39.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.39.0.tgz",
-      "integrity": "sha512-DTsrw/iWVvwHH+9Otxccdyy0Tgiil6TWK/xhfARJZF/QFhwOgZgOIvA2/VIGpM8U7Q8z5nDmdDWC6tuVMJNibw==",
+      "version": "1.40.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
+      "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.23",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.23.tgz",
-      "integrity": "sha512-ROk/m+gMVSrRxTkMlaQOvFmFmYDc7sZgrjjM76abqmd2Cc5fCV7jAMA5XUccEtJ3cYiYdgixUVI+fApc2LkXlw==",
+      "version": "2.1.24",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
+      "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
       "dev": true,
       "requires": {
-        "mime-db": "~1.39.0"
+        "mime-db": "1.40.0"
       }
     },
     "mimic-fn": {
@@ -1376,9 +1396,9 @@
       "dev": true
     },
     "ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
       "dev": true
     },
     "nice-try": {
@@ -1387,12 +1407,6 @@
       "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
       "dev": true
     },
-    "node-eta": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/node-eta/-/node-eta-0.1.1.tgz",
-      "integrity": "sha1-QGYQmzk3HHYccrfr2pqeoKXeEh8=",
-      "dev": true
-    },
     "normalize-package-data": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@@ -1523,12 +1537,6 @@
         }
       }
     },
-    "os-tmpdir": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
-      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
-      "dev": true
-    },
     "p-defer": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
@@ -1654,15 +1662,15 @@
       }
     },
     "process-nextick-args": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
       "dev": true
     },
     "psl": {
-      "version": "1.1.31",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz",
-      "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.2.0.tgz",
+      "integrity": "sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA==",
       "dev": true
     },
     "pump": {
@@ -1771,13 +1779,12 @@
       }
     },
     "request-progress": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-0.4.0.tgz",
-      "integrity": "sha1-wZVOOQhqqFJpxWYLzuAUKmpw1+c=",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-3.0.0.tgz",
+      "integrity": "sha1-TKdUCBx/7GP1BeT6qCWqBs1mnb4=",
       "dev": true,
       "requires": {
-        "node-eta": "^0.1.1",
-        "throttleit": "^0.0.2"
+        "throttleit": "^1.0.0"
       }
     },
     "require-directory": {
@@ -1817,6 +1824,15 @@
         "onetime": "^1.0.0"
       }
     },
+    "rimraf": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
+      "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
+      "dev": true,
+      "requires": {
+        "glob": "^7.1.3"
+      }
+    },
     "run-node": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/run-node/-/run-node-1.0.0.tgz",
@@ -2013,18 +2029,18 @@
       "dev": true
     },
     "throttleit": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-0.0.2.tgz",
-      "integrity": "sha1-z+34jmDADdlpe2H90qg0OptoDq8=",
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz",
+      "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=",
       "dev": true
     },
     "tmp": {
-      "version": "0.0.33",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
-      "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.1.0.tgz",
+      "integrity": "sha512-J7Z2K08jbGcdA1kkQpJSqLF6T0tdQqpR2pnSUXsIchbPdTI9v3e85cLW0d6WDhwuAleOV71j2xWs8qMPfK7nKw==",
       "dev": true,
       "requires": {
-        "os-tmpdir": "~1.0.2"
+        "rimraf": "^2.6.3"
       }
     },
     "tough-cookie": {
@@ -2052,9 +2068,9 @@
       "dev": true
     },
     "tslib": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
-      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==",
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
+      "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==",
       "dev": true
     },
     "tunnel-agent": {

package.json

@@ -2,23 +2,29 @@
   "name": "oodikone",
   "version": "1.0.0",
   "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the docker-compose.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
-  "main": "run.sh",
+  "main": "./scripts/run.sh",
   "scripts": {
-    "cypress:open": "CYPRESS_baseUrl=http://localhost:8081 cypress open",
-    "cypress:run": "CYPRESS_baseUrl=http://localhost:8081 cypress run --config video=false -P ./",
-    "cypress:record": "cypress run --config videoUploadOnPasses=false -P ./",
+    "cli": "./scripts/run.sh",
+    "cypress": "cypress",
+    "cypress:open": "cypress open --config baseUrl=http://localhost:8081/",
+    "cypress:run": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:8081/",
+    "cypress:ci": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:1337/",
     "concurrently": "concurrently",
-    "test": "./run_all_tests.sh",
-    "test_services": "./run_service_tests.sh",
+    "test": "./scripts/run_all_tests.sh",
+    "test_services": "./scripts/run_service_tests.sh",
+    "lint": "./scripts/run_service_lints.sh",
     "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
+    "docker:real": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml",
+    "docker:ci": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml",
     "docker:build": "npm run docker -- build",
     "docker:up": "npm run docker -- up -d",
+    "docker:up:real": "npm run docker:real -- up -d",
     "docker:down": "npm run docker -- down",
     "docker:logs": "npm run docker -- logs -f",
     "docker:logs:backend": "npm run docker -- logs -f backend",
     "docker:restart": "npm run docker -- restart",
     "docker:restart:backend": "npm run docker -- restart backend",
-    "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune",
+    "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune -a",
     "start": "npm run docker:up && npm run docker:logs"
   },
   "husky": {
@@ -37,8 +43,8 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "concurrently": "^4.1.0",
-    "cypress": "^3.2.0",
+    "concurrently": "^4.1.1",
+    "cypress": "^3.4.1",
     "husky": "^1.3.1"
   },
   "dependencies": {}

scripts/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-source scripts.sh
+source ./scripts/scripts.sh
 
 PS3='Please enter your choice: '
 

scripts/run_service_lints.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# stop on first error
+set -e
+docker-compose exec -T backend npm run lint
+docker-compose exec -T updater_writer npm  run lint
+docker-compose exec -T updater_api npm run lint
+docker-compose exec -T updater_scheduler npm run lint
+docker-compose exec -T userservice npm run lint
+docker-compose exec -T frontend npm run lint
+docker-compose exec -T analytics npm run lint
+docker-compose exec -T userservice npm run lint

scripts/run_service_tests.sh

@@ -3,8 +3,11 @@
 # stop on first error
 set -e
 docker-compose exec -T backend npm test
+docker-compose exec -T updater_writer npm test
+docker-compose exec -T updater_api npm test
+docker-compose exec -T updater_scheduler npm test
 docker-compose exec -T userservice npm test
 docker-compose exec -T frontend npm test
 docker-compose exec -T analytics npm test
+docker-compose exec -T userservice npm test
 docker-compose exec -T usageservice npm test
-# docker-compose exec -T updater_writer npm test

scripts/scripts.sh

@@ -32,7 +32,8 @@ echo_path () {
 }
 
 get_oodikone_server_backup() {
-    scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
+    scp -r -o ProxyCommand="ssh -W %h:%p melkinpaasi.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
+    scp -r -o ProxyCommand="ssh -W %h:%p melkinpaasi.cs.helsinki.fi" svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
 }
 
 get_anon_oodikone() {
@@ -50,7 +51,7 @@ get_anon_oodikone() {
 
 restore_psql_from_backup () {
     docker cp $1 $2:/asd.sqz
-    docker exec -it $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
+    docker exec $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
 }
 
 # oodilearn
@@ -113,6 +114,16 @@ reset_db () {
     docker-compose-dev down
 }
 
+reset_db_for_cypress () {
+    # stop any service with connections to DB
+    docker-compose stop backend updater_writer
+    db_anon_setup_full
+    # restart services, run migrations
+    docker-compose restart backend updater_writer
+    # wait until backend is up
+    retry curl --silent --fail localhost:8080/ping
+}
+
 install_cli_npm_packages () {
     npm ci
 }
@@ -164,10 +175,10 @@ run_e2e_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    TAG=$2 docker-compose -f docker-compose.yml -f $1 build
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 build
     echo "Setup oodikone db from dump."
-    TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
     db_anon_setup_full
     echo "Starting services."
-    TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d
+    TAG=$2 SENTRY_RELEASE_VERSION=$3 docker-compose -f docker-compose.yml -f $1 up -d
 }

services/backend/oodikone2-backend/.eslintrc.json

@@ -23,7 +23,6 @@
             "error",
             "never"
         ],
-        "no-console": 0,
-        "max-len": [2, 120]
+        "no-console": 0
     }
 }

services/backend/oodikone2-backend/Dockerfile

@@ -1,22 +1,21 @@
 FROM node:10.15-alpine
 
+WORKDIR /usr/src/app
+EXPOSE 8080
+CMD npm start
+
+COPY ./oodikone2-backend/package.json /usr/src/app/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/app/package-lock.json
+COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
+RUN npm ci
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
+
 COPY ./oodikone2-backend /usr/src/app
 COPY ./shared/models /usr/src/app/src/models
 COPY ./shared/migrations /usr/src/app/src/database/migrations
 COPY ./shared/migrations_kone /usr/src/app/src/database/migrations_kone
 
-WORKDIR /usr/src/app
-RUN npm ci
-
-COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
-RUN chmod +x /wait
-
-EXPOSE 8080
-CMD /wait && npm start

services/backend/oodikone2-backend/package.json

@@ -15,7 +15,8 @@
     "docker_local": "NODE_ENV=docker_local nodemon index.js",
     "dev": "NODE_ENV=dev nodemon --max_old_space_size=4096 index.js",
     "anon": "NODE_ENV=test nodemon index.js",
-    "test": "NODE_ENV=test jest --runInBand --verbose",
+    "test": "npm run lint && npm run jest",
+    "jest": "NODE_ENV=test jest --runInBand --verbose",
     "test_docker": "docker exec -t backend /bin/bash -c 'npm test'",
     "update_database_dev": "NODE_ENV=dev node src/services/doo_api_database_updater/run_updater.js",
     "update_database": "node src/services/doo_api_database_updater/run_updater.js",
@@ -23,7 +24,7 @@
     "init_redis": "node src/util/init_redis.js",
     "seed_redis": "node src/util/add_json_duplicates_to_redis.js && node src/util/map_course_codes.js",
     "studentnumbers_to_file": "NODE_ENV=dev node src/services/doo_api_database_updater/db_studentnumbers_to_file.js",
-    "lint": "eslint --parser babel-eslint src",
+    "lint": "eslint --parser babel-eslint .",
     "force_sync_database": "node scripts/force_sync_database.js",
     "update_studentnumbers_dev": "NODE_ENV=dev node src/services/doo_api_database_updater/student_list_updater.js",
     "anonymize": "node src/anonymizer/anonymizer.js",
@@ -38,41 +39,42 @@
   "author": "Matti Luukkainen",
   "license": "MIT",
   "dependencies": {
+    "async-lock": "^1.2.2",
     "axios": "^0.19.0",
-    "bluebird": "^3.5.3",
-    "body-parser": "^1.18.3",
+    "bluebird": "^3.5.5",
+    "body-parser": "^1.19.0",
     "cors": "^2.8.5",
-    "dotenv": "^6.1.0",
-    "express": "^4.16.4",
-    "jsonwebtoken": "^8.4.0",
-    "kafka-node": "^4.1.1",
-    "lodash": "^4.17.10",
+    "cron": "^1.7.1",
+    "dotenv": "^6.2.0",
+    "express": "^4.17.1",
+    "jsonwebtoken": "^8.5.1",
+    "kafka-node": "^4.1.3",
+    "lodash": "^4.17.15",
     "mkdirp": "^0.5.1",
-    "moment": "^2.22.2",
+    "moment": "^2.24.0",
     "morgan": "^1.9.1",
-    "node-nats-streaming": "^0.2.2",
     "nodemailer": "^4.7.0",
-    "pg": "^7.7.1",
+    "pg": "^7.11.0",
     "raven": "^2.6.3",
     "redis": "^2.8.0",
-    "sequelize": "^5.3.0",
+    "sequelize": "^5.10.2",
     "umzug": "^2.2.0",
     "uuid": "^3.3.2",
     "winston": "^3.0.0",
     "winston-log2gelf": "^2.0.4",
-    "winston-transport": "^4.2.0"
+    "winston-transport": "^4.3.0"
   },
   "devDependencies": {
-    "ava": "^0.25.0",
+    "ava": "^2.2.0",
     "babel-eslint": "^8.2.6",
-    "eslint": "^5.9.0",
+    "eslint": "^5.16.0",
     "faker": "^4.1.0",
-    "jest": "^24.1.0",
-    "nock": "^10.0.2",
+    "jest": "^24.8.0",
+    "nock": "^10.0.6",
     "node-status": "^1.0.0",
-    "nodemon": "^1.18.7",
-    "nyc": "^13.1.0",
-    "supertest": "^3.1.0"
+    "nodemon": "^1.19.1",
+    "nyc": "^13.3.0",
+    "supertest": "^3.4.2"
   },
   "ava": {
     "files": [

services/backend/oodikone2-backend/src/anonymizer/anon_oodi_interface.js

@@ -13,9 +13,10 @@ axios.defaults.auth = {
   username: 'tktl',
 }
 
-axios.defaults.params = {
-  token: process.env.TOKEN
-}
+instance.interceptors.request.use((config) => {
+  config.params = {token: process.env.TOKEN}
+  return config
+})
 
 const getStudent = (studentNumber) => {
   return instance.get(`${process.env.OODI_ADDR}/students/${studentNumber}/info`)

services/backend/oodikone2-backend/src/app.js

@@ -4,33 +4,43 @@ const bodyParser = require('body-parser')
 const Raven = require('raven')
 const conf = require('./conf-backend')
 const routes = require('./routes')
-const { StartNats } = require('./events')
+const { startCron } = require('./events')
 const { PORT } = conf
-const app = express()
+const { initializeDatabaseConnection } = require('./database/connection')
 
-StartNats()
+initializeDatabaseConnection().then(() => {
+  const app = express()
 
-Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
+  startCron()
 
-app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-app.use(bodyParser.json())
+  Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
 
-app.get('/ping', async (req, res) => {
-  res.json({ data: 'pong' })
-})
+  app.use(cors({ credentials: true, origin: conf.frontend_addr }))
+  app.use(bodyParser.json())
 
-let BASE_URL = ''
-if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
-  BASE_URL = '/api'
-}
+  app.get('/ping', async (req, res) => {
+    res.json({ data: 'pong' })
+  })
 
-routes(app, BASE_URL)
+  let BASE_URL = ''
+  if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
+    BASE_URL = '/api'
+  }
 
-app.get('*', async (req, res) => {
-  const results = { error: 'unknown endpoint' }
-  res.status(404).json(results)
-})
+  routes(app, BASE_URL)
 
-module.exports = app.listen(PORT, () => {
-  console.log('Example app listening on port ' + PORT + '!')
+  app.get('*', async (req, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+
+  const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+  process.on('SIGTERM', () => {
+    server.close(() => {
+      console.log('Process terminated')
+    })
+  })
+}).catch(e => {
+  process.exitCode = 1
+  console.log(e)
 })

services/backend/oodikone2-backend/src/database/connection.js

@@ -5,50 +5,65 @@ const conf = require('../conf-backend')
 const sequelize = new Sequelize(conf.DB_URL, {
   schema: conf.DB_SCHEMA,
   searchPath: conf.DB_SCHEMA,
-  logging: false,
-  operatorsAliases: false
+  logging: false
 })
 sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
 
 const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
-  logging: false,
-  operatorsAliases: false
+  logging: false
 })
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
 
-const runMigrationsKone = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelizeKone,
-        tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelizeKone.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/src/database/migrations_kone`,
-        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE
-      },
-    })
-    const migrations = await migrator.up()
+const initializeDatabaseConnection = async () => {
+  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+  const waitSeconds = 60
+  for (let i = 1; i <= waitSeconds; i++) {
+    try {
+      await sequelize.authenticate()
+      await sequelizeKone.authenticate()
+      break
+    } catch (e) {
+      if (i === waitSeconds) {
+        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        throw e
+      }
+      console.log('.')
+      await sleep(1000)
+    }
+  }
+  if (!conf.isTest) {
+    try {
+      const migrator = new Umzug({
+        storage: 'sequelize',
+        storageOptions: {
+          sequelize: sequelizeKone,
+          tableName: 'migrations',
+          schema: conf.DB_SCHEMA_KONE
+        },
+        logging: console.log,
+        migrations: {
+          params: [
+            sequelizeKone.getQueryInterface(),
+            Sequelize
+          ],
+          path: `${process.cwd()}/src/database/migrations_kone`,
+          pattern: /\.js$/,
+          schema: conf.DB_SCHEMA_KONE
+        },
+      })
+      const migrations = await migrator.up()
 
-    console.log('Kone Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Kone Migration error, message:', e)
+      console.log('Kone Migrations up to date', migrations)
+    } catch (e) {
+      console.log('Kone Migration error')
+      throw e
+    }
   }
 }
 
-const migrationPromiseKone = !conf.isTest ? runMigrationsKone() : Promise.resolve()
-
 const forceSyncDatabase = async () => {
   try {
     await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
@@ -67,6 +82,6 @@ const forceSyncDatabase = async () => {
 module.exports = {
   sequelize,
   sequelizeKone,
-  migrationPromiseKone,
+  initializeDatabaseConnection,
   forceSyncDatabase
 }

services/backend/oodikone2-backend/src/events.js

@@ -1,119 +1,101 @@
-var stan = require('node-nats-streaming').connect(
-  'updaterNATS',
-  process.env.HOSTNAME,
-  process.env.NATS_URI
-)
-
+const { CronJob } = require('cron')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studytrack')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
+const { calculateFacultyYearlyStats } = require('./services/faculties')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput, patchFacultyYearlyStats } = require('./services/analyticsService')
 
-const taskStatuses = { }
-const handleMessage = (asyncHandlerFunction) => async (msg) => {
-  const key = msg.getSubject()+'_'+msg.getData()
-  if (taskStatuses[key])
-    return
-  stan.publish('status', key+':RECEIVED', (err) => {
-    if(err) {
-      console.log('publish failed', err)
-    }
-  })
-  taskStatuses[key] = true
-  const success = await asyncHandlerFunction(msg)
-  taskStatuses[key] = false
-  stan.publish('status', key+(success ? ':DONE' : ':ERRORED'), (err) => {
-    if(err) {
-      console.log('publish failed', err)
-    }
-  })
-  msg.ack()
+const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
+
+const refreshFacultyYearlyStats = async () => {
+  try {
+    console.log('Refreshing faculty yearly stats...')
+    const data = await calculateFacultyYearlyStats()
+    await patchFacultyYearlyStats(data)
+  } catch (e) {
+    console.error(e)
+  }
 }
 
-const StartNats = () => {
-  console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-  var opts = stan.subscriptionOptions()
-  opts.setManualAckMode(true)
-  // opts.setAckWait(6*60*60*1000)
-  // opts.setDeliverAllAvailable()
-  // opts.setDurableName('durable')
-  opts.setMaxInFlight(1)
-  stan.on('connect', async function() {
-    stan.subscribe('RefreshStudyrightAssociations', opts).on('message', handleMessage(async () => {
+const refreshStudyrightAssociations = async () => {
+  try {
+    console.log('Refreshing studyright associations...')
+    await refreshAssociationsInRedis()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+const refreshOverview = async () => {
+  try {
+    console.log('Refreshing overview...')
+    const codes = (await getAllProgrammes()).map(p => p.code)
+    let ready = 0
+    for (const code of codes) {
       try {
-        await refreshAssociationsInRedis()
+        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+        const since = new Date().getFullYear() - 5
+        const data = await throughputStatsForStudytrack(
+          code,
+          since
+        )
+        await setThroughput(
+          data
+        )
       } catch (e) {
-        console.error(e)
-        return false
-      }
-      return true
-    }))
-    stan.subscribe('RefreshOverview', opts).on('message', handleMessage(async () => {
-      try {
-        console.log('RefreshOverview starting')
-        const codes = (await getAllProgrammes()).map(p => p.code)
-
-        let ready = 0
-        for(const code of codes) {
-          try {
-            await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-            const since = new Date().getFullYear() - 5
-            const data = await throughputStatsForStudytrack(
-              code,
-              since
-            )
-            await setThroughput(
-              data
-            )
-          } catch (e) {
-            try {
-              await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-            } catch (e) {
-              console.error(
-                e
-              )
-            }
-            console.error(e)
-            console.log(
-              `Failed to update throughput stats for code: ${code}, reason: ${
-                e.message
-              }`
-            )
-          }
-          try {
-            await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-            const since = '2017-08-01'
-            const data = await productivityStatsForStudytrack(code, since)
-            await setProductivity(data)
-
-          } catch (e) {
-            try {
-              await patchProductivity({
-                [code]: { status: 'RECALCULATION ERRORED' }
-              })
-            } catch (e) {
-              console.error(e)
-            }
-            console.error(e)
-            console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-          }
-          ready += 1
-          console.log(
-            `RefreshOverview ${ready}/${
-              codes.length
-            } done`
+        try {
+          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+        } catch (e) {
+          console.error(
+            e
           )
         }
-      } catch (e) {
         console.error(e)
-        return false
+        console.log(
+          `Failed to update throughput stats for code: ${code}, reason: ${
+            e.message
+          }`
+        )
       }
-      return true
-    }))
-  })
-  return stan
+      try {
+        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+        const since = '2017-08-01'
+        const data = await productivityStatsForStudytrack(code, since)
+        await setProductivity(data)
+
+      } catch (e) {
+        try {
+          await patchProductivity({
+            [code]: { status: 'RECALCULATION ERRORED' }
+          })
+        } catch (e) {
+          console.error(e)
+        }
+        console.error(e)
+        console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+      }
+      ready += 1
+      console.log(
+        `RefreshOverview ${ready}/${
+          codes.length
+        } done`
+      )
+    }
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+const startCron = () => {
+  if (process.env.NODE_ENV === 'production') {
+    schedule('0 6 * * *', async () => {
+      await refreshFacultyYearlyStats()
+      await refreshStudyrightAssociations()
+      await refreshOverview()
+    })
+  }
 }
 
 module.exports = {
-  StartNats
+  startCron
 }

services/backend/oodikone2-backend/src/middleware/accesslogger.js

@@ -1,20 +1,9 @@
 const morgan = require('morgan')
-const UsageService = require('../services/usageService')
 
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const message = [
-    req.decodedToken.name, ':',
-    tokens['method'](req, res),
-    tokens['url'](req, res),
-    tokens['status'](req, res),
-    '-',
-    tokens['response-time'](req, res), 'ms'
-  ].join(' ')
   const meta = req.decodedToken
   fields.forEach(field => meta[field] = tokens[field](req, res))
-  UsageService.log(message, meta).catch(console.log)
- 
 })
 
 module.exports = accessLogger

services/backend/oodikone2-backend/src/middleware/auth.js

@@ -1,44 +1,36 @@
 const jwt = require('jsonwebtoken')
 const conf = require('../conf-backend')
 const blacklist = require('../services/blacklist')
+const { getAccessGroupCodesFor, getCodesFromElementDetails } = require('../services/userService')
 const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
 const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
 
-const isShibboUser = (userId, uidHeader) => userId === uidHeader
 const TOKEN_VERSION = 1 // When token structure changes, increment in userservice, backend and frontend
 
 const checkAuth = async (req, res, next) => {
   const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
   const uid = req.headers['uid']
-  if (token) {
-    jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-      if (err) {
-        res.status(403).json(err)
-      } else if (decoded.version !== TOKEN_VERSION) {
-        res.status(401).json({ error: 'Token needs to be refreshed - invalid version', reloadPage: true })
-      } else if (decoded.mockedBy ? isShibboUser(decoded.mockedBy, uid) : isShibboUser(decoded.userId, uid)) {
-        req.decodedToken = decoded
-        next()
-      } else {
-        res.status(403).json({ error: 'User shibboleth id and token id did not match' })
-      }
-    })
-  } else {
-    res.status(403).json({ error: 'No token in headers' })
-  }
+  if (!token) return res.status(403).json({ error: 'No token in headers' })
+  jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
+    if (err) return res.status(403).json(err)
+    if (decoded.version !== TOKEN_VERSION) return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
+    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
+    req.decodedToken = decoded
+    req.roles = await getAccessGroupCodesFor(decoded.userId)
+    req.rights = await getCodesFromElementDetails(decoded.userId)
+    next()
+  })
 }
 
-const roles = requiredRoles => (req, res, next) => {
-  if (req.decodedToken && req.decodedToken.roles != null) {
-    const roles = req.decodedToken.roles.map(r => r.group_code)
-    console.log(`Request has roles: ${roles}`)
+const roles = requiredRoles => async (req, res, next) => {
+  if (req.decodedToken) {
+    const { roles } = req
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
-      console.log(`authorized for ${requiredRoles}`)
       next()
       return
     }
   }
-  console.log(`missing required roles ${requiredRoles}`)
+  console.log(`Missing required roles ${ requiredRoles }`)
   res.status(403).json({ error: 'missing required roles' })
 }
 
@@ -48,8 +40,7 @@ const checkRequiredGroup = async (req, res, next) => {
   const tokenOutdated = req.decodedToken.enabled !== enabled
   if (tokenOutdated) {
     res.status(401).json({
-      error: 'Token needs to be refreshed - enabled doesnt match hy-group requirement',
-      reloadPage: true
+      error: 'Token needs to be refreshed - enabled doesnt match hy-group requirement'
     })
   } else if (!enabled) {
     res.status(403).json({ error: 'User is not enabled' })
@@ -62,7 +53,7 @@ const checkUserBlacklisting = async (req, res, next) => {
   const { userId, createdAt } = req.decodedToken
   const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
   if (isBlacklisted) {
-    res.status(401).json({ error: 'Token needs to be refreshed - blacklisted', reloadPage: true })
+    res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
   } else {
     next()
   }

services/backend/oodikone2-backend/src/routes.js

@@ -23,13 +23,14 @@ const oodi = require('./routes/oodi')
 const task = require('./routes/tasks')
 const feedback = require('./routes/feedback')
 const tags = require('./routes/tags')
+const updater = require('./routes/updater')
 
 module.exports = (app, url) => {
   app.use(url, log)
   app.use(url, login)
+  app.use(`${url}/superlogin`, superlogin)
   app.use(url, ping)
   app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger)
-  app.use(`${url}/superlogin`, auth.roles(['admin']), superlogin)
   app.use(url, elementdetails)
   app.use(url, courses)
   app.use(url, students)
@@ -38,15 +39,16 @@ module.exports = (app, url) => {
   app.use(url, providers)
   app.use(url, faculties)
   app.use(url, semesters)
+  app.use(url, tags)
+  app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['users']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/usage`, auth.roles(['usage']), usage)
   app.use(`${url}/oodilearn`, auth.roles(['oodilearn']), oodilearn)
   app.use(`${url}/course-groups`, auth.roles(['coursegroups']), courseGroups)
-  app.use(`${url}/mandatory_courses`, mandatoryCourses),
-  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels),
+  app.use(`${url}/mandatory_courses`, mandatoryCourses)
+  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)
   app.use(`${url}/oodi`, auth.roles(['dev']), oodi)
   app.use(url, auth.roles(['dev', 'admin']), task)
-  app.use(url, tags)
 }

services/backend/oodikone2-backend/src/routes/courseGroups.js

@@ -1,12 +1,11 @@
 const router = require('express').Router()
-
 const CourseGroupService = require('../services/courseGroups')
 
 router.get('/programme/:programmeId/:force?', async (req, res) => {
   const { programmeId, force } = req.params
-  const { rights, roles } = req.decodedToken
+  const { rights, roles } = req
   const semesterCode = req.query.semester
-  if (rights.includes(programmeId) || (roles && roles.map(r => r.group_code).includes('admin'))) {
+  if (rights.includes(programmeId) || (roles && roles.includes('admin'))) {
     const courseGroups = await CourseGroupService.getCourseGroupsWithTotals(
       programmeId, semesterCode, force === 'force'
     )

services/backend/oodikone2-backend/src/routes/courses.js

@@ -39,8 +39,8 @@ router.get('/coursedisciplines', async (req, res) => {
 
 router.get('/v2/courseyearlystats', async (req, res) => {
   let results = []
-  const { rights, roles } = req.decodedToken
-  const admin = roles.map(r => r.group_code).includes('admin')
+  const { rights, roles } = req
+  const admin = roles.includes('admin')
   if (!admin) {
     if (rights.length <= 0) {
       return res.status(403).json({ error: 'No programmes so no access to course stats' })
@@ -57,19 +57,21 @@ router.get('/v2/courseyearlystats', async (req, res) => {
 
 router.get('/v3/courseyearlystats', async (req, res) => {
   try {
-    const { rights, roles } = req.decodedToken
-    const admin = roles.map(r => r.group_code).includes('admin')
+    const { rights, roles } = req
+    const admin = roles.includes('admin')
     if (!admin) {
+      // If user has rights to see at least one programme, then they are allowed
+      // to see all of them
       if (rights.length <= 0) {
         return res.status(403).json({ error: 'No programmes so no access to course stats' })
       }
     }
-    const { codes, startyearcode, endyearcode, separate: sep } = req.query
+    const { codes, separate: sep } = req.query
     const separate = !sep ? false : JSON.parse(sep)
-    if (!codes || !startyearcode) {
+    if (!codes) {
       res.status(422).send('Missing required query parameters')
     } else {
-      const results = await Course.courseYearlyStats(codes, separate, startyearcode, endyearcode)
+      const results = await Course.courseYearlyStats(codes, separate)
       res.json(results)
     }
   } catch (e) {

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -1,11 +1,20 @@
 const router = require('express').Router()
-const { getAllDegreesAndProgrammes, getAllProgrammes } = require('../services/studyrights')
+const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
 const MandatoryCourses = require('../services/mandatoryCourses')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('../services/studytrack')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
 const { getProductivity, setProductivity, getThroughput, setThroughput,
   patchProductivity, patchThroughput, ping } = require('../services/analyticsService')
 
+router.get('/elementdetails/all', async (req, res) => {
+  try {
+    const elementdetails = await getAllElementDetails()
+    res.json(elementdetails)
+  } catch (e) {
+    res.status(500).json(e)
+  }
+})
+
 router.get('/studyprogrammes', async (req, res) => {
   try {
     const studyrights = await getAllDegreesAndProgrammes()

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -1,9 +1,29 @@
 const router = require('express').Router()
 const { all } = require('../services/organisations')
+const { byType: getElementDetailsByType } = require('../services/elementdetails')
+const { getFacultyYearlyStats } = require('../services/analyticsService')
 
 router.get('/faculties', async (req, res) => {
   const faculties = await all()
   res.json(faculties)
 })
 
+router.get('/faculties/yearlystats', async (req, res) => {
+  try {
+    const facultyYearlyStats = await getFacultyYearlyStats()
+    res.status(200).json(facultyYearlyStats)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
+router.get('/faculties/programmes', async (req, res) => {
+  try {
+    const facultyProgrammes = await getElementDetailsByType(20)
+    res.json(facultyProgrammes)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/language.js

@@ -2,12 +2,13 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/language', async (req, res) => {
-  const { username, language } = req.body
+  const { language } = req.body
+  const { userId } = req.decodedToken
   if (!['fi','sv','en'].includes(language)) {
     return res.status(400).json('invalid language')
   }
   try {
-    const result = await userService.updateUser(username, { language })
+    const result = await userService.updateUser(userId, { language })
     return res.status(200).json(result)
   } catch (e) {
     return res.status(e.response.status).json(e.response.data)

services/backend/oodikone2-backend/src/routes/mandatorycourses.js

@@ -4,56 +4,49 @@ const MandatoryCourses = require('../services/mandatoryCourses')
 router.delete('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.remove(programme, course)
-      res.status(200).json(course)
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.remove(programme, course)
+    res.status(200).json(course)
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 
 router.post('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.create(programme, course)
-      const newCourse = await MandatoryCourses.find(programme, course)
-      res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.create(programme, course)
+    const newCourse = await MandatoryCourses.find(programme, course)
+    if (!newCourse) return res.status(400).end()
+    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 
 router.post('/:programme/label/:course', async (req, res) => {
   const { programme, course } = req.params
   const { label } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.updateLabel(programme, course, label)
-      const updatedCourse = await MandatoryCourses.find(programme, course)
-      res
-        .status(200)
-        .json({
-          name: updatedCourse.course.name,
-          code: updatedCourse.course_code,
-          label: updatedCourse.mandatory_course_label
-        })
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.updateLabel(programme, course, label)
+    const updatedCourse = await MandatoryCourses.find(programme, course)
+    if (!updatedCourse) return res.status(400).end()
+    res
+      .status(200)
+      .json({
+        name: updatedCourse.course.name,
+        code: updatedCourse.course_code,
+        label: updatedCourse.mandatory_course_label
+      })
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 

services/backend/oodikone2-backend/src/routes/population.js

@@ -1,7 +1,8 @@
 const router = require('express').Router()
 const Population = require('../services/populations')
 const Filters = require('../services/filters')
-const { updateStudents }  = require('../services/updaterService')
+const { updateStudents } = require('../services/updaterService')
+const { isValidStudentId } = require('../util/index')
 
 const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
@@ -30,29 +31,37 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
 
     res.json(result)
   } catch (e) {
-    res.status(400).json({ error: e })
+    console.log(e)
+    res.status(500).json({ error: e })
   }
 })
 
 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
   try {
-    if (!req.body.yearcode || !req.body.coursecode) {
+    if (!req.body.yearcode || !req.body.coursecodes) {
       res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
       return
     }
-    const { coursecode, yearcode } = req.body
+    const { coursecodes, yearcode } = req.body
     let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
+    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, yearcode)
+    const { decodedToken: { userId }, roles } = req
 
-    const { roles, userId } = req.decodedToken
-    if (roles && roles.map(r => r.group_code).includes('admin')) {
+    if (roles && roles.includes('admin')) {
       studentnumberlist = studentnumbers
     } else {
       const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
       const codes = unitsUserCanAccess.map(unit => unit.id)
       studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
     }
-    const result = await Population.bottlenecksOf(req.body, studentnumberlist)
+
+    const result = await Population.bottlenecksOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000
+    }, studentnumberlist)
 
     if (result.error) {
       res.status(400).json(result)
@@ -61,7 +70,45 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
 
     res.json(result)
   } catch (e) {
-    res.status(400).json({ error: e })
+    console.log(e)
+    res.status(500).json({ error: e })
+  }
+})
+
+router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
+  try {
+    if (!req.body.studentnumberlist) {
+      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
+      return
+    }
+    let studentnumberlist
+    const { decodedToken: { userId }, roles } = req
+
+    if (roles && roles.includes('admin')) {
+      studentnumberlist = req.body.studentnumberlist
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+    }
+
+    const result = await Population.bottlenecksOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000
+    }, studentnumberlist)
+    if (result.error) {
+      res.status(400).json(result)
+      return
+    }
+
+    res.json(result)
+
+  } catch (err) {
+    console.log(err)
+    res.status(500).json({ error: err })
   }
 })
 
@@ -75,8 +122,9 @@ router.get('/v3/populationstatistics', async (req, res) => {
     let studyRights = null
     try {
       studyRights = JSON.parse(studyRightsJSON)
-      const { roles, rights } = req.decodedToken
-      if (!roles || !roles.map(r => r.group_code).includes('admin')) {
+      const { rights, roles } = req
+
+      if (!roles || !roles.includes('admin')) {
         if (!rights.includes(studyRights.programme)) {
           res.status(403).json([])
           return
@@ -103,19 +151,22 @@ router.get('/v3/populationstatistics', async (req, res) => {
     res.json(result)
   } catch (e) {
     console.log(e)
-    res.status(400).json({ error: e })
+    res.status(500).json({ error: e })
   }
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecode, yearcode } = req.query
+  const { coursecodes, yearcode } = req.query
+  if (!coursecodes || !yearcode) {
+    res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+    return
+  }
   const semesters = ['FALL', 'SPRING']
   let studentnumberlist
-  console.log(coursecode, yearcode)
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
-  console.log(studentnumbers)
-  const { roles, userId } = req.decodedToken
-  if (roles && roles.map(r => r.group_code).includes('admin')) {
+  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), yearcode)
+  const { decodedToken: { userId }, roles } = req
+
+  if (roles && roles.includes('admin')) {
     studentnumberlist = studentnumbers
   } else {
     const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
@@ -127,8 +178,8 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
       startYear: 1900,
       endYear: 2200,
       studyRights: [],
-      semesters, 
-      months: 1000
+      semesters,
+      months: 10000
     }, studentnumberlist)
 
     if (result.error) {
@@ -141,10 +192,39 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     res.json(result)
   } catch (e) {
     console.log(e)
-    res.status(400).json({ error: e })
+    res.status(500).json({ error: e })
   }
 })
 
+router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
+  const { studentnumberlist } = req.body
+  const { roles } = req
+
+  if (!(roles && roles.includes('admin'))) {
+    console.log('no rights')
+    res.status(400).end()
+    return
+  } else {
+    try {
+      const result = await Population.optimizedStatisticsOf({
+        startYear: 1900,
+        endYear: 2200,
+        studyRights: [],
+        semesters: ['FALL', 'SPRING'],
+        months: 10000
+      }, studentnumberlist)
+      if (result.error) {
+        console.log(result.error)
+        res.status(400).end()
+        return
+      }
+      res.status(200).json(result)
+    } catch (err) {
+      console.log(err)
+      res.status(400).end()
+    }
+  }
+})
 
 router.get('/v2/populationstatistics/filters', async (req, res) => {
 
@@ -192,19 +272,24 @@ router.delete('/v2/populationstatistics/filters', async (req, res) => {
 
 router.post('/updatedatabase', async (req, res) => {
   const studentnumbers = req.body
-  console.log(studentnumbers)
-  if (studentnumbers) {
-    await updateStudents(studentnumbers)
-    res.status(200).json('Scheduled')
-  } else {
+  if (!(studentnumbers && studentnumbers.every(sn => isValidStudentId(sn)))) {
     res.status(400).end()
+    return
+  }
+  try {
+    const response = await updateStudents(studentnumbers)
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
   }
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
   try {
-    const { rights, roles } = req.decodedToken
-    if (roles && roles.map(r => r.group_code).includes('admin')) {
+    const { rights, roles } = req
+    if (roles && roles.includes('admin')) {
       const studyrights = await StudyrightService.getAssociations()
       res.json(studyrights)
     } else {

services/backend/oodikone2-backend/src/routes/students.js

@@ -4,8 +4,9 @@ const userService = require('../services/userService')
 const Unit = require('../services/units')
 
 router.get('/students', async (req, res) => {
-  const { roles, userId } = req.decodedToken
-  if (roles && roles.map(r => r.group_code).includes('admin')) {
+  const { roles, decodedToken: { userId } } = req
+
+  if (roles && roles.includes('admin')) {
     let results = []
     if (req.query.searchTerm) {
       results = await Student.bySearchTerm(req.query.searchTerm)
@@ -20,15 +21,21 @@ router.get('/students', async (req, res) => {
 })
 
 router.get('/students/:id', async (req, res) => {
-  const studentId = req.params.id
-  const { roles } = req.decodedToken
-  if (roles && roles.map(r => r.group_code).includes('admin')) {
+  const { id: studentId } = req.params
+  const { roles } = req
+
+  if (roles && roles.includes('admin')) {
     const results = await Student.withId(studentId)
-    return res.json(results)
+    return results.error ?
+      res.status(400).json({ error: 'error finding student' }) :
+      res.json(results)
   }
 
   const uid = req.decodedToken.userId
   const student = await Student.withId(studentId)
+  if (student.error) {
+    return res.status(400).json({ error: 'error finding student' })
+  }
   const units = await userService.getUnitsFromElementDetails(uid)
 
   const rights = await Promise.all(units.map(async unit => {
@@ -39,9 +46,8 @@ router.get('/students/:id', async (req, res) => {
   if (rights.some(right => right !== null)) {
     res.json(student).end()
   } else {
-    res.json([]).end()
+    res.status(400).json({ error: 'error finding student' })
   }
-
 })
 
 module.exports = router

services/backend/oodikone2-backend/src/routes/superlogin.js

@@ -3,12 +3,11 @@ const userService = require('../services/userService')
 
 router.post('/:uid', async (req, res) => {
   try {
-    const asUser = req.params.uid
-    const uid = req.decodedToken.mockedBy == null ? req.headers['uid'] : req.decodedToken.mockedBy
+    const uid = req.headers['uid']
     if (req.headers['shib-session-id'] && uid) {
-      console.log('super')
+      const asUser = req.params.uid
       const token = await userService.superlogin(uid, asUser)
-      res.status(200).json({ token })
+      res.status(200).json(token)
     } else {
       res.status(401).json({
         message: `Not enough headers login, uid:
@@ -17,7 +16,7 @@ router.post('/:uid', async (req, res) => {
     }
   } catch (err) {
     console.log(err.message)
-    res.status(401).json({ message: 'problem with login', err: err.message })
+    res.status(500).json({ message: 'problem with login' })
   }
 })
 

services/backend/oodikone2-backend/src/routes/tags.js

@@ -1,6 +1,8 @@
 const router = require('express').Router()
 const Tags = require('../services/tags')
 const TagStudent = require('../services/tagstudent')
+const Students = require('../services/students')
+const { difference } = require('lodash')
 
 router.get('/tags', async (req, res) => {
   try {
@@ -8,40 +10,54 @@ router.get('/tags', async (req, res) => {
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
-  const { studytrack } = req.params
   try {
+    const { studytrack } = req.params
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     const tags = await Tags.findTagsByStudytrack(studytrack)
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.post('/tags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await Tags.createNewTag(tag)
-    res.status(200).json(result)
+    const { tag } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await Tags.createNewTag(tag)
+    const tags = await Tags.findTagsByStudytrack(tag.studytrack)
+    res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err.message)
+    res.status(500).json(err.message)
   }
 })
 
 router.delete('/tags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await Tags.deleteTag(tag)
-    res.status(200).json(result)
+    const { tag } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await Tags.deleteTag(tag)
+    const t = await Tags.findTagsByStudytrack(tag.studytrack)
+    res.status(200).json(t)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 
 })
@@ -52,52 +68,94 @@ router.get('/studenttags', async (req, res) => {
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/studenttags/:studytrack', async (req, res) => {
-  const { studytrack } = req.params
   try {
+    const { studytrack } = req.params
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
-  const { studentnumber } = req.params
   try {
+    const { studentnumber } = req.params
     const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    console.log(result)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
+  }
+})
+
+router.post('/studenttags/:studentnumber', async (req, res) => {
+  try {
+    const { tag, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await TagStudent.createStudentTag(tag)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(500).json(err)
   }
 })
 
 router.post('/studenttags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await TagStudent.createStudentTag(tag)
+    const { tags, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const existingtags = await Tags.findTagsByStudytrack(studytrack)
+    const existingTagids = existingtags.map(t => t.tag_id)
+    const tagids = [...new Set(tags.map(t => t.tag_id))]
+    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
+
+    const studentnumbers = tags.map(t => t.studentnumber)
+    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+    const missingStudents = difference(studentnumbers, students)
+    if (missingStudents.length !== 0) return res.status(400).json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+
+    await TagStudent.createMultipleStudentTags(tags)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.delete('/studenttags', async (req, res) => {
-  const { id } = req.body
   try {
-    const result = await TagStudent.deleteStudentTag(id)
+    const { tag_id, studentnumber, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+
+    await TagStudent.deleteStudentTag(studentnumber, tag_id)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -2,6 +2,7 @@ const teachers = require('../services/teachers')
 const topteachers = require('../services/topteachers')
 const r = require('./router').routerWithWrapper()
 const { FEATURES } = require('../conf-backend')
+const { mapToProviders } = require('../util/utils')
 
 const router = FEATURES.ERROR_HANDLER ? r.wrapper : r.router
 
@@ -31,28 +32,16 @@ router.get('/top/categories', async (req, res) => {
   res.json(result)
 })
 
-const mapToProviders = rights => rights.map((r) => {
-  if (r.includes('_')) {
-    let newPrefix = ''
-    let newSuffix = ''
-    const split = r.split('_')
-    newPrefix = `${split[0][2]}00`
-    newSuffix = `${split[0][0]}${split[1]}`
-    const providercode = `${newPrefix}-${newSuffix}`
-    return providercode
-  }
-  return r
-})
-
 router.get('/stats', async (req, res) => {
-  const { rights, roles } = req.decodedToken
+  const { rights, roles } = req
+
   const { providers, semesterStart, semesterEnd } = req.query
   if (!providers || !semesterStart) {
     return res.status(422).send('Missing required query parameters.')
   }
   const providerRights = mapToProviders(rights)
 
-  if (!(providers.every(p => providerRights.includes(p)) || roles.map(r => r.group_code).includes('admin'))) {
+  if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')
   }
   const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd||semesterStart + 1)

services/backend/oodikone2-backend/src/routes/updater.js

@@ -0,0 +1,129 @@
+const router = require('express').Router()
+const {
+  updateOldestStudents,
+  getStatuses,
+  updateActiveStudents,
+  updateAllStudents,
+  updateAttainments,
+  updateMetadata,
+  updateStudentlist,
+  rescheduleScheduled,
+  rescheduleFetched,
+  updateNoStudents
+} = require('../services/updaterService')
+
+router.post('/update/oldest', async (req, res) => {
+  const { amount } = req.body
+  if (!Number.isInteger(Number(amount))) {
+    res.status(400).end()
+    return
+  }
+  try {
+    const response = await updateOldestStudents(amount)
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/all', async (req, res) => {
+  try {
+    const response = await updateAllStudents()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/active', async (req, res) => {
+  try {
+    const response = await updateActiveStudents()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/no_student', async (req, res) => {
+  try {
+    const response = await updateNoStudents()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/attainment', async (req, res) => {
+  try {
+    const response = await updateAttainments()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/meta', async (req, res) => {
+  try {
+    const response = await updateMetadata()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/update/studentlist', async (req, res) => {
+  try {
+    const response = await updateStudentlist()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.get('/status', async (req, res) => {
+  try {
+    const statuses = await getStatuses()
+    res.status(200).json(statuses)
+  } catch (err) {
+    console.log(err)
+    res.status(500).json([{ label: 'request status', value: 'request failed' }])
+  }
+})
+
+router.post('/reschedule/scheduled', async (req, res) => {
+  try {
+    const response = await rescheduleScheduled()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+router.post('/reschedule/fetched', async (req, res) => {
+  try {
+    const response = await rescheduleFetched()
+    if (response) {
+      res.status(200).json('Scheduled')
+    }
+  } catch (err) {
+    res.status(500).json(err)
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/users.js

@@ -8,11 +8,6 @@ router.get('/', async (req, res) => {
   res.json(results)
 })
 
-router.get('/enabled', async (req, res) => {
-  const results = await userService.findAllEnabled()
-  res.json(results)
-})
-
 router.get('/access_groups', async (req, res) => {
   const result = await userService.getAccessGroups()
   res.json(result)

services/backend/oodikone2-backend/src/services/analyticsService.js

@@ -38,6 +38,16 @@ const patchThroughput = async (data) => {
   return response.data
 }
 
+const patchFacultyYearlyStats = async (data) => {
+  const response = await client.patch('/facultystats', { data })
+  return response.data
+}
+
+const getFacultyYearlyStats = async (data) => {
+  const response = await client.get('/facultystats', { data })
+  return response.data
+}
+
 module.exports = {
   ping,
   getProductivity,
@@ -45,5 +55,7 @@ module.exports = {
   patchProductivity,
   getThroughput,
   setThroughput,
-  patchThroughput
+  patchThroughput,
+  patchFacultyYearlyStats,
+  getFacultyYearlyStats
 }

services/backend/oodikone2-backend/src/services/blacklist.js

@@ -12,7 +12,6 @@ const isUserBlacklisted = async (userId, tokenCreatedAt) => {
   const isBlacklisted = async () => {
     const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
     const status = !!(tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp))
-    console.log('BLACKLISTED? ', {isBlacklisted: status, userId, tokenCreatedAt, blacklistTimestamp})
     return status
   }
   const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))

services/backend/oodikone2-backend/src/services/course_stats_counter.js

@@ -19,7 +19,8 @@ class CourseStatsCounter {
       failedMany: {},
       improvedPassedGrade: {},
       notParticipated: { ...allstudents },
-      notParticipatedOrFailed: { ...allstudents }
+      notParticipatedOrFailed: { ...allstudents },
+      markedToSemester: {}
     }
     this.stats = {
       students: 0,
@@ -67,6 +68,10 @@ class CourseStatsCounter {
     this.stats.passingSemesters[semester]++
   }
 
+  markToSemester(studentnumber) {
+    this.students.markedToSemester[studentnumber] = true
+  }
+
   markCredit(studentnumber, grade, passed, failed, improved, semester) {
     // studentnumber = `${shajs('sha256').update(process.env.key + studentnumber).digest('hex')}`
     this.markAttempt()
@@ -74,10 +79,11 @@ class CourseStatsCounter {
     this.markGrade(grade, passed, failed, improved)
     this.markToAll(studentnumber)
     if (passed) {
-      this.markPassingGrade(studentnumber)
-      if (!improved) {
+      if (!this.students.markedToSemester[studentnumber]) {
         this.markPassedSemester(semester)
+        this.markToSemester(studentnumber)
       }
+      this.markPassingGrade(studentnumber)
     } else if (improved) {
       this.markImprovedGrade(studentnumber)
     } else if (failed) {
@@ -93,11 +99,15 @@ class CourseStatsCounter {
     return (this.students.passed[studentnumber] !== undefined)
   }
 
-  removeFromFailed (studentnumber) {
+  removeFromFailed(studentnumber) {
     delete this.students.failed[studentnumber]
     delete this.students.failedMany[studentnumber]
   }
 
+  removeFromPassed(studentnumber) {
+    delete this.students.passed[studentnumber]
+  }
+
   markPassingGrade(studentnumber) {
     delete this.students.notParticipatedOrFailed[studentnumber]
     this.students.passed[studentnumber] = true
@@ -148,7 +158,7 @@ class CourseStatsCounter {
     cumulativeStats['0-SPRING'] = cumulativeStats['0-FALL'] + passingSemesters['0-SPRING']
 
     for (let i = 1; i < 7; i++) {
-      cumulativeStats[`${i}-FALL`] = cumulativeStats[`${i-1}-SPRING`] + passingSemesters[`${i}-FALL`]
+      cumulativeStats[`${i}-FALL`] = cumulativeStats[`${i - 1}-SPRING`] + passingSemesters[`${i}-FALL`]
       cumulativeStats[`${i}-SPRING`] = cumulativeStats[`${i}-FALL`] + passingSemesters[`${i}-SPRING`]
     }
 

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js

@@ -8,7 +8,6 @@ const CATEGORY = {
 class CourseYearlyStatsCounter {
 
   constructor() {
-    this.all = []
     this.groups = {}
     this.programmes = {}
     this.history = {
@@ -22,10 +21,11 @@ class CourseYearlyStatsCounter {
     this.programmes[code] = { name, students: [] }
   }
 
-  initGroup(groupcode, name) {
+  initGroup(groupcode, name, coursecode, yearcode) {
     this.groups[groupcode] = {
       code: groupcode,
       name,
+      coursecode,
       attempts: {
         grades: {},
         classes: {
@@ -33,7 +33,8 @@ class CourseYearlyStatsCounter {
           failed: []
         }
       },
-      students: {}
+      students: {},
+      yearcode
     }
   }
 
@@ -44,29 +45,18 @@ class CourseYearlyStatsCounter {
     return { attempted, passed, failed }
   }
 
-  markStudyProgramme(code, name, studentnumber, startdate) {
+  markStudyProgramme(code, name, studentnumber) {
     if (!this.programmes[code]) {
       this.initProgramme(code, name)
     }
-    const prog = this.programmes[code]
-
-    // check if student has earlier studyright wrt course
-    this.all.forEach((student) => {
-      if (student.studentnumber === studentnumber && student.startdate > startdate) {
-        delete this.programmes[student.code]
-        this.all.splice(this.all.indexOf(student), 1)
-      }
-    })
-
-    if (!prog.students.includes(studentnumber)) {
-      this.all.push({ studentnumber, code, startdate })
-      prog.students.push(studentnumber)
+    if (!this.programmes[code].students.includes(studentnumber)) {
+      this.programmes[code].students.push(studentnumber)
     }
   }
 
   markStudyProgrammes(studentnumber, programmes) {
-    programmes.forEach(({ code, name, startdate }) => {
-      this.markStudyProgramme(code, name, studentnumber, startdate)
+    programmes.forEach(({ code, name }) => {
+      this.markStudyProgramme(code, name, studentnumber)
     })
   }
 
@@ -101,9 +91,9 @@ class CourseYearlyStatsCounter {
     }
   }
 
-  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname) {
+  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode) {
     if (!this.groups[groupcode]) {
-      this.initGroup(groupcode, groupname)
+      this.initGroup(groupcode, groupname, coursecode, yearcode)
     }
     this.markCreditToStudents(studentnumber, passed, grade, groupcode)
     this.markCreditToAttempts(studentnumber, passed, grade, groupcode)

services/backend/oodikone2-backend/src/services/courses.js

@@ -39,7 +39,11 @@ const byName = (name, language) => Course.findAll({
         [Op.eq]: name
       }
     }
-  }
+  },
+  order: [
+    ['latest_instance_date', 'DESC']
+  ],
+  limit: 1
 })
 
 const byNameOrCodeTypeAndDiscipline = (searchTerm, type, discipline, language) => {
@@ -330,8 +334,7 @@ const yearlyStatsOf = async (code, year, separate, language) => {
       return b
     }, resultProgrammes)
 
-  const alternatives = await getDuplicateCodes(code)
-  const codes = alternatives ? alternatives : [code]
+  const codes = await alternativeCodes(code)
   const allInstances = await creditsOf(codes)
   const yearInst = allInstances
     .filter(inst => moment(new Date(inst.date)).isBetween(year.start + '-09-01', year.end + '-08-01'))
@@ -479,13 +482,12 @@ const getMainCodeToDuplicates = async () => {
   return maincodeToDuplicates
 }
 
-const getCodeToMainCodeMap = async () => {
+const getCodeToMainCourseMap = async () => {
   try {
     const maincodeToDuplicates = await getMainCodeToDuplicates()
     const codeToMainCode = Object.values(maincodeToDuplicates).reduce((acc, d) => {
-      const maincode = d.maincourse.code
       d.duplicates.forEach((c) => {
-        acc[c.code] = maincode
+        acc[c.code] = d.maincourse
       })
       return acc
     }, {})
@@ -496,37 +498,19 @@ const getCodeToMainCodeMap = async () => {
   return {}
 }
 
-const getMainCodesMap = async () => {
-  return await getCodeToMainCodeMap()
-}
-
-const getMaincodeToDuplicateCodesMap = async () => {
-  try {
-    const codeToMainCodeMap = await getCodeToMainCodeMap()
-    const codes = Object.keys(codeToMainCodeMap)
-    const mainCodeToCoursesMap = codes.reduce((acc, code) => {
-      const maincode = codeToMainCodeMap[code]
-      const duplicates = acc[maincode] || []
-      duplicates.push(code)
-      acc[maincode] = duplicates
-      return acc
-    }, {})
-    return mainCodeToCoursesMap
-  } catch (e) {
-    console.error(e)
-  }
-  return {}
+const getMainCodeToDuplicatesAndCodeToMainCode = async () => {
+  return [await getMainCodeToDuplicates(), await getCodeToMainCourseMap()]
 }
 
 const getMainCourseToCourseMap = async (/*programme*/) => {
   try {
-    const codeToMainCodeMap = await getCodeToMainCodeMap()
+    const codeToMainCodeMap = await getCodeToMainCourseMap()
     const courses = await byCodes(Object.keys(codeToMainCodeMap))
     const mainCodeToCoursesMap = courses.reduce((acc, course) => {
-      const maincode = codeToMainCodeMap[course.code]
-      const duplicates = acc[maincode] || []
+      const maincourse = codeToMainCodeMap[course.code]
+      const duplicates = acc[maincourse.code] || []
       duplicates.push(course)
-      acc[maincode] = duplicates
+      acc[maincourse.code] = duplicates
       return acc
     }, {})
     return mainCodeToCoursesMap
@@ -537,9 +521,10 @@ const getMainCourseToCourseMap = async (/*programme*/) => {
 }
 
 const getDuplicateCodes = async (code) => {
-  const maincodeToDuplicatesMap = await getMaincodeToDuplicateCodesMap()
-  const duplicates = maincodeToDuplicatesMap[code]
-  return duplicates
+  const [mainCodeToDuplicates, codeToMainCourseMap] = await getMainCodeToDuplicatesAndCodeToMainCode()
+  const maincourse = codeToMainCourseMap[code]
+  if (!maincourse) return null
+  return mainCodeToDuplicates[maincourse.code].duplicates.map(e => e.code)
 }
 
 const setDuplicateCode = async (code1, code2) => {
@@ -561,7 +546,7 @@ const setDuplicateCode = async (code1, code2) => {
             { groupid, coursecode: code1 },
             { groupid, coursecode: code2 }
           ],
-            { ignoreDuplicates: true })
+          { ignoreDuplicates: true })
         } else {
           // both have a group, must merge groups
           await CourseDuplicates.update({ groupid: all[code1] }, { where: { groupid: all[code2] } })
@@ -588,7 +573,7 @@ const formatStudyrightElement = ({ code, element_detail, startdate }) => ({
 })
 
 const parseCredit = credit => {
-  const { student, semester, grade } = credit
+  const { student, semester, grade, course_code } = credit
   const { studentnumber, studyright_elements: elements } = student
   const { yearcode, yearname, semestercode, name: semestername } = semester
   return {
@@ -597,6 +582,7 @@ const parseCredit = credit => {
     yearname,
     semestercode,
     semestername,
+    coursecode: course_code,
     grade,
     passed: !Credit.failed(credit) || Credit.passed(credit) || Credit.improved(credit),
     studentnumber,
@@ -604,7 +590,7 @@ const parseCredit = credit => {
   }
 }
 
-const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode) => {
+const yearlyStatsOfNew = async (coursecode, separate) => {
   const codes = await alternativeCodes(coursecode)
   const [credits, course] = await Promise.all([creditsForCourses(codes), Course.findByPk(coursecode)])
   const counter = new CourseYearlyStatsCounter()
@@ -612,24 +598,22 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
     const {
       studentnumber, grade, passed,
       semestercode, semestername,
-      yearcode, yearname, programmes
+      yearcode, yearname, programmes,
+      coursecode
     } = parseCredit(credit)
-    if (startyearcode <= yearcode && yearcode <= endyearcode) {
-      const groupcode = separate ? semestercode : yearcode
-      const groupname = separate ? semestername : yearname
-      const unknownProgramme = [{
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra'
-        }
-      }]
-      counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
-      counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname)
-    } else {
-      counter.markCreditToHistory(studentnumber, passed)
-    }
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+    const unknownProgramme = [{
+      code: 'OTHER',
+      name: {
+        en: 'Other',
+        fi: 'Muu',
+        sv: 'Andra'
+      }
+    }]
+    counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
+    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
+    counter.markCreditToHistory(studentnumber, passed)
   }
   const statistics = counter.getFinalStatistics()
   return {
@@ -640,9 +624,9 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
   }
 }
 
-const courseYearlyStats = async (coursecodes, separate, startyearcode, endyearcode) => {
+const courseYearlyStats = async (coursecodes, separate) => {
   const stats = await Promise.all(coursecodes
-    .map(code => yearlyStatsOfNew(code, separate, startyearcode, endyearcode || startyearcode)))
+    .map(code => yearlyStatsOfNew(code, separate)))
   return stats
 }
 
@@ -670,7 +654,7 @@ const nameLikeTerm = (name) => {
 
 const codeLikeTerm = (code) => !code ? undefined : {
   code: {
-    [Op.iLike]: `${code.trim()}%`
+    [Op.iLike]: `%${code.trim()}%`
   }
 }
 
@@ -709,12 +693,13 @@ module.exports = {
   findDuplicates,
   setDuplicateCode,
   deleteDuplicateCode,
-  getMainCodesMap,
+  getCodeToMainCourseMap,
   getMainCourseToCourseMap,
   getAllCourseTypes,
   getAllDisciplines,
   yearlyStatsOfNew,
   courseYearlyStats,
   byNameAndOrCodeLike,
-  byCodes
+  byCodes,
+  getMainCodeToDuplicatesAndCodeToMainCode
 }

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_data_mapper.js

@@ -250,12 +250,7 @@ const getTransfersFromData = (data, studentnumber) => {
   const studytracks = data.elements.filter(element => element.element_id === 20)
   const sorted = _.sortBy(studytracks, 'end_date')
   let transfers = []
-  let i = 0
-  while (i < sorted.length) {
-    if (i === 0) {
-      i++
-      continue
-    }
+  for (let i = 1; i < sorted.length; ++i) {
     let target = sorted[i].code
     let source = sorted[i - 1].code
     let transferdate = sorted[i - 1].end_date
@@ -266,7 +261,6 @@ const getTransfersFromData = (data, studentnumber) => {
       transferdate,
       studentnumber,
       studyrightid: `${data.studyright_id}`})
-    i++
   }
   return(transfers)
 }

services/backend/oodikone2-backend/src/services/doo_api_database_updater/oodi_interface.js

@@ -5,10 +5,10 @@ const https = require('https')
 const fs = require('fs')
 const base_url = OODI_ADDR
 
-const agent = KEY_PATH && CERT_PATH ? 
+const agent = KEY_PATH && CERT_PATH ?
   new https.Agent({
     cert: fs.readFileSync(process.env.CERT_PATH, 'utf8'),
-    key: fs.readFileSync(process.env.KEY_PATH, 'utf8'), 
+    key: fs.readFileSync(process.env.KEY_PATH, 'utf8'),
   }) :
   new https.Agent({
     rejectUnauthorized: false
@@ -48,9 +48,10 @@ if (process.env.NODE_ENV === 'dev') {
     username: 'tktl',
     password: process.env.OODI_PW
   }
-  axios.defaults.params = {
-    token: process.env.TOKEN
-  }
+  instance.interceptors.request.use((config) => {
+    config.params = {token: process.env.TOKEN}
+    return config
+  })
 
 }
 

services/backend/oodikone2-backend/src/services/elementdetails.js

@@ -1,7 +1,13 @@
 const { ElementDetails } = require('../models/index')
 const { Op } = require('sequelize')
 
-const byId = id =>  ElementDetails.findByPk(id)
+const byId = id => ElementDetails.findByPk(id)
+
+const byType = type => ElementDetails.findAll({
+  where: {
+    type
+  }
+})
 
 const byCodes = codes => ElementDetails.findAll({
   where: {
@@ -11,4 +17,4 @@ const byCodes = codes => ElementDetails.findAll({
   }
 })
 
-module.exports = { byId, byCodes }
+module.exports = { byId, byType, byCodes }

services/backend/oodikone2-backend/src/services/faculties.js

@@ -0,0 +1,72 @@
+const axios = require('axios')
+const moment = require('moment')
+const AsyncLock = require('async-lock')
+const { USERSERVICE_URL } = require('../conf-backend')
+const { Credit, StudyrightElement } = require('../models')
+
+const client = axios.create({ baseURL: USERSERVICE_URL, headers: { 'secret': process.env.USERSERVICE_SECRET } })
+
+const calculateFacultyYearlyStats = async () => {
+  const { data: facultyProgrammes } = await client.get('/faculty_programmes')
+  const res = {}
+  const lock = new AsyncLock()
+
+  let amountDone = 0
+  await Promise.all(facultyProgrammes.map(({ faculty_code, programme_code }) => (
+    new Promise(async (facultyRes) => {
+      if (!res[faculty_code]) res[faculty_code] = {}
+      if (!res[faculty_code][programme_code]) res[faculty_code][programme_code] = {}
+
+      const facultyStudents = await StudyrightElement.findAll({
+        where: {
+          code: programme_code
+        },
+        attributes: ['studentnumber']
+      })
+
+      await Promise.all(facultyStudents.map((student) => (
+        new Promise(async (studentRes) => {
+          const filteredStudentCredits = await Credit.findAll({
+            where: {
+              student_studentnumber: student.studentnumber,
+              credittypecode: [4, 10],
+              isStudyModule: false
+            }
+          })
+
+          filteredStudentCredits.forEach((c) => {
+            const attainmentYear = moment(c.attainment_date).year()
+            if (!res[faculty_code][attainmentYear]) {
+              lock.acquire(faculty_code, (done) => {
+                if (!res[faculty_code][programme_code][attainmentYear]) {
+                  const facultyYearStats = {}
+                  facultyYearStats.studentCredits = 0
+                  facultyYearStats.coursesPassed = 0
+                  facultyYearStats.coursesFailed = 0
+                  res[faculty_code][programme_code][attainmentYear] = facultyYearStats
+                }
+                done()
+              })
+            }
+            const facultyYearStats = res[faculty_code][programme_code][attainmentYear]
+            if (c.credittypecode === 4) {
+              facultyYearStats.studentCredits += c.credits
+              facultyYearStats.coursesPassed += 1
+            } else {
+              facultyYearStats.coursesFailed += 1
+            }
+          })
+          studentRes()
+        })
+      )))
+      amountDone += 1
+      console.log(`Faculty programmes done ${amountDone}/${facultyProgrammes.length}`)
+      facultyRes()
+    })
+  )))
+  return res
+}
+
+module.exports = {
+  calculateFacultyYearlyStats
+}

services/backend/oodikone2-backend/src/services/mailservice.js

@@ -66,7 +66,7 @@ const feedback = (content, user, email, full_name) => {
   return ({
     to: 'Toska <grp-toska@helsinki.fi>',
     replyTo: email,
-    subject: `New message from Oodikone feedback`,
+    subject: 'New message from Oodikone feedback',
     text: `New message from user ${user}`,
     html: `${content}
     <p>sent by ${full_name}, userid: ${user}, email: ${email}</p>

services/backend/oodikone2-backend/src/services/mandatoryCourses.js

@@ -61,12 +61,14 @@ const find = async (studyProgrammeId, code) => {
       }
     }
   })
+  if (!mandatoryCourse) return null
 
   const course = await Course.findOne({
     where: {
       code: mandatoryCourse.course_code
     }
   })
+  if (!course) return null
 
   return { ...mandatoryCourse.get(), course: course.get() }
 }

services/backend/oodikone2-backend/src/services/populations.js

@@ -1,6 +1,6 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
-const { orderBy } = require('lodash')
+
 const {
   Student, Credit, Course, sequelize, Studyright, StudyrightExtent, ElementDetails,
   Discipline, CourseType, SemesterEnrollment, Semester, Transfers, StudyrightElement
@@ -8,7 +8,7 @@ const {
 const {
   Tag, TagStudent
 } = require('../models/models_kone')
-const { getMainCodesMap, byName } = require('./courses')
+const { getCodeToMainCourseMap } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 
@@ -225,7 +225,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
   })
 
   const studentTags = await TagStudent.findAll({
-    attributes: ['id', 'studentnumber'],
+    attributes: ['tag_id', 'studentnumber'],
     include: [
       {
         model: Tag,
@@ -245,20 +245,10 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
   }, {})
 
   students.forEach(student => {
-    student.tags = studentNumberToTags[student.studentnumber]
+    student.tags = studentNumberToTags[student.studentnumber] || []
   })
 
-  if (tag) {
-    const studentsWithSearchedTag = {}
-    students.forEach(student => {
-      student.tags = studentNumberToTags[student.studentnumber]
-      if (student.tags && student.tags.some(t => t.tag.tag_id === tag)) {
-        studentsWithSearchedTag[student.studentnumber] = true
-      }
-    })
-
-    return students.filter(student => studentsWithSearchedTag[student.studentnumber])
-  }
+  if (tag) return students.filter(student => student.tags.some(t => t.tag_id === tag))
   return students
 }
 
@@ -271,7 +261,7 @@ const count = (column, count, distinct = false) => {
   )
 }
 
-const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents) => { // eslint-disable-line
+const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag) => { // eslint-disable-line
   const filteredExtents = []
   let studyrightWhere = {
     extentcode: {
@@ -288,6 +278,21 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
     studyrightWhere.canceldate = null
   }
 
+  let studentWhere = {}
+  if (tag) {
+    const taggedStudentnumbers = await TagStudent.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        tag_id: tag
+      }
+    })
+    studentWhere.where = {
+      student_studentnumber: {
+        [Op.in]: taggedStudentnumbers.map(sn => sn.studentnumber)
+      }
+    }
+  }
+
   const students = await Studyright.findAll({
     attributes: ['student_studentnumber'],
     include: {
@@ -321,15 +326,16 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
         }
       ],
       ...studyrightWhere
+
     },
+    ...studentWhere,
     having: count('studyright_elements.code', studyRights.length, true),
   })
-
   return [...new Set(students.map(s => s.student_studentnumber))]
 }
 
 const parseQueryParams = query => {
-  const { semesters, studentStatuses, studyRights, months, endYear, startYear } = query
+  const { semesters, studentStatuses, studyRights, months, endYear, startYear, tag } = query
   const startDate = semesters.includes('FALL') ?
     `${startYear}-${semesterStart[semesters.find(s => s === 'FALL')]}` :
     `${moment(startYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
@@ -346,7 +352,8 @@ const parseQueryParams = query => {
     studyRights: Array.isArray(studyRights) ? studyRights : Object.values(studyRights),
     months,
     startDate,
-    endDate
+    endDate,
+    tag
   }
 }
 
@@ -393,32 +400,28 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights
     stats.students.push(formatStudentForPopulationStatistics(student, startDate, endDate))
     return stats
   }, {
-      students: [],
-      extents: {},
-      semesters: {},
-      transfers: {
-        targets: {},
-        sources: {}
-      },
-      studyrights: {
-        degrees: [],
-        programmes: []
-      }
-    })
+    students: [],
+    extents: {},
+    semesters: {},
+    transfers: {
+      targets: {},
+      sources: {}
+    },
+    studyrights: {
+      degrees: [],
+      programmes: []
+    }
+  })
 
+  const [momentstart, momentend] = [moment(startDate), moment(endDate)]
   const transferredStudyright = (s) => {
-    const studyright = s.studyrights.find(s => s.studyrightElements
-      .map(d => d.element_detail.code)
-      .includes(studyRights.programme))
-
-    if (studyright) {
-      s.transferredStudyright = moment(startDate).isAfter(moment(studyright.startdate))
-      if (s.transferredStudyright) {
-        const previousRights = studyright.studyrightElements
-          .filter(e => e.element_detail.type === 20 && e.element_detail.code !== studyRights.programme)
-        s.previousRights = previousRights
-      }
-
+    const transferred_from = s.transfers.find(
+      t => t.target.code === studyRights.programme &&
+        moment(t.transferdate).isBetween(momentstart, momentend)
+    )
+    if (transferred_from) {
+      s.transferredStudyright = true
+      s.transferSource = transferred_from.source
     }
     return s
   }
@@ -432,12 +435,24 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights
   }
 }
 
+const formatQueryParamArrays = (query, params) => {
+  let res = { ...query }
+  params.forEach(p => {
+    if (!res[p]) return
+    res[p] = Array.isArray(res[p]) ? res[p] : [res[p]]
+  })
+  return res
+}
+
 const optimizedStatisticsOf = async (query, studentnumberlist) => {
-  if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
+  const formattedQueryParams = formatQueryParamArrays(query, ['semesters', 'studentStatuses'])
+
+  if (!formattedQueryParams.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
     return { error: 'Semester should be either SPRING OR FALL' }
   }
-  if (query.studentStatuses &&
-    !query.studentStatuses.map(
+
+  if (formattedQueryParams.studentStatuses &&
+    !formattedQueryParams.studentStatuses.map(
       status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
     ).every(e => e === true)
   ) {
@@ -445,7 +460,7 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
   }
   const {
     studyRights, startDate, months, endDate, exchangeStudents, cancelledStudents, nondegreeStudents
-  } = parseQueryParams(query)
+  } = parseQueryParams(formattedQueryParams)
 
   const studentnumbers = studentnumberlist ?
     studentnumberlist :
@@ -457,24 +472,23 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     startDate,
     dateMonthsFromNow(startDate, months),
     studyRights,
-    query.tag
+    formattedQueryParams.tag
   )
 
-  const formattedStudents = await formatStudentsForApi(students, startDate, endDate, query)
+  const formattedStudents = await formatStudentsForApi(students, startDate, endDate, formattedQueryParams)
   return formattedStudents
 }
 
 const unifyOpenUniversity = (code) => {
-  if (code[0] === 'A') {
-    return code.substring(code[1] === 'Y' ? 2 : 1)
-  }
-  return code
+  const regexresult = code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+  if (!regexresult) return code
+  return regexresult[1]
 }
 
-const getUnifiedCode = (code, codeduplicates) => {
-  const formattedcode = unifyOpenUniversity(code)
-  const unifiedcode = codeduplicates[formattedcode]
-  return !unifiedcode ? formattedcode : unifiedcode
+const getMainCourse = (course, codeduplicates) => {
+  const formattedcode = unifyOpenUniversity(course.code)
+  const maincourse = codeduplicates[formattedcode]
+  return maincourse || course
 }
 
 const findCourses = (studentnumbers, beforeDate) => {
@@ -492,8 +506,7 @@ const findCourses = (studentnumbers, beforeDate) => {
           attainment_date: {
             [Op.lt]: beforeDate
           }
-        },
-        order: 'attainment_date'
+        }
       },
       {
         model: Discipline
@@ -520,60 +533,70 @@ const parseCreditInfo = credit => ({
 })
 
 const bottlenecksOf = async (query, studentnumberlist) => {
-  let allstudents
-  let courses
-  if (!studentnumberlist) {
-    if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
+  const isValidRequest = async (query, params) => {
+    const { studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents } = params
+    if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
     }
     if (query.studentStatuses &&
-      !query.studentStatuses.map(
+      !query.studentStatuses.every(
         status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
-      ).every(e => e === true)) {
+      )) {
       return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
     }
-    const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query)
-
     if (query.selectedStudents) {
       const allStudents =
         await studentnumbersWithAllStudyrightElements(
-          studyRights, startDate, endDate, exchangeStudents, cancelledStudents
+          studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, query.tag
         )
       const disallowedRequest =
         checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
       if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
     }
-
-    const studentnumbers = query.selectedStudents ||
-      await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents)
-    allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
-    courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
-
-  } else {
-    allstudents = studentnumberlist
-    courses = await findCourses(allstudents, new Date())
+    return null
   }
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
+    if (!studentnumberlist) {
+      const { months, studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag } = params
+      const studentnumbers = selectedStudents ||
+        await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag)
+      const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
+      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+      return [allstudents, courses]
+    } else {
+      const allstudents = studentnumberlist
+      const courses = await findCourses(allstudents, new Date())
+      return [allstudents, courses]
+    }
+  }
+  const params = parseQueryParams(query)
+  const [[allstudents, courses], codeToMainCourse, error] = await Promise.all(
+    [
+      getStudentsAndCourses(query.selectedStudents, studentnumberlist),
+      getCodeToMainCourseMap(),
+      isValidRequest(
+        query,
+        params
+      )
+    ]
+  )
+  if (error) return error
 
   const bottlenecks = {
     disciplines: {},
     coursetypes: {}
   }
 
-  const codeduplicates = await getMainCodesMap()
+  const stats = {}
 
-  const allcoursestatistics = await courses.reduce(async (coursestatistics, course) => {
-    const stats = await coursestatistics
-    let { code, name, disciplines, course_type } = course
-
-    if (name.fi && name.fi.includes('Avoin yo:')) {
-      const courses = await byName(name.fi.split(': ')[1], 'fi')
-      const theOne = courses.length > 0 ? orderBy(courses, ['date'], ['desc'])[0] : { name, code }
-      code = theOne.code
-      name = theOne.name
+  const startYear = parseInt(query.endYear, 10)
+  courses.forEach(course => {
+    let { disciplines, course_type } = course
+    const maincourse = getMainCourse(course, codeToMainCourse)
+    if (!stats[maincourse.code]) {
+      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudents)
     }
-
-    const unifiedcode = getUnifiedCode(code, codeduplicates)
-    const coursestats = stats[unifiedcode] || new CourseStatsCounter(unifiedcode, name, allstudents)
+    const coursestats = stats[maincourse.code]
 
     coursestats.addCourseType(course_type.coursetypecode, course_type.name)
     bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
@@ -584,18 +607,13 @@ const bottlenecksOf = async (query, studentnumberlist) => {
 
     course.credits.forEach(credit => {
       const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
-      const semester = getPassingSemester(parseInt(query.endYear, 10), date)
+      const semester = getPassingSemester(startYear, date)
       coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
     })
-
-    stats[unifiedcode] = coursestats
-
-    return stats
-  }, Promise.resolve({}))
-
-  bottlenecks.coursestatistics = Object.values(allcoursestatistics)
+    stats[maincourse.code] = coursestats
+  })
+  bottlenecks.coursestatistics = Object.values(stats)
     .map(coursestatistics => coursestatistics.getFinalStats())
-
   return bottlenecks
 }
 

services/backend/oodikone2-backend/src/services/semesters.js

@@ -1,6 +1,5 @@
 const sequelize = require('sequelize')
 const { Semester } = require('../models/index')
-const { Op } = sequelize
 
 const getSemestersAndYears = async () => {
   const semesters = await Semester.findAll({})

services/backend/oodikone2-backend/src/services/students.js

@@ -2,6 +2,10 @@ const Sequelize = require('sequelize')
 const { sequelize } = require('../database/connection')
 const moment = require('moment')
 const { Student, Credit, Course, Studyright, StudyrightElement, ElementDetails } = require('../models')
+const {
+  TagStudent,
+  Tag
+} = require('../models/models_kone')
 const Op = Sequelize.Op
 
 const createStudent = student => Student.create(student)
@@ -16,48 +20,73 @@ const updateStudent = student => {
   })
 }
 
-const byId = async (id) => Student.findByPk(id, {
-  include: [
-    {
-      model: Credit,
-      separate: true,
-      include: {
-        model: Course,
-      }
-    },
-    {
-      model: Studyright,
-      include: {
-        model: StudyrightElement,
-        include: {
-          model: ElementDetails,
-          where: {
-            type: {
-              [Op.in]: [10, 20, 30]
+const byId = async (id) => {
+  const [student, tags] = await Promise.all([
+    Student.findByPk(id, {
+      include: [
+        {
+          model: Credit,
+          separate: true,
+          include: {
+            model: Course,
+          }
+        },
+        {
+          model: Studyright,
+          include: {
+            model: StudyrightElement,
+            include: {
+              model: ElementDetails,
+              where: {
+                type: {
+                  [Op.in]: [10, 20, 30]
+                }
+              }
             }
           }
         }
+      ]
+    }),
+    TagStudent.findAll({
+      include: [
+        {
+          model: Tag
+        }
+      ],
+      where: {
+        studentnumber: id
+      }
+    })
+  ])
+  const tagprogrammes = await ElementDetails.findAll({
+    where: {
+      code: {
+        [Op.in]: tags.map(t => t.tag.studytrack)
       }
     }
-  ]
-})
+  })
+  student.tags = tags.map(t => ({
+    ...t.get(),
+    programme: tagprogrammes.find(p => p.code === t.tag.studytrack)
+  }))
+  return student
+}
 
-
-const findByCourseAndSemesters = async (coursecode, yearcode) =>
+const findByCourseAndSemesters = async (coursecodes, yearcode) =>
   sequelize.query(`
     SELECT
       studentnumber, credit.course_code, attainment_date
     FROM student
     INNER JOIN credit ON
       student.studentnumber=credit.student_studentnumber
-    WHERE 
-      course_code=:coursecode AND
+    WHERE
+      course_code IN (:coursecodes) AND
       attainment_date
     BETWEEN
       (select startdate FROM semesters where yearcode=:yearcode ORDER BY semestercode LIMIT 1) AND
       (select enddate FROM semesters where yearcode=:yearcode ORDER BY semestercode DESC LIMIT 1);
   `,
-  { replacements: { coursecode, yearcode }, type: sequelize.QueryTypes.SELECT })
+  { replacements: { coursecodes, yearcode }, type: sequelize.QueryTypes.SELECT })
     .map(st => st.studentnumber)
 
 const byAbreviatedNameOrStudentNumber = (searchTerm) => {
@@ -95,7 +124,8 @@ const formatStudent = ({
   semester_enrollments,
   transfers,
   updatedAt,
-  createdAt
+  createdAt,
+  tags
 }) => {
   const toCourse = ({
     grade,
@@ -173,7 +203,7 @@ const formatStudent = ({
     email,
     semesterenrollments,
     updatedAt: updatedAt || createdAt,
-    tags: []
+    tags
   }
 }
 

services/backend/oodikone2-backend/src/services/studyrights.js

@@ -156,12 +156,12 @@ const getAllStudyrightElementsAndAssociations = async () => {
 }
 
 const getStudyrightElementsAndAssociationsForUser = async username => {
-  const studyrightelements = await getUserElementDetails(username)
-  if (studyrightelements.length === 0) {
+  const studyrightelementcodes = await getUserElementDetails(username)
+  if (studyrightelementcodes.length === 0) {
     return []
   }
-  const associations = await getAssociatedStudyrights(studyrightelements.map(element => element.code))
-  return formatStudyrightElements(studyrightelements, associations)
+  const associations = await getAssociatedStudyrights(studyrightelementcodes)
+  return formatStudyrightElements(studyrightelementcodes, associations)
 }
 
 const getAllDegreesAndProgrammes = async () => {
@@ -187,6 +187,11 @@ const getAllProgrammes = async () => {
   return elementDetails
 }
 
+const getAllElementDetails = async () => {
+  const elementDetails = ElementDetails.findAll()
+  return elementDetails
+}
+
 const associatedStudyrightElements = async (offset, limit) => {
   const studyrights = await Studyright.findAll({
     attributes: [],
@@ -355,8 +360,7 @@ const getFilteredAssociations = async (codes) => {
 }
 
 const getUserAssociations = async (userid) => {
-  const elements = await getUserElementDetails(userid)
-  const codes = elements.map(e => e.code)
+  const codes = await getUserElementDetails(userid)
   const associations = await getFilteredAssociations(codes)
   return associations
 }
@@ -375,5 +379,6 @@ module.exports = {
   getFilteredAssociations,
   getUserAssociations,
   refreshAssociationsInRedis,
-  getAllProgrammes
+  getAllProgrammes,
+  getAllElementDetails
 }

services/backend/oodikone2-backend/src/services/studytrack.js

@@ -3,21 +3,14 @@ const { Op } = sequelize
 const moment = require('moment')
 const { flatMap } = require('lodash')
 const { Credit, Student, Course, Provider, Studyright, StudyrightElement,
-  ElementDetails
+  ElementDetails, Transfers
 } = require('../models')
 const {
   ThesisCourse, ThesisTypeEnums
 } = require('../models/models_kone')
 const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const { semesterStart, semesterEnd } = require('../util/semester')
-const isNumber = str => !Number.isNaN(Number(str))
-
-const studytrackToProviderCode = code => {
-  const [left, right] = code.split('_')
-  const prefix = [...left].filter(isNumber).join('')
-  const suffix = `${left[0]}${right}`
-  return `${prefix}0-${suffix}`
-}
+const { mapToProviders } = require('../util/utils')
 
 const formatCredit = credit => {
   const { id, credits, attainment_date } = credit
@@ -258,7 +251,7 @@ const combineStatistics = (creditStats, studyrightStats, thesisStats, creditsFor
 }
 
 const productivityStatsForStudytrack = async (studytrack, since) => {
-  const providercode = studytrackToProviderCode(studytrack)
+  const providercode = mapToProviders([studytrack])[0]
   const year = new Date(since).getFullYear()
   const startDate = `${year}-${semesterStart['FALL']}`
   const endDate = `${moment(since, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
@@ -380,49 +373,32 @@ const gendersFromClass = async (studentnumbers) => {
   }, {})
 }
 
-const countriesFromClass = async (studentnumbers) => {
+const nationalitiesFromClass = async (studentnumbers) => {
   return Student.findAll({
-    attributes: [[sequelize.fn('count', sequelize.col('home_country_en')), 'count'], 'home_country_en'],
     where: {
       studentnumber: {
         [Op.in]: studentnumbers
-      },
+      }
     },
-    group: ['home_country_en'],
-    raw: true
-  }).reduce((acc, curr) => {
-    acc[curr.country_en] = curr.count
+    attributes: ['home_country_en']
+  }).reduce((acc, { home_country_en }) => {
+    const country = home_country_en || 'Unknown'
+    if (!acc[country]) acc[country] = 0
+    acc[country] += 1
     return acc
   }, {})
 }
 
 const tranferredToStudyprogram = async (studentnumbers, startDate, studytrack, endDate) => {
-  return Studyright.findAndCountAll({
-    include: {
-      include: {
-        model: ElementDetails,
-        where: {
-          type: {
-            [Op.eq]: 20
-          }
-        }
-      },
-      model: StudyrightElement,
-      required: true,
-      where: {
-        code: {
-          [Op.eq]: studytrack
-        },
-        startdate: {
-          [Op.gt]: moment(startDate).add(1, 'days'), // because somehow startdates have a time that is not 00:00
-          [Op.lt]: new Date(endDate)
-        }
-      }
-    },
+  return Transfers.count({
     where: {
-      student_studentnumber: {
+      studentnumber: {
         [Op.in]: studentnumbers
-      }
+      },
+      transferdate: {
+        [Op.between]: [startDate, endDate]
+      },
+      targetcode: studytrack
     }
   })
 }
@@ -477,14 +453,14 @@ const formatCreditsForProductivity = (credits) => {
 }
 
 const productivityCreditsFromStudyprogrammeStudents = async (studytrack, startDate, studentnumbers) => {
-  const providercode = studytrackToProviderCode(studytrack)
+  const providercode = mapToProviders([studytrack])[0]
   const credits = await getCreditsForMajors(providercode, startDate, studentnumbers)
   const formattedStudentCredits = formatCreditsForProductivity(credits)
   return formattedStudentCredits
 }
 
 const transferredCreditsForProductivity = async (studytrack, since) => {
-  const providercode = studytrackToProviderCode(studytrack)
+  const providercode = mapToProviders([studytrack])[0]
   const credits = await getTransferredCredits(providercode, since)
   const formattedCredits = formatCreditsForProductivity(credits)
   return formattedCredits
@@ -496,9 +472,9 @@ const statsForClass = async (studentnumbers, startDate, studytrack, endDate) =>
     graduationsFromClass(studentnumbers, studytrack),
     thesesFromClass(studentnumbers, startDate, studytrack),
     gendersFromClass(studentnumbers),
-    countriesFromClass(studentnumbers),
     tranferredToStudyprogram(studentnumbers, startDate, studytrack, endDate),
-    endedStudyright(studentnumbers, startDate, studytrack, endDate)
+    endedStudyright(studentnumbers, startDate, studytrack, endDate),
+    nationalitiesFromClass(studentnumbers)
   ])
 }
 
@@ -520,13 +496,13 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       mte150: 0,
     },
     genders: {},
-    countries: {},
     thesisM: 0,
     thesisB: 0,
     students: 0,
     graduated: 0,
     inTargetTime: 0,
-    transferred: 0
+    transferred: 0,
+    nationalities: {}
   }
   const years = getYears(since)
   // studyprogramme starts with K if bachelors and M if masters
@@ -550,14 +526,12 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     const studentnumbers = await studentnumbersWithAllStudyrightElements([studytrack], startDate, endDate, false, false)
     const creditsForStudyprogramme =
       await productivityCreditsFromStudyprogrammeStudents(studytrack, startDate, studentnumbers)
-    const [credits, graduated, theses, genders, countries, transferredTo, endedStudyright] =
+    const [credits, graduated, theses, genders, transferredTo, endedStudyright, nationalities] =
       await statsForClass(studentnumbers, startDate, studytrack, endDate)
     //console.log(year)
     //console.log(transferredFrom.rows.map(r => r.get({ plain: true })))
     // theres so much shit in the data that transefferFrom doesnt rly mean anything
     delete genders[null]
-    delete countries[null]
-    delete countries[undefined]
     const creditValues = credits.reduce((acc, curr) => {
       acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
       acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60
@@ -574,10 +548,9 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
         totals.genders[genderKey] + Number(genders[genderKey]) :
         Number(genders[genderKey])
     })
-    Object.keys(countries).forEach(countryKey => {
-      totals.countries[countryKey] = totals.countries[countryKey] ?
-        totals.countries[countryKey] + Number(countries[countryKey]) :
-        Number(countries[countryKey])
+    Object.keys(nationalities).forEach(nationality => {
+      if (!totals.nationalities[nationality]) totals.nationalities[nationality] = 0
+      totals.nationalities[nationality] += nationalities[nationality]
     })
     const graduationTimes = graduated.map(g => moment(g.enddate).diff(g.studystartdate, 'months'))
     const inTargetTime = graduationTimes.filter(time =>
@@ -591,7 +564,7 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     totals.ended = totals.ended + endedStudyright.count,
     totals.medianGraduationTime = median(allGraduationTimes)
     totals.inTargetTime = totals.inTargetTime + inTargetTime
-    totals.transferred = totals.transferred + transferredTo.count
+    totals.transferred = totals.transferred + transferredTo
     return {
       year: `${year}-${year + 1}`,
       credits: credits.map(cr => cr === null ? 0 : cr),
@@ -602,18 +575,16 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       thesisM: theses.MASTER || 0,
       thesisB: theses.BACHELOR || 0,
       genders,
-      countries,
       creditValues,
-      transferred: transferredTo.count,
-      ended: endedStudyright.count
+      transferred: transferredTo,
+      ended: endedStudyright.count,
+      nationalities
     }
   }))
-
   return { id: studytrack, status: null, data: { years: arr, totals } }
 }
 
 module.exports = {
-  studytrackToProviderCode,
   getCreditsForProvider,
   productivityStatsFromCredits,
   productivityStatsForProvider,

services/backend/oodikone2-backend/src/services/studytrack.test.js

@@ -1,5 +1,4 @@
 const {
-  studytrackToProviderCode,
   getCreditsForProvider,
   productivityStatsFromCredits,
   productivityStatsForProvider,
@@ -14,6 +13,7 @@ const {
 const { sequelize } = require('../models')
 const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
 const { readFileSync } = require('fs')
+const { mapToProviders } = require('../util/utils')
 
 const { MASTER, BACHELOR } = ThesisTypeEnums
 
@@ -67,13 +67,13 @@ test('Credits for provider should not contain credit for passed study module', a
 
 test('KH50_005 track code maps to 500-K005 provider code', () => {
   const track = 'KH50_005'
-  const provider = studytrackToProviderCode(track)
+  const provider = mapToProviders([track])[0]
   expect(provider).toBe('500-K005')
 })
 
 test('MH50_010 track code maps to 500-M010 provider code', () => {
   const track = 'MH50_010'
-  const provider = studytrackToProviderCode(track)
+  const provider = mapToProviders([track])[0]
   expect(provider).toBe('500-M010')
 })
 

services/backend/oodikone2-backend/src/services/tags.js

@@ -5,7 +5,7 @@ const Op = Sequelize.Op
 
 const findTags = () => Tag.findAll({})
 
-const findTagsByStudytrack = (studytrack) => {
+const findTagsByStudytrack = async (studytrack) => {
   return Tag.findAll({
     where: {
       studytrack: {
@@ -15,6 +15,17 @@ const findTagsByStudytrack = (studytrack) => {
   })
 }
 
+const findTagsFromStudytrackById = async (studytrack, tag_ids) => {
+  return Tag.findAll({
+    where: {
+      studytrack,
+      tag_id: {
+        [Op.in]: tag_ids
+      }
+    }
+  })
+}
+
 const createNewTag = async (tag) => {
   return Tag.create(tag)
 }
@@ -33,5 +44,6 @@ module.exports = {
   createNewTag,
   findTags,
   findTagsByStudytrack,
+  findTagsFromStudytrackById,
   deleteTag
 }

services/backend/oodikone2-backend/src/services/tagstudent.js

@@ -33,12 +33,15 @@ const createStudentTag = async (tag) => {
   return TagStudent.create(tag)
 }
 
-const deleteStudentTag = async (id) => {
+const createMultipleStudentTags = async (tags) => {
+  return TagStudent.bulkCreate(tags, { ignoreDuplicates: true })
+}
+
+const deleteStudentTag = async (studentnumber, tag_id) => {
   return TagStudent.destroy({
     where: {
-      id: {
-        [Op.eq]: id
-      }
+      tag_id,
+      studentnumber
     }
   })
 }
@@ -48,5 +51,6 @@ module.exports = {
   getStudentTagsByStudentnumber,
   getStudentTagsByStudytrack,
   createStudentTag,
+  createMultipleStudentTags,
   deleteStudentTag
 }

services/backend/oodikone2-backend/src/services/updaterService.js

@@ -14,8 +14,67 @@ const updateStudents = async (studentNumbers) => {
   return response.data
 }
 
+const updateOldestStudents = async (amount) => {
+  const response = await client.post('/update/oldest', { amount })
+  return response.data
+}
+
+const updateAllStudents = async () => {
+  const response = await client.post('/update/all')
+  return response.data
+}
+
+const updateActiveStudents = async () => {
+  const response = await client.post('/update/active')
+  return response.data
+}
+
+const updateNoStudents = async () => {
+  const response = await client.post('/update/no_student')
+  return response.data
+}
+
+const updateMetadata = async () => {
+  const response = await client.post('/update/meta')
+  return response.data
+}
+
+const updateAttainments = async () => {
+  const response = await client.post('/update/attainment')
+  return response.data
+}
+
+const updateStudentlist = async () => {
+  const response = await client.post('/update/studentlist')
+  return response.data
+}
+
+const getStatuses = async () => {
+  const response = await client.get('/statuses')
+  return response.data
+}
+
+const rescheduleScheduled = async () => {
+  const response = await client.post('/reschedule/scheduled')
+  return response.data
+}
+
+const rescheduleFetched = async () => {
+  const response = await client.post('/reschedule/fetched')
+  return response.data
+}
 
 module.exports = {
   ping,
-  updateStudents
+  updateStudents,
+  updateOldestStudents,
+  updateAllStudents,
+  updateActiveStudents,
+  updateMetadata,
+  updateAttainments,
+  updateStudentlist,
+  getStatuses,
+  rescheduleScheduled,
+  rescheduleFetched,
+  updateNoStudents
 }

services/backend/oodikone2-backend/src/services/userService.js

@@ -1,6 +1,7 @@
 const axios = require('axios')
 const { USERSERVICE_URL } = require('../conf-backend')
 const UnitService = require('./units')
+const elementDetailService = require('./elementdetails')
 
 const client = axios.create({ baseURL: USERSERVICE_URL, headers: { 'secret': process.env.USERSERVICE_SECRET } })
 
@@ -15,11 +16,6 @@ const findAll = async () => {
   return response.data
 }
 
-const findAllEnabled = async () => {
-  const response = await client.get('/findallenabled')
-  return response.data
-}
-
 const login = async (uid, full_name, hyGroups, affiliations, email) => {
   const response = await client.post('/login', {
     uid, full_name, hyGroups, affiliations, email,
@@ -43,7 +39,8 @@ const byUsername = async (uid) => {
 const getUserElementDetails = async (username) => {
   const url = `/user/elementdetails/${username}`
   const response = await client.get(url)
-  return response.data
+  const elementdetailcodes = response.data
+  return elementDetailService.byCodes(elementdetailcodes)
 }
 
 const byId = async (id) => {
@@ -89,12 +86,15 @@ const setFaculties = async (uid, faculties) => {
 }
 
 const getUnitsFromElementDetails = async username => {
-  const url = `/user/elementdetails/${username}`
-  const response = await client.get(url)
-  const elementDetails = response.data
+  const elementDetails = await getUserElementDetails(username)
   return elementDetails.map(element => UnitService.parseUnitFromElement(element))
 }
 
+const getCodesFromElementDetails = async uid => {
+  const elementDetails = await getUserElementDetails(uid)
+  return elementDetails.map(e => e.code)
+}
+
 const modifyAccess = async (body) => {
   const response = await client.post('/modifyaccess', body)
   return response.data
@@ -105,6 +105,11 @@ const getAccessGroups = async () => {
   return response.data
 }
 
+const getAccessGroupCodesFor = async (user) => {
+  const response = await client.get(`/get_accessgroupcodes/${user}`)
+  return response.data
+}
+
 module.exports = {
   ping,
   byUsername,
@@ -117,10 +122,11 @@ module.exports = {
   enableElementDetails,
   removeElementDetails,
   findAll,
-  findAllEnabled,
   modifyAccess,
   getAccessGroups,
+  getAccessGroupCodesFor,
   getUnitsFromElementDetails,
+  getCodesFromElementDetails,
   getRolesFor,
   setFaculties
 }

services/backend/oodikone2-backend/src/util/index.js

@@ -17,6 +17,21 @@ const getStudentNumberChecksum = studentNumber => {
   return (10 - (checksum % 10)) % 10
 }
 
+const isValidStudentId = (id) => {
+  if (/^0\d{8}$/.test(id)) {
+    // is a 9 digit number
+    const multipliers = [7, 1, 3, 7, 1, 3, 7]
+    const checksum = id
+      .substring(1, 8)
+      .split('')
+      .reduce((sum, curr, index) => {
+        return (sum + curr * multipliers[index]) % 10
+      }, 0)
+    return (10 - checksum) % 10 == id[8]
+  }
+  return false
+}
+
 // helpers to print complex sequelizedata
 const plainPrint = (sequelizeData) => console.log(JSON.parse(JSON.stringify(sequelizeData)))
 
@@ -33,5 +48,6 @@ module.exports = {
   getStudentNumberChecksum,
   plainPrint,
   plainify,
+  isValidStudentId,
   oldToNew
 }

services/backend/oodikone2-backend/src/util/utils.js

@@ -1,8 +1,19 @@
 const { requiredGroup } = require('../conf-backend')
 
+const mapToProviders = elementDetails => elementDetails.map((r) => {
+  const isNumber = str => !Number.isNaN(Number(str))
+  if (r.includes('_')) {
+    const [left, right] = r.split('_')
+    const prefix = [...left].filter(isNumber).join('')
+    const suffix = `${left[0]}${right}`
+    const providercode = `${prefix}0-${suffix}`
+    return providercode
+  }
+  return r
+})
+
 const hasRequiredGroup = (hyGroups) => {
   const hasGroup = requiredGroup === null || hyGroups.some(e => e === requiredGroup)
-  console.log('has required group?:', hyGroups, requiredGroup, hasGroup)
   return hasGroup
 }
 
@@ -15,6 +26,7 @@ const parseHyGroups = (hyGroups) => {
 }
 
 module.exports = {
+  mapToProviders,
   hasRequiredGroup,
   parseHyGroups
 }

services/backend/oodikone2-backend/test/endpoint/teacher_test.js

@@ -3,19 +3,18 @@ process.env.DB_SCHEMA = schema
 
 const test = require('ava')
 const supertest = require('supertest')
-const jwt = require('jsonwebtoken')
 
 const { sequelize } = require('../../src/models')
 const app = require('../../src/app')
-const conf = require('../../src/conf-backend')
 const api = supertest(app)
 
-const uid = 'tktl', fullname = ''
-const payload = { userId: uid, name: fullname, enabled: true }
-
-const token = jwt.sign(payload, conf.TOKEN_SECRET, {
-  expiresIn: '24h'
-})
+// const jwt = require('jsonwebtoken')
+// const conf = require('../../src/conf-backend')
+// const uid = 'tktl', fullname = ''
+// const payload = { userId: uid, name: fullname, enabled: true }
+// const token = jwt.sign(payload, conf.TOKEN_SECRET, {
+//   expiresIn: '24h'
+// })
 
 test.before(async () => {
   await sequelize.createSchema(schema)

services/backend/oodikone2-backend/test/jest/globals.js

@@ -1,4 +1,5 @@
 const { sequelize } = require('../../src/models/index')
+const { sequelizeKone } = require('../../src/models/models_kone')
 const { redisClient } = require('../../src/services/redis')
 const { forceSyncDatabase } = require('../../src/database/connection')
 const { seedAllMigrations } = require('../../src/database/seed_migrations')
@@ -10,5 +11,7 @@ beforeAll(async () => {
 
 afterAll(async () => {
   await sequelize.close()
-  await redisClient.quitAsync()
+  await sequelizeKone.close()
+  // https://stackoverflow.com/a/54560610
+  await new Promise(res => redisClient.quit(() => setImmediate(res)))
 })

services/backend/oodikone2-backend/test/jest/services/population.test.js

@@ -3,6 +3,7 @@ const {
   Studyright, Credit, StudyrightExtent, Semester
 } = require('../../../src/models/index')
 const { optimizedStatisticsOf } = require('../../../src/services/populations')
+jest.setTimeout(10000)
 
 const langify = name => ({
   en: `${name}_en`,
@@ -138,90 +139,90 @@ describe('optimizedStatisticsOf tests', () => {
     - Two credits in 2011-09-31 and 2012-02-31.
     `, () => {
 
-      beforeAll(async () => {
-        await Semester.bulkCreate([semesters.fall, semesters.spring])
-        await Student.create(student)
-        await Course.create(courses.elements_of_ai)
-        await Credit.bulkCreate([creditFall, creditSpring])
-        await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
-        await StudyrightExtent.create(studyrightextents.bachelors)
-        await Studyright.create(studyright)
-        await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
-      })
-
-      test('Query result for BSc, Fall 2011 for 12 months should contain the student.', async () => {
-        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-      })
-
-      test('Query result for BSc, Fall 2012 for 12 months should contain the student.', async () => {
-        const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.bsc.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-      })
-
-      test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
-        const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-      })
-
-      test('Query result for Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
-        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.maths.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-      })
-
-      test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
-        const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-      })
-
-      test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
-        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
-      })
-
-      test('Query result for BSc and Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
-        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.maths.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-      })
-
-      test('Query result for BSc, Spring 2012 for 12 months should contain the student', async () => {
-        const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.bsc.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-      })
-
-      test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
-        const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-      })
-
-      test('Query result for BSc, Fall 2011 for 4 months should only return the FALL course instance for student. ', async () => {
-        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 4)
-        const { students } = await optimizedStatisticsOf(query)
-        const result = students.find(s => s.studentNumber === student.studentnumber)
-        const courseinstances = result.courses
-        expect(courseinstances.length).toBe(1)
-        expect(
-          courseinstances.some(instance =>
-            (instance.date.getTime() === courseinstanceFall.coursedate.getTime()) &&
-            (instance.course.code === courseinstanceFall.course_code))
-        ).toBe(true)
-      })
-
-      test('Query result for BSc, Fall 2011 for 1 month should return student even though do not have any credits yet. ', async () => {
-        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 1)
-        const { students } = await optimizedStatisticsOf(query)
-        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-      })
-
+    beforeAll(async () => {
+      await Semester.bulkCreate([semesters.fall, semesters.spring])
+      await Student.create(student)
+      await Course.create(courses.elements_of_ai)
+      await Credit.bulkCreate([creditFall, creditSpring])
+      await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
+      await StudyrightExtent.create(studyrightextents.bachelors)
+      await Studyright.create(studyright)
+      await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
     })
 
+    test('Query result for BSc, Fall 2011 for 12 months should contain the student.', async () => {
+      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+    })
+
+    test('Query result for BSc, Fall 2012 for 12 months should contain the student.', async () => {
+      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.bsc.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+    })
+
+    test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
+      const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
+    })
+
+    test('Query result for Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
+      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.maths.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+    })
+
+    test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
+      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
+    })
+
+    test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
+      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
+    })
+
+    test('Query result for BSc and Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
+      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.maths.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+    })
+
+    test('Query result for BSc, Spring 2012 for 12 months should contain the student', async () => {
+      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.bsc.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+    })
+
+    test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
+      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
+    })
+
+    test('Query result for BSc, Fall 2011 for 4 months should only return the FALL course instance for student. ', async () => {
+      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 4)
+      const { students } = await optimizedStatisticsOf(query)
+      const result = students.find(s => s.studentNumber === student.studentnumber)
+      const courseinstances = result.courses
+      expect(courseinstances.length).toBe(1)
+      expect(
+        courseinstances.some(instance =>
+          (instance.date.getTime() === courseinstanceFall.coursedate.getTime()) &&
+            (instance.course.code === courseinstanceFall.course_code))
+      ).toBe(true)
+    })
+
+    test('Query result for BSc, Fall 2011 for 1 month should return student even though do not have any credits yet. ', async () => {
+      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 1)
+      const { students } = await optimizedStatisticsOf(query)
+      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+    })
+
+  })
+
 })

services/backend/oodikone2-backend/test/supertests/api_spec.js

@@ -33,7 +33,7 @@ test.after.always(async () => {
   const tagToAdd = { text: 'mooc-2013' }
   console.log('ennd  bfefore')
 
-  let res = await api
+  await api
     .delete('/api/students/014424850/tags')
     .send(tagToAdd)
     .set('x-access-token', token)

services/backend/shared/migrations/20180803_add_language_to_user.js

@@ -1,12 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('users', 'language',
-      {
-        type: Sequelize.STRING,
-        defaultValue: 'fi'
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('users', 'language')
-  }
-}

services/backend/shared/migrations/20180807_add_attainment_date_column.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('credit', 'attainment_date', { type: Sequelize.DATE })
-  },
-  down: async (queryInterface) => {
-    queryInterface.removeColumn('credit', 'attainment_date')
-  }
-}

services/backend/shared/migrations/20180808_01_add_credit_ courseinstace_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    queryInterface.addIndex('credit', {
-      fields: ['courseinstance_id'],
-      name: 'credit_courseinstance_id'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180808_02_add_credit_course_column.js

@@ -1,19 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn(
-      'credit', 
-      'course_code', 
-      { 
-        type: Sequelize.STRING,
-        references: {
-          model: 'course',
-          key: 'code',
-          foreignKey: 'credit_course_fkey'
-        },
-      }
-    )
-  },
-  down: async (queryInterface) => {
-    await queryInterface.removeColumn('credit', 'course_code')
-  }
-}

services/backend/shared/migrations/20180808_03_update_credit_attainment_and_course_values.js

@@ -1,15 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.sequelize.query(`
-        UPDATE credit
-        SET 
-          attainment_date = ci.coursedate,
-          course_code = ci.course_code
-        FROM courseinstance ci
-        WHERE ci.id = credit.courseinstance_id
-        ;
-    `)
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180808_04_add_credit_course_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.addIndex('credit', {
-      fields: ['course_code'],
-      name: 'credit_course_code'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180903_add_description_to_filters.js

@@ -1,12 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('filters', 'description',
-      {
-        type: Sequelize.STRING,
-        defaultValue: ''
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('filters', 'description')
-  }
-}

services/backend/shared/migrations/20180910_01_add_credit_attainment_date_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.addIndex('credit', {
-      fields: ['attainment_date'],
-      name: 'credit_attainment_date'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_02_add_enrollment_studentnumber_index.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.addIndex('semester_enrollments', {
-      fields: ['studentnumber'],
-      name: 'semester_enrollment_studentnumber'
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_03_add_transfer_studentnumber_index.js

@@ -1,14 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    try {
-      await queryInterface.addIndex('transfers', {
-        fields: ['studentnumber'],
-        name: 'transfers_studentnumber'
-      })
-    } catch (e) {
-      console.log('transfers_studentnumber index in table transfers already existed')
-    }
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_04_add_studyright_studentnumber_index.js

@@ -1,14 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    try {
-      await queryInterface.addIndex('studyright', {
-        fields: ['student_studentnumber'],
-        name: 'studyright_student_studentnumber'
-      })
-    } catch (e) {
-      console.log('studyright_student_studentnumber index in table studyright already existed')
-    }
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_05_drop_courseinstance_columns.js

@@ -1,7 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.removeColumn('credit', 'courseinstance_id')
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_06_drop_courseinstance_and_teacher_tables.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.dropTable('courseteacher')
-    await queryInterface.dropTable('courseinstance')
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_07_add_semester_enrollment_date_column.js

@@ -1,7 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('semester_enrollments', 'enrollment_date', { type: Sequelize.DATE })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_08_create_credit_teacher_table.js

@@ -1,30 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.createTable('credit_teachers', {
-      createdAt: {
-        type: Sequelize.DATE
-      },
-      updatedAt: {
-        type: Sequelize.DATE
-      },
-      credit_id: {
-        type: Sequelize.STRING,
-        primaryKey: true,
-        references: {
-          model: 'credit',
-          key: 'id'
-        }
-      },
-      teacher_id: {
-        type: Sequelize.STRING,
-        primaryKey: true,        
-        references: {
-          model: 'teacher',
-          key: 'id'
-        } 
-      }
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_09_add_semester_code_fkey_to_credit.js

@@ -1,14 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('credit', 'semestercode', {
-      type: Sequelize.INTEGER,
-      references: {
-        model: 'semesters',
-        key: 'semestercode'
-      }
-    }
-    )
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_10_add_year_code_and_name_to_semester.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('semesters', 'yearcode', { type: Sequelize.INTEGER })
-    await queryInterface.addColumn('semesters', 'yearname', { type: Sequelize.STRING })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180910_11_add_email_to_users.js

@@ -1,11 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('users', 'email',
-      {
-        type: Sequelize.STRING
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('users', 'email')
-  }
-}

services/backend/shared/migrations/20180910_12_create_usage_stats_table.js

@@ -1,36 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.createTable('usage_statistics', {
-      id: {
-        type: Sequelize.STRING,
-        primaryKey: true,
-      },
-      username: {
-        type: Sequelize.STRING
-      },
-      name: {
-        type: Sequelize.STRING
-      },
-      time: {
-        type: Sequelize.INTEGER
-      },
-      admin: {
-        type: Sequelize.BOOLEAN
-      },
-      method: {
-        type: Sequelize.STRING
-      },
-      URL: {
-        type: Sequelize.STRING
-      },
-      status: {
-        type: Sequelize.INTEGER
-      },
-      data: {
-        type: Sequelize.JSONB
-      },
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations/20180921_add_new_role_to_users.js

@@ -1,12 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('users', 'czar',
-      {
-        type: Sequelize.BOOLEAN,
-        defaultValue: false
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('users', 'czar')
-  }
-}

services/backend/shared/migrations/20180924_add_admin_to_database.js

@@ -1,12 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.addColumn('users', 'admin',
-      {
-        type: Sequelize.BOOLEAN,
-        defaultValue: false
-      })
-  },
-  down: (queryInterface) => {
-    return queryInterface.removeColumn('users', 'admin')
-  }
-}

services/backend/shared/migrations/20181006_01_add_semester_code_to_missing_credits.js

@@ -1,15 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    return queryInterface.sequelize.query(`
-    UPDATE credit
-    SET 
-      semestercode = semesters.semestercode
-    FROM semesters
-    WHERE credit.semestercode is null
-      AND credit.attainment_date BETWEEN semesters.startdate AND semesters.enddate
-    ;
-`)
-  },
-  down: () => {
-  }
-}

services/backend/shared/migrations/20181006_02_credit_semestercode_notnull.js

@@ -1,14 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.changeColumn(
-      'credit',
-      'semestercode',
-      { 
-        type: Sequelize.INTEGER,
-        allowNull: false
-      }
-    )
-  },
-  down: () => {
-  }
-}

services/backend/shared/migrations/20181101_remove_duplicate_users.js

@@ -1,48 +0,0 @@
-// Commented out because it uses User table definition which doesnt exist.
-// It would be best to avoid using services and use plain SQL instead
-
-// const { User } = require('../../models/index')
-// const users = require('../../services/users')
-
-// const uniques = async () => {
-//   const duplicates = {}
-//   const primary = {}
-//   const dbusers = await User.findAll()
-//   for (let dbuser of dbusers) {
-//     const { username, id } = dbuser
-//     if (!primary[username]) {
-//       primary[username] = id
-//     } else {
-//       const alternatives = duplicates[username] || []
-//       duplicates[username] = alternatives.concat(id)
-//     }
-//   }
-//   return { primary, duplicates}
-// }
-
-// const setElementsToPrimary = async (primary) => {
-//   for (let [username, uid] of Object.entries(primary)) {
-//     const elements = await users.getUserElementDetails(username)
-//     const codes = elements.map(e => e.code)
-//     await users.enableElementDetails(uid, codes)
-//   }
-// }
-
-// const deleteDuplicates = async (duplicates) => {
-//   for (let codes of Object.values(duplicates)) {
-//     for (let uid of codes) {
-//       const user = await User.findById(uid)
-//       await user.destroy()
-//     }
-//   }
-// }
-
-// module.exports = {
-//   up: async () => {
-//     const { primary, duplicates } = await uniques()
-//     await setElementsToPrimary(primary)
-//     await deleteDuplicates(duplicates)
-//   },
-//   down: () => {
-//   }
-// }