Merge pull request #1057 from UniversityOfHelsinkiCS/trunk

custom populations are here

.gitignore

@@ -19,4 +19,11 @@ services/updater_writer/updater/test_assets/testi.bak
 services/updater_writer/updater/test_assets/011120775.json
 services/updater_writer/updater/test_assets/014272112.json
 services/updater_writer/updater/test_assets/014441008.json
-services/updater_writer/updater/test_assets/meta.json
+services/updater_writer/updater/test_assets/meta.json
+services/updater_scheduler/all_student_numbers.txt
+services/updater_scheduler/debug.log
+services/updater_scheduler/active_student_numbers.txt
+services/oodikone2-backend/debug.log
+datastore
+services/updater_api/debug.log
+services/backend/updater_writer/dumps

.travis.yml

@@ -16,6 +16,7 @@ services:
 before_script:
   - docker --version
   - docker-compose --version
+  - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")"
 
 jobs:
   include:
@@ -28,13 +29,16 @@ jobs:
         - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.yml
         - docker ps
         - cat nginx/log
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/staging/); do printf '.'; sleep 5; done
+        - docker-compose logs -f &
+        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
         - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/staging/ npm run cypress:record
+        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
+        - docker ps -a
+        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
       deploy: # deploy is always skipped for PR
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.yml push && curl --fail -k -X POST -H \"Authorization: token ${TRAVIS_ACCESS_TOKEN}\" https://toska.cs.helsinki.fi/toskawatch/release/"
+        script: "docker-compose -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
         on:
           branch: master
     - # in parallel
@@ -46,15 +50,18 @@ jobs:
         - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.production.yml
         - docker ps
         - cat nginx/log
+        - docker-compose logs -f &
         - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
         - time npm run test_services
         - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
+        - docker ps -a
+        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
       deploy: # deploy is always skipped for PR
         provider: script
         skip_cleanup: true
-        script: docker-compose -f docker-compose.lateste2e.production.yml push && echo "Done, please MANUAL DEPLOY"
+        script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
         on:
           tags: true
 
-after_script:
-  - docker-compose logs
+after_failure:
+  - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

cypress.json

@@ -1,9 +1,9 @@
 {
-  "projectId": "c8bfes",
-  "defaultCommandTimeout": 120000,
-  "execTimeout": 120000,
-  "taskTimeout": 120000,
-  "pageLoadTimeout": 120000,
-  "requestTimeout": 120000,
-  "responseTimeout": 120000
+  "projectId": "c3jsph",
+  "defaultCommandTimeout": 60000,
+  "execTimeout": 60000,
+  "taskTimeout": 60000,
+  "pageLoadTimeout": 60000,
+  "requestTimeout": 60000,
+  "responseTimeout": 60000
 }

cypress/integration/Course_statistics.js

@@ -0,0 +1,75 @@
+
+describe('Course Statistics tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+          proxy.xhr.setRequestHeader('uid', 'tktl')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    console.log(Cypress.config().baseUrl)
+    cy.visit(Cypress.config().baseUrl)
+    cy.contains("Course statistics").click()
+    cy.contains("Search for courses")
+  })
+
+  it('Searching single course having duplicate mappings shows course statistics', () => {
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by entering a course code']").type('TKT20003')
+    cy.contains("tr", "TKT20003").click()
+    cy.contains("Fetch statistics").should('be.enabled').click()
+    cy.contains("Search for courses").should('not.exist')
+
+    cy.contains("Käyttöjärjestelmät")
+    cy.contains("TKT20003")
+    cy.contains("582640") // old mapped code
+
+    cy.contains(".tabular.menu a", "Table").click()
+    cy.contains("All")
+    cy.contains(".modeSelectorRow a", "Cumulative").click()
+    cy.contains(".modeSelectorRow a", "Student").click()
+    cy.contains(".modeSelectorRow a", "Grades").click()
+
+    cy.contains(".tabular.menu a", "Pass rate chart").click()
+    cy.get("div.modeSelectorContainer").click()
+    cy.contains("svg", "Pass rate chart")
+
+    cy.contains(".tabular.menu a", "Grade distribution chart").click()
+    cy.get("div.modeSelectorContainer").click()
+    cy.contains("svg", "Grades")
+
+    cy.contains("a", "New query").click()
+    cy.contains("Search for courses")
+  })
+
+  it('Searching multiple courses having duplicate mappings shows course statistics', () => {
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by entering a course code']").type('TKT')
+    cy.contains("tr", "TKT20003").click()
+    cy.contains("tr", "TKT10002").click()
+    cy.contains("Fetch statistics").should('be.enabled').click()
+    cy.contains("Search for courses").should('not.exist')
+
+    cy.contains('.courseNameCell', "Käyttöjärjestelmät").contains("TKT20003").click()
+    cy.contains('.courseNameCell', "Ohjelmoinnin perusteet").should('not.exist');
+    cy.contains("TKT20003")
+    cy.contains("582640") // old mapped code
+    cy.contains("Summary").click()
+
+    cy.contains('.courseNameCell', "Ohjelmoinnin perusteet").contains("TKT10002").click()
+    cy.contains('.courseNameCell', "Käyttöjärjestelmät").should('not.exist');
+    cy.contains("TKT10002")
+    cy.contains("581325") // old mapped code
+    cy.contains("Summary").click()
+
+    cy.contains("a", "New query").click()
+    cy.contains("Search for courses")
+  })
+
+})

cypress/integration/Population_statistics.js

@@ -80,7 +80,7 @@ describe('Population Statistics tests', () => {
 
     cy.contains("add").click()
     cy.contains("Add filters").siblings().within(() => {
-      cy.get(".form").should('have.length', 8)
+      cy.get(".form").should('have.length', 10)
     })
 
     checkAmountOfStudents(26)
@@ -102,15 +102,15 @@ describe('Population Statistics tests', () => {
     cy.contains("DIGI-000A")
     cy.go("back")
 
-   
+
 
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
       cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
       cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '9')
     })
 
-    cy.get(".refresh").eq(1).click()
     cy.contains("Courses of Population").parentsUntil(".ui.segment").parent().within(() => {
+      cy.get(".refresh").click()
       cy.contains("number at least").siblings().within(() => cy.get("input").clear().type("0"))
       cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '1')
     })
@@ -153,6 +153,11 @@ describe('Population Statistics tests', () => {
     cy.contains("Spring 2018").parentsUntil("form").contains("set filter").click()
     cy.contains("Students that were present").should('have.text', "Students that were present during Fall 2018, Spring 2018")
 
+    cy.contains('graduated from Kasvatustieteiden kandiohjelma').contains('have/not').click().siblings().contains('have not').click()
+    cy.contains('graduated from Kasvatustieteiden kandiohjelma').contains('set filter').click()
+
+    cy.contains('Showing students that have not graduated from Kasvatustieteiden kandiohjelma')
+
     checkAmountOfStudents(17)
 
     cy.contains("have/haven't").click().siblings().contains('haven\'t').click()
@@ -200,6 +205,7 @@ describe('Population Statistics tests', () => {
 
     cy.contains('Excluded students that graduated')
 
+
     cy.contains("Students that has").parentsUntil("form").within(() => {
       cy.contains("degree").click().siblings().contains("any degree").click()
       cy.contains("programme").click().siblings().contains("Kasvatustieteiden kandiohjelma")

cypress/integration/Student_statistics.js

@@ -56,6 +56,14 @@ describe('Student Statistics tests', () => {
         cy.contains('Credits').should('not.exist')
     })
 
+    it('Can jump to course', () => {
+        cy.get('.prompt').type('Oinonen')
+        cy.contains('011143561').click()
+        cy.contains('Toinen kotimainen kieli (40061)').siblings().within(() => { cy.get('.level').click() })
+        cy.url().should('include', '/coursestatistics')
+        cy.contains('Toinen kotimainen kieli')
+    })
+
     it('Searching with bad inputs doesnt yield results', () => {
         cy.get('.prompt').type('Oin')
         cy.contains('Student number').should('not.exist')

cypress/integration/Studyprogramme_overview.js

@@ -15,6 +15,30 @@ describe('Studyprogramme overview', () => {
     cy.contains("Study Programme", { timeout: 100000 })
   })
 
+  it('can search for course mappings', () => {
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
+    cy.contains('Code Mapper').click()
+    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').within(($tr) => { cy.get('input').type('582219') })
+    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').within(($tr) => { cy.get('.results').contains("Käyttöjärjestelmät (582219)") })
+    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').within(($tr) => { cy.contains('button', "Add") })
+  })
+
+  it('can view course groups', () => {
+    cy.contains("Kasvatustieteiden kandiohjelma").click()
+    cy.contains('Course Groups').click()
+
+    cy.contains('tr', 'Test course group').get('i.edit').click()
+    cy.contains("Edit group")
+    cy.get('.prompt').type("Professori Pekka")
+    cy.contains("Add teacher").parent().contains("9000960")
+    cy.contains("Teachers in group").parent().contains("9000960")
+    cy.get("i.reply.link.icon").click()
+
+    cy.contains('tr a', 'Test course group').click()
+    cy.contains("Total teachers")
+    cy.get("i.reply.icon").click()
+  })
+
   it('renders progress and productivity tables', () => {
     cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
     cy.get('table').should('have.length', 2)
@@ -23,7 +47,7 @@ describe('Studyprogramme overview', () => {
     cy.contains("2018-2019").siblings().contains('9')
     cy.contains("2017-2018").siblings().contains('12').siblings().contains('7').siblings().contains('2')
     cy.get('table').eq(1).contains('2018').siblings().contains('443')
-    cy.get('table').eq(1).contains('2017').siblings().contains('383')
+    cy.get('table').eq(1).contains('2017').siblings().contains('381')
   })
 
   it('can search for mandatory courses', () => {
@@ -40,5 +64,11 @@ describe('Studyprogramme overview', () => {
     cy.contains('Thesis Courses').click()
     cy.contains('Add thesis course').click()
     cy.contains('No results')
-  })  
+  })
+
+  it('can move to Population statistics page by clickin', () => {
+    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+    cy.get('i.level.up.alternate.icon').eq(0).click()
+    cy.contains('Credit accumulation (for 9 students)')
+  })
 })

cypress/integration/Users.js

@@ -0,0 +1,39 @@
+
+describe('Users tests', () => {
+  beforeEach(() => {
+    cy.server({
+      onAnyRequest: function (route, proxy) {
+        if (Cypress.config().baseUrl.includes("http://localhost:1337/")) {
+          proxy.xhr.setRequestHeader('uid', 'admink')
+          proxy.xhr.setRequestHeader('shib-session-id', 'mock-shibboleth')
+          proxy.xhr.setRequestHeader('hygroupcn', 'grp-oodikone-users')
+          proxy.xhr.setRequestHeader('edupersonaffiliation', 'asdasd')
+        }
+      }
+    })
+    console.log(Cypress.config().baseUrl)
+    cy.visit(Cypress.config().baseUrl)
+    cy.reload()
+    cy.contains("Users").click()
+    cy.contains("Enable or disable access to Oodikone")
+  })
+
+  it("Admin mocking normal user shows only the mocked user's programmes", () => {
+    cy.contains("mocking").should('not.exist')
+    cy.contains("tr", "Normaalikäyttäjä").within(($row) => {
+      cy.contains('button', 'Edit').click()
+    })
+    cy.contains("Access rights").siblings().within(($row) => {
+      cy.get("div[role='listitem']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")
+    })
+
+    cy.get('i.spy').click()
+    cy.contains("mocking as normk")
+    cy.contains("Study programme").click().siblings().contains("Search by class").click()
+    cy.contains("label", "Study programme")
+    cy.contains("label", "Study programme").siblings().within(($row) => {
+      cy.get("div[role='option']").should('have.length', 1).contains("Tietojenkäsittelytieteen kandiohjelma")
+    })
+  })
+
+})

docker-compose.lateste2e.production.yml

@@ -1,6 +1,13 @@
 version: '3'
 
 services:
+  nats:
+    image: nats-streaming
+    command: -cid updaterNATS
+    expose:
+      - "4222"
+    container_name: nats
+
   analytics_db:
     image: postgres:9.6.3
     ports:
@@ -53,25 +60,16 @@ services:
       - ./analytics-redis-data:/data
     container_name: analytics_redis
 
-  mongo_db:
-    image: mongo
-    restart: unless-stopped
-    volumes:
-      - ./backups/mongo/oodilearn:/dump
-      - ./mongodata:/data/db
-    ports:
-      - "27017:27017"
-    container_name: mongo_db
-
   backend:
     image: toska/oodikone2-backend:latest
     build:
-      context: services/oodikone2-backend
-    restart: unless-stopped
+      context: services/backend
+      dockerfile: oodikone2-backend/Dockerfile
     ports:
       - "8080:8080"
       - "9229:9229"
     environment:
+      WAIT_HOSTS: redis:6379,db:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
@@ -79,14 +77,18 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
       USERSERVICE_URL: http://userservice:4567
+      USAGESERVICE_URL: http://usageservice:545
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+      ANALYTICS_URL: http://analytics:4568
+      NATS_URI: nats://nats:4222
+      DB_SCHEMA_KONE: kone_data
     container_name: backend
-    entrypoint: 'node --inspect=0.0.0.0:9229 --max_old_space_size=4096 index.js'
     depends_on:
       - db
       - redis
+      - nats
 
   frontend:
     image: toska/oodikone2-frontend:latest
@@ -94,10 +96,10 @@ services:
       context: services/oodikone2-frontend
       args:
         BASE_PATH: /
-    restart: unless-stopped
     ports:
       - "5000:5000"
     environment:
+      WAIT_HOSTS: user_db:5432
       BACKEND_ADDR: backend
       ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
     container_name: frontend
@@ -106,10 +108,10 @@ services:
     image: toska/oodikone2-userservice:latest
     build:
       context: services/oodikone2-userservice
-    restart: unless-stopped
     ports:
       - 4567:4567
     environment:
+      - WAIT_HOSTS=user_db:5432
       - DB_URL=postgres://postgres@user_db:5432/user_db
       - TEST_DB=postgres://postgres@user_db:5432/user_db_test
       - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -118,14 +120,40 @@ services:
       - user_db
     container_name: userservice
 
+  usageservice:
+    image: toska/oodikone2-usageservice:latest
+    build:
+      context: services/oodikone2-usageservice
+    ports:
+      - 545:545
+    environment:
+      - WAIT_HOSTS=usage_db:5432
+      - PORT=545
+      - DB_URL=postgres://postgres@usage_db:5432/usage_db
+      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+    depends_on:
+      - usage_db
+    container_name: usageservice
+
+  usage_db:
+    image: postgres:9.6.3
+    ports:
+      - "5424:5432"
+    volumes:
+      - ./usage_pgdata:/var/lib/postgresql/data
+    container_name: oodi_usage_db
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=usage_db
+
   analytics:
     image: toska/oodikone2-analytics:latest
     build:
       context: services/oodikone2-analytics
-    restart: unless-stopped
     ports:
       - 4568:4568
     environment:
+      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
       - REDIS=analytics_redis
       - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
       - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
@@ -134,9 +162,63 @@ services:
       - analytics_redis
     container_name: analytics
 
+  updater_api:
+    image: toska/updater_api:latest
+    build:
+      context: services/updater_api
+    depends_on:
+      - nats
+    environment:
+      - WAIT_HOSTS=nats:4222
+      - NATS_URI=nats://nats:4222
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - TOKEN=toskab1gs8cret5
+    container_name: updater_api
+
+  updater_writer:
+    image: toska/updater_writer:latest
+    build:
+      context: services/backend
+      dockerfile: updater_writer/Dockerfile
+    depends_on:
+      - nats
+      - db
+    environment:
+      WAIT_HOSTS: nats:4222,db:5432
+      NATS_URI: nats://nats:4222
+      DB_URL: postgres://postgres@db:5432/tkt_oodi
+      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_SCHEMA: updater_test
+    container_name: updater_writer
+
+  scheduler_mongo:
+    image: mongo
+    volumes:
+      - ./mongodata:/data/db
+    expose:
+      - "27017"
+    container_name: scheduler_db
+
+  updater_scheduler:
+    image: toska/updater_scheduler:latest
+    build:
+      context: services/updater_scheduler
+    command: ["echo", "mluukkai on orjuuttaja"]
+    depends_on:
+      - nats
+      - scheduler_mongo
+    environment:
+      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
+      - NATS_URI=nats://nats:4222
+      - MONGO_URI=mongodb://scheduler_mongo:27017
+      - STUDENT_NUMBERS=./all_student_numbers.txt
+      - ACTIVE_STUDENTS=./active_student_numbers.txt
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - TOKEN=toskab1gs8cret5
+    container_name: updater_scheduler
+
   nginx:
     image: nginx:latest
-    restart: unless-stopped
     volumes:
       - ./nginx.production.conf:/etc/nginx/nginx.conf
       - ./nginx/error.log:/etc/nginx/error_log.log

docker-compose.lateste2e.yml

@@ -1,6 +1,15 @@
 version: '3'
 
 services:
+  nats:
+    image: nats-streaming
+    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
+    expose:
+      - "4222"
+    volumes:
+      - ./datastore:/datastore
+    container_name: nats
+
   analytics_db:
     image: postgres:9.6.3
     ports:
@@ -53,25 +62,16 @@ services:
       - ./analytics-redis-data:/data
     container_name: analytics_redis
 
-  mongo_db:
-    image: mongo
-    restart: unless-stopped
-    volumes:
-      - ./backups/mongo/oodilearn:/dump
-      - ./mongodata:/data/db
-    ports:
-      - "27017:27017"
-    container_name: mongo_db
-
   backend:
     image: toska/oodikone2-backend:staging
     build:
-      context: services/oodikone2-backend
-    restart: unless-stopped
+      context: services/backend
+      dockerfile: oodikone2-backend/Dockerfile
     ports:
       - "8080:8080"
       - "9229:9229"
     environment:
+      WAIT_HOSTS: redis:6379,db:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
@@ -79,37 +79,41 @@ services:
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
       USERSERVICE_URL: http://userservice:4567
+      USAGESERVICE_URL: http://usageservice:545
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+      ANALYTICS_URL: http://analytics:4568
+      NATS_URI: nats://nats:4222
+      DB_SCHEMA_KONE: kone_data
     container_name: backend
-    entrypoint: 'node --inspect=0.0.0.0:9229 --max_old_space_size=4096 index.js'
     depends_on:
       - db
       - redis
+      - nats
 
   frontend:
     image: toska/oodikone2-frontend:staging
     build:
       context: services/oodikone2-frontend
       args:
-        BASE_PATH: /staging/
-    restart: unless-stopped
+        BASE_PATH: /
     ports:
       - "5000:5000"
     environment:
+      WAIT_HOSTS: user_db:5432
       BACKEND_ADDR: backend
       ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
     container_name: frontend
-
+    
   userservice:
     image: toska/oodikone2-userservice:staging
     build:
       context: services/oodikone2-userservice
-    restart: unless-stopped
     ports:
       - 4567:4567
     environment:
+      - WAIT_HOSTS=user_db:5432
       - DB_URL=postgres://postgres@user_db:5432/user_db
       - TEST_DB=postgres://postgres@user_db:5432/user_db_test
       - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -118,14 +122,40 @@ services:
       - user_db
     container_name: userservice
 
+  usageservice:
+    image: toska/oodikone2-usageservice:staging
+    build:
+      context: services/oodikone2-usageservice
+    ports:
+      - 545:545
+    environment:
+      - WAIT_HOSTS=usage_db:5432
+      - PORT=545
+      - DB_URL=postgres://postgres@usage_db:5432/usage_db
+      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+    depends_on:
+      - usage_db
+    container_name: usageservice
+
+  usage_db:
+    image: postgres:9.6.3
+    ports:
+      - "5424:5432"
+    volumes:
+      - ./usage_pgdata:/var/lib/postgresql/data
+    container_name: oodi_usage_db
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=usage_db
+
   analytics:
     image: toska/oodikone2-analytics:staging
     build:
       context: services/oodikone2-analytics
-    restart: unless-stopped
     ports:
       - 4568:4568
     environment:
+      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
       - REDIS=analytics_redis
       - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
       - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
@@ -134,9 +164,63 @@ services:
       - analytics_redis
     container_name: analytics
 
+  updater_api:
+    image: toska/updater_api:staging
+    build:
+      context: services/updater_api
+    depends_on:
+      - nats
+    environment:
+      - WAIT_HOSTS=nats:4222
+      - NATS_URI=nats://nats:4222
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - TOKEN=toskab1gs8cret5
+    container_name: updater_api
+
+  updater_writer:
+    image: toska/updater_writer:staging
+    build:
+      context: services/backend
+      dockerfile: updater_writer/Dockerfile
+    depends_on:
+      - nats
+      - db
+    environment:
+      WAIT_HOSTS: nats:4222,db:5432
+      NATS_URI: nats://nats:4222
+      DB_URL: postgres://postgres@db:5432/tkt_oodi
+      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_SCHEMA: updater_test
+    container_name: updater_writer
+
+  scheduler_mongo:
+    image: mongo
+    volumes:
+      - ./mongodata:/data/db
+    expose:
+      - "27017"
+    container_name: scheduler_db
+
+  updater_scheduler:
+    image: toska/updater_scheduler:staging
+    build:
+      context: services/updater_scheduler
+    command: ["echo", "mluukkai on orjuuttaja"]
+    depends_on:
+      - nats
+      - scheduler_mongo
+    environment:
+      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
+      - NATS_URI=nats://nats:4222
+      - MONGO_URI=mongodb://scheduler_mongo:27017
+      - STUDENT_NUMBERS=./all_student_numbers.txt
+      - ACTIVE_STUDENTS=./active_student_numbers.txt
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - TOKEN=toskab1gs8cret5
+    container_name: updater_scheduler
+
   nginx:
     image: nginx:latest
-    restart: unless-stopped
     volumes:
       - ./nginx.staging.conf:/etc/nginx/nginx.conf
       - ./nginx/error.log:/etc/nginx/error_log.log

docker-compose.yml

@@ -29,9 +29,11 @@ services:
 
   nats:
     image: nats-streaming
-    command: -cid updaterNATS
+    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
     expose:
       - "4222"
+    volumes:
+      - ./datastore:/datastore
     ports:
       - "8222:8222"
       - "4222:4222"
@@ -58,6 +60,7 @@ services:
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+
   usage_db:
     image: postgres:9.6.3
     ports:
@@ -102,14 +105,16 @@ services:
 
   backend:
     build:
-      context: services/oodikone2-backend
+      context: services/backend
+      dockerfile: oodikone2-backend/Dockerfile
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
     restart: unless-stopped
     ports:
       - "8080:8080"
     environment:
+      WAIT_HOSTS: redis:6379,db:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
@@ -121,21 +126,26 @@ services:
       USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_URL: http://analytics:4568
+      NATS_URI: nats://nats:4222
+      DB_SCHEMA_KONE: kone_data
     volumes:
-      - ./services/oodikone2-backend/:/usr/src/app
-      - ./shared/models:/usr/src/app/src/models
-      - ./shared/migrations:/usr/src/app/src/database/migrations
+      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/src/models
+      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+
     container_name: backend
     depends_on:
       - db
       - redis
+      - nats
 
   frontend:
     build:
       context: services/oodikone2-frontend
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker']
     restart: unless-stopped
     ports:
       - "8081:8081"
@@ -171,17 +181,18 @@ services:
       context: services/oodikone2-userservice
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
     restart: unless-stopped
     volumes:
       - ./services/oodikone2-userservice/:/usr/src/app
     ports:
       - 4567:4567
     environment:
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
+      WAIT_HOSTS: user_db:5432
+      DB_URL: postgres://postgres@user_db:5432/user_db
+      TEST_DB: postgres://postgres@user_db:5432/user_db_test
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest
     depends_on:
       - user_db
     container_name: userservice
@@ -191,16 +202,19 @@ services:
       context: services/oodikone2-usageservice
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
     restart: unless-stopped
     ports:
       - 545:545
     environment:
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      WAIT_HOSTS: usage_db:5432
+      PORT: '545'
+      DB_URL: postgres://postgres@usage_db:5432/usage_db
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
     volumes:
       - ./services/oodikone2-usageservice/:/usr/src/app
+    depends_on:
+      - usage_db
     container_name: usageservice
 
   updater_api:
@@ -208,36 +222,80 @@ services:
       context: services/updater_api
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
     restart: unless-stopped
     volumes:
       - ./services/updater_api/:/usr/src/app
     depends_on:
       - nats
     environment:
+      - WAIT_HOSTS=nats:4222
       - NATS_URI=nats://nats:4222
       - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
       - TOKEN=toskab1gs8cret5
+    container_name: updater_api
 
   updater_writer:
     build:
-      context: services/updater_writer
+      context: services/backend
+      dockerfile: updater_writer/Dockerfile
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
     restart: unless-stopped
     volumes:
-      - ./services/updater_writer/:/usr/src/app
-      - ./shared/models:/usr/src/app/models
-      - ./shared/migrations:/usr/src/app/database/migrations
+      - ./services/backend/updater_writer/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/models
+      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
+
     depends_on:
       - nats
       - db
     environment:
+      WAIT_HOSTS: nats:4222,db:5432
       NATS_URI: nats://nats:4222
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_SCHEMA: updater_test
+      DB_SCHEMA_KONE: kone_data
+
+
+
+  scheduler_mongo:
+    image: mongo
+    restart: unless-stopped
+    volumes:
+      - ./mongodata:/data/db
+    expose:
+      - "27017"
+    container_name: scheduler_db
+
+  updater_scheduler:
+    build:
+      context: services/updater_scheduler
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/updater_scheduler/:/usr/src/app
+    depends_on:
+      - nats
+      - scheduler_mongo
+    environment:
+      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
+      - NATS_URI=nats://nats:4222
+      - MONGO_URI=mongodb://scheduler_mongo:27017
+      - STUDENT_NUMBERS=./all_student_numbers.txt
+      - ACTIVE_STUDENTS=./active_student_numbers.txt
+      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
+      - TOKEN=toskab1gs8cret5
+      - LOG_HOST=toska.cs.helsinki.fi
+      - LOG_PORT=9501
+      - LOG_HOSTNAME=updater_scheduler
+
+    container_name: updater_scheduler
 
 
   analytics:
@@ -245,16 +303,17 @@ services:
       context: services/oodikone2-analytics
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
     restart: unless-stopped
     volumes:
       - ./services/oodikone2-analytics/:/usr/src/app
     ports:
       - 4568:4568
     environment:
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
+      WAIT_HOSTS: analytics_db:5432,analytics_redis:6379
+      REDIS: analytics_redis
+      DB_URL: postgres://postgres@analytics_db:5432/analytics_db
+      TEST_DB: postgres://postgres@analytics_db:5432/analytics_db_test
     depends_on:
       - analytics_db
       - analytics_redis

nginx.staging.conf

@@ -8,11 +8,11 @@ http {
   server {
     listen 80;
 
-    location /staging/ {
+    location / {
       proxy_pass http://frontend:5000/;
     }
 
-    location /staging/api/ {
+    location /api/ {
       proxy_pass http://backend:8080/;
     }
   }

run.sh

@@ -22,6 +22,10 @@ options=(
     "e2e setup"
     "Anon setup"
     "Full setup"
+    "Reset anon db"
+    "Reset real db"
+    "Download latest anon db"
+    "Download latest real db"
     "Full reset"
     "Quit"
 )
@@ -40,6 +44,18 @@ do
             mopo
             run_full_setup
             ;;
+        "Reset anon db")
+            reset_db
+            ;;
+        "Reset real db")
+            reset_real_db
+            ;;
+        "Download latest anon db")
+            get_anon_oodikone
+            ;;
+        "Download latest real db")
+            get_oodikone_server_backup
+            ;;
         "Full reset")
             purge
             ;;

run_service_tests.sh

@@ -2,9 +2,9 @@
 
 # stop on first error
 set -e
-npm run --prefix services/oodikone2-backend test_docker
-npm test --prefix services/oodikone2-analytics
-npm run --prefix services/oodikone2-userservice test_docker
-npm test --prefix services/oodikone2-usageservice
-npm run --prefix services/oodikone2-frontend test_docker
-
+docker-compose exec -T backend npm test
+docker-compose exec -T userservice npm test
+docker-compose exec -T frontend npm test
+docker-compose exec -T analytics npm test
+docker-compose exec -T usageservice npm test
+# docker-compose exec -T updater_writer npm test

scripts.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
 DIR_PATH=$(dirname "$0")
+ANONDB_DIR=anonyymioodi
 BACKUP_DIR=backups
-PSQL_DB_BACKUP="$BACKUP_DIR/anon.bak"
-USER_DB_BACKUP="$BACKUP_DIR/user-dump.bak"
-PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.bak"
-USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.bak"
+PSQL_DB_BACKUP="$ANONDB_DIR/anon.sqz"
+USER_DB_BACKUP="$ANONDB_DIR/user-dump.sqz"
+PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
+USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 
 retry () {
-    for i in {1..10}
+    for i in {1..60}
     do
         $@ && break || echo "Retry attempt $i failed, waiting..." && sleep 10;
     done
@@ -44,37 +45,29 @@ get_anon_oodikone() {
     file=./private.key
     if [ -e "$file" ]; then
       echo "Private key exists"
-    else 
+    else
       echo "No private key, echoing from environment variable OODI_KEY"
       echo "$OODI_KEY" | awk  '{gsub("\\\\n","\n")};1' > private.key
       chmod 400 private.key
     fi
+    rm -rf anonyymioodi
     GIT_SSH_COMMAND='ssh -i private.key' git clone git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
-    mv anonyymioodi/anon.bak.bz2 ./$BACKUP_DIR/anon.bak.bz2
-    mv anonyymioodi/user-dump.bak.bz2 ./$BACKUP_DIR/user-dump.bak.bz2
-
-}
-
-unpack_oodikone_server_backup() {
-    bunzip2 -d -v ./$BACKUP_DIR/*.bz2
 }
 
 restore_psql_from_backup () {
-    cat $PSQL_DB_BACKUP | docker exec -i -u postgres oodi_db psql -d tkt_oodi
+    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi -j4 $PSQL_DB_BACKUP
 }
 
 restore_real_psql_from_backup () {
-    cat $PSQL_REAL_DB_BACKUP | docker exec -i -u postgres oodi_db psql -d tkt_oodi_real
+    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi_real -j4 $PSQL_REAL_DB_BACKUP
 }
 
 restore_userdb_from_backup () {
-    docker exec -u postgres oodi_user_db psql -c "CREATE DATABASE user_db"
-    cat $USER_DB_BACKUP | docker exec -i -u postgres oodi_user_db psql -d user_db
+    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db -j4 $USER_DB_BACKUP
 }
 
 restore_real_userdb_from_backup () {
-    docker exec -u postgres oodi_user_db psql -c "CREATE DATABASE user_db_real"
-    cat $USER_REAL_DB_BACKUP | docker exec -i -u postgres oodi_user_db psql -d user_db_real
+    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db_real -j4 $USER_REAL_DB_BACKUP
 }
 
 # oodilearn
@@ -82,34 +75,14 @@ restore_real_userdb_from_backup () {
 #     docker exec -t mongo_db mongorestore -d oodilearn "/dump"
 # }
 
-db_oodikone_reset () {
-    docker exec -u postgres oodi_db dropdb "tkt_oodi" || echo "dbdrop of tkt_oodi failed. "
-    docker exec -u postgres oodi_db createdb "tkt_oodi" || echo "createdb of tkt_oodi failed. "
-    restore_psql_from_backup
-}
-
 ping_psql () {
     echo "Pinging psql in container $1 with db name $2"
-    retry docker exec -u postgres $1 pg_isready
-    docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi" || echo "tkt_oodi DB already exists"
-    echo "Pinging psql in container $1 with db name tkt_oodi_test"
-    docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_test" || echo "tkt_oodi_test DB already exists"
-    retry docker exec -u postgres $1 pg_isready
-}
-
-ping_psql_real () {
-    echo "Pinging psql in container $1 with db name $2"
-    docker exec -u postgres oodi_db psql -c "CREATE DATABASE tkt_oodi_real" || echo "tkt_oodi_real DB already exists"
-    retry docker exec -u postgres $1 pg_isready
+    retry docker exec -u postgres $1 pg_isready --dbname=$2
+    docker exec -u postgres $1 psql -c "CREATE DATABASE $2" || echo "container $1 DB $2 already exists"
 }
 
 db_setup_full () {
-    echo "Getting backups from the Oodikone server, this will prompt you for your password. "
-    get_oodikone_server_backup
-    echo "Unpacking compressed files"
-    unpack_oodikone_server_backup
     echo "Restoring PostgreSQL from backup"
-    ping_psql_real "oodi_db" "tkt_oodi_real"
     retry restore_real_psql_from_backup
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
@@ -120,12 +93,9 @@ db_setup_full () {
 }
 
 db_anon_setup_full () {
-    echo "Getting anon backups from the private repository. "
-    get_anon_oodikone
-    echo "Unpacking compressed files"
-    unpack_oodikone_server_backup
     echo "Restoring PostgreSQL from backup"
     ping_psql "oodi_db" "tkt_oodi"
+    ping_psql "oodi_db" "tkt_oodi_test"
     retry restore_psql_from_backup
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
@@ -135,6 +105,30 @@ db_anon_setup_full () {
     echo "Database setup finished"
 }
 
+reset_real_db () {
+    docker-compose down
+    docker-compose up -d db user_db
+    ping_psql "oodi_db" "tkt_oodi_real"
+    docker exec -u postgres oodi_db dropdb "tkt_oodi_real"
+    ping_psql "oodi_user_db" "user_db_real"
+    docker exec -u postgres oodi_user_db dropdb "user_db_real"
+    db_setup_full
+    docker-compose down
+}
+
+reset_db () {
+    docker-compose down
+    docker-compose up -d db user_db
+    ping_psql "oodi_db" "tkt_oodi"
+    docker exec -u postgres oodi_db dropdb "tkt_oodi"
+    ping_psql "oodi_db" "tkt_oodi_test"
+    docker exec -u postgres oodi_db dropdb "tkt_oodi_test"
+    ping_psql "oodi_user_db" "user_db"
+    docker exec -u postgres oodi_user_db dropdb "user_db"
+    db_anon_setup_full
+    docker-compose down
+}
+
 install_cli_npm_packages () {
     npm ci
 }
@@ -147,35 +141,38 @@ show_instructions () {
     cat ./assets/instructions.txt
 }
 
-docker_restart_backend () {
-    docker-compose restart backend userservice
-}
-
 run_full_setup () {
     echo "Setup npm packages"
     install_cli_npm_packages
     echo "Init dirs"
     init_dirs
-    echo "Building images, starting containers"
-    docker_build
-    echo "Setup oodikone db from dump, this will prompt you for your password."
+    echo "Getting backups from the Oodikone server, this will prompt you for your password. "
+    get_oodikone_server_backup
+    echo "Getting anon backups from the private repository. "
+    get_anon_oodikone
+    echo "Building images"
+    docker-compose build
+    echo "Setup oodikone db from dump."
+    docker-compose up -d db user_db
     db_setup_full
     db_anon_setup_full
-    echo "Restarting Docker backend containers to run migrations, etc."
-    docker_restart_backend
+    docker-compose down
     show_instructions
 }
+
 run_anon_full_setup () {
     echo "Setup npm packages"
     install_cli_npm_packages
     echo "Init dirs"
     init_dirs
-    echo "Building images, starting containers"
-    docker_build
-    echo "Setup oodikone db from dump, this will prompt you for your password."
+    echo "Getting anon backups from the private repository. "
+    get_anon_oodikone
+    echo "Building images"
+    docker-compose build
+    echo "Setup oodikone db from dump."
+    docker-compose up -d db user_db
     db_anon_setup_full
-    echo "Restarting Docker backend containers to run migrations, etc."
-    docker_restart_backend
+    docker-compose down
     show_instructions
 }
 
@@ -184,10 +181,13 @@ run_e2e_setup () {
     install_cli_npm_packages
     echo "Init dirs"
     init_dirs
-    echo "Building images, starting containers"
-    docker-compose -f $1 build && docker-compose -f $1 up -d
-    echo "Setup oodikone db from dump, this will prompt you for your password."
+    echo "Getting anon backups from the private repository. "
+    get_anon_oodikone
+    echo "Building images"
+    docker-compose -f $1 build
+    echo "Setup oodikone db from dump."
+    docker-compose -f $1 up -d db user_db
     db_anon_setup_full
-    echo "Restarting Docker backend containers to run migrations, etc."
-    docker-compose -f $1 restart backend userservice
+    echo "Starting services."
+    docker-compose -f $1 up -d
 }

services/backend/.dockerignore

@@ -0,0 +1,5 @@
+**/node_modules
+updater_writer/**/models
+updater_writer/**/migrations
+oodikone2-backend/**/models
+oodikone2-backend/**/migrations

services/backend/oodikone2-backend/.gitignore

@@ -13,3 +13,6 @@ npm-debug.log
 studentnumbers.txt
 studentnumbers*.txt
 src/anonymized_API
+
+models
+migrations

services/backend/oodikone2-backend/Dockerfile

@@ -0,0 +1,22 @@
+FROM node:10.15-alpine
+
+COPY ./oodikone2-backend /usr/src/app
+COPY ./shared/models /usr/src/app/src/models
+COPY ./shared/migrations /usr/src/app/src/database/migrations
+COPY ./shared/migrations_kone /usr/src/app/src/database/migrations_kone
+
+WORKDIR /usr/src/app
+RUN npm ci
+
+COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
+
+ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
+RUN chmod +x /wait
+
+EXPOSE 8080
+CMD /wait && npm start

services/backend/oodikone2-backend/README.md

@@ -1,9 +1,4 @@
 # oodikone2-backend
-Master:
-[![Build Status](https://travis-ci.org/UniversityOfHelsinkiCS/oodikone2-backend.svg?branch=master)](https://travis-ci.org/UniversityOfHelsinkiCS/oodikone2-backend)
-
-Trunk:
-[![Build Status](https://travis-ci.org/UniversityOfHelsinkiCS/oodikone2-backend.svg?branch=trunk)](https://travis-ci.org/UniversityOfHelsinkiCS/oodikone2-backend)
 
 ## Dev
 
@@ -69,34 +64,6 @@ Fill database with data from test.bak dump file:
 
 `cat test.bak | docker exec -i oodi_db psql -U postgres -d tkt_oodi_test`
 
-Install node
-https://nodejs.org/en/download/
-
-Clone the repository and install the packages in the root of the project:
-
-```
-git clone git@github.com:UniversityOfHelsinkiCS/oodikone2-backend.git`
-cd oodikone2-backend
-npm install
-```
-
-Create .env file containing
-```
-DB_URL=postgres://postgres@localhost:5421/tkt_oodi
-TEST_DB=postgres://postgres@localhost:5421/tkt_oodi_test
-FRONT_URL=http://localhost:8000
-REDIS=localhost
-TOKEN_SECRET=IWannaBeTheVeryBest
-```
-
-To initiate and seed the redis db, run: 
-
-`npm run init_redis`
-
-and
-
-`npm run seed_redis`
-
 ### Running 
 
 #### Dev
@@ -111,7 +78,7 @@ Run `npm test`
 
 Always deploy first to staging! 
 
-When merging from trunk to master Travis builds a docker image for staging and pushes it automatically.
+When merging from trunk to master Travis builds a docker image for staging and pushes it automatically to dockerh-hub. Currently **staging** is also automatically deployed after the image is pushed.
 
 When a release (or tag) is created in master branch Travis builds a docker image for production and pushes it automatically.
 
@@ -188,7 +155,7 @@ To update your database run `docker-compose run -d CONTAINER_NAME npm run update
 
 ### Anonymized data & E2E environment
 
-Use [oodikone2-cli](https://github.com/UniversityOfHelsinkiCS/oodikone2-cli) to setup you're environments.
+Use [oodikone-cli](https://github.com/UniversityOfHelsinkiCS/oodikone) to setup you're environments.
 
 Before running cli make yourself a deployment key for Anonyymioodi private repository and put it in your cli folder as `private.key`. This allows you to download the anonymized dumps from the repository.
 

services/backend/oodikone2-backend/package.json

@@ -38,7 +38,7 @@
   "author": "Matti Luukkainen",
   "license": "MIT",
   "dependencies": {
-    "axios": "^0.18.0",
+    "axios": "^0.19.0",
     "bluebird": "^3.5.3",
     "body-parser": "^1.18.3",
     "cors": "^2.8.5",
@@ -50,6 +50,7 @@
     "mkdirp": "^0.5.1",
     "moment": "^2.22.2",
     "morgan": "^1.9.1",
+    "node-nats-streaming": "^0.2.2",
     "nodemailer": "^4.7.0",
     "pg": "^7.7.1",
     "raven": "^2.6.3",

services/backend/oodikone2-backend/scripts/force_sync_database.js

@@ -0,0 +1,16 @@
+const { seedAllMigrations } = require('../src/database/seed_migrations')
+const { forceSyncDatabase } = require('../src/database/connection')
+
+const sync = async () => {
+  try {
+    await forceSyncDatabase()
+    await seedAllMigrations()
+    console.log('Force sync succeeded. ')
+    process.exit(0)
+  } catch (e) {
+    console.log(`Force sync failed: ${e}`)
+    process.exit(1)
+  }
+}
+
+sync()

services/backend/oodikone2-backend/scripts/seed_migrations.js

@@ -1,8 +1,8 @@
-const { seedMigrations } = require('../src/database/seed_migrations')
+const { seedAllMigrations } = require('../src/database/seed_migrations')
 
 const run = async () => {
   try {
-    await seedMigrations()
+    await seedAllMigrations()
     process.exit(0)
   } catch (e) {
     console.log(`Seeding migrations failed: ${e}`)

services/backend/oodikone2-backend/src/app.js

@@ -4,9 +4,12 @@ const bodyParser = require('body-parser')
 const Raven = require('raven')
 const conf = require('./conf-backend')
 const routes = require('./routes')
+const { StartNats } = require('./events')
 const { PORT } = conf
 const app = express()
 
+StartNats()
+
 Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
 
 app.use(cors({ credentials: true, origin: conf.frontend_addr }))

services/backend/oodikone2-backend/src/conf-backend.js

@@ -2,7 +2,8 @@ require('dotenv').config()
 
 const { NODE_ENV } = process.env
 let DB_URL = process.env.DB_URL
-if (NODE_ENV === 'test') {
+const isTest = process.env.NODE_ENV === 'test'
+if (isTest) {
   DB_URL = process.env.TEST_DB
 }
 else if (NODE_ENV === 'anon') {
@@ -12,13 +13,14 @@ const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
+const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'kone_data'
 const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
 const OODILEARN_URL = process.env.OODILEARN_URL
 const USERSERVICE_URL = process.env.USERSERVICE_URL
 const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
-const PORT = process.env.NODE_ENV === 'test' ? 8079 : 8080
+const PORT = isTest ? 8079 : 8080
 const OODI_SECRET = process.env.OODI_SECRET
 
 const FEATURES = {
@@ -51,14 +53,15 @@ let requiredGroup = 'grp-oodikone-users'
 if (process.env.NODE_ENV === 'staging') {
   requiredGroup = 'grp-oodikone-staging-users'
 }
-if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
+if (process.env.NODE_ENV === 'dev' || isTest) {
   requiredGroup = null
 }
 
 module.exports = {
-  frontend_addr, DB_URL, redis, TOKEN_SECRET, DB_SCHEMA, OODI_ADDR, CERT_PATH, KEY_PATH, FEATURES, OODILEARN_URL,
+  frontend_addr, DB_URL, redis, TOKEN_SECRET, DB_SCHEMA, DB_SCHEMA_KONE, OODI_ADDR, CERT_PATH, KEY_PATH, FEATURES, OODILEARN_URL,
   USERSERVICE_URL: formatURL(USERSERVICE_URL), ACCESS_TOKEN_HEADER_KEY, PORT,
   ANALYTICS_URL: formatURL(ANALYTICS_URL),
   USAGESERVICE_URL,
-  requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY
+  requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY,
+  isTest
 }

services/backend/oodikone2-backend/src/database/connection.js

@@ -0,0 +1,99 @@
+const Sequelize = require('sequelize')
+const Umzug = require('umzug')
+const conf = require('../conf-backend')
+
+const sequelize = new Sequelize(conf.DB_URL, {
+  schema: conf.DB_SCHEMA,
+  searchPath: conf.DB_SCHEMA,
+  logging: false,
+  operatorsAliases: false
+})
+sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
+
+const sequelizeKone = new Sequelize(conf.DB_URL, {
+  schema: conf.DB_SCHEMA_KONE,
+  searchPath: conf.DB_SCHEMA_KONE,
+  logging: false,
+  operatorsAliases: false
+})
+sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
+// See https://github.com/sequelize/sequelize/issues/10875
+
+const runMigrations = async () => {
+  try {
+    const migrator = new Umzug({
+      storage: 'sequelize',
+      storageOptions: {
+        sequelize: sequelize,
+        tableName: 'migrations'
+      },
+      logging: console.log,
+      migrations: {
+        params: [
+          sequelize.getQueryInterface(),
+          Sequelize
+        ],
+        path: `${process.cwd()}/src/database/migrations`,
+        pattern: /\.js$/,
+        schema: conf.DB_SCHEMA
+      }
+    })
+    const migrations = await migrator.up()
+
+    console.log('Migrations up to date', migrations)
+  } catch (e) {
+    console.log('Migration error, message:', e)
+  }
+}
+
+const runMigrationsKone = async () => {
+  try {
+    const migrator = new Umzug({
+      storage: 'sequelize',
+      storageOptions: {
+        sequelize: sequelizeKone,
+        tableName: 'migrations',
+        schema: conf.DB_SCHEMA_KONE
+      },
+      logging: console.log,
+      migrations: {
+        params: [
+          sequelizeKone.getQueryInterface(),
+          Sequelize
+        ],
+        path: `${process.cwd()}/src/database/migrations_kone`,
+        pattern: /\.js$/,
+        schema: conf.DB_SCHEMA_KONE
+      },
+    })
+    const migrations = await migrator.up()
+
+    console.log('Kone Migrations up to date', migrations)
+  } catch (e) {
+    console.log('Kone Migration error, message:', e)
+  }
+}
+
+const migrationPromise = !conf.isTest ? runMigrations().then(() => runMigrationsKone()) : Promise.resolve()
+
+const forceSyncDatabase = async () => {
+  try {
+    await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
+  } catch (e) {
+    // console.log(e)
+  }
+  try {
+    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
+  } catch (e) {
+    // console.log(e)
+  }
+  await sequelize.sync({ force: true })
+  await sequelizeKone.sync({ force: true })
+}
+
+module.exports = {
+  sequelize,
+  sequelizeKone,
+  migrationPromise,
+  forceSyncDatabase
+}

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -0,0 +1,27 @@
+const fs = require('fs')
+const { Migration, MigrationKone } = require('../models/index')
+
+const DEFAULT_PATH = `${process.cwd()}/src/database/migrations`
+const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`
+
+const saveMigrationsToDatabase = Migration => async filenames =>
+  Promise.all(filenames.map(name => Migration.upsert({ name })))
+
+const seedMigrations = async (migrationfilepath=DEFAULT_PATH) => {
+  const filenames = fs.readdirSync(migrationfilepath)
+  await saveMigrationsToDatabase(Migration)(filenames)
+}
+
+const seedMigrationsKone = async (migrationfilepath=DEFAULT_PATH_KONE) => {
+  const filenames = fs.readdirSync(migrationfilepath)
+  await saveMigrationsToDatabase(MigrationKone)(filenames)
+}
+
+const seedAllMigrations = async () => {
+  await seedMigrations()
+  await seedMigrationsKone()
+}
+
+module.exports = {
+  seedAllMigrations
+}

services/backend/oodikone2-backend/src/events.js

@@ -0,0 +1,117 @@
+var stan = require('node-nats-streaming').connect(
+  'updaterNATS',
+  process.env.HOSTNAME,
+  process.env.NATS_URI
+)
+
+const { refreshAssociationsInRedis } = require('./services/studyrights')
+const { getAllProgrammes } = require('./services/studyrights')
+const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studytrack')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
+
+const taskStatuses = { }
+const handleMessage = (asyncHandlerFunction) => async (msg) => {
+  const key = msg.getSubject()+'_'+msg.getData()
+  if (taskStatuses[key])
+    return
+  stan.publish('status', key+':RECEIVED', (err) => {
+    if(err) {
+      console.log('publish failed', err)
+    }
+  })
+  taskStatuses[key] = true
+  const success = await asyncHandlerFunction(msg)
+  taskStatuses[key] = false
+  stan.publish('status', key+(success ? ':DONE' : ':ERRORED'), (err) => {
+    if(err) {
+      console.log('publish failed', err)
+    }
+  })
+  msg.ack()
+}
+
+const StartNats = () => {
+  console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
+  var opts = stan.subscriptionOptions()
+  opts.setManualAckMode(true)
+  // opts.setAckWait(6*60*60*1000)
+  // opts.setDeliverAllAvailable()
+  // opts.setDurableName('durable')
+  opts.setMaxInFlight(1)
+  stan.on('connect', async function() {
+    stan.subscribe('RefreshStudyrightAssociations', opts).on('message', handleMessage(async () => {
+      try {
+        await refreshAssociationsInRedis()
+      } catch (e) {
+        console.error(e)
+        return false
+      }
+      return true
+    }))
+    stan.subscribe('RefreshOverview', opts).on('message', handleMessage(async () => {
+      try {
+        console.log('RefreshOverview starting')
+        const codes = (await getAllProgrammes()).map(p => p.code)
+
+        let ready = 0
+        for(const code of codes) {
+          try {
+            await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+            const since = new Date().getFullYear() - 5
+            const data = await throughputStatsForStudytrack(
+              code,
+              since
+            )
+            await setThroughput(
+              data
+            )
+          } catch (e) {
+            try {
+              await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+            } catch (e) {
+              console.error(
+                e
+              )
+            }
+            console.log(
+              `Failed to update throughput stats for code: ${code}, reason: ${
+                e.message
+              }`
+            )
+          }
+          try {
+            await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+            const since = '2017-08-01'
+            const data = await productivityStatsForStudytrack(code, since)
+            await setProductivity(data)
+
+          } catch (e) {
+            try {
+              await patchProductivity({
+                [code]: { status: 'RECALCULATION ERRORED' }
+              })
+            } catch (e) {
+              console.error(e)
+            }
+            console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+          }
+          ready += 1
+          console.log(
+            `RefreshOverview ${ready}/${
+              codes.length
+            } done`
+          )
+        }
+      } catch (e) {
+        console.error(e)
+        return false
+      }
+      return true
+    }))
+  })
+  return stan
+}
+
+module.exports = {
+  StartNats
+}

services/backend/oodikone2-backend/src/routes.js

@@ -4,6 +4,7 @@ const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
 const login = require('./routes/login')
+const superlogin = require('./routes/superlogin')
 const language = require('./routes/language')
 const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
@@ -11,32 +12,41 @@ const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const usage = require('./routes/usage')
 const providers = require('./routes/providers')
+const faculties = require('./routes/faculties')
 const semesters = require('./routes/semesters')
 const oodilearn = require('./routes/oodilearn')
 const courseGroups = require('./routes/courseGroups')
 const mandatoryCourses = require('./routes/mandatorycourses')
+const mandatoryCourseLabels = require('./routes/mandatorycourselabels')
 const ping = require('./routes/ping')
 const oodi = require('./routes/oodi')
 const task = require('./routes/tasks')
+const feedback = require('./routes/feedback')
+const tags = require('./routes/tags')
 
 module.exports = (app, url) => {
   app.use(url, log)
   app.use(url, login)
   app.use(url, ping)
   app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger)
+  app.use(`${url}/superlogin`, auth.roles(['admin']), superlogin)
   app.use(url, elementdetails)
   app.use(url, courses)
   app.use(url, students)
   app.use(url, population)
   app.use(url, language)
   app.use(url, providers)
+  app.use(url, faculties)
   app.use(url, semesters)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['users']), users)
+  app.use(`${url}/feedback`, feedback)
   app.use(`${url}/usage`, auth.roles(['usage']), usage)
   app.use(`${url}/oodilearn`, auth.roles(['oodilearn']), oodilearn)
   app.use(`${url}/course-groups`, auth.roles(['coursegroups']), courseGroups)
   app.use(`${url}/mandatory_courses`, mandatoryCourses),
+  app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels),
   app.use(`${url}/oodi`, auth.roles(['dev']), oodi)
   app.use(url, auth.roles(['dev', 'admin']), task)
+  app.use(url, tags)
 }

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -21,7 +21,7 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
     res.json(codes)
   } else {
-    res.status(422)
+    res.status(422).end()
   }
 })
 
@@ -30,7 +30,7 @@ router.get('/v2/studyprogrammes/ping', async (req, res) => {
     const result = await ping()
     res.json(result)
   } catch (e) {
-    res.status(500)
+    res.status(500).end()
   }
 })
 
@@ -46,15 +46,15 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
     if (!data) {
       try {
         const since = '2017-08-01'
-        data = await productivityStatsForStudytrack(code, since)
-        setProductivity(data)
+        const stats = await productivityStatsForStudytrack(code, since)
+        data = await setProductivity(stats)
       } catch (e) {
         console.error(e)
       }
     }
-    return res.json({ ...data })
+    return res.json(data)
   } else {
-    res.status(422)
+    res.status(422).end()
   }
 })
 
@@ -110,15 +110,15 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
     if (!data) {
       try {
         const since = req.params.since ? req.params.since : new Date().getFullYear() - 5
-        data = await throughputStatsForStudytrack(req.params.id, since)
-        setThroughput(data)
+        const result = await throughputStatsForStudytrack(req.params.id, since)
+        data = await setThroughput(result)
       } catch (e) {
         console.error(e)
       }
     }
     return res.json({ ...data, thesis: await thesisPromise })
   } else {
-    res.status(422)
+    res.status(422).end()
   }
 })
 
@@ -128,10 +128,8 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   console.log('Throughput stats recalculation starting')
   const codes = code ? [code] : (await getAllProgrammes()).map(p => p.code)
   try {
-    await patchThroughput(codes.reduce((acc, c) => {
-      acc[c] = { status: 'RECALCULATING' }
-      return acc
-    }, {}))
+    const data = codes.reduce((acc, id) => ({ ...acc, [id]: { status: 'RECALCULATING' }}), {})
+    await patchThroughput(data)
     res.status(200).end()
   } catch (e) {
     console.error(e)
@@ -146,9 +144,7 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
       await setThroughput(data)
     } catch (e) {
       try {
-        await patchThroughput({
-          [code]: { status: 'RECALCULATION ERRORED' }
-        })
+        await patchThroughput({ [code]:  { status: 'RECALCULATION ERRORED' }})
       } catch (e) {
         console.error(e)
         return
@@ -166,7 +162,7 @@ router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
     const thesis = await findProgrammeTheses(id)
     res.json(thesis)
   } else {
-    res.status(422)
+    res.status(422).end()
   }
 })
 
@@ -177,7 +173,7 @@ router.post('/v2/studyprogrammes/:id/thesis', async (req, res) => {
     const thesis = await createThesisCourse(id, course, thesisType)
     res.status(201).json(thesis)
   } else {
-    res.status(422)
+    res.status(422).end()
   }
 })
 
@@ -187,8 +183,8 @@ router.delete('/v2/studyprogrammes/:id/thesis/:course', async (req, res) => {
     const deleted = await deleteThesisCourse(id, course)
     res.status(204).json(deleted)
   } else {
-    res.status(422)
+    res.status(422).end()
   }
 })
 
-module.exports = router
+module.exports = router

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -0,0 +1,9 @@
+const router = require('express').Router()
+const { all } = require('../services/organisations')
+
+router.get('/faculties', async (req, res) => {
+  const faculties = await all()
+  res.json(faculties)
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/feedback.js

@@ -0,0 +1,26 @@
+const router = require('express').Router()
+const mailservice = require('../services/mailservice')
+const userService = require('../services/userService')
+
+router.post('/email', async (req, res) => {
+  const { content } = req.body
+  const { uid } = req.headers
+  const { email, full_name } = await userService.byUsername(uid)
+  const formattedEmail = content.split('\n\n').map(line => `<p> ${line} </p>`).join('')
+  const feedback = mailservice.feedback(formattedEmail, uid, email, full_name)
+
+  if (process.env.SMTP !== undefined && content) {
+    await mailservice.transporter.sendMail(feedback, (error) => {
+      if (error) {
+        console.log('Error occured')
+        res.status(400).json(error).end()
+      } else {
+        console.log('Message sent succesfully!')
+        res.status(200).json('success').end()
+      }
+      mailservice.transporter.close()
+    })
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/login.js

@@ -27,7 +27,7 @@ router.post('/login', async (req, res) => {
       const mail = req.headers.mail || ''
       const hyGroups = parseHyGroups(req.headers['hygroupcn'])
       const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-   
+
       console.log(uid, 'trying to login, referring to userservice.')
       let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail)
       isNew && sendEmail(uid)
@@ -44,26 +44,6 @@ router.post('/login', async (req, res) => {
   }
 })
 
-router.post('/superlogin/:uid', async (req, res) => {
-  try {
-    const asUser = req.params.uid
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      console.log('super')
-      const token = await userService.superlogin(uid, asUser)
-      res.status(200).json({ token })
-    } else {
-      res.status(401).json({
-        message: `Not enough headers login, uid:
-        ${req.headers.uid} session-id ${req.headers['shib-session-id']}`
-      }).end()
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(401).json({ message: 'problem with login', err })
-  }
-})
-
 router.delete('/logout', async (req, res) => {
   try {
     const logoutUrl = req.headers.shib_logout_url

services/backend/oodikone2-backend/src/routes/mandatorycourselabels.js

@@ -0,0 +1,70 @@
+const router = require('express').Router()
+const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
+
+router.get('/:programme/labels', async (req, res) => {
+  const { programme } = req.params
+  if (programme) {
+    try {
+      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
+      res.status(200).json(labels)
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.delete('/:programme/labels', async (req, res) => {
+  const { programme } = req.params
+  const { label } = req.body
+  if (programme && label) {
+    try {
+      await MandatoryCourseLabels.destroy(programme, label)
+      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
+      res.status(200).json(labels)
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/:programme/move', async (req, res) => {
+  const { programme } = req.params
+  const { label, direction } = req.body
+  if (programme && label) {
+    try {
+      await MandatoryCourseLabels.move(programme, label, direction)
+      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
+      res.status(200).json(labels)
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/:programme/labels', async (req, res) => {
+  const { programme } = req.params
+  const { label } = req.body
+  if (programme && label) {
+    try {
+      await MandatoryCourseLabels.create(programme, label)
+      const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
+      res.status(200).json(labels)
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/mandatorycourses.js

@@ -0,0 +1,60 @@
+const router = require('express').Router()
+const MandatoryCourses = require('../services/mandatoryCourses')
+
+router.delete('/:programme', async (req, res) => {
+  const { programme } = req.params
+  const { course } = req.body
+  if (programme && course) {
+    try {
+      await MandatoryCourses.remove(programme, course)
+      res.status(200).json(course)
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/:programme', async (req, res) => {
+  const { programme } = req.params
+  const { course } = req.body
+  if (programme && course) {
+    try {
+      await MandatoryCourses.create(programme, course)
+      const newCourse = await MandatoryCourses.find(programme, course)
+      res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.post('/:programme/label/:course', async (req, res) => {
+  const { programme, course } = req.params
+  const { label } = req.body
+  if (programme && course) {
+    try {
+      await MandatoryCourses.updateLabel(programme, course, label)
+      const updatedCourse = await MandatoryCourses.find(programme, course)
+      res
+        .status(200)
+        .json({
+          name: updatedCourse.course.name,
+          code: updatedCourse.course_code,
+          label: updatedCourse.mandatory_course_label
+        })
+    } catch (err) {
+      console.error(err)
+      res.status(400).json(err.message)
+    }
+  } else {
+    res.status(422).end()
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/population.js

@@ -5,7 +5,7 @@ const { updateStudents } = require('../services/doo_api_database_updater/databas
 const StudyrightService = require('../services/studyrights')
 
 // POST instead of GET because of too long params and "sensitive" data
-router.post('/v2/populationstatistics/courses', async (req, res) => { 
+router.post('/v2/populationstatistics/courses', async (req, res) => {
   try {
     if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
       res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
@@ -32,34 +32,36 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
 })
 
 router.get('/v3/populationstatistics', async (req, res) => {
-  console.log(req.query.year, req.query.semesters, req.query.studyRights)
+  const { year, semesters, studyRights: studyRightsJSON } = req.query
   try {
-    if (!req.query.year || !req.query.semesters || !req.query.studyRights) {
-      res.status(400).json({ error: 'The query should have a year, semester and study rights defined' })
+    if (!year || !semesters || !studyRightsJSON) {
+      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
       return
     }
-    if (!Array.isArray(req.query.studyRights)) { // studyRights should always be an array
-      req.query.studyRights = [req.query.studyRights]
-    }
-    req.query.studyRights = req.query.studyRights.filter(sr => sr !== 'undefined')
-    const roles = req.decodedToken.roles
-    if (!roles || !roles.map(r => r.group_code).includes('admin')) {
-      const elements = new Set(req.decodedToken.rights)
-      if (req.query.studyRights.some(code => !elements.has(code))) {
-        res.status(403).json([])
-        return
+    let studyRights = null
+    try {
+      studyRights = JSON.parse(studyRightsJSON)
+      const { roles, rights } = req.decodedToken
+      if (!roles || !roles.map(r => r.group_code).includes('admin')) {
+        if (!rights.includes(studyRights.programme)) {
+          res.status(403).json([])
+          return
+        }
       }
+    } catch (e) {
+      console.error(e)
+      res.status(400).json({ error: 'The query had invalid studyRights' })
+      return
     }
-    
+
     if (req.query.months == null) {
       req.query.months = 12
     }
-
-    const result = await Population.optimizedStatisticsOf(req.query)
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
 
     if (result.error) {
       console.log(result.error)
-      res.status(400)
+      res.status(400).end()
       return
     }
 

services/backend/oodikone2-backend/src/routes/superlogin.js

@@ -0,0 +1,24 @@
+const router = require('express').Router()
+const userService = require('../services/userService')
+
+router.post('/:uid', async (req, res) => {
+  try {
+    const asUser = req.params.uid
+    const uid = req.decodedToken.mockedBy == null ? req.headers['uid'] : req.decodedToken.mockedBy
+    if (req.headers['shib-session-id'] && uid) {
+      console.log('super')
+      const token = await userService.superlogin(uid, asUser)
+      res.status(200).json({ token })
+    } else {
+      res.status(401).json({
+        message: `Not enough headers login, uid:
+        ${req.headers.uid} session-id ${req.headers['shib-session-id']}`
+      }).end()
+    }
+  } catch (err) {
+    console.log(err.message)
+    res.status(401).json({ message: 'problem with login', err: err.message })
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/tags.js

@@ -0,0 +1,104 @@
+const router = require('express').Router()
+const Tags = require('../services/tags')
+const TagStudent = require('../services/tagstudent')
+
+router.get('/tags', async (req, res) => {
+  try {
+    const tags = await Tags.findTags()
+    res.status(200).json(tags)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+router.get('/tags/:studytrack', async (req, res) => {
+  const { studytrack } = req.params
+  try {
+    const tags = await Tags.findTagsByStudytrack(studytrack)
+    res.status(200).json(tags)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+router.post('/tags', async (req, res) => {
+  const { tag } = req.body
+  try {
+    const result = await Tags.createNewTag(tag)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err.message)
+  }
+})
+
+router.delete('/tags', async (req, res) => {
+  const { tag } = req.body
+  try {
+    const result = await Tags.deleteTag(tag)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+
+})
+
+router.get('/studenttags', async (req, res) => {
+  try {
+    const result = await TagStudent.getStudentTags()
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+router.get('/studenttags/:studytrack', async (req, res) => {
+  const { studytrack } = req.params
+  try {
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+router.get('/studenttags/:studentnumber', async (req, res) => {
+  const { studentnumber } = req.params
+  try {
+    const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
+    console.log(result)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+router.post('/studenttags', async (req, res) => {
+  const { tag } = req.body
+  try {
+    const result = await TagStudent.createStudentTag(tag)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+router.delete('/studenttags', async (req, res) => {
+  const { id } = req.body
+  try {
+    const result = await TagStudent.deleteStudentTag(id)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(400).json(err)
+  }
+})
+
+module.exports = router

services/backend/oodikone2-backend/src/routes/teachers.js

@@ -20,6 +20,12 @@ router.get('/top', async (req, res) => {
   res.json(result)
 })
 
+router.post('/top', async (req, res) => {
+  const { startyearcode, endyearcode } = req.body
+  res.status(200).end()
+  await topteachers.findAndSaveTeachers(startyearcode, endyearcode)
+})
+
 router.get('/top/categories', async (req, res) => {
   const result = await topteachers.getCategoriesAndYears()
   res.json(result)

services/backend/oodikone2-backend/src/routes/users.js

@@ -63,4 +63,12 @@ router.delete('/:uid/elements', async (req, res) => {
   res.json(user)
 })
 
+router.post('/:uid/faculties', async (req, res) => {
+  const { uid } = req.params
+  const { faculties } = req.body
+  const user = await userService.setFaculties(uid, faculties)
+  if (user && user.username != req.decodedToken.userId) await blacklist.addUserToBlacklist(user.username)
+  res.json(user)
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/services/courses.js

@@ -327,7 +327,7 @@ const yearlyStatsOf = async (code, year, separate, language) => {
     }, resultProgrammes)
 
   const alternatives = await getDuplicateCodes(code)
-  const codes = alternatives ? [code, ...Object.keys(alternatives.alt)] : [code]
+  const codes = alternatives ? alternatives : [code]
   const allInstances = await creditsOf(codes)
   const yearInst = allInstances
     .filter(inst => moment(new Date(inst.date)).isBetween(year.start + '-09-01', year.end + '-08-01'))
@@ -403,11 +403,22 @@ const getMainCodes = () => {
 }
 
 const deleteDuplicateCode = async (code) => {
-  await CourseDuplicates.destroy({
-    where: {
-      coursecode: code
-    }
-  })
+  try {
+    await CourseDuplicates.destroy({
+      where: {
+        coursecode: code
+      }
+    })
+    await sequelize.query(
+      `DELETE FROM course_duplicates
+      WHERE groupid in ( SELECT groupid FROM course_duplicates
+      group by groupid
+      having count(*) = 1)`,
+      { type: sequelize.QueryTypes.BULKDELETE }
+    )
+  } catch (e) {
+    console.error(e)
+  }
 }
 
 const getDuplicateCodesWithCourses = () => {
@@ -558,7 +569,7 @@ const getAllDisciplines = () => Discipline.findAll()
 
 const alternativeCodes = async code => {
   const alternatives = await getDuplicateCodes(code)
-  return alternatives ? [code, ...Object.keys(alternatives.alt)] : [code]
+  return alternatives ? alternatives : [code]
 }
 
 const formatStudyrightElement = ({ code, element_detail }) => ({