Merge pull request #1089 from UniversityOfHelsinkiCS/trunk

Bug when clearing comparison group in course statistic when "All" selected

.gitignore

@@ -1,5 +1,6 @@
 data
 pgdata
+pgdata_kone
 user_pgdata
 analytics_pgdata
 usage_pgdata
@@ -26,4 +27,4 @@ services/updater_scheduler/active_student_numbers.txt
 services/oodikone2-backend/debug.log
 datastore
 services/updater_api/debug.log
-services/backend/updater_writer/dumps
+services/backend/updater_writer/dumps

.travis.yml

@@ -20,48 +20,53 @@ before_script:
 
 jobs:
   include:
-    - stage: build & test & push & deploy
-      if: NOT tag IS present
+    - stage: build & test & push & deploy staging
       env:
         - IMAGE=STAGING
+        - TAG=staging
       script:
         - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.yml
+        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG}
         - docker ps
         - cat nginx/log
-        - docker-compose logs -f &
+        - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
         - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
         - time npm run test_services
         - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
         - docker ps -a
         - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
+      deploy:
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
+        script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
         on:
           branch: master
-    - # in parallel
-      if: type = pull_request OR NOT branch = master
+    - stage: build & test & push & deploy latest
       env:
         - IMAGE=PRODUCTION
+        - TAG=latest
       script:
         - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source scripts.sh && time run_e2e_setup docker-compose.lateste2e.production.yml
+        - source scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG}
         - docker ps
         - cat nginx/log
-        - docker-compose logs -f &
+        - TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
         - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
         - time npm run test_services
         - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
         - docker ps -a
         - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk '{ sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy: # deploy is always skipped for PR
+      deploy:
         provider: script
         skip_cleanup: true
-        script: "docker-compose -f docker-compose.lateste2e.production.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
+        script: "TAG=${TAG} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
         on:
           tags: true
+stages:
+  - name: build & test & push & deploy staging
+    if: NOT tag IS present
+  - name: build & test & push & deploy latest
+    if: tag IS present
 
 after_failure:
   - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

README.md

@@ -58,27 +58,27 @@ The mapping of ports and environment variables are also defined in the docker-co
 #### With anonymized data:
 
 ```
-docker-compose up -d
+npm run docker:up
 ```
 
 #### With real data:
 
 ```
-docker-compose -f docker-compose.yml -f docker-compose.real.yml up -d
+docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml up -d
 ```
 
 ### Stop the development environment
 
 ```
-docker-compose down
+npm run docker:down
 ```
 
 ### Restart container(s)
 
 ```
-docker-compose restart          # all
+npm run docker:restart          # all
 
-docker-compose restart backend  # just backend
+npm run docker:restart:backend  # just backend
 ```
 
 ### View the containers in the running environment
@@ -90,9 +90,9 @@ docker-compose ps
 
 
 ```
-docker-compose logs -f          # all
+npm run docker:logs          # all
 
-docker-compose logs -f backend  # just backend
+npm run docker:logs:backend  # just backend
 ```
 
 

adminer-plugins/login-servers.php

@@ -6,6 +6,6 @@ require_once('plugins/login-servers.php');
     * @param string
     */
 return new AdminerLoginServers(
-    $servers = array("db" => "db", "user_db" => "user_db", "analytics_db" => "analytics_db"),
+    $servers = array("db" => "db", "db_kone" => "db_kone", "user_db" => "user_db", "analytics_db" => "analytics_db"),
     $driver = "pgsql"
 );

cypress/integration/Users.js

@@ -29,6 +29,7 @@ describe('Users tests', () => {
 
     cy.get('i.spy').click()
     cy.contains("mocking as normk")
+    cy.wait(1000)
     cy.contains("Study programme").click().siblings().contains("Search by class").click()
     cy.contains("label", "Study programme")
     cy.contains("label", "Study programme").siblings().within(($row) => {

docker-compose.lateste2e.production.yml

@@ -1,236 +0,0 @@
-version: '3'
-
-services:
-  nats:
-    image: nats-streaming
-    command: -cid updaterNATS
-    expose:
-      - "4222"
-    container_name: nats
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
-
-  user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=user_db
-
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
-
-  backend:
-    image: toska/oodikone2-backend:latest
-    build:
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    ports:
-      - "8080:8080"
-      - "9229:9229"
-    environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      FRONT_URL: http://localhost:1337
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
-
-  frontend:
-    image: toska/oodikone2-frontend:latest
-    build:
-      context: services/oodikone2-frontend
-      args:
-        BASE_PATH: /
-    ports:
-      - "5000:5000"
-    environment:
-      WAIT_HOSTS: user_db:5432
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
-
-  userservice:
-    image: toska/oodikone2-userservice:latest
-    build:
-      context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      - WAIT_HOSTS=user_db:5432
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
-
-  usageservice:
-    image: toska/oodikone2-usageservice:latest
-    build:
-      context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      - WAIT_HOSTS=usage_db:5432
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  analytics:
-    image: toska/oodikone2-analytics:latest
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-      - analytics_redis
-    container_name: analytics
-
-  updater_api:
-    image: toska/updater_api:latest
-    build:
-      context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - WAIT_HOSTS=nats:4222
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
-
-  updater_writer:
-    image: toska/updater_writer:latest
-    build:
-      context: services/backend
-      dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      WAIT_HOSTS: nats:4222,db:5432
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo
-    volumes:
-      - ./mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_db
-
-  updater_scheduler:
-    image: toska/updater_scheduler:latest
-    build:
-      context: services/updater_scheduler
-    command: ["echo", "mluukkai on orjuuttaja"]
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_scheduler
-
-  nginx:
-    image: nginx:latest
-    volumes:
-      - ./nginx.production.conf:/etc/nginx/nginx.conf
-      - ./nginx/error.log:/etc/nginx/error_log.log
-      - ./nginx/log:/var/log/nginx/localhost.error_log
-      - ./nginx/cache/:/etc/nginx/cache
-      - /etc/letsencrypt/:/etc/letsencrypt/
-    ports:
-      -  1337:80
-    depends_on:
-      - frontend
-      - backend
-      - userservice
-      - analytics
-
-    container_name: nginx

docker-compose.lateste2e.yml

@@ -1,238 +0,0 @@
-version: '3'
-
-services:
-  nats:
-    image: nats-streaming
-    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
-    expose:
-      - "4222"
-    volumes:
-      - ./datastore:/datastore
-    container_name: nats
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - ./analytics_pgdata:/var/lib/postgresql/data
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=analytics_db
-
-  user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - ./user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=user_db
-
-  db:
-    image: postgres:9.6.3
-    ports:
-      - "5421:5432"
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-data:/data
-    container_name: redis
-
-  analytics_redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6380:6379"
-    volumes:
-      - ./analytics-redis-data:/data
-    container_name: analytics_redis
-
-  backend:
-    image: toska/oodikone2-backend:staging
-    build:
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    ports:
-      - "8080:8080"
-      - "9229:9229"
-    environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      FRONT_URL: http://localhost:1337
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      OODILEARN_URL: http://oodilearn:5000
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
-
-  frontend:
-    image: toska/oodikone2-frontend:staging
-    build:
-      context: services/oodikone2-frontend
-      args:
-        BASE_PATH: /
-    ports:
-      - "5000:5000"
-    environment:
-      WAIT_HOSTS: user_db:5432
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
-    
-  userservice:
-    image: toska/oodikone2-userservice:staging
-    build:
-      context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      - WAIT_HOSTS=user_db:5432
-      - DB_URL=postgres://postgres@user_db:5432/user_db
-      - TEST_DB=postgres://postgres@user_db:5432/user_db_test
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      - TOKEN_SECRET=IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
-
-  usageservice:
-    image: toska/oodikone2-usageservice:staging
-    build:
-      context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      - WAIT_HOSTS=usage_db:5432
-      - PORT=545
-      - DB_URL=postgres://postgres@usage_db:5432/usage_db
-      - SECRET=Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - ./usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  analytics:
-    image: toska/oodikone2-analytics:staging
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
-      - REDIS=analytics_redis
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-      - analytics_redis
-    container_name: analytics
-
-  updater_api:
-    image: toska/updater_api:staging
-    build:
-      context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - WAIT_HOSTS=nats:4222
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_api
-
-  updater_writer:
-    image: toska/updater_writer:staging
-    build:
-      context: services/backend
-      dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      WAIT_HOSTS: nats:4222,db:5432
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo
-    volumes:
-      - ./mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_db
-
-  updater_scheduler:
-    image: toska/updater_scheduler:staging
-    build:
-      context: services/updater_scheduler
-    command: ["echo", "mluukkai on orjuuttaja"]
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - STUDENT_NUMBERS=./all_student_numbers.txt
-      - ACTIVE_STUDENTS=./active_student_numbers.txt
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN=toskab1gs8cret5
-    container_name: updater_scheduler
-
-  nginx:
-    image: nginx:latest
-    volumes:
-      - ./nginx.staging.conf:/etc/nginx/nginx.conf
-      - ./nginx/error.log:/etc/nginx/error_log.log
-      - ./nginx/log:/var/log/nginx/localhost.error_log
-      - ./nginx/cache/:/etc/nginx/cache
-      - /etc/letsencrypt/:/etc/letsencrypt/
-    ports:
-      -  1337:80
-    depends_on:
-      - frontend
-      - backend
-      - userservice
-      - analytics
-
-    container_name: nginx

docker-compose.yml

@@ -1,43 +1,17 @@
 version: '3'
 
 services:
-  # zookeeper:
-  #   image: wurstmeister/zookeeper
-  #   ports:
-  #     - "2181:2181"
-  #   environment:
-  #     - KAFKA_BROKER_ID=1
-  #   container_name: zookeeper
-
-  # kafka:
-  #   image: wurstmeister/kafka
-  #   container_name: kafka
-  #   depends_on:
-  #     - zookeeper
-  #   ports:
-  #     - "9092:9092"
-  #   environment:
-  #     - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}'
-  #     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-  #     - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
-  #     - KAFKA_DELETE_TOPIC_ENABLE=true
-  #     - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1
-  #     - KAFKA_LOG_DIRS=/kafka/kafka-logs-1
-  #   volumes:
-  #     - ./kafka/docker.sock:/var/run/docker.sock
-  #     - ./kafka/logs:/kafka/kafka-logs-1
-
   nats:
     image: nats-streaming
-    command: -cid updaterNATS  --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
+    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
     expose:
       - "4222"
-    volumes:
-      - ./datastore:/datastore
     ports:
       - "8222:8222"
       - "4222:4222"
     container_name: nats
+    volumes:
+      - ./datastore:/datastore
 
   analytics_db:
     image: postgres:9.6.3
@@ -59,7 +33,6 @@ services:
     container_name: oodi_user_db
     environment:
       - POSTGRES_USER=postgres
-      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
 
   usage_db:
     image: postgres:9.6.3
@@ -81,7 +54,18 @@ services:
       - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
     container_name: oodi_db
     environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+      - POSTGRES_USER=postgres
+
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - ./pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
       - POSTGRES_USER=postgres
 
   redis:
@@ -93,32 +77,19 @@ services:
       - ./redis-data:/data
     container_name: redis
 
-  # mongo_db:
-  #   image: mongo
-  #   restart: unless-stopped
-  #   volumes:
-  #     - ./backups/mongo/oodilearn:/dump
-  #     - ./mongodata:/data/db
-  #   ports:
-  #     - "27017:27017"
-  #   container_name: mongo_db
-
   backend:
     build:
       context: services/backend
       dockerfile: oodikone2-backend/Dockerfile
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
     ports:
       - "8080:8080"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,nats:4222
+      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      FRONT_URL: http://localhost:8000
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
       TOKEN_SECRET: IWannaBeTheVeryBest
       OODILEARN_URL: http://oodilearn:5000
       USERSERVICE_URL: http://userservice:4567
@@ -127,12 +98,7 @@ services:
       USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
-      DB_SCHEMA_KONE: kone_data
-    volumes:
-      - ./services/backend/oodikone2-backend/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/src/models
-      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+      UPDATER_URL: http://updater_scheduler:3678
 
     container_name: backend
     depends_on:
@@ -143,48 +109,14 @@ services:
   frontend:
     build:
       context: services/oodikone2-frontend
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker']
-    restart: unless-stopped
-    ports:
-      - "8081:8081"
-      - "7777:7777"
     environment:
       BACKEND_ADDR: backend
       ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres
-    volumes:
-      - ./services/oodikone2-frontend/:/usr/src/app
     container_name: frontend
 
-  # oodilearn:
-  #   build:
-  #     context: services/oodilearn/server
-  #     dockerfile: Dockerfile
-  #   restart: unless-stopped
-  #   volumes:
-  #     - ./services/oodilearn/data:/data
-  #     - ./services/oodilearn/server:/src
-  #     - ./services/oodilearn/models:/models
-  #   ports:
-  #     - "5000:5000"
-  #   environment:
-  #     - ENV=development
-  #     - MONGO_URI=mongodb://mongo_db:27017/oodilearn
-  #   container_name: oodilearn
-
   userservice:
     build:
       context: services/oodikone2-userservice
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/oodikone2-userservice/:/usr/src/app
     ports:
       - 4567:4567
     environment:
@@ -200,10 +132,6 @@ services:
   usageservice:
     build:
       context: services/oodikone2-usageservice
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
     ports:
       - 545:545
     environment:
@@ -211,8 +139,6 @@ services:
       PORT: '545'
       DB_URL: postgres://postgres@usage_db:5432/usage_db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    volumes:
-      - ./services/oodikone2-usageservice/:/usr/src/app
     depends_on:
       - usage_db
     container_name: usageservice
@@ -220,12 +146,6 @@ services:
   updater_api:
     build:
       context: services/updater_api
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/updater_api/:/usr/src/app
     depends_on:
       - nats
     environment:
@@ -239,16 +159,6 @@ services:
     build:
       context: services/backend
       dockerfile: updater_writer/Dockerfile
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/backend/updater_writer/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/models
-      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
-
     depends_on:
       - nats
       - db
@@ -258,28 +168,19 @@ services:
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
       TEST_DB_SCHEMA: updater_test
-      DB_SCHEMA_KONE: kone_data
-
-
+    container_name: updater_writer
 
   scheduler_mongo:
     image: mongo
-    restart: unless-stopped
     volumes:
       - ./mongodata:/data/db
     expose:
       - "27017"
-    container_name: scheduler_db
+    container_name: scheduler_mongo
 
   updater_scheduler:
     build:
       context: services/updater_scheduler
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/updater_scheduler/:/usr/src/app
     depends_on:
       - nats
       - scheduler_mongo
@@ -294,26 +195,19 @@ services:
       - LOG_HOST=toska.cs.helsinki.fi
       - LOG_PORT=9501
       - LOG_HOSTNAME=updater_scheduler
-
+      - PORT=3678
     container_name: updater_scheduler
 
-
   analytics:
     build:
       context: services/oodikone2-analytics
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
-    volumes:
-      - ./services/oodikone2-analytics/:/usr/src/app
     ports:
       - 4568:4568
     environment:
-      WAIT_HOSTS: analytics_db:5432,analytics_redis:6379
-      REDIS: analytics_redis
-      DB_URL: postgres://postgres@analytics_db:5432/analytics_db
-      TEST_DB: postgres://postgres@analytics_db:5432/analytics_db_test
+      - WAIT_HOSTS=analytics_db:5432,analytics_redis:6379
+      - REDIS=analytics_redis
+      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
+      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
     depends_on:
       - analytics_db
       - analytics_redis
@@ -327,14 +221,3 @@ services:
     volumes:
       - ./analytics-redis-data:/data
     container_name: analytics_redis
-
-  adminer:
-    image: adminer:4.6.2
-    environment:
-      - ADMINER_DESIGN=pepa-linha
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-    ports:
-      - 5050:8080
-    restart: always
-    container_name: adminer

docker/docker-compose.dev.real.yml

@@ -4,6 +4,7 @@ services:
   backend:
     environment:
       DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
   userservice:
     environment:
       DB_URL: postgres://postgres@user_db:5432/user_db_real

docker/docker-compose.dev.yml

@@ -0,0 +1,170 @@
+version: '3'
+
+services:
+  user_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+
+  backend:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    environment:
+      FRONT_URL: http://localhost:8000
+    volumes:
+      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/src/models
+      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+
+  frontend:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker']
+    restart: unless-stopped
+    ports:
+      - "8081:8081"
+      - "7777:7777"
+    environment:
+      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
+      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+      USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres
+      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
+    volumes:
+      - ./services/oodikone2-frontend/:/usr/src/app
+
+  userservice:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/oodikone2-userservice/:/usr/src/app
+
+  usageservice:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/oodikone2-usageservice/:/usr/src/app
+
+  updater_api:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/updater_api/:/usr/src/app
+
+  updater_writer:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/backend/updater_writer/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/models
+      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
+    environment:
+      DB_SCHEMA_KONE: kone_data
+
+  scheduler_mongo:
+    restart: unless-stopped
+
+  updater_scheduler:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/updater_scheduler/:/usr/src/app
+    environment:
+      - LOG_HOST=toska.cs.helsinki.fi
+      - LOG_PORT=9501
+      - LOG_HOSTNAME=updater_scheduler
+
+  analytics:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
+    restart: unless-stopped
+    volumes:
+      - ./services/oodikone2-analytics/:/usr/src/app
+
+  adminer:
+    image: adminer:4.6.2
+    environment:
+      - ADMINER_DESIGN=pepa-linha
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+    ports:
+      - 5050:8080
+    restart: always
+    container_name: adminer
+
+  # zookeeper:
+  #   image: wurstmeister/zookeeper
+  #   ports:
+  #     - "2181:2181"
+  #   environment:
+  #     - KAFKA_BROKER_ID=1
+  #   container_name: zookeeper
+
+  # kafka:
+  #   image: wurstmeister/kafka
+  #   container_name: kafka
+  #   depends_on:
+  #     - zookeeper
+  #   ports:
+  #     - "9092:9092"
+  #   environment:
+  #     - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}'
+  #     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+  #     - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+  #     - KAFKA_DELETE_TOPIC_ENABLE=true
+  #     - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1
+  #     - KAFKA_LOG_DIRS=/kafka/kafka-logs-1
+  #   volumes:
+  #     - ./kafka/docker.sock:/var/run/docker.sock
+  #     - ./kafka/logs:/kafka/kafka-logs-1
+
+  # mongo_db:
+  #   image: mongo
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./backups/mongo/oodilearn:/dump
+  #     - ./mongodata:/data/db
+  #   ports:
+  #     - "27017:27017"
+  #   container_name: mongo_db
+
+  # oodilearn:
+  #   build:
+  #     context: services/oodilearn/server
+  #     dockerfile: Dockerfile
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./services/oodilearn/data:/data
+  #     - ./services/oodilearn/server:/src
+  #     - ./services/oodilearn/models:/models
+  #   ports:
+  #     - "5000:5000"
+  #   environment:
+  #     - ENV=development
+  #     - MONGO_URI=mongodb://mongo_db:27017/oodilearn
+  #   container_name: oodilearn

docker/docker-compose.lateste2e.yml

@@ -0,0 +1,62 @@
+version: '3'
+
+services:
+  user_db:
+    environment:
+      - POSTGRES_DB=user_db
+
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
+
+  backend:
+    image: toska/oodikone2-backend:${TAG}
+    ports:
+      - "9229:9229"
+    environment:
+      FRONT_URL: http://localhost:1337
+      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+
+  frontend:
+    image: toska/oodikone2-frontend:${TAG}
+    build:
+      args:
+        BASE_PATH: /
+    ports:
+      - "5000:5000"
+
+  userservice:
+    image: toska/oodikone2-userservice:${TAG}
+
+  usageservice:
+    image: toska/oodikone2-usageservice:${TAG}
+
+  updater_api:
+    image: toska/updater_api:${TAG}
+
+  updater_writer:
+    image: toska/updater_writer:${TAG}
+
+  updater_scheduler:
+    image: toska/updater_scheduler:${TAG}
+    command: ["echo", "mluukkai on orjuuttaja"]
+
+  analytics:
+    image: toska/oodikone2-analytics:${TAG}
+
+  nginx:
+    image: nginx:latest
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/error.log:/etc/nginx/error_log.log
+      - ./nginx/log:/var/log/nginx/localhost.error_log
+      - ./nginx/cache/:/etc/nginx/cache
+      - /etc/letsencrypt/:/etc/letsencrypt/
+    ports:
+      -  1337:80
+    depends_on:
+      - frontend
+      - backend
+      - userservice
+      - analytics
+    container_name: nginx

nginx.staging.conf

@@ -1,19 +0,0 @@
-events { worker_connections 1024; }
-
-http {
-
-  rewrite_log on;
-  error_log /var/log/nginx/localhost.error_log notice;
-
-  server {
-    listen 80;
-
-    location / {
-      proxy_pass http://frontend:5000/;
-    }
-
-    location /api/ {
-      proxy_pass http://backend:8080/;
-    }
-  }
-}

package.json

@@ -9,7 +9,17 @@
     "cypress:record": "cypress run --config videoUploadOnPasses=false -P ./",
     "concurrently": "concurrently",
     "test": "./run_all_tests.sh",
-    "test_services": "./run_service_tests.sh"
+    "test_services": "./run_service_tests.sh",
+    "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
+    "docker:build": "npm run docker -- build",
+    "docker:up": "npm run docker -- up -d",
+    "docker:down": "npm run docker -- down",
+    "docker:logs": "npm run docker -- logs -f",
+    "docker:logs:backend": "npm run docker -- logs -f backend",
+    "docker:restart": "npm run docker -- restart",
+    "docker:restart:backend": "npm run docker -- restart backend",
+    "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune",
+    "start": "npm run docker:up && npm run docker:logs"
   },
   "husky": {
     "hooks": {

run.sh

@@ -26,7 +26,6 @@ options=(
     "Reset real db"
     "Download latest anon db"
     "Download latest real db"
-    "Full reset"
     "Quit"
 )
 select opt in "${options[@]}"
@@ -34,7 +33,7 @@ do
     case $opt in
         "e2e setup")
             mopo
-            run_e2e_setup docker-compose.lateste2e.yml
+            run_e2e_setup ./docker/docker-compose.lateste2e.yml staging
             ;;
         "Anon setup")
             mopo
@@ -56,9 +55,6 @@ do
         "Download latest real db")
             get_oodikone_server_backup
             ;;
-        "Full reset")
-            purge
-            ;;
         "Quit")
             break
             ;;

scripts.sh

@@ -5,9 +5,15 @@ ANONDB_DIR=anonyymioodi
 BACKUP_DIR=backups
 PSQL_DB_BACKUP="$ANONDB_DIR/anon.sqz"
 USER_DB_BACKUP="$ANONDB_DIR/user-dump.sqz"
+KONE_DB_BACKUP="$ANONDB_DIR/anon_kone.sqz"
 PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
+KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz"
 USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 
+docker-compose-dev () {
+    docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@"
+}
+
 retry () {
     for i in {1..60}
     do
@@ -25,18 +31,6 @@ echo_path () {
     echo $(pwd)
 }
 
-purge () {
-    docker-compose down || echo "docker-compose down failed"
-    git clean -f -fdX
-}
-
-megapurge () {
-    git clean -f -fdX
-    docker stop $(docker ps -q)
-    docker container prune
-    docker rmi $(docker images -q)
-}
-
 get_oodikone_server_backup() {
     scp -r -o ProxyCommand="ssh -W %h:%p melkki.cs.helsinki.fi" oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
 }
@@ -55,19 +49,8 @@ get_anon_oodikone() {
 }
 
 restore_psql_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi -j4 $PSQL_DB_BACKUP
-}
-
-restore_real_psql_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5421 --no-owner -F c --dbname=tkt_oodi_real -j4 $PSQL_REAL_DB_BACKUP
-}
-
-restore_userdb_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db -j4 $USER_DB_BACKUP
-}
-
-restore_real_userdb_from_backup () {
-    time pg_restore -U postgres -h localhost -p 5422 --no-owner -F c --dbname=user_db_real -j4 $USER_REAL_DB_BACKUP
+    docker cp $1 $2:/asd.sqz
+    docker exec -it $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
 }
 
 # oodilearn
@@ -76,19 +59,28 @@ restore_real_userdb_from_backup () {
 # }
 
 ping_psql () {
-    echo "Pinging psql in container $1 with db name $2"
+    drop_psql $1 $2
+    echo "Creating psql in container $1 with db name $2"
     retry docker exec -u postgres $1 pg_isready --dbname=$2
     docker exec -u postgres $1 psql -c "CREATE DATABASE $2" || echo "container $1 DB $2 already exists"
 }
 
+drop_psql () {
+    echo "Dropping psql in container $1 with db name $2"
+    retry docker exec -u postgres $1 pg_isready --dbname=$2
+    docker exec -u postgres $1 psql -c "DROP DATABASE $2" || echo "container $1 DB $2 doesn't exists"
+}
+
 db_setup_full () {
     echo "Restoring PostgreSQL from backup"
-    retry restore_real_psql_from_backup
+    ping_psql "oodi_db" "tkt_oodi_real"
+    restore_psql_from_backup $PSQL_REAL_DB_BACKUP oodi_db tkt_oodi_real
+    ping_psql "db_kone" "db_kone_real"
+    restore_psql_from_backup $KONE_REAL_DB_BACKUP db_kone db_kone_real
+    ping_psql "oodi_user_db" "user_db_real"
+    restore_psql_from_backup $USER_REAL_DB_BACKUP oodi_user_db user_db_real
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
-    echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db_real"
-    retry restore_real_userdb_from_backup
     echo "Database setup finished"
 }
 
@@ -96,47 +88,35 @@ db_anon_setup_full () {
     echo "Restoring PostgreSQL from backup"
     ping_psql "oodi_db" "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
-    retry restore_psql_from_backup
+    restore_psql_from_backup $PSQL_DB_BACKUP oodi_db tkt_oodi
+    ping_psql "db_kone" "db_kone"
+    ping_psql "db_kone" "db_kone_test"
+    restore_psql_from_backup $KONE_DB_BACKUP db_kone db_kone
+    ping_psql "oodi_user_db" "user_db"
+    restore_psql_from_backup $USER_DB_BACKUP oodi_user_db user_db
     # echo "Restoring MongoDB from backup"
     # retry restore_mongodb_from_backup
-    echo "Restore user db from backup"
-    ping_psql "oodi_user_db" "user_db"
-    retry restore_userdb_from_backup
     echo "Database setup finished"
 }
 
 reset_real_db () {
-    docker-compose down
-    docker-compose up -d db user_db
-    ping_psql "oodi_db" "tkt_oodi_real"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi_real"
-    ping_psql "oodi_user_db" "user_db_real"
-    docker exec -u postgres oodi_user_db dropdb "user_db_real"
+    docker-compose-dev down
+    docker-compose-dev up -d db user_db db_kone
     db_setup_full
-    docker-compose down
+    docker-compose-dev down
 }
 
 reset_db () {
-    docker-compose down
-    docker-compose up -d db user_db
-    ping_psql "oodi_db" "tkt_oodi"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi"
-    ping_psql "oodi_db" "tkt_oodi_test"
-    docker exec -u postgres oodi_db dropdb "tkt_oodi_test"
-    ping_psql "oodi_user_db" "user_db"
-    docker exec -u postgres oodi_user_db dropdb "user_db"
+    docker-compose-dev down
+    docker-compose-dev up -d db user_db db_kone
     db_anon_setup_full
-    docker-compose down
+    docker-compose-dev down
 }
 
 install_cli_npm_packages () {
     npm ci
 }
 
-docker_build () {
-    docker-compose up -d --build
-}
-
 show_instructions () {
     cat ./assets/instructions.txt
 }
@@ -151,12 +131,12 @@ run_full_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose build
+    docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose up -d db user_db
+    docker-compose-dev up -d db user_db db_kone
     db_setup_full
     db_anon_setup_full
-    docker-compose down
+    docker-compose-dev down
     show_instructions
 }
 
@@ -168,11 +148,11 @@ run_anon_full_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose build
+    docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose up -d db user_db
+    docker-compose-dev up -d db user_db db_kone
     db_anon_setup_full
-    docker-compose down
+    docker-compose-dev down
     show_instructions
 }
 
@@ -184,10 +164,10 @@ run_e2e_setup () {
     echo "Getting anon backups from the private repository. "
     get_anon_oodikone
     echo "Building images"
-    docker-compose -f $1 build
+    TAG=$2 docker-compose -f docker-compose.yml -f $1 build
     echo "Setup oodikone db from dump."
-    docker-compose -f $1 up -d db user_db
+    TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d db user_db db_kone
     db_anon_setup_full
     echo "Starting services."
-    docker-compose -f $1 up -d
+    TAG=$2 docker-compose -f docker-compose.yml -f $1 up -d
 }

services/backend/oodikone2-backend/src/conf-backend.js

@@ -1,23 +1,27 @@
 require('dotenv').config()
 
 const { NODE_ENV } = process.env
-let DB_URL = process.env.DB_URL
 const isTest = process.env.NODE_ENV === 'test'
+let DB_URL = process.env.DB_URL
+let DB_URL_KONE = process.env.DB_URL_KONE
 if (isTest) {
   DB_URL = process.env.TEST_DB
+  DB_URL_KONE = process.env.TEST_DB_KONE
 }
 else if (NODE_ENV === 'anon') {
   DB_URL = process.env.ANON_DB
+  DB_URL_KONE = process.env.ANON_DB_KONE
 }
 const frontend_addr = process.env.FRONT_URL
 const redis = process.env.REDIS
 const TOKEN_SECRET = process.env.TOKEN_SECRET
 const DB_SCHEMA = process.env.DB_SCHEMA || 'public'
-const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'kone_data'
+const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
 const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
 const OODILEARN_URL = process.env.OODILEARN_URL
 const USERSERVICE_URL = process.env.USERSERVICE_URL
+const UPDATER_URL = process.env.UPDATER_URL
 const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
 const PORT = isTest ? 8079 : 8080
@@ -58,10 +62,26 @@ if (process.env.NODE_ENV === 'dev' || isTest) {
 }
 
 module.exports = {
-  frontend_addr, DB_URL, redis, TOKEN_SECRET, DB_SCHEMA, DB_SCHEMA_KONE, OODI_ADDR, CERT_PATH, KEY_PATH, FEATURES, OODILEARN_URL,
-  USERSERVICE_URL: formatURL(USERSERVICE_URL), ACCESS_TOKEN_HEADER_KEY, PORT,
+  frontend_addr,
+  DB_URL,
+  DB_URL_KONE,
+  redis,
+  TOKEN_SECRET,
+  DB_SCHEMA,
+  DB_SCHEMA_KONE,
+  OODI_ADDR,
+  CERT_PATH,
+  KEY_PATH,
+  FEATURES,
+  OODILEARN_URL,
+  USERSERVICE_URL: formatURL(USERSERVICE_URL),
+  ACCESS_TOKEN_HEADER_KEY,
+  PORT,
   ANALYTICS_URL: formatURL(ANALYTICS_URL),
   USAGESERVICE_URL,
-  requiredGroup, OODI_SECRET, OODI_SECRET_HEADER_KEY,
-  isTest
+  requiredGroup,
+  OODI_SECRET,
+  OODI_SECRET_HEADER_KEY,
+  isTest,
+  UPDATER_URL
 }

services/backend/oodikone2-backend/src/database/connection.js

@@ -10,41 +10,14 @@ const sequelize = new Sequelize(conf.DB_URL, {
 })
 sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
 
-const sequelizeKone = new Sequelize(conf.DB_URL, {
+const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
   schema: conf.DB_SCHEMA_KONE,
   searchPath: conf.DB_SCHEMA_KONE,
   logging: false,
   operatorsAliases: false
 })
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
-// See https://github.com/sequelize/sequelize/issues/10875
 
-const runMigrations = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelize,
-        tableName: 'migrations'
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelize.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/src/database/migrations`,
-        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA
-      }
-    })
-    const migrations = await migrator.up()
-
-    console.log('Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Migration error, message:', e)
-  }
-}
 
 const runMigrationsKone = async () => {
   try {
@@ -74,7 +47,7 @@ const runMigrationsKone = async () => {
   }
 }
 
-const migrationPromise = !conf.isTest ? runMigrations().then(() => runMigrationsKone()) : Promise.resolve()
+const migrationPromiseKone = !conf.isTest ? runMigrationsKone() : Promise.resolve()
 
 const forceSyncDatabase = async () => {
   try {
@@ -94,6 +67,6 @@ const forceSyncDatabase = async () => {
 module.exports = {
   sequelize,
   sequelizeKone,
-  migrationPromise,
+  migrationPromiseKone,
   forceSyncDatabase
 }

services/backend/oodikone2-backend/src/database/seed_migrations.js

@@ -1,5 +1,6 @@
 const fs = require('fs')
-const { Migration, MigrationKone } = require('../models/index')
+const { Migration } = require('../models/index')
+const { MigrationKone } = require('../models/models_kone')
 
 const DEFAULT_PATH = `${process.cwd()}/src/database/migrations`
 const DEFAULT_PATH_KONE = `${process.cwd()}/src/database/migrations_kone`

services/backend/oodikone2-backend/src/events.js

@@ -73,6 +73,7 @@ const StartNats = () => {
                 e
               )
             }
+            console.error(e)
             console.log(
               `Failed to update throughput stats for code: ${code}, reason: ${
                 e.message
@@ -93,6 +94,7 @@ const StartNats = () => {
             } catch (e) {
               console.error(e)
             }
+            console.error(e)
             console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
           }
           ready += 1

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -89,6 +89,7 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
         console.error(e)
         return
       }
+      console.error(e)
       console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
     }
     ready += 1
@@ -149,6 +150,7 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
         console.error(e)
         return
       }
+      console.error(e)
       console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
     }
     ready += 1

services/backend/oodikone2-backend/src/routes/population.js

@@ -1,13 +1,16 @@
 const router = require('express').Router()
 const Population = require('../services/populations')
 const Filters = require('../services/filters')
-const { updateStudents } = require('../services/doo_api_database_updater/database_updater')
+const { updateStudents }  = require('../services/updaterService')
+
+const Student = require('../services/students')
 const StudyrightService = require('../services/studyrights')
+const UserService = require('../services/userService')
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
   try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
+    if (!req.body.startYear || !req.body.semesters || !req.body.studyRights) {
       res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
       return
     }
@@ -31,10 +34,41 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
   }
 })
 
-router.get('/v3/populationstatistics', async (req, res) => {
-  const { year, semesters, studyRights: studyRightsJSON } = req.query
+router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
   try {
-    if (!year || !semesters || !studyRightsJSON) {
+    if (!req.body.yearcode || !req.body.coursecode) {
+      res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
+      return
+    }
+    const { coursecode, yearcode } = req.body
+    let studentnumberlist
+    const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
+
+    const { roles, userId } = req.decodedToken
+    if (roles && roles.map(r => r.group_code).includes('admin')) {
+      studentnumberlist = studentnumbers
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+    }
+    const result = await Population.bottlenecksOf(req.body, studentnumberlist)
+
+    if (result.error) {
+      res.status(400).json(result)
+      return
+    }
+
+    res.json(result)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
+router.get('/v3/populationstatistics', async (req, res) => {
+  const { startYear, semesters, studyRights: studyRightsJSON } = req.query
+  try {
+    if (!startYear || !semesters || !studyRightsJSON) {
       res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
       return
     }
@@ -48,7 +82,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
           return
         }
       }
-    } catch(e) {
+    } catch (e) {
       console.error(e)
       res.status(400).json({ error: 'The query had invalid studyRights' })
       return
@@ -57,8 +91,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
     if (req.query.months == null) {
       req.query.months = 12
     }
-
-    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights})
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
 
     if (result.error) {
       console.log(result.error)
@@ -74,6 +107,45 @@ router.get('/v3/populationstatistics', async (req, res) => {
   }
 })
 
+router.get('/v3/populationstatisticsbycourse', async (req, res) => {
+  const { coursecode, yearcode } = req.query
+  const semesters = ['FALL', 'SPRING']
+  let studentnumberlist
+  console.log(coursecode, yearcode)
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
+  console.log(studentnumbers)
+  const { roles, userId } = req.decodedToken
+  if (roles && roles.map(r => r.group_code).includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+  try {
+    const result = await Population.optimizedStatisticsOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters, 
+      months: 1000
+    }, studentnumberlist)
+
+    if (result.error) {
+      console.log(result.error)
+      res.status(400).end()
+      return
+    }
+
+    console.log(`request completed ${new Date()}`)
+    res.json(result)
+  } catch (e) {
+    console.log(e)
+    res.status(400).json({ error: e })
+  }
+})
+
+
 router.get('/v2/populationstatistics/filters', async (req, res) => {
 
   let results = []
@@ -122,8 +194,8 @@ router.post('/updatedatabase', async (req, res) => {
   const studentnumbers = req.body
   console.log(studentnumbers)
   if (studentnumbers) {
-    await updateStudents(studentnumbers, 128)
-    res.status(200).json('Updated')
+    await updateStudents(studentnumbers)
+    res.status(200).json('Scheduled')
   } else {
     res.status(400).end()
   }

services/backend/oodikone2-backend/src/routes/semesters.js

@@ -2,8 +2,7 @@ const router = require('express').Router()
 const { getSemestersAndYears } = require('../services/semesters')
 
 router.get('/semesters/codes', async(req, res) => {
-  const before = new Date()
-  const providers = await getSemestersAndYears(before)
+  const providers = await getSemestersAndYears()
   res.json(providers)
 })
 

services/backend/oodikone2-backend/src/services/courseGroups.js

@@ -1,5 +1,8 @@
 const { redisClient } = require('../services/redis')
-const { Teacher, CourseGroup } = require('../models')
+const { Teacher } = require('../models')
+const { CourseGroup, TeacherCourseGroup } = require('../models/models_kone')
+const { Op } = require('sequelize')
+const { flatMap } = require('lodash')
 
 const {
   getCurrentAcademicYear,
@@ -26,18 +29,27 @@ const getCurrentAcademicYearSemesterCode = async () => {
   return academicYear[0].semestercode
 }
 
-const getTeachersForCourseGroup = (courseGroupId) => {
-  return Teacher.findAll({
-    attributes: ['name', 'code', 'id'],
+const getTeachersForCourseGroup = async (courseGroupId) => {
+  const courseGroups = await CourseGroup.findAll({
+    model: CourseGroup,
     include: {
-      model: CourseGroup,
-      attributes: [],
+      model: TeacherCourseGroup,
       required: true,
-      where: {
-        id: courseGroupId
-      },
+    },
+    where: {
+      id: courseGroupId
     },
   })
+  const teacherIds = flatMap(courseGroups, e => e.teacher_course_groups.map(tcg => tcg.teacher_id))
+  const teachers = await Teacher.findAll({
+    attributes: ['name', 'code', 'id'],
+    where: {
+      id: {
+        [Op.in]: teacherIds
+      }
+    },
+  })
+  return teachers
 }
 
 const createCourseGroup = (name) => CourseGroup.create({ name })
@@ -141,17 +153,13 @@ const getCoursesByTeachers = async (teacherIds, semesterCode) => {
 }
 
 const addTeacher = async (courseGroupId, teacherid) => {
-  const group = await CourseGroup.findByPk(courseGroupId)
   const teacher = await Teacher.findByPk(teacherid)
-  if (!group || !teacher) return
-  return await group.addTeacher(teacher)
+  if (!teacher) return
+  return TeacherCourseGroup.create({ teacher_id: teacherid, course_group_id: courseGroupId })
 }
 
-const removeTeacher = async (courseGroupId, teacherid) => {
-  const group = await CourseGroup.findByPk(courseGroupId)
-  const teacher = await Teacher.findByPk(teacherid)
-  if (!group || !teacher) return
-  return await group.removeTeacher(teacher)
+const removeTeacher = (courseGroupId, teacherid) => {
+  return TeacherCourseGroup.destroy({ where: { teacher_id: teacherid, course_group_id: courseGroupId } })
 }
 
 module.exports = {

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js

@@ -8,6 +8,7 @@ const CATEGORY = {
 class CourseYearlyStatsCounter {
 
   constructor() {
+    this.all = []
     this.groups = {}
     this.programmes = {}
     this.history = {
@@ -43,17 +44,29 @@ class CourseYearlyStatsCounter {
     return { attempted, passed, failed }
   }
 
-  markStudyProgramme(code, name, studentnumber) {
+  markStudyProgramme(code, name, studentnumber, startdate) {
     if (!this.programmes[code]) {
       this.initProgramme(code, name)
     }
     const prog = this.programmes[code]
-    prog.students.push(studentnumber)
+
+    // check if student has earlier studyright wrt course
+    this.all.forEach((student) => {
+      if (student.studentnumber === studentnumber && student.startdate > startdate) {
+        delete this.programmes[student.code]
+        this.all.splice(this.all.indexOf(student), 1)
+      }
+    })
+
+    if (!prog.students.includes(studentnumber)) {
+      this.all.push({ studentnumber, code, startdate })
+      prog.students.push(studentnumber)
+    }
   }
 
   markStudyProgrammes(studentnumber, programmes) {
-    programmes.forEach(({ code, name }) => {
-      this.markStudyProgramme(code, name, studentnumber)
+    programmes.forEach(({ code, name, startdate }) => {
+      this.markStudyProgramme(code, name, studentnumber, startdate)
     })
   }
 
@@ -114,12 +127,14 @@ class CourseYearlyStatsCounter {
   formatStudentStatistics(students) {
     const grades = {}
     const classes = {}
+    const studentnumbers = []
     Object.entries(students).forEach(([studentnumber, stat]) => {
       const { grade, category } = stat
       grades[grade] = grades[grade] ? grades[grade].concat(studentnumber) : [studentnumber]
       classes[category] = classes[category] ? classes[category].concat(studentnumber) : [studentnumber]
+      studentnumbers.includes(studentnumber) ? null : studentnumbers.push(studentnumber)
     })
-    return { grades, classes }
+    return { grades, classes, studentnumbers }
   }
 
   formatGroupStatistics() {

services/backend/oodikone2-backend/src/services/courses.js

@@ -4,8 +4,11 @@ const {
   sequelize, Student, Credit,
   Course, CourseType, Discipline,
   ElementDetails, StudyrightElement,
-  Studyright, Semester, CourseDuplicates,
+  Studyright, Semester,
 } = require('../models')
+const {
+  sequelizeKone, CourseDuplicates,
+} = require('../models/models_kone')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('../services/course_yearly_stats_counter')
 const _ = require('lodash')
@@ -97,7 +100,7 @@ const creditsForCourses = (codes) => Credit.findAll({
       attributes: ['studentnumber'],
       include: {
         model: StudyrightElement,
-        attributes: ['code'],
+        attributes: ['code', 'startdate'],
         include: [
           {
             model: ElementDetails,
@@ -193,6 +196,7 @@ const creditsOf = async (codes) => {
       fail: credits.filter(Credit.failed).length,
       pass: credits.filter(Credit.passed).length,
       students: credits.length,
+      studentnumbers: credits.map(cr => cr.student_studentnumber)
     }
   }
 
@@ -235,7 +239,7 @@ const oneYearStats = (instances, year, separate, allInstancesUntilYear) => {
     return {
       studentsThatPassedThisYear, studentsThatFailedThisYear, allStudentsThatFailedEver,
       passedStudentsThatFailedBefore, passedStudentsOnFirstTry, failedStudentsThatFailedBefore,
-      failedStudentsOnFirstTry, gradeDistribution
+      failedStudentsOnFirstTry, gradeDistribution, studentnumbers: thisSemester.studentnumbers
     }
   }
   const stats = []
@@ -409,7 +413,7 @@ const deleteDuplicateCode = async (code) => {
         coursecode: code
       }
     })
-    await sequelize.query(
+    await sequelizeKone.query(
       `DELETE FROM course_duplicates
       WHERE groupid in ( SELECT groupid FROM course_duplicates
       group by groupid
@@ -421,12 +425,17 @@ const deleteDuplicateCode = async (code) => {
   }
 }
 
-const getDuplicateCodesWithCourses = () => {
-  return CourseDuplicates.findAll({
-    include: {
-      model: Course,
+const getDuplicateCodesWithCourses = async () => {
+  const courseDuplicates = await CourseDuplicates.findAll()
+  const courses = await Course.findAll({
+    where: {
+      code: {
+        [Op.in]: courseDuplicates.map(e => e.coursecode)
+      }
     }
   })
+  const codeToCourse = courses.reduce((acc, c) => { acc[c.code] = c; return acc }, {})
+  return courseDuplicates.map(cd => ({ ...cd.get(), course: codeToCourse[cd.coursecode] }))
 }
 
 const getDuplicatesToIdMap = () => {
@@ -463,7 +472,7 @@ const getMainCodeToDuplicates = async () => {
     )[0]
     acc[main.code] = {
       maincourse: { code: main.code, name: main.name },
-      duplicates: courses.map( c => ({ code: c.code, name: c.name }))
+      duplicates: courses.map(c => ({ code: c.code, name: c.name }))
     }
     return acc
   }, {})
@@ -541,18 +550,18 @@ const setDuplicateCode = async (code1, code2) => {
       if (course1 && course2) {
         const all = await getDuplicatesToIdMap()
         // make sure both dont have a group
-        if ([all[code1], all[code2]].filter(e=>e).length <= 1) {
+        if ([all[code1], all[code2]].filter(e => e).length <= 1) {
           let groupid = all[code1] || all[code2]
           if (!groupid) {
             // neither has a group, make one
-            groupid = Math.max(0, ...Object.values(all).filter(e=>e))
-            groupid = groupid && !isNaN(groupid) ? groupid+1 : 1
+            groupid = Math.max(0, ...Object.values(all).filter(e => e))
+            groupid = groupid && !isNaN(groupid) ? groupid + 1 : 1
           }
           await CourseDuplicates.bulkCreate([
             { groupid, coursecode: code1 },
             { groupid, coursecode: code2 }
           ],
-          { ignoreDuplicates: true })
+            { ignoreDuplicates: true })
         } else {
           // both have a group, must merge groups
           await CourseDuplicates.update({ groupid: all[code1] }, { where: { groupid: all[code2] } })
@@ -572,9 +581,10 @@ const alternativeCodes = async code => {
   return alternatives ? alternatives : [code]
 }
 
-const formatStudyrightElement = ({ code, element_detail }) => ({
+const formatStudyrightElement = ({ code, element_detail, startdate }) => ({
   code,
-  name: element_detail.name
+  name: element_detail.name,
+  startdate
 })
 
 const parseCredit = credit => {
@@ -604,7 +614,6 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
       semestercode, semestername,
       yearcode, yearname, programmes
     } = parseCredit(credit)
-
     if (startyearcode <= yearcode && yearcode <= endyearcode) {
       const groupcode = separate ? semestercode : yearcode
       const groupname = separate ? semestername : yearname
@@ -634,7 +643,6 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
 const courseYearlyStats = async (coursecodes, separate, startyearcode, endyearcode) => {
   const stats = await Promise.all(coursecodes
     .map(code => yearlyStatsOfNew(code, separate, startyearcode, endyearcode || startyearcode)))
-
   return stats
 }
 

services/backend/oodikone2-backend/src/services/filters.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { Filters } = require('../models')
+const { Filters } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 

services/backend/oodikone2-backend/src/services/mandatoryCourseLabels.js

@@ -1,10 +1,7 @@
 const Sequelize = require('sequelize')
 const _ = require('lodash')
-const { MandatoryCourseLabels } = require('../models')
+const { MandatoryCourseLabels, sequelizeKone } = require('../models/models_kone')
 const { Op } = Sequelize
-const {
-  sequelize
-} = require('../models/index')
 
 const create = async (studyprogramme_id, { label }) => {
   const labels = await labelsByStudyprogramme(studyprogramme_id)
@@ -23,7 +20,7 @@ const move = async (studyprogramme_id, { id }, direction) => {
   if (indexToSwap < 0 || indexToSwap >= orderedlabels.length) return null;
   [orderedlabels[indexToMove], orderedlabels[indexToSwap]] = [orderedlabels[indexToSwap], orderedlabels[indexToMove]]
   const newOrder = orderedlabels.map((e, index) => ({ ...e, orderNumber: index }))
-  await sequelize.transaction(async transaction => {
+  await sequelizeKone.transaction(async transaction => {
     for (let e of newOrder) {
       await MandatoryCourseLabels.update(
         {orderNumber: e.orderNumber}, { where: { studyprogramme_id, id: e.id }, transaction }

services/backend/oodikone2-backend/src/services/mandatoryCourses.js

@@ -1,15 +1,12 @@
 const Sequelize = require('sequelize')
-const { Course, MandatoryCourse, MandatoryCourseLabels } = require('../models')
+const { Course } = require('../models')
+const { MandatoryCourse, MandatoryCourseLabels } = require('../models/models_kone')
 const { Op } = Sequelize
 
-const byStudyprogramme = (studyProgrammeId) => {
-  const courses = MandatoryCourse.findAll({
+const byStudyprogramme = async (studyProgrammeId) => {
+  const mandatoryCourses = await MandatoryCourse.findAll({
     attributes: ['course_code'],
     include: [
-      {
-        model: Course,
-        attributes: ['name'],
-      },
       {
         model: MandatoryCourseLabels,
         attributes: ['id', 'label', 'orderNumber'],
@@ -22,10 +19,20 @@ const byStudyprogramme = (studyProgrammeId) => {
     }
   })
 
-  return courses.map(course => ({
-    name: course.course.name,
-    code: course.course_code,
-    label: course.mandatory_course_label
+  const courses = await Course.findAll({
+    model: Course,
+    where: {
+      code: {
+        [Op.in]: mandatoryCourses.map(c => c.course_code)
+      }
+    }
+  })
+  const courseCodeToCourse = courses.reduce((acc, c) => { acc[c.code] = c; return acc }, {})
+
+  return mandatoryCourses.map(mc => ({
+    name: courseCodeToCourse[mc.course_code].name,
+    code: mc.course_code,
+    label: mc.mandatory_course_label
   }))
 }
 
@@ -36,14 +43,10 @@ const create = (studyProgrammeId, code) => {
   })
 }
 
-const find = (studyProgrammeId, code) => {
-  return MandatoryCourse.findOne({
+const find = async (studyProgrammeId, code) => {
+  const mandatoryCourse = await MandatoryCourse.findOne({
     attributes: ['course_code', 'label'],
     include: [
-      {
-        model: Course,
-        attributes: ['name'],
-      },
       {
         model: MandatoryCourseLabels,
         attributes: ['id', 'label', 'orderNumber'],
@@ -58,6 +61,14 @@ const find = (studyProgrammeId, code) => {
       }
     }
   })
+
+  const course = await Course.findOne({
+    where: {
+      code: mandatoryCourse.course_code
+    }
+  })
+
+  return { ...mandatoryCourse.get(), course: course.get() }
 }
 
 const remove = (studyProgrammeId, code) => {

services/backend/oodikone2-backend/src/services/populations.js

@@ -3,9 +3,11 @@ const moment = require('moment')
 const { orderBy } = require('lodash')
 const {
   Student, Credit, Course, sequelize, Studyright, StudyrightExtent, ElementDetails,
-  Discipline, CourseType, SemesterEnrollment, Semester, Transfers, StudyrightElement,
-  Tag, TagStudent
+  Discipline, CourseType, SemesterEnrollment, Semester, Transfers, StudyrightElement
 } = require('../models')
+const {
+  Tag, TagStudent
+} = require('../models/models_kone')
 const { getMainCodesMap, byName } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
@@ -25,7 +27,7 @@ const formatStudentForPopulationStatistics = ({
   firstnames, lastname, studentnumber, dateofuniversityenrollment, creditcount,
   matriculationexamination, credits, abbreviatedname, email, studyrights,
   semester_enrollments, transfers, updatedAt, createdAt, gender_code,
-  gender_fi, gender_sv, gender_en, tag_students
+  gender_fi, gender_sv, gender_en, tags
 }, startDate, endDate) => {
 
   const toCourse = ({ grade, attainment_date, credits, course, credittypecode, isStudyModule }) => {
@@ -99,7 +101,7 @@ const formatStudentForPopulationStatistics = ({
     email,
     semesterenrollments,
     updatedAt: updatedAt || createdAt,
-    tags: tag_students || [],
+    tags: tags || [],
     studyrightStart: startDate,
     starting: moment(started).isBetween(startDate, endDate, null, '[]')
   }
@@ -107,7 +109,8 @@ const formatStudentForPopulationStatistics = ({
 
 const dateMonthsFromNow = (date, months) => moment(date).add(months, 'months').format('YYYY-MM-DD')
 
-const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDate, studyright) => {
+const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDate, studyright, tag) => {
+
   const creditsOfStudentOther = {
     student_studentnumber: {
       [Op.in]: studentnumbers
@@ -138,9 +141,9 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
 
   const creditsOfStudent = ['320001', 'MH30_001'].includes(studyright[0]) ?
     creditsOfStudentLaakis : creditsOfStudentOther
-  
+
   const students = await Student.findAll({
-    attributes: ['firstnames', 'lastname', 'studentnumber',
+    attributes: ['firstnames', 'lastname', 'studentnumber', 'home_country_en',
       'dateofuniversityenrollment', 'creditcount', 'matriculationexamination',
       'abbreviatedname', 'email', 'updatedAt', 'gender_code', 'gender_fi', 'gender_sv', 'gender_en'],
     include: [
@@ -213,17 +216,21 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
           }
         }
       },
-      {
-        model: TagStudent,
-        attributes: ['id'],
-        include: [
-          {
-            model: Tag,
-            attributes: ['tag_id','tagname']
-          }
-        ],
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers
       }
+    },
+  })
 
+  const studentTags = await TagStudent.findAll({
+    attributes: ['id', 'studentnumber'],
+    include: [
+      {
+        model: Tag,
+        attributes: ['tag_id', 'tagname'],
+      }
     ],
     where: {
       studentnumber: {
@@ -231,6 +238,27 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       }
     }
   })
+  const studentNumberToTags = studentTags.reduce((acc, t) => {
+    acc[t.studentnumber] = acc[t.studentnumber] || []
+    acc[t.studentnumber].push(t)
+    return acc
+  }, {})
+
+  students.forEach(student => {
+    student.tags = studentNumberToTags[student.studentnumber]
+  })
+
+  if (tag) {
+    const studentsWithSearchedTag = {}
+    students.forEach(student => {
+      student.tags = studentNumberToTags[student.studentnumber]
+      if (student.tags && student.tags.some(t => t.tag.tag_id === tag)) {
+        studentsWithSearchedTag[student.studentnumber] = true
+      }
+    })
+
+    return students.filter(student => studentsWithSearchedTag[student.studentnumber])
+  }
   return students
 }
 
@@ -301,13 +329,13 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
 }
 
 const parseQueryParams = query => {
-  const { semesters, studentStatuses, year, studyRights, months } = query
+  const { semesters, studentStatuses, studyRights, months, endYear, startYear } = query
   const startDate = semesters.includes('FALL') ?
-    `${year}-${semesterStart[semesters.find(s => s === 'FALL')]}` :
-    `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
+    `${startYear}-${semesterStart[semesters.find(s => s === 'FALL')]}` :
+    `${moment(startYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
   const endDate = semesters.includes('SPRING') ?
-    `${moment(year, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}` :
-    `${year}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
+    `${moment(endYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd[semesters.find(s => s === 'SPRING')]}` :
+    `${endYear}-${semesterEnd[semesters.find(s => s === 'FALL')]}`
   const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
   const cancelledStudents = studentStatuses && studentStatuses.includes('CANCELLED')
   const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
@@ -404,7 +432,7 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights
   }
 }
 
-const optimizedStatisticsOf = async (query) => {
+const optimizedStatisticsOf = async (query, studentnumberlist) => {
   if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
     return { error: 'Semester should be either SPRING OR FALL' }
   }
@@ -415,17 +443,22 @@ const optimizedStatisticsOf = async (query) => {
   ) {
     return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
   }
-
   const {
-    studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents, nondegreeStudents
+    studyRights, startDate, months, endDate, exchangeStudents, cancelledStudents, nondegreeStudents
   } = parseQueryParams(query)
 
-  const studentnumbers =
+  const studentnumbers = studentnumberlist ?
+    studentnumberlist :
     await studentnumbersWithAllStudyrightElements(
       studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents
     )
-  const students =
-    await getStudentsIncludeCoursesBetween(studentnumbers, startDate, dateMonthsFromNow(startDate, months), studyRights)
+  const students = await getStudentsIncludeCoursesBetween(
+    studentnumbers,
+    startDate,
+    dateMonthsFromNow(startDate, months),
+    studyRights,
+    query.tag
+  )
 
   const formattedStudents = await formatStudentsForApi(students, startDate, endDate, query)
   return formattedStudents
@@ -486,26 +519,39 @@ const parseCreditInfo = credit => ({
   date: credit.attainment_date
 })
 
-const bottlenecksOf = async (query) => {
-  if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
-    return { error: 'Semester should be either SPRING OR FALL' }
-  }
-  if (query.studentStatuses &&
-    !query.studentStatuses.map(
-      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
-    ).every(e => e === true)) {
-    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
-  }
-  const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query)
+const bottlenecksOf = async (query, studentnumberlist) => {
+  let allstudents
+  let courses
+  if (!studentnumberlist) {
+    if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
+      return { error: 'Semester should be either SPRING OR FALL' }
+    }
+    if (query.studentStatuses &&
+      !query.studentStatuses.map(
+        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
+      ).every(e => e === true)) {
+      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
+    }
+    const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query)
 
-  if (query.selectedStudents) {
-    const allStudents =
-      await studentnumbersWithAllStudyrightElements(
-        studyRights, startDate, endDate, exchangeStudents, cancelledStudents
-      )
-    const disallowedRequest =
-      checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
-    if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
+    if (query.selectedStudents) {
+      const allStudents =
+        await studentnumbersWithAllStudyrightElements(
+          studyRights, startDate, endDate, exchangeStudents, cancelledStudents
+        )
+      const disallowedRequest =
+        checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
+      if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
+    }
+
+    const studentnumbers = query.selectedStudents ||
+      await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents)
+    allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
+    courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+
+  } else {
+    allstudents = studentnumberlist
+    courses = await findCourses(allstudents, new Date())
   }
 
   const bottlenecks = {
@@ -515,11 +561,6 @@ const bottlenecksOf = async (query) => {
 
   const codeduplicates = await getMainCodesMap()
 
-  const studentnumbers = query.selectedStudents ||
-    await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents)
-  const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
-  const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
-
   const allcoursestatistics = await courses.reduce(async (coursestatistics, course) => {
     const stats = await coursestatistics
     let { code, name, disciplines, course_type } = course
@@ -543,7 +584,7 @@ const bottlenecksOf = async (query) => {
 
     course.credits.forEach(credit => {
       const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
-      const semester = getPassingSemester(parseInt(query.year, 10), date)
+      const semester = getPassingSemester(parseInt(query.endYear, 10), date)
       coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
     })
 

services/backend/oodikone2-backend/src/services/semesters.js

@@ -2,14 +2,8 @@ const sequelize = require('sequelize')
 const { Semester } = require('../models/index')
 const { Op } = sequelize
 
-const getSemestersAndYears = async before => {
-  const semesters = await Semester.findAll({
-    where: {
-      startdate: {
-        [Op.lt]: before
-      }
-    }
-  })
+const getSemestersAndYears = async () => {
+  const semesters = await Semester.findAll({})
   const result = semesters.reduce((acc, semester) => {
     const { semestercode, name, yearcode, yearname }  = semester
     const semesters = { ...acc.semesters, [semestercode]: { semestercode, name, yearcode }}

services/backend/oodikone2-backend/src/services/students.js

@@ -1,4 +1,5 @@
 const Sequelize = require('sequelize')
+const { sequelize } = require('../database/connection')
 const moment = require('moment')
 const { Student, Credit, Course, Studyright, StudyrightElement, ElementDetails } = require('../models')
 const Op = Sequelize.Op
@@ -41,6 +42,24 @@ const byId = async (id) => Student.findByPk(id, {
   ]
 })
 
+
+const findByCourseAndSemesters = async (coursecode, yearcode) =>
+  sequelize.query(`
+    SELECT
+      studentnumber, credit.course_code, attainment_date
+    FROM student
+    INNER JOIN credit ON
+      student.studentnumber=credit.student_studentnumber
+    WHERE 
+      course_code=:coursecode AND
+      attainment_date
+    BETWEEN
+      (select startdate FROM semesters where yearcode=:yearcode ORDER BY semestercode LIMIT 1) AND
+      (select enddate FROM semesters where yearcode=:yearcode ORDER BY semestercode DESC LIMIT 1);
+  `,
+  { replacements: { coursecode, yearcode }, type: sequelize.QueryTypes.SELECT })
+    .map(st => st.studentnumber)
+
 const byAbreviatedNameOrStudentNumber = (searchTerm) => {
   return Student.findAll({
     where: {
@@ -280,10 +299,36 @@ const bySearchTermAndElementsNew = async (searchterm, codes) => {
   return matches.map(formatStudent)
 }
 
+const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
+  const students = await Student.findAll({
+    include: {
+      model: StudyrightElement,
+      required: true,
+      where: {
+        code: {
+          [Op.in]: codes
+        }
+      }
+    },
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers
+      }
+    }
+  })
+  return students.map(student => student.studentnumber)
+}
+
 const NEW_SEARCH = true
 const bySearchTerm = NEW_SEARCH ? bySearchTermNew : bySearchTermOld
 const bySearchTermAndElements = NEW_SEARCH ? bySearchTermAndElementsNew : bySearchTermAndElementsOld
 
 module.exports = {
-  withId, bySearchTerm, createStudent, updateStudent, bySearchTermAndElements
+  withId,
+  bySearchTerm,
+  createStudent,
+  updateStudent,
+  bySearchTermAndElements,
+  filterStudentnumbersByAccessrights,
+  findByCourseAndSemesters
 }

services/backend/oodikone2-backend/src/services/studytrack.js

@@ -1,9 +1,13 @@
 const sequelize = require('sequelize')
 const { Op } = sequelize
 const moment = require('moment')
+const { flatMap } = require('lodash')
 const { Credit, Student, Course, Provider, Studyright, StudyrightElement,
-  ElementDetails, ThesisCourse, ThesisTypeEnums
+  ElementDetails
 } = require('../models')
+const {
+  ThesisCourse, ThesisTypeEnums
+} = require('../models/models_kone')
 const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const { semesterStart, semesterEnd } = require('../util/semester')
 const isNumber = str => !Number.isNaN(Number(str))
@@ -169,30 +173,39 @@ const graduatedStatsForStudytrack = async (studytrack, since) => {
   return graduatedStatsFromStudyrights(studyrights)
 }
 
-const findProgrammeThesisCredits = code => Credit.findAll({
-  include: {
-    model: Course,
-    required: true,
+const findProgrammeThesisCredits = async code => {
+  const thesiscourses = await ThesisCourse.findAll({
+    model: ThesisCourse,
+    where: {
+      programmeCode: code
+    }
+  })
+  const courseCodeToThesisCourse = thesiscourses.reduce((acc, tc) => { acc[tc.courseCode] = tc; return acc }, {})
+
+  const credits = await Credit.findAll({
     include: {
-      model: ThesisCourse,
+      model: Course,
       required: true,
       where: {
-        programmeCode: code
+        code: {
+          [Op.in]: thesiscourses.map(tc => tc.courseCode)
+        }
+      }
+    },
+    where: {
+      credittypecode: {
+        [Op.ne]: 10
       }
     }
-  },
-  where: {
-    credittypecode: {
-      [Op.ne]: 10
-    }
-  }
-}).map(credit => {
-  const { id, course, attainment_date } = credit
-  const { code, thesis_courses } = course
-  const { thesisType: type } = thesis_courses[0]
-  const year = attainment_date && attainment_date.getFullYear()
-  return { id, code, type, year }
-})
+  })
+
+  return credits.map(credit => {
+    const { id, course, attainment_date } = credit
+    const { code } = course
+    const year = attainment_date && attainment_date.getFullYear()
+    return { id, code, type: courseCodeToThesisCourse[code].thesisType, year }
+  })
+}
 
 const thesisProductivityFromCredits = credits => {
   const stats = {}
@@ -215,9 +228,25 @@ const thesisProductivityForStudytrack = async code => {
 }
 
 const combineStatistics = (creditStats, studyrightStats, thesisStats, creditsForMajors, transferredCredits) => {
-  const stats = { ...creditStats }
-  Object.keys(stats).forEach(year => {
+  const allYears = [
+    ...new Set(
+      flatMap(
+        [
+          creditStats,
+          studyrightStats,
+          thesisStats,
+          creditsForMajors,
+          transferredCredits
+        ],
+        Object.keys
+      )
+    )
+  ]
+  const stats = {}
+  allYears.forEach(year => {
     const thesis = thesisStats[year] || {}
+    stats[year] = {}
+    stats[year] = creditStats[year] || { credits: 0, year }
     stats[year].graduated = studyrightStats[year] ? studyrightStats[year].graduated : 0
     // stats[year].medianGraduationTime = studyrightStats[year] ? studyrightStats[year].medianGraduationTime : 0
     stats[year].bThesis = thesis.bThesis || 0
@@ -273,16 +302,18 @@ const creditsAfter = (studentnumbers, startDate) => {
 }
 
 const thesesFromClass = async (studentnumbers, startDate, code) => {
-  return Credit.findAll({
+  const thesiscourses = await ThesisCourse.findAll({
+    where: {
+      programmeCode: code
+    }
+  })
+  const credits = await Credit.findAll({
     include: {
       model: Course,
       required: true,
-      include: {
-        attributes: ['thesisType'],
-        model: ThesisCourse,
-        required: true,
-        where: {
-          programmeCode: code
+      where: {
+        code: {
+          [Op.in]: thesiscourses.map(tc => tc.courseCode)
         }
       }
     },
@@ -297,12 +328,16 @@ const thesesFromClass = async (studentnumbers, startDate, code) => {
         [Op.in]: studentnumbers
       }
     }
-  }).reduce((acc, curr) => {
-    curr.course.thesis_courses.map(th => {
-      acc[th.thesisType] = acc[th.thesisType] ? acc[th.thesisType] + 1 : 1
-    })
+  })
+
+  const courseCodeToThesisCourse = thesiscourses.reduce((acc, tc) => { acc[tc.courseCode] = tc; return acc }, {})
+  const courseCodes = credits.map(c => c.course.code)
+  const theses = courseCodes.reduce((acc, code) => {
+    const thesisType = courseCodeToThesisCourse[code].thesisType
+    acc[thesisType] = acc[thesisType] ? acc[thesisType] + 1 : 1
     return acc
   }, {})
+  return theses
 }
 
 const graduationsFromClass = async (studentnumbers, studytrack) => {
@@ -347,13 +382,13 @@ const gendersFromClass = async (studentnumbers) => {
 
 const countriesFromClass = async (studentnumbers) => {
   return Student.findAll({
-    attributes: [[sequelize.fn('count', sequelize.col('country_en')), 'count'], 'country_en'],
+    attributes: [[sequelize.fn('count', sequelize.col('home_country_en')), 'count'], 'home_country_en'],
     where: {
       studentnumber: {
         [Op.in]: studentnumbers
       },
     },
-    group: ['country_en'],
+    group: ['home_country_en'],
     raw: true
   }).reduce((acc, curr) => {
     acc[curr.country_en] = curr.count
@@ -515,7 +550,6 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     const studentnumbers = await studentnumbersWithAllStudyrightElements([studytrack], startDate, endDate, false, false)
     const creditsForStudyprogramme =
       await productivityCreditsFromStudyprogrammeStudents(studytrack, startDate, studentnumbers)
-
     const [credits, graduated, theses, genders, countries, transferredTo, endedStudyright] =
       await statsForClass(studentnumbers, startDate, studytrack, endDate)
     //console.log(year)
@@ -523,6 +557,7 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     // theres so much shit in the data that transefferFrom doesnt rly mean anything
     delete genders[null]
     delete countries[null]
+    delete countries[undefined]
     const creditValues = credits.reduce((acc, curr) => {
       acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
       acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60

services/backend/oodikone2-backend/src/services/studytrack.test.js

@@ -11,7 +11,8 @@ const {
   thesisProductivityFromCredits,
   thesisProductivityForStudytrack
 } = require('./studytrack')
-const { sequelize, ThesisTypeEnums } = require('../models')
+const { sequelize } = require('../models')
+const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
 const { readFileSync } = require('fs')
 
 const { MASTER, BACHELOR } = ThesisTypeEnums
@@ -19,9 +20,15 @@ const { MASTER, BACHELOR } = ThesisTypeEnums
 const provider = '500-M010'
 const studytrack = 'MH50_010'
 
+const thesisCourse = {
+  // from studytrack.test.sql
+  programmeCode: 'MH50_010', courseCode: 'THESIS_01', thesisType: MASTER
+}
+
 beforeAll(async () => {
   const query = readFileSync('./src/services/studytrack.test.sql', 'utf8')
   await sequelize.query(query)
+  await ThesisCourse.create(thesisCourse)
 })
 
 test('Credits for provider should contain passed course credit', async () => {

services/backend/oodikone2-backend/src/services/studytrack.test.sql

@@ -16,7 +16,7 @@ BEGIN
   VALUES
   (5, ts, ts)
   ;
-  
+
   INSERT INTO course
   ("code", "name", "is_study_module")
   VALUES
@@ -25,14 +25,14 @@ BEGIN
   (course_02, '{ "en": "COURSE_EN" }', 'false'),
   (thesis, '{ "en": "Bachelor thesis" }', 'false')
   ;
-  
+
   INSERT INTO credit_types
   ("credittypecode", "createdAt", "updatedAt")
   VALUES
   (10, ts, ts),
   (4, ts, ts)
   ;
-  
+
   INSERT INTO credit
   ("id", "createddate", "lastmodifieddate", "semestercode", "course_code", "credittypecode", "attainment_date", "credits")
   VALUES
@@ -44,14 +44,14 @@ BEGIN
   ('CREDIT_06', ts, ts, 5, course_01, 4, date_2015, 5),
   ('CREDIT_07', ts, ts, 5, thesis, 4, date_2015, 30)
   ;
-  
+
   INSERT INTO providers
   ("providercode", "createdAt", "updatedAt")
   VALUES
   (provider_01, ts, ts),
   (provider_02, ts, ts)
   ;
-  
+
   INSERT INTO course_providers
   ("coursecode", "providercode", "createdAt", "updatedAt")
   VALUES
@@ -84,8 +84,4 @@ BEGIN
   (12, studytrk_02, ts, ts)
   ;
 
-  INSERT INTO thesis_courses
-  ("programmeCode", "courseCode", "thesisType", "createdAt", "updatedAt")
-  VALUES
-  (studytrk_01, thesis, 'MASTER', ts, ts);
-END $$;
+END $$;

services/backend/oodikone2-backend/src/services/tags.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { Tag } = require('../models')
+const { Tag } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 

services/backend/oodikone2-backend/src/services/tagstudent.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { TagStudent, Tag } = require('../models')
+const { TagStudent, Tag } = require('../models/models_kone')
 
 const Op = Sequelize.Op
 

services/backend/oodikone2-backend/src/services/teachers.js

@@ -1,5 +1,5 @@
 const { Op } = require('sequelize')
-const { Teacher, Credit, Course, Semester, Provider, CourseGroup } = require('../models/index')
+const { Teacher, Credit, Course, Semester, Provider } = require('../models/index')
 
 const splitByEmptySpace = str => str.replace(/\s\s+/g, ' ').split(' ')
 
@@ -40,11 +40,6 @@ const bySearchTerm = async (rawTerm) => {
         codeLike(terms),
         matchesId(searchTerm)
       ]
-    },
-    include: {
-      model: CourseGroup,
-      attributes: ['id', 'name'],
-      required: false
     }
   })
 }
@@ -93,7 +88,7 @@ const markCredit = (stats, passed, failed, credits, transferred) => {
       ...stats,
       credits: transferred ? stats.credits : stats.credits + credits,
       passed: stats.passed + 1,
-      transferred: transferred ? stats.transferred + credits : stats.transferred 
+      transferred: transferred ? stats.transferred + credits : stats.transferred
     }
   } else if (failed) {
     return {

services/backend/oodikone2-backend/src/services/thesis.js

@@ -1,4 +1,4 @@
-const { ThesisCourse } = require('../models/index')
+const { ThesisCourse } = require('../models/models_kone')
 
 const createThesisCourse = (programme, course, type) => ThesisCourse.create({
   programmeCode: programme,

services/backend/oodikone2-backend/src/services/thesis.test.js

@@ -1,5 +1,6 @@
 const { forceSyncDatabase } = require('../database/connection')
-const { Course, ElementDetails, ThesisTypeEnums, ThesisCourse } = require('../models')
+const { Course, ElementDetails } = require('../models')
+const { ThesisTypeEnums, ThesisCourse } = require('../models/models_kone')
 const { createThesisCourse, deleteThesisCourse, findProgrammeTheses }  = require('./thesis')
 
 const courseOne = {

services/backend/oodikone2-backend/src/services/updaterService.js

@@ -0,0 +1,21 @@
+const axios = require('axios')
+const { UPDATER_URL } = require('../conf-backend')
+
+const client = axios.create({ baseURL: UPDATER_URL })
+
+const ping = async () => {
+  const url = '/ping'
+  const response = await axios.get(url)
+  return response.data
+}
+
+const updateStudents = async (studentNumbers) => {
+  const response = await client.post('/update', studentNumbers)
+  return response.data
+}
+
+
+module.exports = {
+  ping,
+  updateStudents
+}

services/backend/oodikone2-backend/src/services/usageStatistics.js

@@ -1,6 +1,6 @@
 const _ = require('lodash')
 const moment = require('moment')
-const { UsageStatistic } = require('../models')
+const { UsageStatistic } = require('../models/models_kone')
 const Sequelize = require('sequelize')
 const { Op } = Sequelize
 

services/backend/oodikone2-backend/src/util/logSaver.js

@@ -1,6 +1,6 @@
 const Transport = require('winston-transport')
 const uuid = require('uuid')
-const { UsageStatistic } = require('../models')
+const { UsageStatistic } = require('../models/models_kone')
 
 module.exports = class LogSaverTransport extends Transport {
   log(payload, callback) {
@@ -16,7 +16,7 @@ module.exports = class LogSaverTransport extends Transport {
         status: payload.status,
         data: payload
       }
-      
+
       UsageStatistic.create(object)
     }
     callback()

services/backend/oodikone2-backend/test/jest/services/population.test.js

@@ -58,7 +58,8 @@ const semesters = {
 
 const createQueryObject = (year, semester, codes, months) => ({
   studyRights: codes,
-  year,
+  endYear: year,
+  startYear: year,
   semesters: [semester],
   months
 })
@@ -137,90 +138,90 @@ describe('optimizedStatisticsOf tests', () => {
     - Two credits in 2011-09-31 and 2012-02-31.
     `, () => {
 
-    beforeAll(async () => {
-      await Semester.bulkCreate([semesters.fall, semesters.spring])
-      await Student.create(student)
-      await Course.create(courses.elements_of_ai)
-      await Credit.bulkCreate([creditFall, creditSpring])
-      await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
-      await StudyrightExtent.create(studyrightextents.bachelors)
-      await Studyright.create(studyright)
-      await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
-    })
+      beforeAll(async () => {
+        await Semester.bulkCreate([semesters.fall, semesters.spring])
+        await Student.create(student)
+        await Course.create(courses.elements_of_ai)
+        await Credit.bulkCreate([creditFall, creditSpring])
+        await ElementDetails.bulkCreate([elementdetails.bsc, elementdetails.maths, elementdetails.cs])
+        await StudyrightExtent.create(studyrightextents.bachelors)
+        await Studyright.create(studyright)
+        await StudyrightElement.bulkCreate([studyrightelements.bsc, studyrightelements.maths])
+      })
 
-    test('Query result for BSc, Fall 2011 for 12 months should contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-    })
+      test('Query result for BSc, Fall 2011 for 12 months should contain the student.', async () => {
+        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+      })
 
-    test('Query result for BSc, Fall 2012 for 12 months should contain the student.', async () => {
-      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.bsc.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-    })
+      test('Query result for BSc, Fall 2012 for 12 months should contain the student.', async () => {
+        const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.bsc.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+      })
 
-    test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
+      test('Query result for Mathematics, Fall 2010 for 12 months should not contain the student.', async () => {
+        const query = createQueryObject('2010', SEMESTER.FALL, [elementdetails.maths.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
+      })
 
-    test('Query result for Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-    })
+      test('Query result for Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
+        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.maths.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+      })
 
-    test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
+      test('Query result for Mathematics, Fall 2012 for 12 months should not contain the student.', async () => {
+        const query = createQueryObject('2012', SEMESTER.FALL, [elementdetails.maths.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
+      })
 
-    test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
-    })
+      test('Query result for BSc and Computer Science, Fall 2011 for 12 months should not contain the student.', async () => {
+        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.cs.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.student_studentnumber)).toBe(false)
+      })
 
-    test('Query result for BSc and Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-    })
+      test('Query result for BSc and Mathematics, Fall 2011 for 12 months should contain the student.', async () => {
+        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code, elementdetails.maths.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+      })
 
-    test('Query result for BSc, Spring 2012 for 12 months should contain the student', async () => {
-      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.bsc.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-    })
+      test('Query result for BSc, Spring 2012 for 12 months should contain the student', async () => {
+        const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.bsc.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+      })
 
-    test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
-      const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
-    })
+      test('Query result for Mathematics, Spring 2012 for 12 months should not contain the student', async () => {
+        const query = createQueryObject('2012', SEMESTER.SPRING, [elementdetails.maths.code], 12)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(false)
+      })
 
-    test('Query result for BSc, Fall 2011 for 4 months should only return the FALL course instance for student. ', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 4)
-      const { students } = await optimizedStatisticsOf(query)
-      const result = students.find(s => s.studentNumber === student.studentnumber)
-      const courseinstances = result.courses
-      expect(courseinstances.length).toBe(1)
-      expect(
-        courseinstances.some(instance =>
-          (instance.date.getTime() === courseinstanceFall.coursedate.getTime()) &&
-          (instance.course.code === courseinstanceFall.course_code))
-      ).toBe(true)
-    })
+      test('Query result for BSc, Fall 2011 for 4 months should only return the FALL course instance for student. ', async () => {
+        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 4)
+        const { students } = await optimizedStatisticsOf(query)
+        const result = students.find(s => s.studentNumber === student.studentnumber)
+        const courseinstances = result.courses
+        expect(courseinstances.length).toBe(1)
+        expect(
+          courseinstances.some(instance =>
+            (instance.date.getTime() === courseinstanceFall.coursedate.getTime()) &&
+            (instance.course.code === courseinstanceFall.course_code))
+        ).toBe(true)
+      })
 
-    test('Query result for BSc, Fall 2011 for 1 month should return student even though do not have any credits yet. ', async () => {
-      const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 1)
-      const { students } = await optimizedStatisticsOf(query)
-      expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
-    })
+      test('Query result for BSc, Fall 2011 for 1 month should return student even though do not have any credits yet. ', async () => {
+        const query = createQueryObject('2011', SEMESTER.FALL, [elementdetails.bsc.code], 1)
+        const { students } = await optimizedStatisticsOf(query)
+        expect(students.some(s => s.studentNumber === student.studentnumber)).toBe(true)
+      })
 
-  })
+    })
 
 })

services/backend/oodikone2-backend/test/supertests/api_spec.js

@@ -2,7 +2,7 @@ const test = require('ava')
 const supertest = require('supertest')
 const app = require('../../src/app')
 const api = supertest(app)
-const { sequelize } = require('../../src/models')
+const { sequelizeKone } = require('../../src/models/models_kone')
 const conf = require('../../src/conf-backend')
 const jwt = require('jsonwebtoken')
 
@@ -22,9 +22,9 @@ test.before(async () => {
     expiresIn: '24h'
   })
   //console.log(token)
-  sequelize.query(
-    `DELETE 
-      FROM tag_student 
+  sequelizeKone.query(
+    `DELETE
+      FROM tag_student
       WHERE tags_tagname = 'test3'`
   )
 })
@@ -186,7 +186,6 @@ test.only('new api populations can be fetched', async t => {
     .expect('Content-Type', /application\/json/)
 
   const stats = res.body
-  console.log(stats)
   t.is(stats.length, 6)
 
 })

services/backend/shared/migrations/20190701_01_add_home_country_to_student.js

@@ -0,0 +1,12 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('student', 'home_country_en', { type: Sequelize.STRING })
+    await queryInterface.addColumn('student', 'home_country_fi', { type: Sequelize.STRING })
+    await queryInterface.addColumn('student', 'home_country_sv', { type: Sequelize.STRING })
+  },
+  down: async () => {
+    await queryInterface.removeColumn('student', 'home_country_en')
+    await queryInterface.removeColumn('student', 'home_country_fi')
+    await queryInterface.removeColumn('student', 'home_country_sv')
+  }
+}

services/backend/shared/migrations/20190701_02_remove_unused_old_country_from_student.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.removeColumn('student', 'country')
+  },
+  down: async () => {
+    await queryInterface.addColumn('student', 'country', { type: Sequelize.STRING })
+
+  }
+}

services/backend/shared/migrations/20190702_01_drop_old_tables.js

@@ -0,0 +1,10 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.dropTable('course_duplicates', { transaction })
+      await queryInterface.dropTable('thesis_courses', { transaction })
+    })
+  },
+  down: async () => {
+  }
+}

services/backend/shared/migrations/20190702_02_drop_schema_kone.js

@@ -0,0 +1,9 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.dropSchema('kone_data', { transaction })
+    })
+  },
+  down: async () => {
+  }
+}

services/backend/shared/migrations_kone/20190528_add_s.js

@@ -1,10 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return Sequelize.transaction(async transaction => {
-      await queryInterface.renameTable('teacher_course_group', 'teacher_course_groups', { transaction })
-    })
-  },
-
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190604_courselabels.js

@@ -1,57 +0,0 @@
-const conf = require('../../conf-backend')
-
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    return queryInterface.sequelize.transaction(async transaction => {
-      await queryInterface.createTable(
-        'mandatory_course_labels',
-        {
-          studyprogramme_id: {
-            type: Sequelize.STRING
-          },
-          id: {
-            primaryKey: true,
-            type: Sequelize.BIGINT,
-            autoIncrement: true
-          },
-          label: {
-            type: Sequelize.STRING
-          },
-          orderNumber: {
-            type: Sequelize.INTEGER
-          },
-          createdAt: {
-            type: Sequelize.DATE
-          },
-          updatedAt: {
-            type: Sequelize.DATE
-          }
-        },
-        { transaction }
-      )
-      await queryInterface.addIndex('mandatory_course_labels', ['studyprogramme_id'], { transaction })
-      // Temporarily hardcode schema here since sequelize has issues:
-      // https://github.com/sequelize/sequelize/issues/10875
-      await queryInterface.removeColumn({ tableName: 'mandatory_courses', schema: conf.DB_SCHEMA_KONE },
-        'label', { transaction, }
-      )
-      await queryInterface.addColumn(
-        { tableName: 'mandatory_courses', schema: 'kone_data' },
-        'label',
-        {
-          type: Sequelize.BIGINT,
-          references: {
-            model: 'mandatory_course_labels',
-            key: 'id'
-          },
-          onDelete: 'SET NULL',
-          onUpdate: 'CASCADE'
-        },
-        { transaction }
-      )
-    })
-  },
-
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190606_01_delete_legacy_tags.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface) => {
-    await queryInterface.dropTable('tag')
-    await queryInterface.dropTable('tag_student')
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190606_02_create_tags.js

@@ -1,26 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.createTable('tag', {
-      createdAt: {
-        type: Sequelize.DATE
-      },
-      updatedAt: {
-        type: Sequelize.DATE
-      },
-      tagname: {
-        type: Sequelize.STRING,
-      },
-      tag_id: {
-        type: Sequelize.BIGINT,
-        autoIncrement: true,
-        primaryKey: true
-      },
-      studytrack: {
-        type: Sequelize.STRING,
-        primaryKey: true,
-      }
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190606_03_add_unique_constraint_to_tag.js

@@ -1,8 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.removeColumn({ tableName: 'tag', schema: 'kone_data' }, 'tag_id')
-    await queryInterface.addColumn({ tableName: 'tag', schema: 'kone_data' }, 'tag_id', { type: Sequelize.BIGINT, primaryKey: true, allowNull: false, autoIncrement: true, unique:true })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190606_04_create_student_tags.js

@@ -1,25 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.createTable('tag_student', {
-      createdAt: {
-        type: Sequelize.DATE
-      },
-      updatedAt: {
-        type: Sequelize.DATE
-      },
-      tag_id: {
-        type: Sequelize.BIGINT,
-        primaryKey: true,
-        reference: {
-          model: 'tag',
-          key: 'tag_id'
-        }
-      },
-      studentnumber: {
-        type: Sequelize.STRING
-      }
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190611_01_add_column_to_tag_student.js

@@ -1,17 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    await queryInterface.removeColumn({ tableName: 'tag_student', schema: 'kone_data' }, 'tag_id')
-    await queryInterface.addColumn({ tableName: 'tag_student', schema: 'kone_data' }, 'id', { type: Sequelize.BIGINT, primaryKey: true, allowNull: false, autoIncrement: true })
-    await queryInterface.sequelize.query(`ALTER TABLE tag_student ADD PRIMARY KEY (id);`)
-    await queryInterface.addColumn({ tableName: 'tag_student', schema: 'kone_data' }, 'tag_id', {
-      type: Sequelize.BIGINT,
-      allowNull: false,
-      references: {
-        model: 'tag',
-        key: 'tag_id'
-      },
-    })
-  },
-  down: async () => {
-  }
-}

services/backend/shared/migrations_kone/20190701_00_Initialize_database.js

@@ -0,0 +1,135 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.sequelize.query(`
+-- Adminer 4.6.2 PostgreSQL dump
+
+CREATE TABLE "public"."course_duplicates" (
+    "groupid" integer NOT NULL,
+    "coursecode" character varying(255) NOT NULL,
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    CONSTRAINT "course_duplicates_pkey" PRIMARY KEY ("coursecode")
+) WITH (oids = false);
+
+
+CREATE TYPE enum_thesis_courses_thesisType AS ENUM ('MASTER', 'BACHELOR');
+
+
+CREATE TABLE "public"."thesis_courses" (
+    "programmeCode" character varying(255) NOT NULL,
+    "courseCode" character varying(255) NOT NULL,
+    "thesisType" enum_thesis_courses_thesisType,
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    CONSTRAINT "thesis_courses_pkey" PRIMARY KEY ("programmeCode", "courseCode")
+) WITH (oids = false);
+
+
+-- 2019-07-01 15:28:04.971883+00
+
+-- Adminer 4.6.2 PostgreSQL dump
+
+CREATE SEQUENCE course_groups_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;
+
+CREATE TABLE "public"."course_groups" (
+    "id" bigint DEFAULT nextval('course_groups_id_seq') NOT NULL,
+    "name" character varying(255),
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    "programmeid" character varying(255),
+    CONSTRAINT "course_groups_name_key" UNIQUE ("name"),
+    CONSTRAINT "course_groups_pkey" PRIMARY KEY ("id")
+) WITH (oids = false);
+
+
+CREATE TABLE "public"."filters" (
+    "id" character varying(255) NOT NULL,
+    "name" character varying(255),
+    "filters" jsonb,
+    "population" jsonb,
+    "createdAt" timestamptz NOT NULL,
+    "updatedAt" timestamptz NOT NULL,
+    "description" character varying(255) DEFAULT '',
+    CONSTRAINT "filters_pkey" PRIMARY KEY ("id")
+) WITH (oids = false);
+
+
+CREATE SEQUENCE mandatory_course_labels_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;
+
+CREATE TABLE "public"."mandatory_course_labels" (
+    "studyprogramme_id" character varying(255),
+    "id" bigint DEFAULT nextval('mandatory_course_labels_id_seq') NOT NULL,
+    "label" character varying(255),
+    "orderNumber" integer,
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    CONSTRAINT "mandatory_course_labels_pkey" PRIMARY KEY ("id")
+) WITH (oids = false);
+
+CREATE INDEX "mandatory_course_labels_studyprogramme_id" ON "public"."mandatory_course_labels" USING btree ("studyprogramme_id");
+
+
+CREATE SEQUENCE mandatory_courses_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;
+
+CREATE TABLE "public"."mandatory_courses" (
+    "id" bigint DEFAULT nextval('mandatory_courses_id_seq') NOT NULL,
+    "course_code" character varying(255),
+    "studyprogramme_id" character varying(255),
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    "label" bigint,
+    CONSTRAINT "mandatory_courses_pkey" PRIMARY KEY ("id"),
+    CONSTRAINT "mandatory_courses_label_fkey" FOREIGN KEY (label) REFERENCES mandatory_course_labels(id) ON UPDATE CASCADE ON DELETE SET NULL NOT DEFERRABLE
+) WITH (oids = false);
+
+
+CREATE SEQUENCE tag_tag_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;
+
+CREATE TABLE "public"."tag" (
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    "tagname" character varying(255),
+    "studytrack" character varying(255) NOT NULL,
+    "tag_id" bigint DEFAULT nextval('tag_tag_id_seq') NOT NULL,
+    CONSTRAINT "tag_tag_id_key" UNIQUE ("tag_id")
+) WITH (oids = false);
+
+
+CREATE SEQUENCE tag_student_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;
+
+CREATE TABLE "public"."tag_student" (
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    "studentnumber" character varying(255),
+    "id" bigint DEFAULT nextval('tag_student_id_seq') NOT NULL,
+    "tag_id" bigint NOT NULL,
+    CONSTRAINT "tag_student_pkey" PRIMARY KEY ("id"),
+    CONSTRAINT "tag_student_tag_id_fkey" FOREIGN KEY (tag_id) REFERENCES tag(tag_id) NOT DEFERRABLE
+) WITH (oids = false);
+
+
+CREATE SEQUENCE teacher_course_group_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;
+
+CREATE TABLE "public"."teacher_course_groups" (
+    "id" bigint DEFAULT nextval('teacher_course_group_id_seq') NOT NULL,
+    "teacher_id" character varying(255),
+    "course_group_id" bigint,
+    "createdAt" timestamptz,
+    "updatedAt" timestamptz,
+    CONSTRAINT "teacher_course_group_pkey" PRIMARY KEY ("id")
+) WITH (oids = false);
+
+
+CREATE TABLE "public"."unit_tag" (
+    "tags_tagname" character varying(255) NOT NULL,
+    "unit_id" bigint NOT NULL,
+    CONSTRAINT "unit_tag_pkey" PRIMARY KEY ("tags_tagname", "unit_id")
+) WITH (oids = false);
+
+
+-- 2019-07-01 15:26:14.147541+00
+`)
+  },
+  down: async () => {
+  }
+}

services/backend/shared/migrations_kone/20190703_00_add_usage_table.js

@@ -0,0 +1,25 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.sequelize.query(`
+-- Adminer 4.6.2 PostgreSQL dump
+
+CREATE TABLE IF NOT EXISTS "public"."usage_statistics" (
+    "id" character varying(255) NOT NULL,
+    "username" character varying(255),
+    "name" character varying(255),
+    "time" integer,
+    "admin" boolean,
+    "method" character varying(255),
+    "URL" character varying(255),
+    "status" integer,
+    "data" jsonb,
+    CONSTRAINT "usage_statistics_pkey" PRIMARY KEY ("id")
+) WITH (oids = false);
+
+
+-- 2019-07-03 15:39:09.675736+00
+`)
+  },
+  down: async () => {
+  }
+}

services/backend/shared/models/index.js

@@ -1,186 +1,5 @@
 const Sequelize = require('sequelize')
-const { sequelize, sequelizeKone, migrationPromise } = require('../database/connection')
-const conf = require('../conf-backend')
-
-const ThesisTypeEnums = {
-  MASTER: 'MASTER',
-  BACHELOR: 'BACHELOR'
-}
-
-const CourseGroup = sequelize.define('course_groups', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    autoIncrement: true
-  },
-  name: {
-    type: Sequelize.STRING,
-    unique: true
-  }
-},
-  {
-    schema: conf.DB_SCHEMA_KONE
-  })
-
-const TeacherCourseGroup = sequelize.define('teacher_course_groups', {
-  course_group_id: {
-    type: Sequelize.BIGINT,
-  },
-  teacher_id: {
-    type: Sequelize.STRING,
-  }
-},
-  {
-    schema: conf.DB_SCHEMA_KONE
-  })
-
-const MandatoryCourse = sequelize.define('mandatory_courses', {
-  course_code: {
-    type: Sequelize.STRING,
-  },
-  studyprogramme_id: {
-    type: Sequelize.STRING
-  },
-  label: {
-    type: Sequelize.BIGINT
-  }
-},
-  {
-    schema: conf.DB_SCHEMA_KONE
-  })
-
-const Filters = sequelize.define('filters',
-  {
-    id: {
-      type: Sequelize.STRING,
-      primaryKey: true
-    },
-    name: {
-      type: Sequelize.STRING
-    },
-    description: {
-      type: Sequelize.STRING
-    },
-    filters: {
-      type: Sequelize.JSONB
-    },
-    population: {
-      type: Sequelize.JSONB
-    }
-  },
-  {
-    schema: conf.DB_SCHEMA_KONE
-  }
-)
-
-const UsageStatistic = sequelize.define('usage_statistics', {
-  id: {
-    primaryKey: true,
-    type: Sequelize.STRING
-  },
-  username: {
-    type: Sequelize.STRING
-  },
-  name: {
-    type: Sequelize.STRING
-  },
-  time: {
-    type: Sequelize.INTEGER
-  },
-  admin: {
-    type: Sequelize.BOOLEAN
-  },
-  method: {
-    type: Sequelize.STRING
-  },
-  URL: {
-    type: Sequelize.STRING
-  },
-  status: {
-    type: Sequelize.INTEGER
-  },
-  data: {
-    type: Sequelize.JSONB
-  },
-
-}, {
-    timestamps: false,
-    schema: conf.DB_SCHEMA_KONE
-  })
-
-const TagStudent = sequelize.define('tag_student',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.BIGINT,
-      autoIncrement: true
-    },
-    studentnumber: {
-      type: Sequelize.STRING,
-    },
-    tag_id: {
-      type: Sequelize.BIGINT,
-    },
-  },
-  {
-    tableName: 'tag_student',
-    schema: conf.DB_SCHEMA_KONE
-  }
-)
-
-const Tag = sequelize.define('tag',
-  {
-    tagname: {
-      type: Sequelize.STRING
-    },
-    tag_id: {
-      primaryKey: true,
-      type: Sequelize.BIGINT,
-      autoIncrement: true
-    },
-    studytrack: {
-      primaryKey: true,
-      type: Sequelize.STRING
-    }
-  },
-  {
-    tableName: 'tag',
-    schema: conf.DB_SCHEMA_KONE
-  }
-)
-
-const MandatoryCourseLabels = sequelize.define('mandatory_course_labels',
-  {
-    id: {
-      primaryKey: true,
-      type: Sequelize.BIGINT,
-      autoIncrement: true
-    },
-    studyprogramme_id: {
-      type: Sequelize.STRING,
-    },
-    label: {
-      type: Sequelize.STRING
-    },
-    orderNumber: {
-      type: Sequelize.INTEGER,
-    },
-  },
-  {
-    schema: conf.DB_SCHEMA_KONE
-  }
-)
-
-const MigrationKone = sequelize.define('migrationsKone', {
-  name: {
-    type: Sequelize.STRING,
-    primaryKey: true
-  }
-}, {
-    tableName: 'migrations',
-    timestamps: false,
-    schema: conf.DB_SCHEMA_KONE
-  })
+const { sequelize, migrationPromise } = require('../database/connection')
 
 const Student = sequelize.define('student',
   {
@@ -193,7 +12,6 @@ const Student = sequelize.define('student',
     abbreviatedname: { type: Sequelize.STRING },
     birthdate: { type: Sequelize.DATE },
     communicationlanguage: { type: Sequelize.STRING },
-    country: { type: Sequelize.STRING },
     creditcount: { type: Sequelize.INTEGER },
     dateofuniversityenrollment: { type: Sequelize.DATE },
     matriculationexamination: { type: Sequelize.STRING },
@@ -214,6 +32,9 @@ const Student = sequelize.define('student',
     country_fi: { type: Sequelize.STRING },
     country_sv: { type: Sequelize.STRING },
     country_en: { type: Sequelize.STRING },
+    home_country_fi: { type: Sequelize.STRING },
+    home_country_sv: { type: Sequelize.STRING },
+    home_country_en: { type: Sequelize.STRING },
     gender_code: { type: Sequelize.INTEGER },
     gender_fi: { type: Sequelize.STRING },
     gender_sv: { type: Sequelize.STRING },
@@ -600,42 +421,6 @@ const CreditTeacher = sequelize.define('credit_teacher', {
   }
 })
 
-const ThesisCourse = sequelize.define('thesis_courses', {
-  programmeCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-    references: {
-      model: ElementDetails,
-      key: 'code'
-    }
-  },
-  courseCode: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-    references: {
-      model: Course,
-      key: 'code'
-    }
-  },
-  thesisType: {
-    type: Sequelize.ENUM([ThesisTypeEnums.BACHELOR, ThesisTypeEnums.MASTER])
-  }
-})
-
-const CourseDuplicates = sequelize.define('course_duplicates', {
-  groupid: {
-    primaryKey: true,
-    type: Sequelize.INTEGER
-  },
-  coursecode: {
-    type: Sequelize.STRING,
-    references: {
-      model: Course,
-      key: 'code'
-    }
-  }
-})
-
 const ErrorData = sequelize.define('error_data', {
   id: {
     primaryKey: true,
@@ -652,17 +437,6 @@ Student.hasMany(Credit, { foreignKey: 'student_studentnumber', sourceKey: 'stude
 Credit.belongsTo(Course, { foreignKey: 'course_code' })
 Course.hasMany(Credit, { foreignKey: 'course_code' })
 
-MandatoryCourse.belongsTo(Course, { foreignKey: 'course_code' })
-MandatoryCourse.belongsTo(MandatoryCourseLabels, { foreignKey: 'label', sourceKey: 'id' })
-MandatoryCourseLabels.hasMany(MandatoryCourse, { foreignKey: 'label', sourceKey: 'id' })
-
-TagStudent.belongsTo(Tag, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
-TagStudent.belongsTo(Student, { foreignKey: 'studentnumber' })
-Student.hasMany(TagStudent, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
-
-Tag.hasOne(ElementDetails, { foreignKey: 'code', sourceKey: 'studytrack' })
-Tag.hasMany(TagStudent, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
-
 Studyright.belongsTo(Student, { foreignKey: 'student_studentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'student_studentnumber', sourceKey: 'studentnumber' })
 
@@ -713,37 +487,18 @@ Student.belongsToMany(CourseRealisation, { through: CourseEnrollment, foreignKey
 Credit.belongsToMany(Teacher, { through: CreditTeacher, foreignKey: 'credit_id' })
 Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id' })
 
-Teacher.belongsToMany(CourseGroup, { through: TeacherCourseGroup, foreignKey: 'teacher_id' })
-CourseGroup.belongsToMany(Teacher, { through: TeacherCourseGroup, foreignKey: 'course_group_id' })
-CourseGroup.belongsTo(ElementDetails, { foreignKey: 'programmeid' })
-
 Credit.belongsTo(Semester, { foreignKey: { name: 'semestercode', allowNull: false } })
 
-Course.hasMany(ThesisCourse, { foreignKey: 'courseCode', sourceKey: 'code' })
-ThesisCourse.belongsTo(Course, { foreignKey: 'courseCode', targetKey: 'code' })
-
-ElementDetails.hasMany(ThesisCourse, { foreignKey: 'programmeCode', sourceKey: 'code' })
-ThesisCourse.belongsTo(ElementDetails, { foreignKey: 'programmeCode', targetKey: 'code' })
-
-CourseDuplicates.belongsTo(Course, { foreignKey: 'coursecode' })
-
 module.exports = {
-  MandatoryCourse,
   Student,
   Credit,
   Studyright,
   Course,
-  TagStudent,
-  Tag,
   Teacher,
-  sequelize,
-  sequelizeKone,
-  migrationPromise,
   Organisation,
   StudentList,
   StudyrightElement,
   ElementDetails,
-  Filters,
   StudyrightExtent,
   CourseType,
   CreditType,
@@ -757,14 +512,9 @@ module.exports = {
   CourseRealisationType,
   CourseRealisation,
   CourseEnrollment,
-  Migration,
-  MigrationKone,
   CreditTeacher,
-  UsageStatistic,
-  CourseGroup,
-  ThesisCourse,
-  CourseDuplicates,
-  ThesisTypeEnums,
-  MandatoryCourseLabels,
-  ErrorData
+  ErrorData,
+  Migration,
+  migrationPromise,
+  sequelize
 }

services/backend/shared/models/models_kone.js

@@ -0,0 +1,239 @@
+const Sequelize = require('sequelize')
+const {
+  sequelizeKone,
+  migrationPromiseKone
+} = require('../database/connection')
+
+const ThesisTypeEnums = {
+  MASTER: 'MASTER',
+  BACHELOR: 'BACHELOR'
+}
+
+const CourseGroup = sequelizeKone.define(
+  'course_groups',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true
+    },
+    name: {
+      type: Sequelize.STRING,
+      unique: true
+    },
+    programmeid: {
+      type: Sequelize.STRING,
+    }
+  }
+)
+
+const TeacherCourseGroup = sequelizeKone.define('teacher_course_groups', {
+  course_group_id: {
+    type: Sequelize.BIGINT
+  },
+  teacher_id: {
+    type: Sequelize.STRING
+  }
+})
+
+const MandatoryCourse = sequelizeKone.define('mandatory_courses', {
+  course_code: {
+    type: Sequelize.STRING
+  },
+  studyprogramme_id: {
+    type: Sequelize.STRING
+  },
+  label: {
+    type: Sequelize.BIGINT
+  }
+})
+
+const Filters = sequelizeKone.define('filters', {
+  id: {
+    type: Sequelize.STRING,
+    primaryKey: true
+  },
+  name: {
+    type: Sequelize.STRING
+  },
+  description: {
+    type: Sequelize.STRING
+  },
+  filters: {
+    type: Sequelize.JSONB
+  },
+  population: {
+    type: Sequelize.JSONB
+  }
+})
+
+const UsageStatistic = sequelizeKone.define(
+  'usage_statistics',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.STRING
+    },
+    username: {
+      type: Sequelize.STRING
+    },
+    name: {
+      type: Sequelize.STRING
+    },
+    time: {
+      type: Sequelize.INTEGER
+    },
+    admin: {
+      type: Sequelize.BOOLEAN
+    },
+    method: {
+      type: Sequelize.STRING
+    },
+    URL: {
+      type: Sequelize.STRING
+    },
+    status: {
+      type: Sequelize.INTEGER
+    },
+    data: {
+      type: Sequelize.JSONB
+    }
+  },
+  {
+    timestamps: false
+  }
+)
+
+const TagStudent = sequelizeKone.define(
+  'tag_student',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true
+    },
+    studentnumber: {
+      type: Sequelize.STRING
+    },
+    tag_id: {
+      type: Sequelize.BIGINT
+    }
+  },
+  {
+    tableName: 'tag_student'
+  }
+)
+
+const Tag = sequelizeKone.define(
+  'tag',
+  {
+    tagname: {
+      type: Sequelize.STRING
+    },
+    tag_id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true
+    },
+    studytrack: {
+      primaryKey: true,
+      type: Sequelize.STRING
+    }
+  },
+  {
+    tableName: 'tag'
+  }
+)
+
+const MandatoryCourseLabels = sequelizeKone.define('mandatory_course_labels', {
+  id: {
+    primaryKey: true,
+    type: Sequelize.BIGINT,
+    autoIncrement: true
+  },
+  studyprogramme_id: {
+    type: Sequelize.STRING
+  },
+  label: {
+    type: Sequelize.STRING
+  },
+  orderNumber: {
+    type: Sequelize.INTEGER
+  }
+})
+
+const MigrationKone = sequelizeKone.define(
+  'migrations',
+  {
+    name: {
+      type: Sequelize.STRING,
+      primaryKey: true
+    }
+  },
+  {
+    tableName: 'migrations',
+    timestamps: false
+  }
+)
+
+const CourseDuplicates = sequelizeKone.define('course_duplicates', {
+  groupid: {
+    primaryKey: true,
+    type: Sequelize.INTEGER
+  },
+  coursecode: {
+    type: Sequelize.STRING,
+  }
+})
+
+const ThesisCourse = sequelizeKone.define('thesis_courses', {
+  programmeCode: {
+    primaryKey: true,
+    type: Sequelize.STRING
+  },
+  courseCode: {
+    primaryKey: true,
+    type: Sequelize.STRING
+  },
+  thesisType: {
+    type: Sequelize.ENUM([ThesisTypeEnums.BACHELOR, ThesisTypeEnums.MASTER])
+  }
+})
+
+MandatoryCourse.belongsTo(MandatoryCourseLabels, {
+  foreignKey: 'label',
+  sourceKey: 'id'
+})
+MandatoryCourseLabels.hasMany(MandatoryCourse, {
+  foreignKey: 'label',
+  sourceKey: 'id'
+})
+
+TeacherCourseGroup.belongsTo(CourseGroup, {
+  foreignKey: 'course_group_id',
+  sourceKey: 'id'
+})
+CourseGroup.hasMany(TeacherCourseGroup, {
+  foreignKey: 'course_group_id',
+  sourceKey: 'id'
+})
+
+TagStudent.belongsTo(Tag, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
+Tag.hasMany(TagStudent, { foreignKey: 'tag_id', sourceKey: 'tag_id' })
+
+module.exports = {
+  TeacherCourseGroup,
+  MandatoryCourse,
+  TagStudent,
+  Tag,
+  Filters,
+  UsageStatistic,
+  CourseGroup,
+  CourseDuplicates,
+  ThesisCourse,
+  ThesisTypeEnums,
+  MandatoryCourseLabels,
+  MigrationKone,
+  migrationPromiseKone,
+  sequelizeKone
+}

services/backend/shared/models/queries.js

@@ -1,4 +1,4 @@
-const { sequelize } = require('../database/connection')
+const { sequelize, sequelizeKone } = require('../database/connection')
 const conf = require('../conf-backend')
 
 const getStudentNumbers = () => sequelize.query(
@@ -111,29 +111,52 @@ const getAcademicYearCoursesByTeacherIds = (teacherIds, startSemester) => {
 
 const getAcademicYearStatisticsForStudyProgramme = async (programmeid, startSemester) => {
   const endSemester = startSemester + 1
-  const result = await sequelize.query(
+  const courseGroupData = await sequelizeKone.query(
     `
-SELECT cg.id AS id,cg.name AS name,
-    COUNT(DISTINCT c.course_code) AS courses,
-    COALESCE(SUM(credits), 0) AS credits,
-    COUNT(DISTINCT student_studentnumber) AS students
-FROM   ${conf.DB_SCHEMA_KONE}.course_groups cg
-    left join ${conf.DB_SCHEMA_KONE}.teacher_course_groups tcg
+SELECT cg.id AS id, cg.name AS name, tcg.teacher_id as teacher_id, cg.programmeid as programmeid
+FROM   course_groups cg
+    left join teacher_course_groups tcg
            ON tcg.course_group_id = cg.id
-    left join credit_teachers ct
-           ON ct.teacher_id = tcg.teacher_id
-    left join credit c
-           ON c.id = ct.credit_id
-              AND c.semestercode >= :startSemester
-              AND c.semestercode <= :endSemester
 WHERE cg.programmeid = :programmeid
-GROUP BY cg.programmeid,cg.id,cg.name`,
+`,
     {
-      replacements: { startSemester, endSemester, programmeid },
+      replacements: { programmeid },
       type: sequelize.QueryTypes.SELECT,
     }
   )
-  return result
+
+  const courseGroupIdToCGData = courseGroupData.reduce((acc, cgd) => {
+    acc[cgd.id] = acc[cgd.id] || { name: cgd.name, id: cgd.id, programmeid: cgd.programmeid, teacherids: [] }
+    if (cgd.teacher_id) acc[cgd.id].teacherids.push(cgd.teacher_id)
+    return acc
+  }, {})
+
+  const resultPromises = Object.keys(courseGroupIdToCGData).map(async cgid => {
+    const { name, id, programmeid, teacherids } = courseGroupIdToCGData[cgid]
+    if (teacherids.length === 0) return { name, id, programmeid, courses: 0, credits: 0, students: 0 }
+    const stats = await sequelize.query(
+      `
+SELECT
+    COUNT(DISTINCT c.course_code) AS courses,
+    COALESCE(SUM(credits), 0) AS credits,
+    COUNT(DISTINCT student_studentnumber) AS students
+FROM   credit_teachers ct
+    left join credit c
+            ON c.id = ct.credit_id
+              AND c.semestercode >= :startSemester
+              AND c.semestercode <= :endSemester
+WHERE ct.teacher_id IN (:teacherids)
+  `,
+      {
+        replacements: { startSemester, endSemester, teacherids },
+        type: sequelize.QueryTypes.SELECT,
+      }
+    )
+    if (stats.length !== 1) throw "expecting single row"
+    const { courses, credits, students } = stats[0]
+    return { name, id, programmeid, courses, credits, students }
+  })
+  return Promise.all(resultPromises)
 }
 
 module.exports = {

services/backend/updater_writer/database/connection.js

@@ -1,4 +1,6 @@
 const Sequelize = require('sequelize')
+const Umzug = require('umzug')
+
 const conf = require('../conf-backend')
 
 const sequelize = new Sequelize(conf.DB_URL, {
@@ -9,15 +11,34 @@ const sequelize = new Sequelize(conf.DB_URL, {
 })
 sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
 
-const sequelizeKone = new Sequelize(conf.DB_URL, {
-  schema: conf.DB_SCHEMA_KONE,
-  searchPath: conf.DB_SCHEMA_KONE,
-  logging: false,
-  operatorsAliases: false
-})
-sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
-// See https://github.com/sequelize/sequelize/issues/10875
 
+const runMigrations = async () => {
+  try {
+    const migrator = new Umzug({
+      storage: 'sequelize',
+      storageOptions: {
+        sequelize: sequelize,
+        tableName: 'migrations'
+      },
+      logging: console.log,
+      migrations: {
+        params: [
+          sequelize.getQueryInterface(),
+          Sequelize
+        ],
+        path: `${process.cwd()}/database/migrations`,
+        pattern: /\.js$/,
+        schema: conf.DB_SCHEMA
+      }
+    })
+    const migrations = await migrator.up()
+
+    console.log('Migrations up to date', migrations)
+  } catch (e) {
+    console.log('Migration error, message:', e)
+  }
+}
+const migrationPromise = !conf.isTest ? runMigrations() : Promise.resolve()
 
 const forceSyncDatabase = async () => {
   try {
@@ -25,19 +46,12 @@ const forceSyncDatabase = async () => {
   } catch (e) {
     // console.log(e)
   }
-  try {
-    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
-  } catch (e) {
-    // console.log(e)
-  }
   console.log('OODI SYNC')
   await sequelize.sync({ force: true })
-  console.log('KONE SYNC')
-  await sequelizeKone.sync({ force: true })
 }
 
 module.exports = {
   sequelize,
-  sequelizeKone,
+  migrationPromise,
   forceSyncDatabase
 }

services/backend/updater_writer/database/dump_database.js

@@ -1,6 +1,5 @@
 const spawn = require('child_process').spawn
 const fs = require('fs')
-const { sync } = require('./force_sync_database')
 const { DB_SCHEMA } = require('../conf-backend')
 
 const dumpDatabase = async () => {
@@ -19,8 +18,7 @@ const dumpDatabase = async () => {
     if (code !== 0) {
       throw new Error('pg_dump: Bad exit code (' + code + ')');
     } else {
-      console.log('database dump created, force syncing database')
-      await sync()
+      console.log('database dump created')
     }
   })
 }

services/backend/updater_writer/index.js

@@ -14,7 +14,7 @@ stan.on('connect', function () {
 
   const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
   const attSub = stan.subscribe('UpdateAttainmentDates', opts)
-  const dumpSub = stan.subscribe('DumpDatabase')
+  const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
 
   sub.on('message', async (msg) => {
     const data = JSON.parse(msg.getData())
@@ -30,14 +30,10 @@ stan.on('connect', function () {
     await updateAttainmentMeta()
     msg.ack()
   })
-  dumpSub.on('message', async (_) => {
-    await dumpDatabase()
-    stan.publish('ScheduleAll', null, (err, guid) => {
-      if (err) {
-        console.log(err)
-      } else {
-        console.log('calling re-scheduling database')
-      }
-    })
+  prioSub.on('message', async (msg) => {
+    const data = JSON.parse(msg.getData())
+    await updateStudent(data, stan)
+    msg.ack()
+    stan.publish('status', `${data.studentInfo.studentnumber}:DONE`, (err) => { if (err) console.log(err) })
   })
 })

services/backend/updater_writer/package-lock.json

@@ -676,6 +676,15 @@
         "babel-plugin-jest-hoist": "^24.6.0"
       }
     },
+    "babel-runtime": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
+      "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
+      "requires": {
+        "core-js": "^2.4.0",
+        "regenerator-runtime": "^0.11.0"
+      }
+    },
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
@@ -1049,6 +1058,11 @@
       "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
       "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40="
     },
+    "core-js": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+      "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A=="
+    },
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -4160,6 +4174,11 @@
         "util.promisify": "^1.0.0"
       }
     },
+    "regenerator-runtime": {
+      "version": "0.11.1",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
+      "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg=="
+    },
     "regex-not": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
@@ -4986,6 +5005,15 @@
         }
       }
     },
+    "umzug": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.2.0.tgz",
+      "integrity": "sha512-xZLW76ax70pND9bx3wqwb8zqkFGzZIK8dIHD9WdNy/CrNfjWcwQgQkGCuUqcuwEBvUm+g07z+qWvY+pxDmMEEw==",
+      "requires": {
+        "babel-runtime": "^6.23.0",
+        "bluebird": "^3.5.3"
+      }
+    },
     "undefsafe": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.2.tgz",

services/backend/updater_writer/package.json

@@ -19,6 +19,7 @@
     "node-nats-streaming": "^0.2.2",
     "nodemon": "^1.19.0",
     "pg": "^7.10.0",
-    "sequelize": "^5.8.5"
+    "sequelize": "^5.8.5",
+    "umzug": "^2.2.0"
   }
 }

services/backend/updater_writer/updater/update_attainment_dates.js

@@ -2,20 +2,26 @@ const { sequelize } = require('../database/connection')
 
 const updateAttainmentDates = async () => {
   const { schema } = sequelize.options
-  await sequelize.query(
-    `UPDATE  ${schema}.course
+  const transaction = await sequelize.transaction()
+  try {
+    await sequelize.query(
+      `UPDATE  ${schema}.course
      SET max_attainment_date = cr.max
      FROM (select course_code, max(attainment_date) from ${schema}.credit group by course_code) cr
      WHERE course.code=cr.course_code`,
-    { type: sequelize.QueryTypes.UPDATE }
-  )
-  await sequelize.query(
-    `UPDATE  ${schema}.course
+      { type: sequelize.QueryTypes.UPDATE, transaction, lock: transaction.LOCK.UPDATE }
+    )
+    await sequelize.query(
+      `UPDATE  ${schema}.course
      SET min_attainment_date = cr.min
      FROM (select course_code, min(attainment_date) from ${schema}.credit group by course_code) cr
      WHERE course.code=cr.course_code`,
-    { type: sequelize.QueryTypes.UPDATE }
-  )
+      { type: sequelize.QueryTypes.UPDATE, transaction, lock: transaction.LOCK.UPDATE }
+    )
+    transaction.commit()
+  } catch (e) {
+    transaction.rollback()
+  }
 }
 
 module.exports = {

services/oodikone2-frontend/.eslintrc.json

@@ -17,7 +17,7 @@
       }
     ],
     "import/no-unresolved": 2,
-    "max-len": [1, 120],
+    "max-len": [1, 160],
     "react/jsx-no-bind": 2,
     "react/no-did-mount-set-state": 0,
     "react/prefer-stateless-function": 2,

services/oodikone2-frontend/package.json

@@ -7,8 +7,8 @@
     "url": "git+https://github.com/UniversityOfHelsinkiCS/oodikone2-frontend.git"
   },
   "scripts": {
-    "dev": "webpack-dev-server --mode=development --hot",
-    "docker": "webpack-dev-server --mode=development --hot --host 0.0.0.0",
+    "dev": "webpack-dev-server --mode=development --hot --inline",
+    "docker": "webpack-dev-server --mode=development --hot --inline --host 0.0.0.0",
     "dist": "webpack --mode=production",
     "lint": "concurrently 'npm run lint:js' 'npm run lint:css'",
     "lint:css": "stylelint 'src/**/*.css'",

services/oodikone2-frontend/src/apiConnection/index.js

@@ -1,9 +1,8 @@
 import axios from 'axios'
-
-import { getToken, setToken, getAsUserWithoutRefreshToken } from '../common'
+import { getToken, setToken, getAsUserWithoutRefreshToken, getTokenWithoutRefresh } from '../common'
 import { API_BASE_PATH, TOKEN_NAME, BASE_PATH } from '../constants'
+import { login as loginAction } from '../redux/auth'
 
-const getAxios = () => axios.create({ baseURL: API_BASE_PATH })
 const isTestEnv = BASE_PATH === '/testing/'
 const isDevEnv = process.env.NODE_ENV === 'development'
 const devOptions = {
@@ -12,7 +11,6 @@ const devOptions = {
     displayName: 'Development Kayttaja',
     'shib-session-id': 'mock-session'
   }
-
 }
 
 const testOptions = {
@@ -34,18 +32,13 @@ const getDefaultConfig = () => {
   }
 }
 
-const createDefaultAxiosConfig = async () => {
+const createDefaultAxiosConfig = () => {
   const config = getDefaultConfig()
-  const token = await getToken()
   config.baseURL = API_BASE_PATH
-  config.headers['x-access-token'] = token
   return config
 }
 
-export const createConfiguredAxios = async () => {
-  const config = await createDefaultAxiosConfig()
-  return axios.create(config)
-}
+export const api = axios.create({ ...createDefaultAxiosConfig() })
 
 const types = {
   attempt: prefix => `${prefix}ATTEMPT`,
@@ -68,10 +61,17 @@ export const login = async () => {
   if (isTestEnv) {
     options = testOptions
   }
-  const response = await getAxios().post('/login', null, options)
+  const response = await api.post('/login', null, options)
   return response.data.token
 }
 
+export const logout = async () => {
+  const returnUrl = window.location.origin
+  const response = await api.delete('/logout', { data: { returnUrl } })
+  localStorage.removeItem(TOKEN_NAME)
+  window.location = response.data.logoutUrl
+}
+
 export const returnToSelf = async () => {
   const token = await login()
   await setToken(token)
@@ -85,18 +85,16 @@ export const callApi = async (url, method = 'get', data, params, timeout = 0) =>
   if (isTestEnv) {
     options = testOptions
   }
-  const token = await getToken()
-  options.headers['x-access-token'] = token
 
   switch (method) {
     case 'get':
-      return getAxios().get(url, { ...options, params })
+      return api.get(url, { ...options, params })
     case 'post':
-      return getAxios().post(url, data, options)
+      return api.post(url, data, options)
     case 'put':
-      return getAxios().put(url, data, options)
+      return api.put(url, data, options)
     case 'delete':
-      return getAxios().delete(url, { headers: options.headers, data })
+      return api.delete(url, { headers: options.headers, data })
     default:
       return Promise.reject(new Error('Invalid http method'))
   }
@@ -104,8 +102,10 @@ export const callApi = async (url, method = 'get', data, params, timeout = 0) =>
 
 export const superLogin = async (uid) => {
   const response = await callApi(`/superlogin/${uid}`, 'post')
-  console.log(`Setting new token ${response.data.token}`)
-  await setToken(response.data.token)
+  const { token } = response.data
+  console.log(`Setting new token ${token}`)
+  await setToken(token)
+  return token
 }
 
 export const callController = (route, prefix, data, method = 'get', query, params) => {
@@ -132,29 +132,57 @@ export const handleRequest = store => next => async (action) => {
       store.dispatch({ type: `${prefix}SUCCESS`, response: res.data, query })
     } catch (e) {
       // Something failed. Assume it's the token and try again.
-      try {
-        const mock = getAsUserWithoutRefreshToken()
-        if (!mock) {
-          await getToken(true)
-          if (e.response.data.reloadPage) {
-            setTimeout(() => { window.location.reload() }, 1000)
-            return
-          }
-        }
-        const res = await callApi(route, method, data, params)
-        store.dispatch({ type: `${prefix}SUCCESS`, response: res.data, query })
-      } catch (err) {
-        store.dispatch({ type: `${prefix}FAILURE`, response: err, query })
+      if (e.response.data.reloadPage) {
+        setTimeout(() => { window.location.reload() }, 1000)
+        return
       }
+      store.dispatch(loginAction(true, requestSettings))
     }
   }
 }
 
-export const logout = async () => {
-  const returnUrl = window.location.origin
-  const response = await getAxios().delete('/logout', { data: { returnUrl } })
-  localStorage.removeItem(TOKEN_NAME)
-  window.location = response.data.logoutUrl
+export const handleAuth = store => next => async (action) => {
+  next(action)
+  const { type, force = false, retryRequestSettings, uid, refresh } = action
+  const {
+    route, method, data, prefix, query, params
+  } = retryRequestSettings || {}
+  if (type === 'LOGIN_SUCCESS' && refresh) window.location.reload()
+  if (type === 'LOGIN_ATTEMPT') {
+    try {
+      const mock = await getAsUserWithoutRefreshToken()
+      let token
+      if (mock && !uid) token = getTokenWithoutRefresh()
+      else if (uid) token = await superLogin(uid)
+      else token = await getToken(force)
+
+      api.defaults.headers.common = {
+        ...api.defaults.headers.common,
+        'x-access-token': token
+      }
+      store.dispatch({ type: 'LOGIN_SUCCESS', token, refresh: uid })
+      try {
+        if (retryRequestSettings) {
+          const res = await callApi(route, method, data, params)
+          store.dispatch({ type: `${prefix}SUCCESS`, response: res.data, query })
+        }
+      } catch (err) {
+        store.dispatch({ type: `${prefix}FAILURE`, response: err, query })
+      }
+    } catch (err) {
+      store.dispatch({ type: 'LOGIN_FAILURE' })
+      if (retryRequestSettings) {
+        store.dispatch({ type: `${prefix}FAILURE`, response: err, query })
+      }
+    }
+  } else if (type === 'LOGOUT_ATTEMPT') {
+    try {
+      await logout()
+      store.dispatch({ type: 'LOGOUT_SUCCESSFUL' })
+    } catch (err) {
+      store.dispatch({ type: 'LOGOUT_FAILURE' })
+    }
+  }
 }
 
 export const sendLog = async data => callApi('/log', 'post', data)

services/oodikone2-frontend/src/components/AccessDenied/index.jsx

@@ -1,9 +1,10 @@
 import React, { useState, useEffect } from 'react'
 import { Transition, Dimmer, Header, Image, Container, Button } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { bool, func } from 'prop-types'
 import Highcharts from 'highcharts'
 import ReactHighchart from 'react-highcharts'
-import { logout } from '../../apiConnection'
+import { connect } from 'react-redux'
+import { logout as logoutAction } from '../../redux/auth'
 import { log, images } from '../../common'
 
 import MulticolorBarChart from '../MulticolorBarChart'
@@ -15,7 +16,7 @@ const dummyData = [
   { text: 'ttuotila', value: 59.7 }
 ]
 
-const AccessDenied = ({ notEnabled }) => {
+const AccessDenied = ({ notEnabled, logout }) => {
   const header = notEnabled ? 'Welcome to Oodikone!' : 'Something broke'
   const subheader = notEnabled ? `You're currently not allowed to enter 
   but you will get an email when you're authorized`
@@ -116,7 +117,15 @@ const AccessDenied = ({ notEnabled }) => {
 }
 
 AccessDenied.propTypes = {
-  notEnabled: bool.isRequired
+  notEnabled: bool.isRequired,
+  logout: func.isRequired
 }
 
-export default AccessDenied
+const mapDispatchToProps = {
+  logout: logoutAction
+}
+
+export default connect(
+  null,
+  mapDispatchToProps
+)(AccessDenied)

services/oodikone2-frontend/src/components/CourseGroups/CourseGroupAddTeacher/index.jsx

@@ -64,8 +64,7 @@ class CourseGroupAddTeacher extends Component {
     const searchResultColumns = [
       { key: 'teacherid', title: 'Teacher ID', getRowVal: s => s.id, headerProps: { onClick: null, sorted: null } },
       { key: 'username', title: 'Username', getRowVal: s => s.code, headerProps: { onClick: null, sorted: null } },
-      { key: 'name', title: 'Name', getRowVal: s => s.name, headerProps: { onClick: null, sorted: null } },
-      { key: 'coursegroup', title: 'Course groups', getRowVal: s => s.course_groups.map(cg => cg.name).join(), headerProps: { onClick: null, sorted: null, colSpan: 2 } },
+      { key: 'name', title: 'Name', getRowVal: s => s.name, headerProps: { onClick: null, sorted: null, colSpan: 2 } },
       { key: 'icon', getRowVal: () => (<Icon name="add user" />), cellProps: { collapsing: true }, headerProps: { onClick: null, sorted: null } }
     ]
 

services/oodikone2-frontend/src/components/CourseStatistics/CourseTable/index.jsx

@@ -17,8 +17,9 @@ const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, ma
   )
 
   const toCourseRow = course => (
-    <Table.Row style={{ cursor: 'pointer' }} key={course.code} onClick={() => (course.min_attainment_date || mandatory ? onSelectCourse(course) : null)} >
-      <Table.Cell width={10}>
+    // kinda hax solution for the problem, after fetching course course is array of selected course code letters
+    <Table.Row style={{ cursor: 'pointer' }} key={course.code ? course.code : course[0]} onClick={() => (course.min_attainment_date || mandatory ? onSelectCourse(course) : null)} >
+      <Table.Cell key={course.code} width={10}>
         <div>{course.name}</div>
         <div>{getActiveYears(course)}</div>
       </Table.Cell>
@@ -26,6 +27,7 @@ const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, ma
     </Table.Row>
   )
 
+
   return (
     !hidden && (
       <Segment basic style={{ padding: '0' }} >

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/student.jsx

@@ -3,7 +3,7 @@ import { Header } from 'semantic-ui-react'
 import { shape, string, number, oneOfType, arrayOf } from 'prop-types'
 import SortableTable from '../../../../SortableTable'
 
-const formatPercentage = p => `${p.toFixed(2) * 100} %`
+const formatPercentage = p => `${(p * 100).toFixed(2)} %`
 
 const StudentTable = ({ stats, name }) => {
   const formatted = stats.map((statistic) => {

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/passRate.jsx

@@ -33,14 +33,12 @@ const getPassRateCumSeriesFromStats = (stats) => {
     ],
     relative: [
       // eslint-disable-next-line no-unused-vars
-      getDataObject(`all`, all.map(_ => 1), 'a'),
       getDataObject(`passed`, passed.map(absoluteToRelative(all)), 'b'),
       getDataObject(`failed`, failed.map(absoluteToRelative(all)), 'c')
     ]
   }
 }
 
-
 const getPassRateStudSeriesFromStats = (stats) => {
   const all = []
   const failedFirst = []
@@ -73,7 +71,6 @@ const getPassRateStudSeriesFromStats = (stats) => {
     ],
     relative: [
       // eslint-disable-next-line no-unused-vars
-      getDataObject(`all`, all.map(_ => 1), 'a'),
       getDataObject(` passed on first try`, passedFirst.map(absoluteToRelative(all)), 'b'),
       getDataObject(`passed after retry`, passedRetry.map(absoluteToRelative(all)), 'b'),
       getDataObject(`failed on first try`, failedFirst.map(absoluteToRelative(all)), 'c'),
@@ -95,8 +92,8 @@ const PassRate = ({ primary, comparison, viewMode, isRelative = false }) => {
 
   const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
   const graphOptionsFn = isCumulativeMode ? passRateCumGraphOptions : passRateStudGraphOptions
-  const primaryGraphOptions = comparison ? graphOptionsFn(statYears, maxPassRateVal, 'Primary pass rate chart') : graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart')
-  const comparisonGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Comparison pass rate chart')
+  const primaryGraphOptions = comparison ? graphOptionsFn(statYears, maxPassRateVal, 'Primary pass rate chart', isRelative) : graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
+  const comparisonGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Comparison pass rate chart', isRelative)
 
   return (
     <>

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/index.jsx

@@ -41,7 +41,7 @@ class ResultTabs extends Component {
             comparison={comparison}
             primary={primary}
             viewMode={viewMode}
-            isRelative={isRelative}
+            isRelative={isRelative && comparison}
           />)
       },
       {
@@ -51,7 +51,7 @@ class ResultTabs extends Component {
             comparison={comparison}
             primary={primary}
             viewMode={viewMode}
-            isRelative={isRelative}
+            isRelative={isRelative && comparison}
           />)
       }
     ]
@@ -125,6 +125,7 @@ class ResultTabs extends Component {
               <label className="toggleLabel">Absolute</label>
               <Radio
                 toggle
+                checked={this.state.isRelative}
                 onChange={() => this.setState({ isRelative: !this.state.isRelative })}
               />
               <label className="toggleLabel">Relative</label>

services/oodikone2-frontend/src/components/CourseStatistics/SearchForm/YearFilter/index.jsx

@@ -2,7 +2,7 @@ import React, { Fragment } from 'react'
 import { arrayOf, bool, func, number, shape } from 'prop-types'
 import { Form } from 'semantic-ui-react'
 
-const YearFilter = ({ years, fromYear, toYear, handleChange, separate, onToggleCheckbox }) => (
+const YearFilter = ({ years, fromYear, toYear, handleChange, separate, onToggleCheckbox, showCheckbox }) => (
   <Fragment>
     <Form.Group widths="equal" inline>
       <Form.Dropdown
@@ -26,12 +26,16 @@ const YearFilter = ({ years, fromYear, toYear, handleChange, separate, onToggleC
         value={toYear}
       />
     </Form.Group>
-    <Form.Checkbox
-      label="Separate statistics for Spring and Fall semesters"
-      name="separate"
-      onChange={onToggleCheckbox}
-      checked={separate}
-    />
+    {
+      showCheckbox && (
+        <Form.Checkbox
+          label="Separate statistics for Spring and Fall semesters"
+          name="separate"
+          onChange={onToggleCheckbox}
+          checked={separate}
+        />
+      )
+    }
   </Fragment>
 )
 
@@ -40,13 +44,15 @@ YearFilter.propTypes = {
   fromYear: number,
   toYear: number,
   handleChange: func.isRequired,
-  onToggleCheckbox: func.isRequired,
-  separate: bool.isRequired
+  onToggleCheckbox: func,
+  separate: bool.isRequired,
+  showCheckbox: bool.isRequired
 }
 
 YearFilter.defaultProps = {
   fromYear: undefined,
-  toYear: undefined
+  toYear: undefined,
+  onToggleCheckbox: () => null
 }
 
 export default YearFilter

services/oodikone2-frontend/src/components/CourseStatistics/SearchForm/index.jsx

@@ -33,10 +33,10 @@ class SearchForm extends Component {
 
   componentDidMount() {
     const { location } = this.props
+    this.props.getSemesters()
     if (location.search) {
       this.fetchStatisticsFromUrlParams()
     } else {
-      this.props.getSemesters()
       this.props.clearCourses()
     }
   }
@@ -199,6 +199,7 @@ class SearchForm extends Component {
                   separate={separate}
                   handleChange={this.handleChange}
                   onToggleCheckbox={this.onToggleCheckbox}
+                  showCheckbox
                 />
                 <Form.Button
                   type="button"

services/oodikone2-frontend/src/components/CourseStatistics/SingleCourseStats/index.jsx

@@ -2,9 +2,12 @@ import React, { Component } from 'react'
 import { Segment, Header, Form } from 'semantic-ui-react'
 import { shape, string, arrayOf, objectOf, oneOfType, number } from 'prop-types'
 import { connect } from 'react-redux'
+import { withRouter } from 'react-router'
+import qs from 'query-string'
 import ResultTabs from '../ResultTabs'
 import ProgrammeDropdown from '../ProgrammeDropdown'
 import selectors, { ALL } from '../../../selectors/courseStats'
+import YearFilter from '../SearchForm/YearFilter'
 
 const countFilteredStudents = (stat, filter) => Object.entries(stat).reduce((acc, entry) => {
   const [category, students] = entry
@@ -17,7 +20,15 @@ const countFilteredStudents = (stat, filter) => Object.entries(stat).reduce((acc
 class SingleCourseStats extends Component {
   state = {
     primary: ALL.value,
-    comparison: null
+    comparison: null,
+    separate: null
+  }
+
+  componentDidMount = () => {
+    const { location } = this.props
+    if (location.search) {
+      this.setState({ ...this.parseQueryFromUrl() })
+    }
   }
 
   getProgrammeName = (progcode) => {
@@ -28,6 +39,18 @@ class SingleCourseStats extends Component {
     return name.fi || name.en || name.sv
   }
 
+  parseQueryFromUrl = () => {
+    const { location } = this.props
+    const { separate, fromYear, toYear } = qs.parse(location.search)
+    return {
+      separate: JSON.parse(separate),
+      fromYear: JSON.parse(fromYear),
+      fromYearInitial: JSON.parse(fromYear),
+      toYear: JSON.parse(toYear),
+      toYearInitial: JSON.parse(toYear)
+    }
+  }
+
   belongsToProgramme = (code) => {
     if (code === ALL.value) {
       return () => true
@@ -46,6 +69,33 @@ class SingleCourseStats extends Component {
     return programmes[code] || (code === ALL.value)
   }
 
+  filteredYearsAndSemesters = (useInitialValues = false) => {
+    const { years, semesters } = this.props
+    const { fromYearInitial, fromYear, toYearInitial, toYear } = this.state
+    if (!fromYearInitial || !toYearInitial) {
+      return {
+        filteredYears: years,
+        filteredSemesters: semesters
+      }
+    }
+    const timeFilter = ({ value }) => (
+      value >= (useInitialValues ? fromYearInitial : fromYear) &&
+      value <= (useInitialValues ? toYearInitial : toYear)
+    )
+    return {
+      filteredYears: years.filter(timeFilter),
+      filteredSemesters: semesters.filter(timeFilter)
+    }
+  }
+
+  isStatInYearRange = ({ name }) => {
+    const { separate } = this.state
+    const { filteredYears, filteredSemesters } = this.filteredYearsAndSemesters()
+    return separate ?
+      filteredSemesters.find(year => year.texts.includes(name)) :
+      filteredYears.find(year => year.text === name)
+  }
+
   statsForProgramme = (progcode) => {
     const { statistics } = this.props.stats
     const filter = this.belongsToProgramme(progcode)
@@ -59,7 +109,7 @@ class SingleCourseStats extends Component {
         categories: countFilteredStudents(allstudents.classes, filter)
       }
       return { code, name, cumulative, students }
-    })
+    }).filter(this.isStatInYearRange)
     return {
       code: progcode,
       name: this.getProgrammeName(progcode),
@@ -87,14 +137,16 @@ class SingleCourseStats extends Component {
     const { primary: p, comparison: c } = this.state
     const { programmes } = this.props.stats
     const primary = !programmes[p] ? ALL.value : p
-    const comparison = !programmes[c] ? undefined : c
+    const comparison = (!programmes[c] && c !== ALL.value) ? undefined : c
     return { primary, comparison }
   }
 
   render() {
     const { programmes } = this.props
+    const { fromYear, toYear } = this.state
     const { primary, comparison } = this.selectedProgrammes()
     const statistics = this.filteredProgrammeStatistics()
+    const { filteredYears } = this.filteredYearsAndSemesters(true)
     return (
       <div>
         <Segment>
@@ -121,6 +173,19 @@ class SingleCourseStats extends Component {
             </Form.Group>
           </Form>
         </Segment>
+        <Segment>
+          <Form>
+            <Header content="Filter statistics by time range" as="h4" />
+            <YearFilter
+              years={filteredYears}
+              fromYear={fromYear}
+              toYear={toYear}
+              handleChange={this.handleChange}
+              showCheckbox={false}
+              separate={false}
+            />
+          </Form>
+        </Segment>
         <ResultTabs
           primary={statistics.primary}
           comparison={statistics.comparison}
@@ -148,11 +213,27 @@ SingleCourseStats.propTypes = {
     name: string,
     coursecode: string
   }).isRequired,
-  programmes: arrayOf(shape({})).isRequired
+  programmes: arrayOf(shape({})).isRequired,
+  years: arrayOf(shape({})).isRequired,
+  semesters: arrayOf(shape({})).isRequired,
+  location: shape({}).isRequired
 }
 
-const mapStateToProps = state => ({
-  programmes: selectors.getAllStudyProgrammes(state)
-})
+const mapStateToProps = (state) => {
+  const { semesters = [], years = [] } = state.semesters.data
+  return {
+    programmes: selectors.getAllStudyProgrammes(state),
+    years: Object.values(years).map(({ yearcode, yearname }) => ({
+      key: yearcode,
+      text: yearname,
+      value: yearcode
+    })).reverse(),
+    semesters: Object.values(semesters).map(({ semestercode, name, yearcode }) => ({
+      key: semestercode,
+      texts: Object.values(name),
+      value: yearcode
+    })).reverse()
+  }
+}
 
-export default connect(mapStateToProps)(SingleCourseStats)
+export default withRouter(connect(mapStateToProps)(SingleCourseStats))

services/oodikone2-frontend/src/components/CourseStatistics/index.jsx

@@ -69,7 +69,7 @@ class CourseStatistics extends Component {
   }
 
   render() {
-    const { statsIsEmpty } = this.props
+    const { statsIsEmpty, history } = this.props
     const panes = this.getPanes()
     return (
       <div className="container">
@@ -77,7 +77,7 @@ class CourseStatistics extends Component {
           Course Statistics
         </Header>
         <Segment className="contentSegment" >
-          {statsIsEmpty ? <SearchForm /> : (
+          {statsIsEmpty || history.location.search === '' ? <SearchForm /> : (
             <Tab
               menu={{ attached: false, borderless: false }}
               panes={panes}

services/oodikone2-frontend/src/components/Faculty/FacultySelector/index.jsx

@@ -0,0 +1,52 @@
+import React, { useEffect } from 'react'
+import { withRouter } from 'react-router'
+import { connect } from 'react-redux'
+import { string, func, arrayOf, shape } from 'prop-types'
+import { getFaculties } from '../../../redux/faculties'
+import { getTextIn } from '../../../common'
+import SortableTable from '../../SortableTable'
+
+const FacultySelector = ({ language, handleSelect, dispatchGetFaculties, faculties }) => {
+  const fetchFaculties = async () => {
+    await dispatchGetFaculties()
+  }
+  useEffect(() => {
+    fetchFaculties()
+  }, [])
+
+  if (!faculties) return null
+  const headers = [
+    {
+      key: 'name',
+      title: 'name',
+      getRowVal: faculty => getTextIn(faculty.name, language)
+    },
+    {
+      key: 'code',
+      title: 'code',
+      getRowVal: faculty => faculty.code
+    }
+  ]
+
+  return (
+    <SortableTable
+      columns={headers}
+      getRowKey={faculty => faculty.code}
+      getRowProps={faculty => ({ onClick: () => handleSelect(faculty.code), style: { cursor: 'pointer' } })}
+      data={faculties}
+    />
+  )
+}
+FacultySelector.propTypes = {
+  language: string.isRequired,
+  handleSelect: func.isRequired,
+  dispatchGetFaculties: func.isRequired,
+  faculties: arrayOf(shape({})).isRequired
+}
+
+const mapStateToProps = ({ faculties, settings }) => ({
+  faculties: faculties.data,
+  language: settings.language
+})
+
+export default connect(mapStateToProps, { dispatchGetFaculties: getFaculties })(withRouter(FacultySelector))

services/oodikone2-frontend/src/components/Faculty/index.jsx

@@ -0,0 +1,17 @@
+import React from 'react'
+import { withRouter } from 'react-router'
+import { Header, Segment } from 'semantic-ui-react'
+import FacultySelector from './FacultySelector'
+
+const Faculty = () => (
+  <div className="segmentContainer">
+    <Header className="segmentTitle" size="large">
+      Faculties
+    </Header>
+    <Segment className="contentSegment" >
+      <FacultySelector />
+    </Segment>
+  </div>
+)
+
+export default withRouter(Faculty)

services/oodikone2-frontend/src/components/Main/index.jsx

@@ -1,23 +1,58 @@
-import React from 'react'
+import React, { useEffect } from 'react'
 import { localize } from 'react-localize-redux'
 import { BrowserRouter as Router } from 'react-router-dom'
+import { connect } from 'react-redux'
+import { bool, func, shape, string } from 'prop-types'
 import { BASE_PATH } from '../../constants'
+import { login as loginAction } from '../../redux/auth'
 
 import Header from '../Header'
 import ErrorContainer from '../ErrorContainer'
 import Routes from '../Routes'
 import './main.css'
 
-const Main = () => (
-  <div className="appContainer">
-    <Router basename={BASE_PATH}>
-      <main className="routeViewContainer">
-        <Header />
-        <ErrorContainer />
-        <Routes />
-      </main>
-    </Router>
-  </div>
-)
+const Main = ({ auth: { token, error }, login }) => {
+  useEffect(() => {
+    login()
+  }, [])
+  if (!token && !error) return <div />
+  return (
+    <div className="appContainer">
+      <Router basename={BASE_PATH}>
+        <main className="routeViewContainer">
+          <Header />
+          <ErrorContainer />
+          <Routes />
+        </main>
+      </Router>
+    </div>
+  )
+}
 
-export default localize(Main, 'locale')
+Main.propTypes = {
+  auth: shape({
+    token: string,
+    error: bool
+  }),
+  login: func.isRequired
+}
+
+Main.defaultProps = {
+  auth: {
+    token: null,
+    error: false
+  }
+}
+
+const mapStateToProps = state => ({
+  auth: state.auth
+})
+
+const mapDispatchToProps = {
+  login: loginAction
+}
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(localize(Main, 'locale'))

services/oodikone2-frontend/src/components/NavigationBar/index.jsx

@@ -5,12 +5,12 @@ import { func, shape, string } from 'prop-types'
 import { connect } from 'react-redux'
 import { withRouter } from 'react-router'
 import { routes } from '../../constants'
-import { userRoles, userRights } from '../../common'
+import { userRoles, userRights, getAsUserWithoutRefreshToken } from '../../common'
 import './navigationBar.css'
-import { logout, login, returnToSelf } from '../../apiConnection'
+import { login as loginAction, logout as logoutAction } from '../../redux/auth'
 import LanguageChooser from '../LanguageChooser'
 
-const { USER_ADMINER_URL, ADMINER_URL, ANALYTICS_ADMINER_URL, USAGE_ADMINER_URL } = process.env
+const { USER_ADMINER_URL, ADMINER_URL, ANALYTICS_ADMINER_URL, USAGE_ADMINER_URL, KONE_ADMINER_URL } = process.env
 
 class NavigationBar extends Component {
   state = {
@@ -21,9 +21,11 @@ class NavigationBar extends Component {
     await this.setNavigationRoutes()
   }
 
-  async componentWillReceiveProps() {
-    await this.setNavigationRoutes()
-    this.render()
+  async componentWillReceiveProps(nextProps) {
+    if (this.props.token !== nextProps.token) {
+      await this.setNavigationRoutes()
+      this.render()
+    }
   }
 
   setNavigationRoutes = async () => {
@@ -44,13 +46,12 @@ class NavigationBar extends Component {
   }
 
   returnToSelf = () => async () => {
-    await returnToSelf()
+    this.props.logout()
     this.props.history.push('/')
-    window.location.reload()
   }
 
   renderUserMenu = (itemWidth) => {
-    const { translate } = this.props
+    const { translate, login, logout } = this.props
     if (process.env.NODE_ENV === 'development') {
       const testUsers = ['tktl']
       return (
@@ -91,6 +92,16 @@ class NavigationBar extends Component {
                 icon="database"
               />
             )}
+            {KONE_ADMINER_URL && (
+              <Dropdown.Item
+                onClick={() => {
+                  const win = window.open(KONE_ADMINER_URL, '_blank')
+                  win.focus()
+                }}
+                text="Kone database"
+                icon="database"
+              />
+            )}
             {USAGE_ADMINER_URL && (
               <Dropdown.Item
                 onClick={() => {
@@ -127,7 +138,8 @@ class NavigationBar extends Component {
   }
 
   render() {
-    const { translate: t, asUser } = this.props
+    const { translate: t } = this.props
+    const asUser = getAsUserWithoutRefreshToken()
     const { navigationRoutes } = this.state
     const menuWidth = asUser ? Object.keys(navigationRoutes).length + 3 : Object.keys(navigationRoutes).length + 2
     const itemWidth = 100 / menuWidth
@@ -205,18 +217,28 @@ class NavigationBar extends Component {
 
 NavigationBar.propTypes = {
   translate: func.isRequired,
-  asUser: string,
   history: shape({
     push: func.isRequired
-  }).isRequired
+  }).isRequired,
+  token: string,
+  login: func.isRequired,
+  logout: func.isRequired
 }
 
 NavigationBar.defaultProps = {
-  asUser: null
+  token: null
 }
 
-const mapStateToProps = ({ settings }) => ({
-  asUser: settings.asUser
+const mapStateToProps = ({ auth }) => ({
+  token: auth.token
 })
 
-export default connect(mapStateToProps)(withRouter(NavigationBar))
+const mapDispatchToProps = {
+  login: loginAction,
+  logout: logoutAction
+}
+
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(withRouter(NavigationBar))

services/oodikone2-frontend/src/components/PopulationCourseStats/index.jsx

@@ -6,9 +6,10 @@ import { getTranslate } from 'react-localize-redux'
 import _ from 'lodash'
 import { withRouter } from 'react-router-dom'
 import moment from 'moment'
+import qs from 'query-string'
 
 import { setPopulationFilter, removePopulationFilterOfCourse } from '../../redux/populationFilters'
-import { getCourseStats, clearCourseStats } from '../../redux/coursestats'
+import { clearCourseStats } from '../../redux/coursestats'
 
 import { courseParticipation } from '../../populationFilters'
 import PassingSemesters from './PassingSemesters'
@@ -89,7 +90,6 @@ class PopulationCourseStats extends Component {
     selectedCourses: arrayOf(object).isRequired,
     removePopulationFilterOfCourse: func.isRequired,
     history: shape({}).isRequired,
-    getCourseStats: func.isRequired,
     clearCourseStats: func.isRequired,
     language: string.isRequired,
     query: shape({}).isRequired,
@@ -130,7 +130,6 @@ class PopulationCourseStats extends Component {
       .filter(studentAmountFilter)
       .filter(c => !codeFilter || courseCodeFilter(c))
 
-
     const lodashSortOrder = reversed ? lodashSortOrderTypes.DESC : lodashSortOrderTypes.ASC
 
     const courseStatistics = _.orderBy(
@@ -187,24 +186,18 @@ class PopulationCourseStats extends Component {
     const {
       history,
       query,
-      getCourseStats: getStatsFn,
       clearCourseStats: clearCourseStatsfn,
       years
     } = this.props
-
     const yearCode = year => Object.values(years).find(yearObject =>
       yearObject.yearname.slice(0, 4).includes(year)).yearcode
-    const { year, months } = query
-    const fromYear = yearCode(year)
-    const toYear = yearCode(moment(moment(year, 'YYYY').add(months, 'months')).format('YYYY'))
-    history.push('/coursestatistics/')
+    const { startYear, months } = query
+    const fromYear = yearCode(startYear)
+    const toYear = yearCode(moment(moment(startYear, 'YYYY').add(months, 'months')).format('YYYY'))
+    const queryObject = { toYear, fromYear, separate: false, courseCodes: JSON.stringify([code]) }
+    const searchString = qs.stringify(queryObject)
+    history.push(`/coursestatistics?${searchString}`)
     clearCourseStatsfn()
-    getStatsFn({
-      courseCodes: [code],
-      fromYear,
-      toYear,
-      separate: false
-    })
   }
 
   onCourseNameCellClick = (courseStats) => {
@@ -517,7 +510,6 @@ export default connect(
   {
     setPopulationFilter,
     removePopulationFilterOfCourse,
-    getCourseStats,
     clearCourseStats
   }
 )(withRouter(PopulationCourseStats))

services/oodikone2-frontend/src/components/PopulationFilters/SimpleExtentGraduated.jsx

@@ -84,7 +84,6 @@ const SimpleExtentGraduated = (props) => {
   )
 }
 
-
 const mapStateToProps = ({ settings, populations, populationDegreesAndProgrammes }) => {
   const code = populations.query.studyRights.programme
   const studyrightName = populationDegreesAndProgrammes.data.programmes[code].name

services/oodikone2-frontend/src/components/PopulationFilters/TagFilter.jsx

@@ -7,11 +7,10 @@ import _ from 'lodash'
 import { removePopulationFilter, setPopulationFilter } from '../../redux/populationFilters'
 import { tagFilter } from '../../populationFilters'
 
-
-
 const TagFilter = ({ setPopulationFilterAction, removePopulationFilterAction, filter, samples }) => {
   const [options, setOptions] = useState([])
   const [selectedTag, setSelectedTag] = useState()
+  const [selectedComp, setSelectedComp] = useState()
 
   const createOptions = () => {
     const tags = samples.map(s => s.tags.map(t => ({ tagname: t.tag.tagname, tag_id: t.tag.tag_id })))
@@ -26,13 +25,16 @@ const TagFilter = ({ setPopulationFilterAction, removePopulationFilterAction, fi
   }, [])
 
   const handleFilter = () => {
-    setPopulationFilterAction(tagFilter({ tag: selectedTag }))
+    setPopulationFilterAction(tagFilter({ tag: selectedTag, comp: selectedComp }))
+  }
+
+  const handleCompChange = (e, { value }) => {
+    setSelectedComp(value)
   }
 
   const handleChange = (e, { value }) => {
-    // any better solutions?
-    const selection = options.filter(tag => tag.value === value)
-    setSelectedTag(selection[0])
+    const selection = options.find(tag => tag.value === value)
+    setSelectedTag(selection)
   }
   const clearFilter = () => {
     removePopulationFilterAction(filter.id)
@@ -47,7 +49,17 @@ const TagFilter = ({ setPopulationFilterAction, removePopulationFilterAction, fi
           />
           <Form.Group inline>
             <Form.Field>
-              <label>Select students that have tag </label>
+              <label>Select students that </label>
+            </Form.Field>
+            <Form.Field>
+              <Dropdown
+                placeholder="select"
+                options={[{ key: 1, text: 'have', value: true }, { key: 2, text: 'don\'t have', value: false }]}
+                onChange={handleCompChange}
+              />
+            </Form.Field>
+            <Form.Field>
+              <label>a tag </label>
             </Form.Field>
             <Form.Field>
               <Dropdown
@@ -69,7 +81,7 @@ const TagFilter = ({ setPopulationFilterAction, removePopulationFilterAction, fi
   }
   return (
     <Segment>
-      Students that have a tag {filter.params.text}
+      Students that {filter.params.comp ? 'have' : 'don\'t have'} a tag {filter.params.text}
       <span style={{ float: 'right' }}>
         <Icon name="remove" onClick={clearFilter} />
       </span>

services/oodikone2-frontend/src/components/PopulationFilters/index.jsx

@@ -32,7 +32,6 @@ import {
 import { presetFilter, getFilterFunction } from '../../populationFilters'
 import { getTextIn, userIsAdmin } from '../../common'
 
-
 const componentFor = {
   CreditsAtLeast,
   CreditsLessThan,

services/oodikone2-frontend/src/components/PopulationQueryCard/index.jsx

@@ -23,7 +23,7 @@ const PopulationQueryCard =
       history.push('/populations')
       removeSampleFn(uuid)
     }
-    const { uuid, year, semesters, months, studentStatuses } = query
+    const { uuid, startYear, endYear, semesters, months, studentStatuses, tag } = query
     const { students } = population
     if (students.length > 0) {
       return (
@@ -39,20 +39,23 @@ const PopulationQueryCard =
           <Card.Meta>
             <div className="dateItem">
               <Icon name="calendar" size="small" />
-              {`${semesters.map(s => translate(`populationStatistics.${s}`))}/
-                ${year}-${Number(year) + 1}, showing ${months} months.`}
+              {tag ? `${semesters.map(s => translate(`populationStatistics.${s}`))}/
+                ${startYear}-${endYear}, showing ${months} months.` :
+                `${semesters.map(s => translate(`populationStatistics.${s}`))}/
+                ${startYear}-${Number(startYear) + 1}, showing ${months} months.`}
             </div>
+            {tag ? (<div>{`Tagged with: ${tag.tagname}`}</div>) : null}
             <div>
               {`${translate('populationStatistics.sampleSize', { amount: students.length })} `}
             </div>
             <div>
               {`Updated at ${reformatDate(_.minBy(students, 'updatedAt').updatedAt, DISPLAY_DATE_FORMAT)} `}
             </div>
-            <div>{studentStatuses.includes('EXCHANGE') ? 'Includes' : 'Excludes' } exchange students</div>
-            <div>{studentStatuses.includes('CANCELLED') ? 'Includes ' : 'Excludes ' }
+            <div>{studentStatuses.includes('EXCHANGE') ? 'Includes' : 'Excludes'} exchange students</div>
+            <div>{studentStatuses.includes('CANCELLED') ? 'Includes ' : 'Excludes '}
               students with cancelled study right
             </div>
-            <div>{studentStatuses.includes('NONDEGREE') ? 'Includes ' : 'Excludes ' }
+            <div>{studentStatuses.includes('NONDEGREE') ? 'Includes ' : 'Excludes '}
               students with non-degree study right
             </div>
             {updating ?
@@ -83,7 +86,7 @@ const PopulationQueryCard =
         <Card.Meta>
           <div className="dateItem">
             <Icon name="calendar" size="small" />
-            {`${semesters.map(s => translate(`populationStatistics.${s}`))}/${year}-${Number(year) + 1},
+            {`${semesters.map(s => translate(`populationStatistics.${s}`))}/${startYear}-${Number(startYear) + 1},
              showing ${months} months.`}
           </div>
           <div>
@@ -109,7 +112,7 @@ PopulationQueryCard.propTypes = {
   translate: func.isRequired,
   population: shape({ students: arrayOf(object), extents: arrayOf(object) }).isRequired,
   query: shape({
-    year: oneOfType([string, number]),
+    startYear: oneOfType([string, number]),
     semester: string,
     studyRights: shape({ programme: string, degree: string, studyTrack: string }),
     uuid: string

services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx

@@ -2,7 +2,7 @@ import React, { Component } from 'react'
 import { connect } from 'react-redux'
 import { withRouter } from 'react-router'
 import qs from 'query-string'
-import { func, arrayOf, shape, bool, string, object } from 'prop-types'
+import { func, arrayOf, shape, bool, string, object, oneOfType } from 'prop-types'
 import { Form, Button, Message, Icon, Grid } from 'semantic-ui-react'
 import { getTranslate } from 'react-localize-redux'
 import uuidv4 from 'uuid/v4'
@@ -18,7 +18,8 @@ import { getSemesters } from '../../redux/semesters'
 import { transferTo } from '../../populationFilters'
 
 import { getDegreesAndProgrammes } from '../../redux/populationDegreesAndProgrammes'
-import { momentFromFormat, reformatDate, textAndDescriptionSearch, getTextIn } from '../../common'
+import { getTagsByStudytrackAction } from '../../redux/tags'
+import { momentFromFormat, reformatDate, textAndDescriptionSearch, getTextIn, userIsAdmin } from '../../common'
 import { setLoading } from '../../redux/graphSpinner'
 import './populationSearchForm.css'
 import { dropdownType } from '../../constants/types'
@@ -48,7 +49,12 @@ class PopulationSearchForm extends Component {
     getSemesters: func.isRequired,
     semesters: shape({}).isRequired,
     history: shape({}).isRequired,
-    location: shape({}).isRequired
+    location: shape({}).isRequired,
+    getTagsByStudytrackAction: func.isRequired,
+    tags: oneOfType([
+      arrayOf(shape({ tag_id: string, tagname: string })),
+      object
+    ]).isRequired
   }
 
   constructor() {
@@ -60,11 +66,13 @@ class PopulationSearchForm extends Component {
       isLoading: false,
       showAdvancedSettings: false,
       momentYear: Datetime.moment('2017-01-01'),
-      floatMonths: this.months('2017', 'FALL')
+      floatMonths: this.months('2017', 'FALL'),
+      selectedTag: '',
+      isAdmin: false
     }
   }
 
-  componentDidMount() {
+  async componentDidMount() {
     const { studyProgrammes, semesters, location } = this.props
     if (!studyProgrammes || Object.values(studyProgrammes).length === 0) {
       this.setState({ query: this.initialQuery() }) // eslint-disable-line
@@ -76,6 +84,8 @@ class PopulationSearchForm extends Component {
     if (location.search) {
       this.fetchPopulationFromUrlParams()
     }
+    const admin = await userIsAdmin()
+    this.setState({ isAdmin: admin })
   }
 
   componentDidUpdate(prevProps) {
@@ -133,14 +143,14 @@ class PopulationSearchForm extends Component {
   }
 
   fetchPopulation = (query) => {
+    const { selectedTag } = this.state
     const queryCodes = Object.values(query.studyRights).filter(e => e != null)
-
     const uuid = uuidv4()
     const request = { ...query, studyRights: queryCodes, uuid }
     this.setState({ isLoading: true })
     this.props.setLoading()
     Promise.all([
-      this.props.getPopulationStatistics({ ...query, uuid }),
+      this.props.getPopulationStatistics({ ...query, uuid, tag: selectedTag }),
       this.props.getPopulationCourses(request),
       this.props.getPopulationFilters(request),
       this.props.getMandatoryCourses(query.studyRights.programme)
@@ -149,13 +159,14 @@ class PopulationSearchForm extends Component {
         this.props.setPopulationFilter(transferTo(false))
       }
       this.setState({ isLoading: false })
+      // not a good solution FIX
+      this.setState({ selectedTag: '' })
     })
   }
 
   handleYearSelection = (momentYear) => {
     const { query } = this.state
     const { studyProgrammes } = this.props
-
     if (!moment.isMoment(momentYear)) {
       this.setState({
         momentYear: null,
@@ -193,12 +204,12 @@ class PopulationSearchForm extends Component {
         }
       }
     }
-
     this.setState({
       momentYear,
       query: {
         ...query,
-        year: reformatDate(momentYear, YEAR_DATE_FORMAT),
+        endYear: reformatDate(momentYear, YEAR_DATE_FORMAT),
+        startYear: reformatDate(momentYear, YEAR_DATE_FORMAT),
         months: this.months(
           reformatDate(momentYear, YEAR_DATE_FORMAT),
           this.state.query.semesters.includes('FALL') ? 'FALL' : 'SPRING'
@@ -212,15 +223,35 @@ class PopulationSearchForm extends Component {
     })
   }
 
+  handleTagSearch = (event, { value }) => {
+    const tag = this.props.tags.find(t => t.tag_id === value)
+    this.setState({
+      selectedTag: tag
+    })
+  }
+
+  handleTagYearSelect = (momentYear) => {
+    const { query } = this.state
+    const months = this.getMonths(reformatDate(momentYear, YEAR_DATE_FORMAT), new Date(), 'FALL')
+    this.setState({
+      query: {
+        ...query,
+        startYear: reformatDate(momentYear, YEAR_DATE_FORMAT),
+        endYear: reformatDate(new Date(), YEAR_DATE_FORMAT),
+        months
+      }
+    })
+  }
+
   addYear = () => {
-    const { year } = this.state.query
-    const nextYear = momentFromFormat(year, YEAR_DATE_FORMAT).add(1, 'year')
+    const { startYear } = this.state.query
+    const nextYear = momentFromFormat(startYear, YEAR_DATE_FORMAT).add(1, 'year')
     this.handleYearSelection(nextYear)
   }
 
   subtractYear = () => {
-    const { year } = this.state.query
-    const previousYear = momentFromFormat(year, YEAR_DATE_FORMAT).subtract(1, 'year')
+    const { startYear } = this.state.query
+    const previousYear = momentFromFormat(startYear, YEAR_DATE_FORMAT).subtract(1, 'year')
     this.handleYearSelection(previousYear)
   }
 
@@ -232,7 +263,7 @@ class PopulationSearchForm extends Component {
       query: {
         ...query,
         semesters,
-        months: this.months(this.state.query.year, semesters.includes('FALL') ? 'FALL' : 'SPRING')
+        months: this.months(this.state.query.startYear, semesters.includes('FALL') ? 'FALL' : 'SPRING')
       }
     })
   }
@@ -269,6 +300,7 @@ class PopulationSearchForm extends Component {
 
   handleProgrammeChange = (e, { value }) => {
     const programme = value
+    this.props.getTagsByStudytrackAction(value)
     if (programme === '') {
       this.handleClear('programme')
       return
@@ -304,7 +336,7 @@ class PopulationSearchForm extends Component {
 
   handleMonthsChange = (value) => {
     const { query } = this.state
-    const months = this.getMonths(query.year, value, this.state.query.semesters.includes('FALL') ? 'FALL' : 'SPRING')
+    const months = this.getMonths(query.startYear, value, this.state.query.semesters.includes('FALL') ? 'FALL' : 'SPRING')
     this.setState({
       query: {
         ...query,
@@ -326,17 +358,17 @@ class PopulationSearchForm extends Component {
     })
   }
 
-  getMonths = (year, end, term) => {
+  getMonths = (startYear, end, term) => {
     const lastDayOfMonth = moment(end).endOf('month')
-    const start = term === 'FALL' ? `${year}-08-01` : `${year}-01-01`
+    const start = term === 'FALL' ? `${startYear}-08-01` : `${startYear}-01-01`
     this.setState({
       floatMonths: moment.duration(moment(lastDayOfMonth).diff(moment(start))).asMonths()
     })
     return Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(start))).asMonths())
   }
 
-  getMonthValue = (year, months) => {
-    const start = `${year}-08-01`
+  getMonthValue = (startYear, months) => {
+    const start = `${startYear}-08-01`
     return moment(start).add(months - 1, 'months').format('MMMM YY')
   }
 
@@ -350,7 +382,7 @@ class PopulationSearchForm extends Component {
     return this.props.studyProgrammes[this.state.query.studyRights.programme].enrollmentStartYears[momentYear.year()] != null
   }
 
-  getMinSelection = (year, semester) => (semester === 'FALL' ? `${year}-08-01` : `${year}-01-01`)
+  getMinSelection = (startYear, semester) => (semester === 'FALL' ? `${startYear}-08-01` : `${startYear}-01-01`)
 
   fetchPopulationFromUrlParams() {
     const query = this.parseQueryFromUrl()
@@ -359,7 +391,8 @@ class PopulationSearchForm extends Component {
   }
 
   initialQuery = () => ({
-    year: Datetime.moment('2017-01-01').year(),
+    endYear: Datetime.moment('2017-01-01').year(),
+    startYear: Datetime.moment('2017-01-01').year(),
     semesters: ['FALL', 'SPRING'],
     studentStatuses: [],
     studyRights: [],
@@ -380,7 +413,7 @@ class PopulationSearchForm extends Component {
 
   renderEnrollmentDateSelector = () => {
     const { query, momentYear } = this.state
-    const { semesters, year } = query
+    const { semesters, startYear } = query
     return (
       <Form.Group key="year" className="enrollmentSelectorGroup">
         <Form.Field error={!this.validYearCheck(momentYear)} className="yearSelect">
@@ -390,9 +423,9 @@ class PopulationSearchForm extends Component {
             control={Datetime}
             dateFormat={YEAR_DATE_FORMAT}
             timeFormat={false}
-            renderYear={(props, selectableYear) => <td {...props}>{`${selectableYear}-${selectableYear + 1}`}</td>}
+            renderYear={(props, selectableYear) => <td {...props} >{`${selectableYear}`}-<br />{`${selectableYear + 1}`}</td>}
             closeOnSelect
-            value={`${year}-${moment().year(year).add(1, 'years').format('YYYY')}`}
+            value={`${startYear}-${moment().year(startYear).add(1, 'years').format('YYYY')}`}
             isValidDate={this.validYearCheck}
             onChange={this.handleYearSelection}
           />
@@ -407,10 +440,10 @@ class PopulationSearchForm extends Component {
           <label>Statistics until</label>
           <Datetime
             dateFormat="MMMM YYYY"
-            defaultValue={this.getMonthValue(this.state.query.year, this.state.floatMonths)}
+            defaultValue={this.getMonthValue(this.state.query.startYear, this.state.floatMonths)}
             onChange={value => this.handleMonthsChange(value)}
             isValidDate={current => current.isBefore(moment()) &&
-              current.isAfter(this.getMinSelection(year, semesters[1] || semesters[0]))}
+              current.isAfter(this.getMinSelection(startYear, semesters[1] || semesters[0]))}
           />
         </Form.Field>
       </Form.Group>
@@ -471,10 +504,10 @@ class PopulationSearchForm extends Component {
       return (
         <Form.Group>
           <Form.Field width={8}>
-            { degreesToRender && degreesToRender.length > 1 ? renderableDegrees() : null }
+            {degreesToRender && degreesToRender.length > 1 ? renderableDegrees() : null}
           </Form.Field>
           <Form.Field width={8}>
-            { studyTracksToRender && studyTracksToRender.length > 0 ? renderableTracks() : null }
+            {studyTracksToRender && studyTracksToRender.length > 0 ? renderableTracks() : null}
           </Form.Field>
         </Form.Group>
       )
@@ -506,7 +539,6 @@ class PopulationSearchForm extends Component {
       const sortedStudyProgrammes = _.sortBy(studyProgrammes, s => getTextIn(s.name, language))
       programmesToRender = this.renderableList(sortedStudyProgrammes)
     }
-
     let degreesToRender
     let studyTracksToRender
     if (studyRights.programme && this.validYearCheck(momentYear)) {
@@ -536,9 +568,12 @@ class PopulationSearchForm extends Component {
     if (!this.state.showAdvancedSettings) {
       return null
     }
-    const { translate } = this.props
+
+    const { translate, tags } = this.props
     const { query } = this.state
     const { semesters, studentStatuses } = query
+    const options = this.state.isAdmin ? tags.map(tag => ({ key: tag.tag_id, text: tag.tagname, value: tag.tag_id })) : []
+
     return (
       <div>
         <Form.Group>
@@ -596,6 +631,37 @@ class PopulationSearchForm extends Component {
             />
           </Form.Field>
         </Form.Group>
+        {this.state.isAdmin ? (
+          <Form.Group>
+            <Form.Field>
+              <label>Select tag</label>
+              <Form.Dropdown
+                placeholder="select tag"
+                fluid
+                selection
+                options={options}
+                onChange={this.handleTagSearch}
+              />
+              <label>Select starting year</label>
+              <Datetime
+                className="yearSelectInput"
+                control={Datetime}
+                dateFormat={YEAR_DATE_FORMAT}
+                timeFormat={false}
+                renderYear={(props, selectableYear) => <td {...props}>{selectableYear}</td>}
+                closeOnSelect
+                value={null}
+                isValidDate={this.validYearCheck}
+                onChange={this.handleTagYearSelect}
+              />
+              <Button
+                disabled={!this.state.selectedTag}
+                onClick={this.handleSubmit}
+              >
+                Search by tag
+              </Button>
+            </Form.Field>
+          </Form.Group>) : null}
       </div>)
   }
 
@@ -605,7 +671,8 @@ class PopulationSearchForm extends Component {
   }
 
   render() {
-    if (!this.shouldRenderSearchForm()) {
+    const { location } = this.props
+    if (!this.shouldRenderSearchForm() && location.search !== '') {
       return null
     }
     const { translate } = this.props
@@ -661,7 +728,7 @@ class PopulationSearchForm extends Component {
   }
 }
 
-const mapStateToProps = ({ semesters, settings, populations, populationDegreesAndProgrammes, locale }) => {
+const mapStateToProps = ({ semesters, settings, populations, populationDegreesAndProgrammes, locale, tags }) => {
   const { language } = settings
   const { pending } = populationDegreesAndProgrammes
   return ({
@@ -671,7 +738,8 @@ const mapStateToProps = ({ semesters, settings, populations, populationDegreesAn
     translate: getTranslate(locale),
     studyProgrammes: populationDegreesAndProgrammes.data.programmes || {},
     pending,
-    extents: populations.data.extents || []
+    extents: populations.data.extents || [],
+    tags: tags.data
   })
 }
 
@@ -684,5 +752,6 @@ export default withRouter(connect(mapStateToProps, {
   getDegreesAndProgrammes,
   clearPopulations,
   setLoading,
-  getSemesters
+  getSemesters,
+  getTagsByStudytrackAction
 })(PopulationSearchForm))

services/oodikone2-frontend/src/components/PopulationStatistics/index.jsx

@@ -1,7 +1,7 @@
 import React, { PureComponent } from 'react'
 import { connect } from 'react-redux'
 import { getActiveLanguage, getTranslate } from 'react-localize-redux'
-import { func, bool } from 'prop-types'
+import { func, bool, shape } from 'prop-types'
 import { Header, Segment, Divider } from 'semantic-ui-react'
 
 import PopulationSearchForm from '../PopulationSearchForm'
@@ -16,11 +16,12 @@ class PopulationStatistics extends PureComponent {
   static propTypes = {
     translate: func.isRequired,
     populationFound: bool.isRequired,
-    loading: bool.isRequired
+    loading: bool.isRequired,
+    location: shape({}).isRequired
   }
 
   renderPopulationSearch = () => {
-    const { translate, populationFound, loading } = this.props
+    const { translate, populationFound, loading, location } = this.props
     const { Main } = infoTooltips.PopulationStatistics
     const title = populationFound ?
       translate('populationStatistics.foundTitle') :
@@ -33,20 +34,20 @@ class PopulationStatistics extends PureComponent {
         </Header>
         <PopulationSearchForm />
         <Divider />
-        <PopulationSearchHistory />
+        {location.search !== '' ? (<PopulationSearchHistory />) : null}
         <SegmentDimmer translate={translate} isLoading={loading} />
       </Segment>
     )
   }
 
   render() {
-    const { translate } = this.props
+    const { translate, location } = this.props
     return (
       <div className="segmentContainer">
         <Header className="segmentTitle" size="large">{translate('populationStatistics.header')}</Header>
         <Segment className="contentSegment">
-          { this.renderPopulationSearch() }
-          <PopulationDetails />
+          {this.renderPopulationSearch()}
+          {location.search !== '' ? (<PopulationDetails />) : null}
         </Segment>
       </div>
     )

services/oodikone2-frontend/src/components/PopulationStudents/index.jsx

@@ -1,7 +1,7 @@
 import React, { Component, Fragment } from 'react'
 import { connect } from 'react-redux'
 import { string, arrayOf, object, func, bool, shape } from 'prop-types'
-import { Header, Segment, Button, Icon, Popup, Tab, Grid, Checkbox } from 'semantic-ui-react'
+import { Header, Segment, Button, Icon, Popup, Tab, Grid, Checkbox, List } from 'semantic-ui-react'
 import { withRouter } from 'react-router-dom'
 import _ from 'lodash'
 import XLSX from 'xlsx'
@@ -22,7 +22,6 @@ import TagPopulation from '../TagPopulation'
 
 const popupTimeoutLength = 1000
 
-
 class PopulationStudents extends Component {
   state = {
     containsStudyTracks: false,
@@ -360,7 +359,7 @@ class PopulationStudents extends Component {
             <div style={{ overflowX: 'hidden' }}><div style={{ width: 0 }}>{e.label}</div></div>
           ),
           parent: true,
-          headerProps: { colSpan: labelToMandatoryCourses[e.label].length, title: e.label }
+          headerProps: { colSpan: labelToMandatoryCourses[e.label].length, title: e.label, ordernumber: e.orderNumber }
         }))
       )
     }
@@ -393,16 +392,22 @@ class PopulationStudents extends Component {
         const check = this.state.checkedStudents.find(c => c.studentnumber === s.studentNumber) || false
         return (
           <div key={s.studentNumber}>
-            <Checkbox
-              checked={check.checked}
-              onChange={() => this.handleSingleCheck(s.studentNumber)}
-            />
-            <TagStudent
-              tags={this.props.tags}
-              studentnumber={s.studentNumber}
-              studentstags={s.tags}
-              studytrack={this.props.queryStudyrights[0]}
-            />
+            <List horizontal>
+              <List.Item>
+                <Checkbox
+                  checked={check.checked}
+                  onChange={() => this.handleSingleCheck(s.studentNumber)}
+                />
+              </List.Item>
+              <List.Item>
+                <TagStudent
+                  tags={this.props.tags}
+                  studentnumber={s.studentNumber}
+                  studentstags={s.tags}
+                  studytrack={this.props.queryStudyrights[0]}
+                />
+              </List.Item>
+            </List>
           </div>)
       })
 

services/oodikone2-frontend/src/components/Postman/index.jsx

@@ -1,7 +1,7 @@
 import React, { Component } from 'react'
 import { Button, Form } from 'semantic-ui-react'
 import KeyValueTable from './KeyValueTable'
-import { createConfiguredAxios } from '../../apiConnection'
+import { api } from '../../apiConnection'
 
 class Postman extends Component {
     state={
@@ -13,16 +13,14 @@ class Postman extends Component {
     }
 
     doOodiApiRequest = async () => {
-      const axios = await createConfiguredAxios()
       const { route, secret } = this.state
       const headers = { 'x-oodi-secret': secret }
-      const { status, data } = await axios.post('/oodi', { route }, { headers })
+      const { status, data } = await api.post('/oodi', { route }, { headers })
       this.setState({ data: { status, data }, pending: false })
     }
 
     doBackendRequest = async () => {
-      const axios = await createConfiguredAxios()
-      const { status, data } = await axios.get(this.state.route)
+      const { status, data } = await api.get(this.state.route)
       this.setState({ data: { status, data }, pending: false })
     }
 

services/oodikone2-frontend/src/components/Routes/index.jsx

@@ -15,6 +15,7 @@ const Sandbox = React.lazy(() => import('../Sandbox'))
 const UsageStatistics = React.lazy(() => import('../UsageStatistics'))
 const OodiLearn = React.lazy(() => import('../OodiLearn'))
 const Feedback = React.lazy(() => import('../Feedback'))
+const Faculty = React.lazy(() => import('../Faculty'))
 
 const Routes = () => (
   <Suspense fallback={<Loader active inline="centered" />}>
@@ -25,6 +26,7 @@ const Routes = () => (
       <Route exact path="/study-programme/:studyProgrammeId/course-group/:courseGroupId" component={StudyProgramme} />
       <Route exact path={routes.students.route} component={StudentStatistics} />
       <Route exact path={routes.courseStatistics.route} component={CourseStatistics} />
+      <Route exact path={routes.faculty.route} component={Faculty} />
       <Route exact path={routes.settings.route} component={Settings} />
       <Route exact path={routes.users.route} component={EnableUsers} />
       <Route exact path={routes.teachers.route} component={Teachers} />

services/oodikone2-frontend/src/components/Sandbox/UpdateTopTeachers.jsx

@@ -1,6 +1,6 @@
 import React, { Component } from 'react'
 import { Button, Progress, Message } from 'semantic-ui-react'
-import { createConfiguredAxios } from '../../apiConnection'
+import { api } from '../../apiConnection'
 import KeyValueTable from '../Postman/KeyValueTable'
 
 const TASK_URL = '/tasks/topteachers'
@@ -21,8 +21,7 @@ class UpdateTopTeachers extends Component {
   checkStatus = async () => {
     this.setState({ fetching: true })
     try {
-      const axios = await createConfiguredAxios()
-      const { data } = await axios.get(TASK_URL)
+      const { data } = await api.get(TASK_URL)
       const { progress, computing, ...rest } = data
       this.setState({ status: rest, progress, computing })
     } catch (error) {
@@ -33,8 +32,7 @@ class UpdateTopTeachers extends Component {
 
   doUpdate = async () => {
     this.setState({ computing: true, progress: 0 })
-    const axios = await createConfiguredAxios()
-    await axios.post(TASK_URL)
+    await api.post(TASK_URL)
     await this.checkStatus()
   }
 

services/oodikone2-frontend/src/components/SortableTable/index.jsx

@@ -56,28 +56,30 @@ class SortableTable extends Component {
     </div>
   )
 
-
   render() {
     const calculateSkippedColumns = () => {
       const { columns, showNames } = this.props
       const { collapsed } = this.state
       const defaultColumnCount = showNames ? 6 : 3
       if (collapsed) {
-        const collapsedKeys = collapsed.map(c => c.key)
-        return collapsedKeys.reduce((acc, curr) => {
-          const previousCols = columns.filter(c => c.key < curr)
+        const collapsedOrderNumbers = collapsed.map(c => c.headerProps.ordernumber)
+        return collapsedOrderNumbers.reduce((acc, curr) => {
+          const previousCols = columns.filter(c => c.headerProps && c.headerProps.ordernumber < curr)
           const sumOfPreviousColSpans = previousCols.reduce((a, b) => a + b.headerProps.colSpan, 0)
           return [
             ...acc,
-            sumOfPreviousColSpans + columns.find(c => c.key === curr).headerProps.colSpan + defaultColumnCount]
+            sumOfPreviousColSpans + columns.find(c => c.headerProps && c.headerProps.ordernumber === curr).headerProps.colSpan + defaultColumnCount]
         }, [])
       }
       return []
     }
     const { tableProps, getRowProps, columns, getRowKey, collapsingHeaders } = this.props
     const { selected, direction, collapsed } = this.state
-    const columnsWithCollapsedHeaders = collapsingHeaders ? [...columns.filter(c => (c.headerProps && (!collapsed.map(cell => cell.headerProps.title).includes(c.headerProps.title) && !c.collapsed))), ...this.state.collapsed].sort((a, b) => a.key - b.key) : columns
-
+    const columnsWithCollapsedHeaders = collapsingHeaders ? [...columns.filter(c => (
+      c.headerProps && (!collapsed.map(cell => cell.headerProps.title).includes(c.headerProps.title) && !c.collapsed))),
+    ...this.state.collapsed].sort((a, b) => a.headerProps.ordernumber - b.headerProps.ordernumber)
+      :
+      columns
     const sortDirection = name => (selected === name ? direction : null)
 
     return (
@@ -88,7 +90,9 @@ class SortableTable extends Component {
               <Table.HeaderCell
                 key={c.key}
                 content={c.title}
-                onClick={c.parent && collapsingHeaders ? this.handleCollapse({ title: this.verticalTitle(<Fragment>{c.headerProps.title}</Fragment>), headerProps: { ...c.headerProps, colSpan: 1, rowSpan: 2 }, key: c.key, collapsed: true, parent: c.parent }) : this.handleSort(c.key)}
+                onClick={c.parent && collapsingHeaders && c.key !== 'general' ?
+                  this.handleCollapse({ title: this.verticalTitle(<Fragment>{c.headerProps.title}</Fragment>), headerProps: { ...c.headerProps, colSpan: 1, rowSpan: 2 }, key: c.key, collapsed: true, parent: c.parent }) :
+                  this.handleSort(c.key)}
                 sorted={c.parent ? undefined : sortDirection(c.key)}
                 {...c.headerProps}
               />

services/oodikone2-frontend/src/components/StudentDetails/index.jsx

@@ -13,7 +13,6 @@ import SearchResultTable from '../SearchResultTable'
 import { byDateDesc, reformatDate, getTextIn } from '../../common'
 import { clearCourseStats } from '../../redux/coursestats'
 
-
 class StudentDetails extends Component {
   componentDidMount() {
     this.props.history.listen((location, action) => {
@@ -36,8 +35,7 @@ class StudentDetails extends Component {
     const queryObject = { ...rest, courseCodes: JSON.stringify(courseCodes) }
     const searchString = qs.stringify(queryObject)
     this.props.clearCourseStats()
-    history.push('/coursestatistics/')
-    history.push({ search: searchString })
+    history.push(`/coursestatistics?${searchString}`)
   }
 
   renderCreditsGraph = () => {
@@ -75,7 +73,8 @@ class StudentDetails extends Component {
       } else {
         icon = <Icon name="circle outline" color="red" />
       }
-      const year = -(moment(new Date('1.1.1950')).diff(new Date(date), 'years')) // :D
+      const year = (moment(new Date(date)).diff(new Date('1950-1-1'), 'years'))
+
       return [
         reformatDate(date, 'DD.MM.YYYY'),
         `${isStudyModuleCredit ? `${getTextIn(course.name, language)} [Study Module]` : getTextIn(course.name, language)} (${course.code})`,

services/oodikone2-frontend/src/components/StudyProgramme/ProductivityTable/index.jsx

@@ -2,7 +2,7 @@ import React from 'react'
 import moment from 'moment'
 import { Table, Header, Loader, Button, Grid } from 'semantic-ui-react'
 import { connect } from 'react-redux'
-import { shape, number, arrayOf, bool, string, func } from 'prop-types'
+import { shape, number, arrayOf, bool, string, func, oneOfType } from 'prop-types'
 import { callApi } from '../../../apiConnection'
 import { getProductivity } from '../../../redux/productivity'
 
@@ -94,7 +94,7 @@ ProductivityTable.propTypes = {
     lastUpdated: string,
     status: string,
     data: arrayOf(shape({
-      year: number,
+      year: oneOfType([number, string]),
       credits: number,
       mThesis: number,
       bThesis: number,

services/oodikone2-frontend/src/components/StudyProgramme/ThroughputTable/index.jsx

@@ -22,7 +22,7 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
     const year = Number(yearLabel.slice(0, 4))
     const months = Math.ceil(moment.duration(moment().diff(`${year}-08-01`)).asMonths())
     history.push(`/populations?months=${months}&semesters=FALL&semesters=` +
-      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&year=${year}`)
+      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&startYear=${year}&endYear=${year}`)
   }
   if (error) return <h1>Oh no so error {error}</h1>
   let GRADUATED_FEATURE_TOGGLED_ON = false
@@ -31,9 +31,10 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
   }
   const data = throughput && throughput.data ? throughput.data.filter(year => year.credits.length > 0) : []
   const genders = data.length > 0 ? uniq(flatten(data.map(year => Object.keys(year.genders)))) : []
-  const countries = data.length > 0 && throughput.totals.countries ? uniq(flatten(data.map(year => Object.keys(year.countries)))).sort() : []
+  const countries = data.length > 0 && throughput.totals.countries ?
+    uniq(flatten(data.map(year => Object.keys(year.countries)))).sort() : []
   const renderGenders = genders.length > 0
-  const renderCountries = false // countries.length > 0, disabled for now
+  const renderCountries = countries.length > 0
   let thesisTypes = []
   if (thesis) {
     thesisTypes = thesis.map(t => t.thesisType)
@@ -132,7 +133,7 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                 </Table.Cell>
                 <Table.Cell>{year.credits.length}</Table.Cell>
                 {genders.map(gender => (
-                  <Table.Cell key={year.year + year.genders[gender]}>
+                  <Table.Cell key={`${year.year} gender:${gender}`}>
                     {`${year.genders[gender] || 0} (${Math.floor((year.genders[gender] / year.credits.length) * 100) || 0}%)`}
                   </Table.Cell>
                 ))}
@@ -145,12 +146,12 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                 }
                 <Table.Cell>{year.transferred}</Table.Cell>
                 {renderCountries ? countries.map(country => (
-                  <Table.Cell key={year.year + country}>
+                  <Table.Cell key={`${year.year} country:${country}`}>
                     {year.countries[country] || 0}
                   </Table.Cell>
                 )) : null}
                 {Object.keys(year.creditValues).map(creditKey => (
-                  <Table.Cell key={creditKey}>{year.creditValues[creditKey]}
+                  <Table.Cell key={`${year.year} credit:${creditKey}`}>{year.creditValues[creditKey]}
                   </Table.Cell>
                 ))}
                 {thesisTypes.includes('MASTER') ? (