Merge pull request #1317 from UniversityOfHelsinkiCS/trunk

fix studyprogramme column 820ab42

.circleci/config.yml

@@ -0,0 +1,93 @@
+version: 2.1
+
+executors:
+  my-executor:
+    working_directory: ~/app
+    machine:
+      image: ubuntu-1604:201903-01
+
+aliases:
+  - &check_exit_statuses
+    run:
+      name: check_exit_statuses
+      command: |
+        docker ps -a
+        if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then
+          echo 'Some process had nonzero exit code';
+          npm run docker:ci -- logs
+          exit 1;
+        fi
+
+jobs:
+  build:
+    executor: my-executor
+    steps:
+      - run:
+          name: set_env_variables
+          command: |
+            echo $CIRCLE_BRANCH;
+            echo $CIRCLE_TAG;
+            if [[ -n "$CIRCLE_TAG" ]]; then
+              echo 'export TAG=latest' >> $BASH_ENV;
+              echo 'export DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=staging' >> $BASH_ENV;
+              echo 'USING RELEASE CONFIG';
+            elif [[ "$CIRCLE_BRANCH" = "master" && -z "$CIRCLE_PULL_REQUEST" ]]; then
+              echo 'export TAG=staging' >> $BASH_ENV;
+              echo 'export DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'USING STAGING CONFIG';
+            else
+              echo 'export TAG=trunk' >> $BASH_ENV;
+              echo 'export DEPLOY=true' >> $BASH_ENV;
+              echo 'export CACHE_TAG=trunk' >> $BASH_ENV;
+              echo 'USING TRUNK CONFIG';
+            fi
+      - checkout
+      - restore_cache:
+          keys:
+            - npmcache-v2-{{ checksum "package-lock.json" }}
+      - run: npm ci
+      - run: npm run cypress -- verify
+      - save_cache:
+          key: npmcache-v2-{{ checksum "package-lock.json" }}
+          paths:
+            - ~/.cache
+      - run: TAG=${CACHE_TAG} npm run docker:ci -- pull --ignore-pull-failures
+      - run: npm run docker:ci -- build
+      - run: npm run docker:ci -- up -d db user_db db_kone
+      - *check_exit_statuses
+      - run: git clone --depth 1 git@github.com:UniversityOfHelsinkiCS/anonyymioodi.git
+      - run: source ./scripts/scripts.sh && init_dirs && db_anon_setup_full
+      - *check_exit_statuses
+      - run: npm run docker:ci -- up -d --no-recreate
+      - run:
+          name: log_in_background
+          background: true
+          command: |
+            docker ps -a && npm run docker:ci -- logs -f
+      - *check_exit_statuses
+      - run: npm run test_services
+      - *check_exit_statuses
+      - run: timeout 5m bash -c 'until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf .; sleep 5; done'
+      - run: npm run cypress:ci
+      - *check_exit_statuses
+      - store_artifacts:
+          path: cypress/videos
+      - store_artifacts:
+          path: cypress/screenshots
+      - deploy:
+          command: |
+            if [ "${DEPLOY}" == "true" ]; then
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              npm run docker:ci -- push
+            fi
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/

.travis.yml

@@ -1,75 +0,0 @@
-language: node_js
-
-node_js:
-  # Node 10.3+ includes npm@6 which has good "npm ci" command
-  - 10.8
-
-cache:
-  # cache both npm modules and Cypress binary
-  npm: true
-  directories:
-    - ~/.cache
-
-services:
-  - docker
-
-before_script:
-  - docker --version
-  - docker-compose --version
-  - export AUTHOR_NAME="$(git log -1 ${TRAVIS_COMMIT} --pretty="%aN")"
-  - if [[ -n "$TRAVIS_TAG" || ("$TRAVIS_BRANCH" = "master" && "$TRAVIS_PULL_REQUEST" = "false") ]];
-    then export SENTRY_RELEASE_VERSION=${IMAGE}-${TRAVIS_COMMIT};
-    fi
-
-jobs:
-  include:
-    - stage: build & test & push & deploy staging
-      env:
-        - IMAGE=STAGING
-        - TAG=staging
-      script:
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source ./scripts/scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} ${SENTRY_RELEASE_VERSION}
-        - docker ps
-        - cat nginx/log
-        - TAG=${TAG} SENTRY_RELEASE_VERSION=${SENTRY_RELEASE_VERSION} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
-        - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
-        - docker ps -a
-        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy:
-        provider: script
-        skip_cleanup: true
-        script: "TAG=${TAG} SENTRY_RELEASE_VERSION=${SENTRY_RELEASE_VERSION} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"Staging auto deployment started!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          branch: master
-    - stage: build & test & push & deploy latest
-      env:
-        - IMAGE=PRODUCTION
-        - TAG=latest
-      script:
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - source ./scripts/scripts.sh && time run_e2e_setup ./docker/docker-compose.lateste2e.yml ${TAG} ${SENTRY_RELEASE_VERSION}
-        - docker ps
-        - cat nginx/log
-        - TAG=${TAG} SENTRY_RELEASE_VERSION=${SENTRY_RELEASE_VERSION} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml logs -f &
-        - until $(curl --output /dev/null --silent --fail http://localhost:1337/); do printf '.'; sleep 5; done
-        - time npm run test_services
-        - time CYPRESS_baseUrl=http://localhost:1337/ npm run cypress:record
-        - docker ps -a
-        - if [[ $(docker ps --all | grep -Eo '(Exited|Restarting) \([0-9]+\)' | grep -Eo '[0-9]+' | awk 'BEGIN {sum=0} { sum += $1 } END { print sum }') != '0' ]]; then echo 'Some process had nonzero exit code'; exit 1; fi
-      deploy:
-        provider: script
-        skip_cleanup: true
-        script: "TAG=${TAG} SENTRY_RELEASE_VERSION=${SENTRY_RELEASE_VERSION} docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml push && curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${TRAVIS_TAG} release started auto deployment!\\\"}\" ${SLACKBOT_URL}"
-        on:
-          tags: true
-stages:
-  - name: build & test & push & deploy staging
-    if: NOT tag IS present
-  - name: build & test & push & deploy latest
-    if: tag IS present
-
-after_failure:
-  - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"${IMAGE} buildi paskana. STOP AND FIX tai tulee kenkää! *${AUTHOR_NAME}* ${TRAVIS_BUILD_WEB_URL}\\\"}\" ${SLACKBOT_URL}"

cypress/integration/Course_statistics.js

@@ -51,8 +51,8 @@ describe('Course Statistics tests', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
     cy.get("input[placeholder='Search by entering a course code']").type('TKT')
-    cy.contains("tr", "TKT20003").click()
-    cy.contains("tr", "TKT10002").click()
+    cy.contains("td", "TKT20003").click()
+    cy.contains("td", /^TKT10002/).click()
     cy.contains("Fetch statistics").should('be.enabled').click()
     cy.contains("Search for courses").should('not.exist')
 

cypress/integration/Population_statistics.js

@@ -116,7 +116,7 @@ describe('Population Statistics tests', () => {
       cy.contains("Matematiikan didaktiikka").siblings().eq(2).should("have.text", '1')
     })
 
-    cy.get("button").contains("show").click()
+    cy.contains("button", "show").click()
     cy.contains("Student names hidden").click()
     cy.contains("Oinonen").siblings().eq(2).click()
     cy.contains("Oinonen").invoke('text').then((text) => expect(text).to.equal('Oinonen Heidi Eeva Elisabet, 014473717'))
@@ -125,7 +125,7 @@ describe('Population Statistics tests', () => {
   it('Student list checking works as intended', () => {
     cy.contains("Select study programme").click().siblings().contains("Kasvatustieteiden kandiohjelma").click()
     cy.contains("See population").click()
-    cy.get("button").contains("show").click()
+    cy.contains("button", "show").click()
     cy.contains("010483918")
     cy.contains("666666666").should('not.exist')
     cy.contains('button', "Check studentnumbers").click()
@@ -209,8 +209,7 @@ describe('Population Statistics tests', () => {
     cy.contains("and at least").parentsUntil("form").contains("set filter").click()
 
     cy.contains("Filters").siblings().within(() => {
-      cy.contains('Kehittävä').should('have.text', 'Kehittävä työntutkimus')
-      cy.contains('Kvalitatiiviset').should('have.text', 'Kvalitatiiviset tutkimusmenetelmät I')
+      cy.contains('Kehittävä').should('have.text', 'Avoin yo: Kehittävä työntutkimus')
     })
 
     checkAmountOfStudents(1)
@@ -236,7 +235,7 @@ describe('Population Statistics tests', () => {
       cy.contains("set filter").click()
     })
 
-    checkAmountOfStudents(1)
+    checkAmountOfStudents(3)
 
     cy.contains("Basic filters").parentsUntil("form").contains("set filter").click()
 

cypress/integration/Users.js

@@ -15,7 +15,7 @@ describe('Users tests', () => {
     cy.visit(Cypress.config().baseUrl)
     cy.reload()
     cy.contains("Users").click()
-    cy.contains("Enable or disable access to Oodikone")
+    cy.contains("Oodikone users")
   })
 
   it("Admin mocking normal user shows only the mocked user's programmes", () => {

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.2'
 
 services:
   nats:
@@ -84,7 +84,6 @@ services:
     ports:
       - "8080:8080"
     environment:
-      WAIT_HOSTS: redis:6379,db:5432,db_kone:5432,nats:4222
       REDIS: redis
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
@@ -99,7 +98,6 @@ services:
       ANALYTICS_URL: http://analytics:4568
       NATS_URI: nats://nats:4222
       UPDATER_URL: http://updater_scheduler:3678
-
     container_name: backend
     depends_on:
       - db
@@ -120,7 +118,6 @@ services:
     ports:
       - 4567:4567
     environment:
-      WAIT_HOSTS: user_db:5432
       DB_URL: postgres://postgres@user_db:5432/user_db
       TEST_DB: postgres://postgres@user_db:5432/user_db_test
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -135,7 +132,6 @@ services:
     ports:
       - 545:545
     environment:
-      WAIT_HOSTS: usage_db:5432
       PORT: '545'
       DB_URL: postgres://postgres@usage_db:5432/usage_db
       SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
@@ -149,7 +145,6 @@ services:
     depends_on:
       - nats
     environment:
-      - WAIT_HOSTS=nats:4222
       - NATS_URI=nats://nats:4222
       - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
       - TOKEN=jotaindiibadaabaa
@@ -163,7 +158,6 @@ services:
       - nats
       - db
     environment:
-      WAIT_HOSTS: nats:4222,db:5432
       NATS_URI: nats://nats:4222
       DB_URL: postgres://postgres@db:5432/tkt_oodi
       TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
@@ -185,7 +179,6 @@ services:
       - nats
       - scheduler_mongo
     environment:
-      - WAIT_HOSTS=nats:4222,scheduler_mongo:27017
       - NATS_URI=nats://nats:4222
       - MONGO_URI=mongodb://scheduler_mongo:27017
       - PORT=3678
@@ -197,7 +190,6 @@ services:
     ports:
       - 4568:4568
     environment:
-      - WAIT_HOSTS=analytics_db:5432
       - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
       - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
     depends_on:

docker/docker-compose.dev.real.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.2'
 
 services:
   backend:

docker/docker-compose.dev.yml

@@ -1,20 +1,21 @@
-version: '3'
+version: '3.2'
 
 services:
   user_db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+    restart: unless-stopped
 
   db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+    restart: unless-stopped
 
   backend:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     environment:
       FRONT_URL: http://localhost:8000
     volumes:
@@ -22,13 +23,13 @@ services:
       - ./services/backend/shared/models:/usr/src/app/src/models
       - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
       - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+    restart: unless-stopped
 
   frontend:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run docker']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
     ports:
       - "8081:8081"
       - "7777:7777"
@@ -39,40 +40,40 @@ services:
       KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
     volumes:
       - ./services/oodikone2-frontend/:/usr/src/app
+    restart: unless-stopped
 
   userservice:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/oodikone2-userservice/:/usr/src/app
+    restart: unless-stopped
 
   usageservice:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/oodikone2-usageservice/:/usr/src/app
+    restart: unless-stopped
 
   updater_api:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/updater_api/:/usr/src/app
+    restart: unless-stopped
 
   updater_writer:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/backend/updater_writer/:/usr/src/app
       - ./services/backend/shared/models:/usr/src/app/models
@@ -80,27 +81,25 @@ services:
       - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
     environment:
       DB_SCHEMA_KONE: kone_data
-
-  scheduler_mongo:
     restart: unless-stopped
 
   updater_scheduler:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/updater_scheduler/:/usr/src/app
+    restart: unless-stopped
 
   analytics:
     build:
       args:
         NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && /wait && npm run dev']
-    restart: unless-stopped
+    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
     volumes:
       - ./services/oodikone2-analytics/:/usr/src/app
+    restart: unless-stopped
 
   adminer:
     image: adminer:4.6.2
@@ -110,8 +109,8 @@ services:
       - ./adminer-plugins:/var/www/html/plugins-enabled
     ports:
       - 5050:8080
-    restart: always
     container_name: adminer
+    restart: unless-stopped
 
   # zookeeper:
   #   image: wurstmeister/zookeeper
@@ -141,7 +140,6 @@ services:
 
   # mongo_db:
   #   image: mongo
-  #   restart: unless-stopped
   #   volumes:
   #     - ./backups/mongo/oodilearn:/dump
   #     - ./mongodata:/data/db
@@ -153,7 +151,6 @@ services:
   #   build:
   #     context: services/oodilearn/server
   #     dockerfile: Dockerfile
-  #   restart: unless-stopped
   #   volumes:
   #     - ./services/oodilearn/data:/data
   #     - ./services/oodilearn/server:/src

docker/docker-compose.lateste2e.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.2'
 
 services:
   user_db:
@@ -11,6 +11,11 @@ services:
 
   backend:
     image: toska/oodikone2-backend:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-backend:trunk
+        - toska/oodikone2-backend:staging
+        - toska/oodikone2-backend:latest
     ports:
       - "9229:9229"
     environment:
@@ -22,31 +27,64 @@ services:
     build:
       args:
         BASE_PATH: /
-        SENTRY_RELEASE_VERSION: ${SENTRY_RELEASE_VERSION}
         SENTRY_URL: ${SENTRY_URL}
         SENTRY_AUTH_TOKEN: ${SENTRY_AUTH_TOKEN}
         SENTRY_ORG: sentry
         SENTRY_PROJECT: oodifront
+      cache_from:
+        - toska/oodikone2-frontend:trunk
+        - toska/oodikone2-frontend:staging
+        - toska/oodikone2-frontend:latest
     ports:
       - "5000:5000"
 
   userservice:
     image: toska/oodikone2-userservice:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-userservice:trunk
+        - toska/oodikone2-userservice:staging
+        - toska/oodikone2-userservice:latest
 
   usageservice:
     image: toska/oodikone2-usageservice:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-usageservice:trunk
+        - toska/oodikone2-usageservice:staging
+        - toska/oodikone2-usageservice:latest
 
   updater_api:
     image: toska/updater_api:${TAG}
+    build:
+      cache_from:
+        - toska/updater_api:trunk
+        - toska/updater_api:staging
+        - toska/updater_api:latest
 
   updater_writer:
     image: toska/updater_writer:${TAG}
+    build:
+      cache_from:
+        - toska/updater_writer:trunk
+        - toska/updater_writer:staging
+        - toska/updater_writer:latest
 
   updater_scheduler:
     image: toska/updater_scheduler:${TAG}
+    build:
+      cache_from:
+        - toska/updater_scheduler:trunk
+        - toska/updater_scheduler:staging
+        - toska/updater_scheduler:latest
 
   analytics:
     image: toska/oodikone2-analytics:${TAG}
+    build:
+      cache_from:
+        - toska/oodikone2-analytics:trunk
+        - toska/oodikone2-analytics:staging
+        - toska/oodikone2-analytics:latest
 
   nginx:
     image: nginx:latest
@@ -55,7 +93,6 @@ services:
       - ./nginx/error.log:/etc/nginx/error_log.log
       - ./nginx/log:/var/log/nginx/localhost.error_log
       - ./nginx/cache/:/etc/nginx/cache
-      - /etc/letsencrypt/:/etc/letsencrypt/
     ports:
       -  1337:80
     depends_on:

nginx.conf

@@ -4,6 +4,7 @@ http {
 
   rewrite_log on;
   error_log /var/log/nginx/localhost.error_log notice;
+  access_log off;
 
   server {
     listen 80;

package.json

@@ -4,16 +4,18 @@
   "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the docker-compose.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
   "main": "./scripts/run.sh",
   "scripts": {
+    "cli": "./scripts/run.sh",
     "cypress": "cypress",
-    "cypress:open": "CYPRESS_baseUrl=http://localhost:8081 cypress open",
-    "cypress:run": "CYPRESS_baseUrl=http://localhost:8081 cypress run --config video=true --config videoCompression=false -P ./",
-    "cypress:record": "cypress run --config videoUploadOnPasses=false --config videoCompression=false -P ./",
+    "cypress:open": "cypress open --config baseUrl=http://localhost:8081/",
+    "cypress:run": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:8081/",
+    "cypress:ci": "cypress run --config video=true,videoCompression=false,baseUrl=http://localhost:1337/",
     "concurrently": "concurrently",
     "test": "./scripts/run_all_tests.sh",
     "test_services": "./scripts/run_service_tests.sh",
     "lint": "./scripts/run_service_lints.sh",
     "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
     "docker:real": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml",
+    "docker:ci": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml",
     "docker:build": "npm run docker -- build",
     "docker:up": "npm run docker -- up -d",
     "docker:up:real": "npm run docker:real -- up -d",

services/backend/oodikone2-backend/Dockerfile

@@ -1,22 +1,21 @@
 FROM node:10.15-alpine
 
+WORKDIR /usr/src/app
+EXPOSE 8080
+CMD npm start
+
+COPY ./oodikone2-backend/package.json /usr/src/app/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/app/package-lock.json
+COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
+COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
+RUN npm ci
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
+
 COPY ./oodikone2-backend /usr/src/app
 COPY ./shared/models /usr/src/app/src/models
 COPY ./shared/migrations /usr/src/app/src/database/migrations
 COPY ./shared/migrations_kone /usr/src/app/src/database/migrations_kone
 
-WORKDIR /usr/src/app
-RUN npm ci
-
-COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
-RUN chmod +x /wait
-
-EXPOSE 8080
-CMD /wait && npm start

services/backend/oodikone2-backend/src/app.js

@@ -6,31 +6,41 @@ const conf = require('./conf-backend')
 const routes = require('./routes')
 const { startCron } = require('./events')
 const { PORT } = conf
-const app = express()
+const { initializeDatabaseConnection } = require('./database/connection')
 
-startCron()
+initializeDatabaseConnection().then(() => {
+  const app = express()
 
-Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
+  startCron()
 
-app.use(cors({ credentials: true, origin: conf.frontend_addr }))
-app.use(bodyParser.json())
+  Raven.config(process.env.SENTRY_ADDR, { captureUnhandledRejections: true }).install()
 
-app.get('/ping', async (req, res) => {
-  res.json({ data: 'pong' })
-})
+  app.use(cors({ credentials: true, origin: conf.frontend_addr }))
+  app.use(bodyParser.json())
 
-let BASE_URL = ''
-if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
-  BASE_URL = '/api'
-}
+  app.get('/ping', async (req, res) => {
+    res.json({ data: 'pong' })
+  })
 
-routes(app, BASE_URL)
+  let BASE_URL = ''
+  if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'test') {
+    BASE_URL = '/api'
+  }
 
-app.get('*', async (req, res) => {
-  const results = { error: 'unknown endpoint' }
-  res.status(404).json(results)
-})
+  routes(app, BASE_URL)
 
-module.exports = app.listen(PORT, () => {
-  console.log('Example app listening on port ' + PORT + '!')
+  app.get('*', async (req, res) => {
+    const results = { error: 'unknown endpoint' }
+    res.status(404).json(results)
+  })
+
+  const server = app.listen(PORT, () => console.log(`Backend listening on port ${PORT}!`))
+  process.on('SIGTERM', () => {
+    server.close(() => {
+      console.log('Process terminated')
+    })
+  })
+}).catch(e => {
+  process.exitCode = 1
+  console.log(e)
 })

services/backend/oodikone2-backend/src/database/connection.js

@@ -17,36 +17,53 @@ const sequelizeKone = new Sequelize(conf.DB_URL_KONE, {
 sequelizeKone.query(`SET SESSION search_path to ${conf.DB_SCHEMA_KONE}`)
 
 
-const runMigrationsKone = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelizeKone,
-        tableName: 'migrations',
-        schema: conf.DB_SCHEMA_KONE
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelizeKone.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/src/database/migrations_kone`,
-        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA_KONE
-      },
-    })
-    const migrations = await migrator.up()
+const initializeDatabaseConnection = async () => {
+  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+  const waitSeconds = 60
+  for (let i = 1; i <= waitSeconds; i++) {
+    try {
+      await sequelize.authenticate()
+      await sequelizeKone.authenticate()
+      break
+    } catch (e) {
+      if (i === waitSeconds) {
+        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        throw e
+      }
+      console.log('.')
+      await sleep(1000)
+    }
+  }
+  if (!conf.isTest) {
+    try {
+      const migrator = new Umzug({
+        storage: 'sequelize',
+        storageOptions: {
+          sequelize: sequelizeKone,
+          tableName: 'migrations',
+          schema: conf.DB_SCHEMA_KONE
+        },
+        logging: console.log,
+        migrations: {
+          params: [
+            sequelizeKone.getQueryInterface(),
+            Sequelize
+          ],
+          path: `${process.cwd()}/src/database/migrations_kone`,
+          pattern: /\.js$/,
+          schema: conf.DB_SCHEMA_KONE
+        },
+      })
+      const migrations = await migrator.up()
 
-    console.log('Kone Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Kone Migration error, message:', e)
+      console.log('Kone Migrations up to date', migrations)
+    } catch (e) {
+      console.log('Kone Migration error')
+      throw e
+    }
   }
 }
 
-const migrationPromiseKone = !conf.isTest ? runMigrationsKone() : Promise.resolve()
-
 const forceSyncDatabase = async () => {
   try {
     await sequelize.getQueryInterface().createSchema(conf.DB_SCHEMA)
@@ -65,6 +82,6 @@ const forceSyncDatabase = async () => {
 module.exports = {
   sequelize,
   sequelizeKone,
-  migrationPromiseKone,
+  initializeDatabaseConnection,
   forceSyncDatabase
 }

services/backend/oodikone2-backend/src/events.js

@@ -87,11 +87,13 @@ const refreshOverview = async () => {
 }
 
 const startCron = () => {
-  schedule('0 6 * * *', async () => {
-    await refreshFacultyYearlyStats()
-    await refreshStudyrightAssociations()
-    await refreshOverview()
-  })
+  if (process.env.NODE_ENV === 'production') {
+    schedule('0 6 * * *', async () => {
+      await refreshFacultyYearlyStats()
+      await refreshStudyrightAssociations()
+      await refreshOverview()
+    })
+  }
 }
 
 module.exports = {

services/backend/oodikone2-backend/src/middleware/accesslogger.js

@@ -1,20 +1,9 @@
 const morgan = require('morgan')
-const UsageService = require('../services/usageService')
 
 const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
-  const message = [
-    req.decodedToken.name, ':',
-    tokens['method'](req, res),
-    tokens['url'](req, res),
-    tokens['status'](req, res),
-    '-',
-    tokens['response-time'](req, res), 'ms'
-  ].join(' ')
   const meta = req.decodedToken
   fields.forEach(field => meta[field] = tokens[field](req, res))
-  UsageService.log(message, meta).catch(console.log)
- 
 })
 
 module.exports = accessLogger

services/backend/oodikone2-backend/src/routes/courses.js

@@ -66,12 +66,12 @@ router.get('/v3/courseyearlystats', async (req, res) => {
         return res.status(403).json({ error: 'No programmes so no access to course stats' })
       }
     }
-    const { codes, startyearcode, endyearcode, separate: sep } = req.query
+    const { codes, separate: sep } = req.query
     const separate = !sep ? false : JSON.parse(sep)
-    if (!codes || !startyearcode) {
+    if (!codes) {
       res.status(422).send('Missing required query parameters')
     } else {
-      const results = await Course.courseYearlyStats(codes, separate, startyearcode, endyearcode)
+      const results = await Course.courseYearlyStats(codes, separate)
       res.json(results)
     }
   } catch (e) {

services/backend/oodikone2-backend/src/routes/faculties.js

@@ -1,5 +1,6 @@
 const router = require('express').Router()
 const { all } = require('../services/organisations')
+const { byType: getElementDetailsByType } = require('../services/elementdetails')
 const { getFacultyYearlyStats } = require('../services/analyticsService')
 
 router.get('/faculties', async (req, res) => {
@@ -16,4 +17,13 @@ router.get('/faculties/yearlystats', async (req, res) => {
   }
 })
 
+router.get('/faculties/programmes', async (req, res) => {
+  try {
+    const facultyProgrammes = await getElementDetailsByType(20)
+    res.json(facultyProgrammes)
+  } catch (e) {
+    res.status(400).json({ error: e })
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/mandatorycourses.js

@@ -4,56 +4,49 @@ const MandatoryCourses = require('../services/mandatoryCourses')
 router.delete('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.remove(programme, course)
-      res.status(200).json(course)
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.remove(programme, course)
+    res.status(200).json(course)
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 
 router.post('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.create(programme, course)
-      const newCourse = await MandatoryCourses.find(programme, course)
-      res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.create(programme, course)
+    const newCourse = await MandatoryCourses.find(programme, course)
+    if (!newCourse) return res.status(400).end()
+    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 
 router.post('/:programme/label/:course', async (req, res) => {
   const { programme, course } = req.params
   const { label } = req.body
-  if (programme && course) {
-    try {
-      await MandatoryCourses.updateLabel(programme, course, label)
-      const updatedCourse = await MandatoryCourses.find(programme, course)
-      res
-        .status(200)
-        .json({
-          name: updatedCourse.course.name,
-          code: updatedCourse.course_code,
-          label: updatedCourse.mandatory_course_label
-        })
-    } catch (err) {
-      console.error(err)
-      res.status(400).json(err.message)
-    }
-  } else {
-    res.status(422).end()
+  if (!programme || !course) return res.status(400).end()
+  try {
+    await MandatoryCourses.updateLabel(programme, course, label)
+    const updatedCourse = await MandatoryCourses.find(programme, course)
+    if (!updatedCourse) return res.status(400).end()
+    res
+      .status(200)
+      .json({
+        name: updatedCourse.course.name,
+        code: updatedCourse.course_code,
+        label: updatedCourse.mandatory_course_label
+      })
+  } catch (err) {
+    console.error(err)
+    res.status(500).json(err.message)
   }
 })
 

services/backend/oodikone2-backend/src/routes/population.js

@@ -31,20 +31,20 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
 
     res.json(result)
   } catch (e) {
-    res.status(400).json({ error: e })
+    console.log(e)
+    res.status(500).json({ error: e })
   }
 })
 
 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
   try {
-    if (!req.body.yearcode || !req.body.coursecode) {
+    if (!req.body.yearcode || !req.body.coursecodes) {
       res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
       return
     }
-    const { coursecode, yearcode } = req.body
+    const { coursecodes, yearcode } = req.body
     let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
-
+    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, yearcode)
     const { decodedToken: { userId }, roles } = req
 
     if (roles && roles.includes('admin')) {
@@ -54,7 +54,14 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
       const codes = unitsUserCanAccess.map(unit => unit.id)
       studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
     }
-    const result = await Population.bottlenecksOf(req.body, studentnumberlist)
+
+    const result = await Population.bottlenecksOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000
+    }, studentnumberlist)
 
     if (result.error) {
       res.status(400).json(result)
@@ -63,7 +70,45 @@ router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) =>
 
     res.json(result)
   } catch (e) {
-    res.status(400).json({ error: e })
+    console.log(e)
+    res.status(500).json({ error: e })
+  }
+})
+
+router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
+  try {
+    if (!req.body.studentnumberlist) {
+      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
+      return
+    }
+    let studentnumberlist
+    const { decodedToken: { userId }, roles } = req
+
+    if (roles && roles.includes('admin')) {
+      studentnumberlist = req.body.studentnumberlist
+    } else {
+      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+      const codes = unitsUserCanAccess.map(unit => unit.id)
+      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+    }
+
+    const result = await Population.bottlenecksOf({
+      startYear: 1900,
+      endYear: 2200,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000
+    }, studentnumberlist)
+    if (result.error) {
+      res.status(400).json(result)
+      return
+    }
+
+    res.json(result)
+
+  } catch (err) {
+    console.log(err)
+    res.status(500).json({ error: err })
   }
 })
 
@@ -106,20 +151,19 @@ router.get('/v3/populationstatistics', async (req, res) => {
     res.json(result)
   } catch (e) {
     console.log(e)
-    res.status(400).json({ error: e })
+    res.status(500).json({ error: e })
   }
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
-  const { coursecode, yearcode } = req.query
-  if (!coursecode || !yearcode) {
+  const { coursecodes, yearcode } = req.query
+  if (!coursecodes || !yearcode) {
     res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
     return
   }
   const semesters = ['FALL', 'SPRING']
   let studentnumberlist
-  const studentnumbers = await Student.findByCourseAndSemesters(coursecode, yearcode)
-
+  const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), yearcode)
   const { decodedToken: { userId }, roles } = req
 
   if (roles && roles.includes('admin')) {
@@ -148,10 +192,39 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     res.json(result)
   } catch (e) {
     console.log(e)
-    res.status(400).json({ error: e })
+    res.status(500).json({ error: e })
   }
 })
 
+router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
+  const { studentnumberlist } = req.body
+  const { roles } = req
+
+  if (!(roles && roles.includes('admin'))) {
+    console.log('no rights')
+    res.status(400).end()
+    return
+  } else {
+    try {
+      const result = await Population.optimizedStatisticsOf({
+        startYear: 1900,
+        endYear: 2200,
+        studyRights: [],
+        semesters: ['FALL', 'SPRING'],
+        months: 10000
+      }, studentnumberlist)
+      if (result.error) {
+        console.log(result.error)
+        res.status(400).end()
+        return
+      }
+      res.status(200).json(result)
+    } catch (err) {
+      console.log(err)
+      res.status(400).end()
+    }
+  }
+})
 
 router.get('/v2/populationstatistics/filters', async (req, res) => {
 

services/backend/oodikone2-backend/src/routes/tags.js

@@ -1,6 +1,8 @@
 const router = require('express').Router()
 const Tags = require('../services/tags')
 const TagStudent = require('../services/tagstudent')
+const Students = require('../services/students')
+const { difference } = require('lodash')
 
 router.get('/tags', async (req, res) => {
   try {
@@ -8,42 +10,54 @@ router.get('/tags', async (req, res) => {
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
-  const { studytrack } = req.params
   try {
+    const { studytrack } = req.params
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     const tags = await Tags.findTagsByStudytrack(studytrack)
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.post('/tags', async (req, res) => {
-  const { tag } = req.body
   try {
+    const { tag } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     await Tags.createNewTag(tag)
-    const tags = await Tags.findTagsByStudytrack(tag.studytrack) 
+    const tags = await Tags.findTagsByStudytrack(tag.studytrack)
     res.status(200).json(tags)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err.message)
+    res.status(500).json(err.message)
   }
 })
 
 router.delete('/tags', async (req, res) => {
-  const { tag } = req.body
   try {
+    const { tag } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(tag.studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     await Tags.deleteTag(tag)
     const t = await Tags.findTagsByStudytrack(tag.studytrack)
     res.status(200).json(t)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 
 })
@@ -54,52 +68,94 @@ router.get('/studenttags', async (req, res) => {
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/studenttags/:studytrack', async (req, res) => {
-  const { studytrack } = req.params
   try {
+    const { studytrack } = req.params
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
     const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
-  const { studentnumber } = req.params
   try {
+    const { studentnumber } = req.params
     const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    console.log(result)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
+  }
+})
+
+router.post('/studenttags/:studentnumber', async (req, res) => {
+  try {
+    const { tag, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    await TagStudent.createStudentTag(tag)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+    res.status(200).json(result)
+  } catch (err) {
+    console.log(err)
+    res.status(500).json(err)
   }
 })
 
 router.post('/studenttags', async (req, res) => {
-  const { tag } = req.body
   try {
-    const result = await TagStudent.createStudentTag(tag)
+    const { tags, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const existingtags = await Tags.findTagsByStudytrack(studytrack)
+    const existingTagids = existingtags.map(t => t.tag_id)
+    const tagids = [...new Set(tags.map(t => t.tag_id))]
+    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
+
+    const studentnumbers = tags.map(t => t.studentnumber)
+    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+    const missingStudents = difference(studentnumbers, students)
+    if (missingStudents.length !== 0) return res.status(400).json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+
+    await TagStudent.createMultipleStudentTags(tags)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 
 router.delete('/studenttags', async (req, res) => {
-  const { id } = req.body
   try {
-    const result = await TagStudent.deleteStudentTag(id)
+    const { tag_id, studentnumber, studytrack } = req.body
+    const { rights, roles } = req
+
+    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+
+    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+
+    await TagStudent.deleteStudentTag(studentnumber, tag_id)
+    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
     res.status(200).json(result)
   } catch (err) {
     console.log(err)
-    res.status(400).json(err)
+    res.status(500).json(err)
   }
 })
 

services/backend/oodikone2-backend/src/services/course_yearly_stats_counter.js

@@ -8,7 +8,6 @@ const CATEGORY = {
 class CourseYearlyStatsCounter {
 
   constructor() {
-    this.all = []
     this.groups = {}
     this.programmes = {}
     this.history = {
@@ -22,7 +21,7 @@ class CourseYearlyStatsCounter {
     this.programmes[code] = { name, students: [] }
   }
 
-  initGroup(groupcode, name, coursecode) {
+  initGroup(groupcode, name, coursecode, yearcode) {
     this.groups[groupcode] = {
       code: groupcode,
       name,
@@ -34,7 +33,8 @@ class CourseYearlyStatsCounter {
           failed: []
         }
       },
-      students: {}
+      students: {},
+      yearcode
     }
   }
 
@@ -45,29 +45,18 @@ class CourseYearlyStatsCounter {
     return { attempted, passed, failed }
   }
 
-  markStudyProgramme(code, name, studentnumber, startdate) {
+  markStudyProgramme(code, name, studentnumber) {
     if (!this.programmes[code]) {
       this.initProgramme(code, name)
     }
-    const prog = this.programmes[code]
-
-    // check if student has earlier studyright wrt course
-    this.all.forEach((student) => {
-      if (student.studentnumber === studentnumber && student.startdate > startdate) {
-        delete this.programmes[student.code]
-        this.all.splice(this.all.indexOf(student), 1)
-      }
-    })
-
-    if (!prog.students.includes(studentnumber)) {
-      this.all.push({ studentnumber, code, startdate })
-      prog.students.push(studentnumber)
+    if (!this.programmes[code].students.includes(studentnumber)) {
+      this.programmes[code].students.push(studentnumber)
     }
   }
 
   markStudyProgrammes(studentnumber, programmes) {
-    programmes.forEach(({ code, name, startdate }) => {
-      this.markStudyProgramme(code, name, studentnumber, startdate)
+    programmes.forEach(({ code, name }) => {
+      this.markStudyProgramme(code, name, studentnumber)
     })
   }
 
@@ -102,9 +91,9 @@ class CourseYearlyStatsCounter {
     }
   }
 
-  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode) {
+  markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode) {
     if (!this.groups[groupcode]) {
-      this.initGroup(groupcode, groupname, coursecode)
+      this.initGroup(groupcode, groupname, coursecode, yearcode)
     }
     this.markCreditToStudents(studentnumber, passed, grade, groupcode)
     this.markCreditToAttempts(studentnumber, passed, grade, groupcode)

services/backend/oodikone2-backend/src/services/courses.js

@@ -334,8 +334,7 @@ const yearlyStatsOf = async (code, year, separate, language) => {
       return b
     }, resultProgrammes)
 
-  const alternatives = await getDuplicateCodes(code)
-  const codes = alternatives ? alternatives : [code]
+  const codes = await alternativeCodes(code)
   const allInstances = await creditsOf(codes)
   const yearInst = allInstances
     .filter(inst => moment(new Date(inst.date)).isBetween(year.start + '-09-01', year.end + '-08-01'))
@@ -483,13 +482,12 @@ const getMainCodeToDuplicates = async () => {
   return maincodeToDuplicates
 }
 
-const getCodeToMainCodeMap = async () => {
+const getCodeToMainCourseMap = async () => {
   try {
     const maincodeToDuplicates = await getMainCodeToDuplicates()
     const codeToMainCode = Object.values(maincodeToDuplicates).reduce((acc, d) => {
-      const maincode = d.maincourse.code
       d.duplicates.forEach((c) => {
-        acc[c.code] = maincode
+        acc[c.code] = d.maincourse
       })
       return acc
     }, {})
@@ -500,37 +498,19 @@ const getCodeToMainCodeMap = async () => {
   return {}
 }
 
-const getMainCodesMap = async () => {
-  return await getCodeToMainCodeMap()
-}
-
-const getMaincodeToDuplicateCodesMap = async () => {
-  try {
-    const codeToMainCodeMap = await getCodeToMainCodeMap()
-    const codes = Object.keys(codeToMainCodeMap)
-    const mainCodeToCoursesMap = codes.reduce((acc, code) => {
-      const maincode = codeToMainCodeMap[code]
-      const duplicates = acc[maincode] || []
-      duplicates.push(code)
-      acc[maincode] = duplicates
-      return acc
-    }, {})
-    return mainCodeToCoursesMap
-  } catch (e) {
-    console.error(e)
-  }
-  return {}
+const getMainCodeToDuplicatesAndCodeToMainCode = async () => {
+  return [await getMainCodeToDuplicates(), await getCodeToMainCourseMap()]
 }
 
 const getMainCourseToCourseMap = async (/*programme*/) => {
   try {
-    const codeToMainCodeMap = await getCodeToMainCodeMap()
+    const codeToMainCodeMap = await getCodeToMainCourseMap()
     const courses = await byCodes(Object.keys(codeToMainCodeMap))
     const mainCodeToCoursesMap = courses.reduce((acc, course) => {
-      const maincode = codeToMainCodeMap[course.code]
-      const duplicates = acc[maincode] || []
+      const maincourse = codeToMainCodeMap[course.code]
+      const duplicates = acc[maincourse.code] || []
       duplicates.push(course)
-      acc[maincode] = duplicates
+      acc[maincourse.code] = duplicates
       return acc
     }, {})
     return mainCodeToCoursesMap
@@ -541,9 +521,10 @@ const getMainCourseToCourseMap = async (/*programme*/) => {
 }
 
 const getDuplicateCodes = async (code) => {
-  const maincodeToDuplicatesMap = await getMaincodeToDuplicateCodesMap()
-  const duplicates = maincodeToDuplicatesMap[code]
-  return duplicates
+  const [mainCodeToDuplicates, codeToMainCourseMap] = await getMainCodeToDuplicatesAndCodeToMainCode()
+  const maincourse = codeToMainCourseMap[code]
+  if (!maincourse) return null
+  return mainCodeToDuplicates[maincourse.code].duplicates.map(e => e.code)
 }
 
 const setDuplicateCode = async (code1, code2) => {
@@ -609,7 +590,7 @@ const parseCredit = credit => {
   }
 }
 
-const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode) => {
+const yearlyStatsOfNew = async (coursecode, separate) => {
   const codes = await alternativeCodes(coursecode)
   const [credits, course] = await Promise.all([creditsForCourses(codes), Course.findByPk(coursecode)])
   const counter = new CourseYearlyStatsCounter()
@@ -620,22 +601,19 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
       yearcode, yearname, programmes,
       coursecode
     } = parseCredit(credit)
-    if (startyearcode <= yearcode && yearcode <= endyearcode) {
-      const groupcode = separate ? semestercode : yearcode
-      const groupname = separate ? semestername : yearname
-      const unknownProgramme = [{
-        code: 'OTHER',
-        name: {
-          en: 'Other',
-          fi: 'Muu',
-          sv: 'Andra'
-        }
-      }]
-      counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
-      counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode)
-    } else {
-      counter.markCreditToHistory(studentnumber, passed)
-    }
+    const groupcode = separate ? semestercode : yearcode
+    const groupname = separate ? semestername : yearname
+    const unknownProgramme = [{
+      code: 'OTHER',
+      name: {
+        en: 'Other',
+        fi: 'Muu',
+        sv: 'Andra'
+      }
+    }]
+    counter.markStudyProgrammes(studentnumber, programmes.length === 0 ? unknownProgramme : programmes)
+    counter.markCreditToGroup(studentnumber, passed, grade, groupcode, groupname, coursecode, yearcode)
+    counter.markCreditToHistory(studentnumber, passed)
   }
   const statistics = counter.getFinalStatistics()
   return {
@@ -646,9 +624,9 @@ const yearlyStatsOfNew = async (coursecode, separate, startyearcode, endyearcode
   }
 }
 
-const courseYearlyStats = async (coursecodes, separate, startyearcode, endyearcode) => {
+const courseYearlyStats = async (coursecodes, separate) => {
   const stats = await Promise.all(coursecodes
-    .map(code => yearlyStatsOfNew(code, separate, startyearcode, endyearcode || startyearcode)))
+    .map(code => yearlyStatsOfNew(code, separate)))
   return stats
 }
 
@@ -676,7 +654,7 @@ const nameLikeTerm = (name) => {
 
 const codeLikeTerm = (code) => !code ? undefined : {
   code: {
-    [Op.iLike]: `${code.trim()}%`
+    [Op.iLike]: `%${code.trim()}%`
   }
 }
 
@@ -715,12 +693,13 @@ module.exports = {
   findDuplicates,
   setDuplicateCode,
   deleteDuplicateCode,
-  getMainCodesMap,
+  getCodeToMainCourseMap,
   getMainCourseToCourseMap,
   getAllCourseTypes,
   getAllDisciplines,
   yearlyStatsOfNew,
   courseYearlyStats,
   byNameAndOrCodeLike,
-  byCodes
+  byCodes,
+  getMainCodeToDuplicatesAndCodeToMainCode
 }

services/backend/oodikone2-backend/src/services/elementdetails.js

@@ -1,7 +1,13 @@
 const { ElementDetails } = require('../models/index')
 const { Op } = require('sequelize')
 
-const byId = id =>  ElementDetails.findByPk(id)
+const byId = id => ElementDetails.findByPk(id)
+
+const byType = type => ElementDetails.findAll({
+  where: {
+    type
+  }
+})
 
 const byCodes = codes => ElementDetails.findAll({
   where: {
@@ -11,4 +17,4 @@ const byCodes = codes => ElementDetails.findAll({
   }
 })
 
-module.exports = { byId, byCodes }
+module.exports = { byId, byType, byCodes }

services/backend/oodikone2-backend/src/services/faculties.js

@@ -7,14 +7,15 @@ const { Credit, StudyrightElement } = require('../models')
 const client = axios.create({ baseURL: USERSERVICE_URL, headers: { 'secret': process.env.USERSERVICE_SECRET } })
 
 const calculateFacultyYearlyStats = async () => {
-  const { data: faculties } = await client.get('/faculty_programmes')
+  const { data: facultyProgrammes } = await client.get('/faculty_programmes')
   const res = {}
   const lock = new AsyncLock()
 
   let amountDone = 0
-  await Promise.all(faculties.map(({ faculty_code, programme_code }) => (
+  await Promise.all(facultyProgrammes.map(({ faculty_code, programme_code }) => (
     new Promise(async (facultyRes) => {
       if (!res[faculty_code]) res[faculty_code] = {}
+      if (!res[faculty_code][programme_code]) res[faculty_code][programme_code] = {}
 
       const facultyStudents = await StudyrightElement.findAll({
         where: {
@@ -37,17 +38,17 @@ const calculateFacultyYearlyStats = async () => {
             const attainmentYear = moment(c.attainment_date).year()
             if (!res[faculty_code][attainmentYear]) {
               lock.acquire(faculty_code, (done) => {
-                if (!res[faculty_code][attainmentYear]) {
+                if (!res[faculty_code][programme_code][attainmentYear]) {
                   const facultyYearStats = {}
                   facultyYearStats.studentCredits = 0
                   facultyYearStats.coursesPassed = 0
                   facultyYearStats.coursesFailed = 0
-                  res[faculty_code][attainmentYear] = facultyYearStats
+                  res[faculty_code][programme_code][attainmentYear] = facultyYearStats
                 }
                 done()
               })
             }
-            const facultyYearStats = res[faculty_code][attainmentYear]
+            const facultyYearStats = res[faculty_code][programme_code][attainmentYear]
             if (c.credittypecode === 4) {
               facultyYearStats.studentCredits += c.credits
               facultyYearStats.coursesPassed += 1
@@ -59,7 +60,7 @@ const calculateFacultyYearlyStats = async () => {
         })
       )))
       amountDone += 1
-      console.log(`Faculties done ${amountDone}/${faculties.length}`)
+      console.log(`Faculty programmes done ${amountDone}/${facultyProgrammes.length}`)
       facultyRes()
     })
   )))

services/backend/oodikone2-backend/src/services/mandatoryCourses.js

@@ -61,12 +61,14 @@ const find = async (studyProgrammeId, code) => {
       }
     }
   })
+  if (!mandatoryCourse) return null
 
   const course = await Course.findOne({
     where: {
       code: mandatoryCourse.course_code
     }
   })
+  if (!course) return null
 
   return { ...mandatoryCourse.get(), course: course.get() }
 }

services/backend/oodikone2-backend/src/services/populations.js

@@ -1,6 +1,6 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
-const AsyncLock = require('async-lock')
+
 const {
   Student, Credit, Course, sequelize, Studyright, StudyrightExtent, ElementDetails,
   Discipline, CourseType, SemesterEnrollment, Semester, Transfers, StudyrightElement
@@ -8,7 +8,7 @@ const {
 const {
   Tag, TagStudent
 } = require('../models/models_kone')
-const { getMainCodesMap, byName } = require('./courses')
+const { getCodeToMainCourseMap } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 
@@ -225,7 +225,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
   })
 
   const studentTags = await TagStudent.findAll({
-    attributes: ['id', 'studentnumber'],
+    attributes: ['tag_id', 'studentnumber'],
     include: [
       {
         model: Tag,
@@ -245,20 +245,10 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
   }, {})
 
   students.forEach(student => {
-    student.tags = studentNumberToTags[student.studentnumber]
+    student.tags = studentNumberToTags[student.studentnumber] || []
   })
 
-  if (tag) {
-    const studentsWithSearchedTag = {}
-    students.forEach(student => {
-      student.tags = studentNumberToTags[student.studentnumber]
-      if (student.tags && student.tags.some(t => t.tag.tag_id === tag)) {
-        studentsWithSearchedTag[student.studentnumber] = true
-      }
-    })
-
-    return students.filter(student => studentsWithSearchedTag[student.studentnumber])
-  }
+  if (tag) return students.filter(student => student.tags.some(t => t.tag_id === tag))
   return students
 }
 
@@ -271,7 +261,7 @@ const count = (column, count, distinct = false) => {
   )
 }
 
-const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents) => { // eslint-disable-line
+const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag) => { // eslint-disable-line
   const filteredExtents = []
   let studyrightWhere = {
     extentcode: {
@@ -288,6 +278,21 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
     studyrightWhere.canceldate = null
   }
 
+  let studentWhere = {}
+  if (tag) {
+    const taggedStudentnumbers = await TagStudent.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        tag_id: tag
+      }
+    })
+    studentWhere.where = {
+      student_studentnumber: {
+        [Op.in]: taggedStudentnumbers.map(sn => sn.studentnumber)
+      }
+    }
+  }
+
   const students = await Studyright.findAll({
     attributes: ['student_studentnumber'],
     include: {
@@ -321,15 +326,16 @@ const studentnumbersWithAllStudyrightElements = async (studyRights, startDate, e
         }
       ],
       ...studyrightWhere
+
     },
+    ...studentWhere,
     having: count('studyright_elements.code', studyRights.length, true),
   })
-
   return [...new Set(students.map(s => s.student_studentnumber))]
 }
 
 const parseQueryParams = query => {
-  const { semesters, studentStatuses, studyRights, months, endYear, startYear } = query
+  const { semesters, studentStatuses, studyRights, months, endYear, startYear, tag } = query
   const startDate = semesters.includes('FALL') ?
     `${startYear}-${semesterStart[semesters.find(s => s === 'FALL')]}` :
     `${moment(startYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterStart[semesters.find(s => s === 'SPRING')]}`
@@ -346,7 +352,8 @@ const parseQueryParams = query => {
     studyRights: Array.isArray(studyRights) ? studyRights : Object.values(studyRights),
     months,
     startDate,
-    endDate
+    endDate,
+    tag
   }
 }
 
@@ -429,7 +436,7 @@ const formatStudentsForApi = async (students, startDate, endDate, { studyRights
 }
 
 const formatQueryParamArrays = (query, params) => {
-  let res = {...query}
+  let res = { ...query }
   params.forEach(p => {
     if (!res[p]) return
     res[p] = Array.isArray(res[p]) ? res[p] : [res[p]]
@@ -473,16 +480,15 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
 }
 
 const unifyOpenUniversity = (code) => {
-  if (code[0] === 'A') {
-    return code.substring(code[1] === 'Y' ? 2 : 1)
-  }
-  return code
+  const regexresult = code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+  if (!regexresult) return code
+  return regexresult[1]
 }
 
-const getUnifiedCode = (code, codeduplicates) => {
-  const formattedcode = unifyOpenUniversity(code)
-  const unifiedcode = codeduplicates[formattedcode]
-  return !unifiedcode ? formattedcode : unifiedcode
+const getMainCourse = (course, codeduplicates) => {
+  const formattedcode = unifyOpenUniversity(course.code)
+  const maincourse = codeduplicates[formattedcode]
+  return maincourse || course
 }
 
 const findCourses = (studentnumbers, beforeDate) => {
@@ -500,8 +506,7 @@ const findCourses = (studentnumbers, beforeDate) => {
           attainment_date: {
             [Op.lt]: beforeDate
           }
-        },
-        order: 'attainment_date'
+        }
       },
       {
         model: Discipline
@@ -528,81 +533,85 @@ const parseCreditInfo = credit => ({
 })
 
 const bottlenecksOf = async (query, studentnumberlist) => {
-  let allstudents
-  let courses
-  if (!studentnumberlist) {
-    if (!query.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)) {
+  const isValidRequest = async (query, params) => {
+    const { studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents } = params
+    if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
     }
     if (query.studentStatuses &&
-      !query.studentStatuses.map(
+      !query.studentStatuses.every(
         status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE'
-      ).every(e => e === true)) {
+      )) {
       return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
     }
-    const { studyRights, startDate, endDate, months, exchangeStudents, cancelledStudents } = parseQueryParams(query)
     if (query.selectedStudents) {
       const allStudents =
         await studentnumbersWithAllStudyrightElements(
-          studyRights, startDate, endDate, exchangeStudents, cancelledStudents
+          studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, query.tag
         )
       const disallowedRequest =
         checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
       if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
     }
-    const studentnumbers = query.selectedStudents ||
-      await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents)
-    allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
-    courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
-  } else {
-    allstudents = studentnumberlist
-    courses = await findCourses(allstudents, new Date())
+    return null
   }
+  const getStudentsAndCourses = async (selectedStudents, studentnumberlist) => {
+    if (!studentnumberlist) {
+      const { months, studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag } = params
+      const studentnumbers = selectedStudents ||
+        await studentnumbersWithAllStudyrightElements(studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents, tag)
+      const allstudents = studentnumbers.reduce((numbers, num) => ({ ...numbers, [num]: true }), {})
+      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months))
+      return [allstudents, courses]
+    } else {
+      const allstudents = studentnumberlist
+      const courses = await findCourses(allstudents, new Date())
+      return [allstudents, courses]
+    }
+  }
+  const params = parseQueryParams(query)
+  const [[allstudents, courses], codeToMainCourse, error] = await Promise.all(
+    [
+      getStudentsAndCourses(query.selectedStudents, studentnumberlist),
+      getCodeToMainCourseMap(),
+      isValidRequest(
+        query,
+        params
+      )
+    ]
+  )
+  if (error) return error
+
   const bottlenecks = {
     disciplines: {},
     coursetypes: {}
   }
-  const codeduplicates = await getMainCodesMap()
 
-  const lock = new AsyncLock()
   const stats = {}
 
-  await Promise.all(
-    courses.map(course => (
-      new Promise(async res => {
-        let { code, name, disciplines, course_type } = course
-        if (name.fi && name.fi.includes('Avoin yo:')) {
-          const courses = await byName(name.fi.split(': ')[1], 'fi')
-          const theOne = courses.length > 0 ? courses[0] : { name, code }
-          code = theOne.code
-          name = theOne.name
-        }
-        const unifiedcode = getUnifiedCode(code, codeduplicates)
-        if (!stats[unifiedcode]) {
-          lock.acquire('stats', done => {
-            if (!stats[unifiedcode]) stats[unifiedcode] = new CourseStatsCounter(unifiedcode, name, allstudents)
-            done()
-          })
-        }
-        const coursestats = stats[unifiedcode]
+  const startYear = parseInt(query.endYear, 10)
+  courses.forEach(course => {
+    let { disciplines, course_type } = course
+    const maincourse = getMainCourse(course, codeToMainCourse)
+    if (!stats[maincourse.code]) {
+      stats[maincourse.code] = new CourseStatsCounter(maincourse.code, maincourse.name, allstudents)
+    }
+    const coursestats = stats[maincourse.code]
 
-        coursestats.addCourseType(course_type.coursetypecode, course_type.name)
-        bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
-        disciplines.forEach(({ discipline_id, name }) => {
-          coursestats.addCourseDiscipline(discipline_id, name)
-          bottlenecks.disciplines[discipline_id] = name
-        })
+    coursestats.addCourseType(course_type.coursetypecode, course_type.name)
+    bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
+    disciplines.forEach(({ discipline_id, name }) => {
+      coursestats.addCourseDiscipline(discipline_id, name)
+      bottlenecks.disciplines[discipline_id] = name
+    })
 
-        course.credits.forEach(credit => {
-          const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
-          const semester = getPassingSemester(parseInt(query.endYear, 10), date)
-          coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
-        })
-        stats[unifiedcode] = coursestats
-        res()
-      })
-    ))
-  )
+    course.credits.forEach(credit => {
+      const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
+      const semester = getPassingSemester(startYear, date)
+      coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
+    })
+    stats[maincourse.code] = coursestats
+  })
   bottlenecks.coursestatistics = Object.values(stats)
     .map(coursestatistics => coursestatistics.getFinalStats())
   return bottlenecks

services/backend/oodikone2-backend/src/services/students.js

@@ -42,8 +42,7 @@ const byId = async (id) => Student.findByPk(id, {
   ]
 })
 
-
-const findByCourseAndSemesters = async (coursecode, yearcode) =>
+const findByCourseAndSemesters = async (coursecodes, yearcode) =>
   sequelize.query(`
     SELECT
       studentnumber, credit.course_code, attainment_date
@@ -51,13 +50,13 @@ const findByCourseAndSemesters = async (coursecode, yearcode) =>
     INNER JOIN credit ON
       student.studentnumber=credit.student_studentnumber
     WHERE 
-      course_code=:coursecode AND
+      course_code IN (:coursecodes) AND
       attainment_date
     BETWEEN
       (select startdate FROM semesters where yearcode=:yearcode ORDER BY semestercode LIMIT 1) AND
       (select enddate FROM semesters where yearcode=:yearcode ORDER BY semestercode DESC LIMIT 1);
   `,
-  { replacements: { coursecode, yearcode }, type: sequelize.QueryTypes.SELECT })
+  { replacements: { coursecodes, yearcode }, type: sequelize.QueryTypes.SELECT })
     .map(st => st.studentnumber)
 
 const byAbreviatedNameOrStudentNumber = (searchTerm) => {

services/backend/oodikone2-backend/src/services/studytrack.js

@@ -380,18 +380,18 @@ const gendersFromClass = async (studentnumbers) => {
   }, {})
 }
 
-const countriesFromClass = async (studentnumbers) => {
+const nationalitiesFromClass = async (studentnumbers) => {
   return Student.findAll({
-    attributes: [[sequelize.fn('count', sequelize.col('home_country_en')), 'count'], 'home_country_en'],
     where: {
       studentnumber: {
         [Op.in]: studentnumbers
-      },
+      }
     },
-    group: ['home_country_en'],
-    raw: true
-  }).reduce((acc, curr) => {
-    acc[curr.country_en] = curr.count
+    attributes: ['home_country_en']
+  }).reduce((acc, { home_country_en }) => {
+    const country = home_country_en || 'Unknown'
+    if (!acc[country]) acc[country] = 0
+    acc[country] += 1
     return acc
   }, {})
 }
@@ -479,9 +479,9 @@ const statsForClass = async (studentnumbers, startDate, studytrack, endDate) =>
     graduationsFromClass(studentnumbers, studytrack),
     thesesFromClass(studentnumbers, startDate, studytrack),
     gendersFromClass(studentnumbers),
-    countriesFromClass(studentnumbers),
     tranferredToStudyprogram(studentnumbers, startDate, studytrack, endDate),
-    endedStudyright(studentnumbers, startDate, studytrack, endDate)
+    endedStudyright(studentnumbers, startDate, studytrack, endDate),
+    nationalitiesFromClass(studentnumbers)
   ])
 }
 
@@ -503,13 +503,13 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       mte150: 0,
     },
     genders: {},
-    countries: {},
     thesisM: 0,
     thesisB: 0,
     students: 0,
     graduated: 0,
     inTargetTime: 0,
-    transferred: 0
+    transferred: 0,
+    nationalities: {}
   }
   const years = getYears(since)
   // studyprogramme starts with K if bachelors and M if masters
@@ -533,14 +533,12 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
     const studentnumbers = await studentnumbersWithAllStudyrightElements([studytrack], startDate, endDate, false, false)
     const creditsForStudyprogramme =
       await productivityCreditsFromStudyprogrammeStudents(studytrack, startDate, studentnumbers)
-    const [credits, graduated, theses, genders, countries, transferredTo, endedStudyright] =
+    const [credits, graduated, theses, genders, transferredTo, endedStudyright, nationalities] =
       await statsForClass(studentnumbers, startDate, studytrack, endDate)
     //console.log(year)
     //console.log(transferredFrom.rows.map(r => r.get({ plain: true })))
     // theres so much shit in the data that transefferFrom doesnt rly mean anything
     delete genders[null]
-    delete countries[null]
-    delete countries[undefined]
     const creditValues = credits.reduce((acc, curr) => {
       acc.mte30 = curr >= 30 ? acc.mte30 + 1 : acc.mte30
       acc.mte60 = curr >= 60 ? acc.mte60 + 1 : acc.mte60
@@ -557,10 +555,9 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
         totals.genders[genderKey] + Number(genders[genderKey]) :
         Number(genders[genderKey])
     })
-    Object.keys(countries).forEach(countryKey => {
-      totals.countries[countryKey] = totals.countries[countryKey] ?
-        totals.countries[countryKey] + Number(countries[countryKey]) :
-        Number(countries[countryKey])
+    Object.keys(nationalities).forEach(nationality => {
+      if (!totals.nationalities[nationality]) totals.nationalities[nationality] = 0
+      totals.nationalities[nationality] += nationalities[nationality]
     })
     const graduationTimes = graduated.map(g => moment(g.enddate).diff(g.studystartdate, 'months'))
     const inTargetTime = graduationTimes.filter(time =>
@@ -585,13 +582,12 @@ const throughputStatsForStudytrack = async (studytrack, since) => {
       thesisM: theses.MASTER || 0,
       thesisB: theses.BACHELOR || 0,
       genders,
-      countries,
       creditValues,
       transferred: transferredTo,
-      ended: endedStudyright.count
+      ended: endedStudyright.count,
+      nationalities
     }
   }))
-
   return { id: studytrack, status: null, data: { years: arr, totals } }
 }
 

services/backend/oodikone2-backend/src/services/tags.js

@@ -5,7 +5,7 @@ const Op = Sequelize.Op
 
 const findTags = () => Tag.findAll({})
 
-const findTagsByStudytrack = (studytrack) => {
+const findTagsByStudytrack = async (studytrack) => {
   return Tag.findAll({
     where: {
       studytrack: {
@@ -15,6 +15,17 @@ const findTagsByStudytrack = (studytrack) => {
   })
 }
 
+const findTagsFromStudytrackById = async (studytrack, tag_ids) => {
+  return Tag.findAll({
+    where: {
+      studytrack,
+      tag_id: {
+        [Op.in]: tag_ids
+      }
+    }
+  })
+}
+
 const createNewTag = async (tag) => {
   return Tag.create(tag)
 }
@@ -33,5 +44,6 @@ module.exports = {
   createNewTag,
   findTags,
   findTagsByStudytrack,
+  findTagsFromStudytrackById,
   deleteTag
 }

services/backend/oodikone2-backend/src/services/tagstudent.js

@@ -33,12 +33,15 @@ const createStudentTag = async (tag) => {
   return TagStudent.create(tag)
 }
 
-const deleteStudentTag = async (id) => {
+const createMultipleStudentTags = async (tags) => {
+  return TagStudent.bulkCreate(tags, { ignoreDuplicates: true })
+}
+
+const deleteStudentTag = async (studentnumber, tag_id) => {
   return TagStudent.destroy({
     where: {
-      id: {
-        [Op.eq]: id
-      }
+      tag_id,
+      studentnumber
     }
   })
 }
@@ -48,5 +51,6 @@ module.exports = {
   getStudentTagsByStudentnumber,
   getStudentTagsByStudytrack,
   createStudentTag,
+  createMultipleStudentTags,
   deleteStudentTag
 }

services/backend/shared/migrations_kone/20190821_00_tag_student_foreignkeys_and_primarykeys_refactor.js

@@ -0,0 +1,21 @@
+module.exports = {
+  up: async (queryInterface) => {
+    return queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.sequelize.query('ALTER TABLE tag_student DROP CONSTRAINT "tag_student_tag_id_fkey"', { transaction })
+      await queryInterface.sequelize.query('DELETE FROM tag_student WHERE tag_id IS NULL', { transaction })
+      await queryInterface.removeConstraint('tag', 'tag_tag_id_key', { transaction })
+      await queryInterface.addConstraint('tag', ['tag_id'], {
+        type: 'primary key',
+        transaction
+      })
+      await queryInterface.sequelize.query('ALTER TABLE tag_student ADD CONSTRAINT "tag_student_tag_id_fkey" FOREIGN KEY (tag_id) REFERENCES tag(tag_id) ON UPDATE CASCADE ON DELETE CASCADE NOT DEFERRABLE', { transaction })
+      await queryInterface.removeColumn('tag_student', 'id', { transaction })
+      await queryInterface.addConstraint('tag_student', ['studentnumber', 'tag_id'], {
+        type: 'primary key',
+        transaction
+      })
+    })
+  },
+  down: async () => {
+  }
+}

services/backend/shared/models/index.js

@@ -1,5 +1,5 @@
 const Sequelize = require('sequelize')
-const { sequelize, migrationPromise } = require('../database/connection')
+const { sequelize } = require('../database/connection')
 
 const Student = sequelize.define('student',
   {
@@ -531,6 +531,5 @@ module.exports = {
   CreditTeacher,
   ErrorData,
   Migration,
-  migrationPromise,
   sequelize
 }

services/backend/shared/models/models_kone.js

@@ -1,7 +1,6 @@
 const Sequelize = require('sequelize')
 const {
-  sequelizeKone,
-  migrationPromiseKone
+  sequelizeKone
 } = require('../database/connection')
 
 const ThesisTypeEnums = {
@@ -107,15 +106,12 @@ const UsageStatistic = sequelizeKone.define(
 const TagStudent = sequelizeKone.define(
   'tag_student',
   {
-    id: {
-      primaryKey: true,
-      type: Sequelize.BIGINT,
-      autoIncrement: true
-    },
     studentnumber: {
+      primaryKey: true,
       type: Sequelize.STRING
     },
     tag_id: {
+      primaryKey: true,
       type: Sequelize.BIGINT
     }
   },
@@ -136,7 +132,6 @@ const Tag = sequelizeKone.define(
       autoIncrement: true
     },
     studytrack: {
-      primaryKey: true,
       type: Sequelize.STRING
     }
   },
@@ -234,6 +229,5 @@ module.exports = {
   ThesisTypeEnums,
   MandatoryCourseLabels,
   MigrationKone,
-  migrationPromiseKone,
   sequelizeKone
 }

services/backend/updater_writer/Dockerfile

@@ -1,22 +1,21 @@
 FROM node:10.15-alpine
 
+WORKDIR /usr/src/app
+CMD npm start
+
 RUN apk add postgresql-client
+COPY ./updater_writer/package.json /usr/src/app/package.json
+COPY ./updater_writer/package-lock.json /usr/src/app/package-lock.json
+COPY ./updater_writer/package.json /usr/src/cache/package.json
+COPY ./updater_writer/package-lock.json /usr/src/cache/package-lock.json
+RUN npm ci
+
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
 COPY ./updater_writer /usr/src/app
 COPY ./shared/models /usr/src/app/models
 COPY ./shared/migrations /usr/src/app/database/migrations
 COPY ./shared/migrations_kone /usr/src/app/database/migrations_kone
 
-WORKDIR /usr/src/app
-RUN npm ci
-COPY ./updater_writer/package.json /usr/src/cache/package.json
-COPY ./updater_writer/package-lock.json /usr/src/cache/package-lock.json
-
-ARG NODE_ENV=production
-ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
-
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
-RUN chmod +x /wait
-
-CMD /wait && npm start

services/backend/updater_writer/database/connection.js

@@ -11,33 +11,50 @@ const sequelize = new Sequelize(conf.DB_URL, {
 sequelize.query(`SET SESSION search_path to ${conf.DB_SCHEMA}`)
 
 
-const runMigrations = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelize,
-        tableName: 'migrations'
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelize.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/database/migrations`,
-        pattern: /\.js$/,
-        schema: conf.DB_SCHEMA
+const initializeDatabaseConnection = async () => {
+  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+  const waitSeconds = 60
+  for (let i = 1; i <= waitSeconds; i++) {
+    try {
+      await sequelize.authenticate()
+      break
+    } catch (e) {
+      if (i === waitSeconds) {
+        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        throw e
       }
-    })
-    const migrations = await migrator.up()
+      console.log('.')
+      await sleep(1000)
+    }
+  }
+  if (!conf.isTest) {
+    try {
+      const migrator = new Umzug({
+        storage: 'sequelize',
+        storageOptions: {
+          sequelize: sequelize,
+          tableName: 'migrations'
+        },
+        logging: console.log,
+        migrations: {
+          params: [
+            sequelize.getQueryInterface(),
+            Sequelize
+          ],
+          path: `${process.cwd()}/database/migrations`,
+          pattern: /\.js$/,
+          schema: conf.DB_SCHEMA
+        }
+      })
+      const migrations = await migrator.up()
 
-    console.log('Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Migration error, message:', e)
+      console.log('Migrations up to date', migrations)
+    } catch (e) {
+      console.log('Kone Migration error')
+      throw e
+    }
   }
 }
-const migrationPromise = !conf.isTest ? runMigrations() : Promise.resolve()
 
 const forceSyncDatabase = async () => {
   try {
@@ -51,6 +68,6 @@ const forceSyncDatabase = async () => {
 
 module.exports = {
   sequelize,
-  migrationPromise,
+  initializeDatabaseConnection,
   forceSyncDatabase
 }

services/backend/updater_writer/index.js

@@ -1,54 +1,62 @@
-const stan = require('node-nats-streaming').connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
+const natsStreaming = require('node-nats-streaming')
 const { updateStudent, updateMeta, updateAttainmentMeta } = require('./updater/database_updater')
 const logger = require('./logger')
+const { initializeDatabaseConnection } = require('./database/connection')
 
-console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
-const opts = stan.subscriptionOptions()
-opts.setManualAckMode(true)
-opts.setAckWait(10 * 60 * 1000) // some students have taken over 5 min to write!
-// opts.setDeliverAllAvailable()
-// opts.setDurableName('durable')
-opts.setMaxInFlight(1)
+initializeDatabaseConnection().then(() => {
+  const stan = natsStreaming.connect('updaterNATS', process.env.HOSTNAME, process.env.NATS_URI)
 
-stan.on('connect', function () {
+  console.log(`STARTING WITH ${process.env.HOSTNAME} as id`)
+  const opts = stan.subscriptionOptions()
+  opts.setManualAckMode(true)
+  opts.setAckWait(10 * 60 * 1000) // some students have taken over 5 min to write!
+  // opts.setDeliverAllAvailable()
+  // opts.setDurableName('durable')
+  opts.setMaxInFlight(1)
 
-  const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
-  const attSub = stan.subscribe('UpdateAttainmentDates', 'updater.workers.attainments', opts)
-  const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
+  stan.on('connect', function () {
 
-  const writeStudent = async (msg) => {
-    let data = null
-    try {
-      const start = new Date()
-      data = JSON.parse(msg.getData())
-      if (data.task === 'meta') {
-        await updateMeta(data.data)
-      } else {
-        await updateStudent(data.data)
-      }
-      stan.publish('status', JSON.stringify({ task: data.task, status: 'DONE', timems: new Date() - start }), (err) => { if (err) console.log(err) })
-    } catch (err) {
-      console.log('update failed', data.task, err)
-      logger.info('failure', { service: 'WRITER' })
-    }
-    msg.ack()
-  }
+    const sub = stan.subscribe('UpdateWrite', 'updater.workers', opts)
+    const attSub = stan.subscribe('UpdateAttainmentDates', 'updater.workers.attainments', opts)
+    const prioSub = stan.subscribe('PriorityWrite', 'updater.workers.prio', opts)
 
-  sub.on('message', writeStudent)
-  prioSub.on('message', writeStudent)
-
-  attSub.on('message', async (msg) => {
-    try {
-      await updateAttainmentMeta()
-      stan.publish('status', JSON.stringify({ task: 'attainment', status: 'DONE' }), (err) => {
-        if (err) {
-          console.log('publish failed')
+    const writeStudent = async (msg) => {
+      let data = null
+      try {
+        const start = new Date()
+        data = JSON.parse(msg.getData())
+        if (data.task === 'meta') {
+          await updateMeta(data.data)
+        } else {
+          await updateStudent(data.data)
         }
-      })
-    } catch (err) {
-      console.log('attainment meta update failed', err)
-      logger.info('failure', { service: 'WRITER' })
+        stan.publish('status', JSON.stringify({ task: data.task, status: 'DONE', timems: new Date() - start }), (err) => { if (err) console.log(err) })
+      } catch (err) {
+        console.log('update failed', data.task, err)
+        logger.info('failure', { service: 'WRITER' })
+      }
+      msg.ack()
     }
-    msg.ack()
+
+    sub.on('message', writeStudent)
+    prioSub.on('message', writeStudent)
+
+    attSub.on('message', async (msg) => {
+      try {
+        await updateAttainmentMeta()
+        stan.publish('status', JSON.stringify({ task: 'attainment', status: 'DONE' }), (err) => {
+          if (err) {
+            console.log('publish failed')
+          }
+        })
+      } catch (err) {
+        console.log('attainment meta update failed', err)
+        logger.info('failure', { service: 'WRITER' })
+      }
+      msg.ack()
+    })
   })
-})
+}).catch(e => {
+  process.exitCode = 1
+  console.log(e)
+})

services/oodikone2-analytics/Dockerfile

@@ -1,18 +1,18 @@
 FROM node:10.15-alpine
 
-COPY . /usr/src/app
 WORKDIR /usr/src/app
-RUN npm ci
+EXPOSE 4568
+CMD npm start
 
+COPY package.json /usr/src/app/package.json
+COPY package-lock.json /usr/src/app/package-lock.json
 COPY package.json /usr/src/cache/package.json
 COPY package-lock.json /usr/src/cache/package-lock.json
+RUN npm ci
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
 RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
-RUN chmod +x /wait
+COPY . /usr/src/app
 
-EXPOSE 4568
-CMD /wait && npm start

services/oodikone2-analytics/index.js

@@ -1,84 +1,95 @@
 const express = require('express')
 const morgan = require('morgan')
-const { Productivity, Throughput, FacultyStats } = require('./src/models')
-
-const app = express()
-const port = 4568
 const bodyParser = require('body-parser')
+const { Productivity, Throughput, FacultyStats } = require('./src/models')
+const { initializeDatabaseConnection } = require('./src/database/connection')
 
-app.use(bodyParser.json())
-app.use(morgan('tiny'))
+initializeDatabaseConnection().then(() => {
+  const app = express()
+  const port = 4568
 
-app.get('/ping', (req, res) => res.json({ message: 'pong'}))
+  app.use(bodyParser.json({ limit: '50mb' }))
+  app.use(morgan('tiny'))
 
-const formatProductivity = stats => {
-  const { id, updatedAt: lastUpdated, data, status } = stats
-  return { [id] : { lastUpdated, data, status } }
-}
+  app.get('/ping', (req, res) => res.json({ message: 'pong'}))
 
-const formatThroughput = stats => {
-  const { id, updatedAt: lastUpdated, data, status } = stats
-  return { [id] : { lastUpdated, data: data.years, status, totals: data.totals } }
-}
-
-app.get('/productivity/:id', async (req, res) => {
-  const { id } = req.params
-  const result = await Productivity.findByPk(id)
-  if (!result) {
-    res.json(null)
-  } else {
-    res.json(formatProductivity(result))
+  const formatProductivity = stats => {
+    const { id, updatedAt: lastUpdated, data, status } = stats
+    return { [id] : { lastUpdated, data, status } }
   }
-})
 
-app.post('/productivity', async (req, res) => {
-  const { data } = req.body
-  const [saved] = await Productivity.upsert({ ...data, status: 'DONE' }, { returning: true })
-  const result = formatProductivity(saved)
-  res.json(result)
-})
-
-app.patch('/productivity', async (req, res) => {
-  for (let [id, data] of Object.entries(req.body.data)) {
-    await Productivity.upsert({ ...data, id })
+  const formatThroughput = stats => {
+    const { id, updatedAt: lastUpdated, data, status } = stats
+    return { [id] : { lastUpdated, data: data.years, status, totals: data.totals } }
   }
-  res.status(200).end()
-})
 
-app.get('/throughput/:id', async (req, res) => {
-  const { id } = req.params
-  const result = await Throughput.findByPk(id)
-  if (!result) {
-    res.json(null)
-  } else {
-    res.json(formatThroughput(result))
-  }
-})
+  app.get('/productivity/:id', async (req, res) => {
+    const { id } = req.params
+    const result = await Productivity.findByPk(id)
+    if (!result) {
+      res.json(null)
+    } else {
+      res.json(formatProductivity(result))
+    }
+  })
 
-app.post('/throughput', async (req, res) => {
-  const { data } = req.body
-  const [saved] = await Throughput.upsert({ ...data, status: 'DONE' }, { returning: true })
-  const result = formatThroughput(saved)
-  res.json(result)
-})
+  app.post('/productivity', async (req, res) => {
+    const { data } = req.body
+    const [saved] = await Productivity.upsert({ ...data, status: 'DONE' }, { returning: true })
+    const result = formatProductivity(saved)
+    res.json(result)
+  })
 
-app.patch('/throughput', async (req, res) => {
-  for (let [id, data] of Object.entries(req.body.data)) {
-    await Throughput.upsert({ ...data, id })
-  }
-  res.status(200).end()
-})
+  app.patch('/productivity', async (req, res) => {
+    for (let [id, data] of Object.entries(req.body.data)) {
+      await Productivity.upsert({ ...data, id })
+    }
+    res.status(200).end()
+  })
 
-app.get('/facultystats', async (req, res) => {
-  const results = await FacultyStats.findAll()
-  res.status(200).json(results)
-})
+  app.get('/throughput/:id', async (req, res) => {
+    const { id } = req.params
+    const result = await Throughput.findByPk(id)
+    if (!result) {
+      res.json(null)
+    } else {
+      res.json(formatThroughput(result))
+    }
+  })
 
-app.patch('/facultystats', async (req, res) => {
-  for (let [id, data] of Object.entries(req.body.data)) {
-    await FacultyStats.upsert({ id, data })
-  }
-  res.status(200).end()
-})
+  app.post('/throughput', async (req, res) => {
+    const { data } = req.body
+    const [saved] = await Throughput.upsert({ ...data, status: 'DONE' }, { returning: true })
+    const result = formatThroughput(saved)
+    res.json(result)
+  })
 
-app.listen(port, () => console.log(`Analytics listening on port ${port}!`))
+  app.patch('/throughput', async (req, res) => {
+    for (let [id, data] of Object.entries(req.body.data)) {
+      await Throughput.upsert({ ...data, id })
+    }
+    res.status(200).end()
+  })
+
+  app.get('/facultystats', async (req, res) => {
+    const results = await FacultyStats.findAll()
+    res.status(200).json(results)
+  })
+
+  app.patch('/facultystats', async (req, res) => {
+    for (let [id, data] of Object.entries(req.body.data)) {
+      await FacultyStats.upsert({ id, data })
+    }
+    res.status(200).end()
+  })
+
+  const server = app.listen(port, () => console.log(`Analytics listening on port ${port}!`))
+  process.on('SIGTERM', () => {
+    server.close(() => {
+      console.log('Process terminated')
+    })
+  })
+}).catch(e => {
+  process.exitCode = 1
+  console.log(e)
+})

services/oodikone2-analytics/src/database/connection.js

@@ -6,36 +6,53 @@ const sequelize = new Sequelize(conf.DB_URL, {
   schema: conf.DB_SCHEMA,
   logging: false
 })
-const runMigrations = async () => {
-  try {
-    const migrator = new Umzug({
-      storage: 'sequelize',
-      storageOptions: {
-        sequelize: sequelize,
-        tableName: 'migrations'
-      },
-      logging: console.log,
-      migrations: {
-        params: [
-          sequelize.getQueryInterface(),
-          Sequelize
-        ],
-        path: `${process.cwd()}/src/database/migrations`,
-        pattern: /\.js$/,
-      }
-    })
-    const migrations = await migrator.up()
 
-    console.log('Migrations up to date', migrations)
-  } catch (e) {
-    console.log('Migration error, message:', e)
+const initializeDatabaseConnection = async () => {
+  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+  const waitSeconds = 60
+  for (let i = 1; i <= waitSeconds; i++) {
+    try {
+      await sequelize.authenticate()
+      break
+    } catch (e) {
+      if (i === waitSeconds) {
+        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        throw e
+      }
+      console.log('.')
+      await sleep(1000)
+    }
+  }
+  if (process.env.NODE_ENV != 'test') {
+    try {
+      const migrator = new Umzug({
+        storage: 'sequelize',
+        storageOptions: {
+          sequelize: sequelize,
+          tableName: 'migrations'
+        },
+        logging: console.log,
+        migrations: {
+          params: [
+            sequelize.getQueryInterface(),
+            Sequelize
+          ],
+          path: `${process.cwd()}/src/database/migrations`,
+          pattern: /\.js$/,
+        }
+      })
+      const migrations = await migrator.up()
+
+      console.log('Migrations up to date', migrations)
+    } catch (e) {
+      console.log('Migration error')
+      throw e
+    }
   }
 }
-const migrationPromise = conf.DB_SCHEMA === 'public' && process.env.NODE_ENV != 'test' ? runMigrations()
-  : Promise.resolve()
 
 const forceSyncDatabase = async () => {
   await sequelize.sync({ force: true })
 }
 
-module.exports = { sequelize, migrationPromise, forceSyncDatabase }
+module.exports = { sequelize, initializeDatabaseConnection, forceSyncDatabase }

services/oodikone2-analytics/src/database/migrations/190814_00_empty_throughput_database.js

@@ -0,0 +1,7 @@
+module.exports = {
+  up: async (queryInterface) => {
+    await queryInterface.sequelize.query('DELETE FROM throughput')
+  },
+  down: () => {
+  }
+}

services/oodikone2-analytics/src/database/migrations/190815_00_empty_facultystats.js

@@ -0,0 +1,7 @@
+module.exports = {
+  up: async (queryInterface) => {
+    await queryInterface.sequelize.query('DELETE FROM facultystats')
+  },
+  down: () => {
+  }
+}

services/oodikone2-frontend/Dockerfile

@@ -1,30 +1,28 @@
 FROM node:10.15-alpine
 
-ARG BASE_PATH
-ENV BASE_PATH=$BASE_PATH
-
-COPY . /usr/src/app
 WORKDIR /usr/src/app
-RUN npm ci
+EXPOSE 5000
+CMD serve -s -l 5000 dist
 
+COPY package.json /usr/src/app/package.json
+COPY package-lock.json /usr/src/app/package-lock.json
 COPY package.json /usr/src/cache/package.json
 COPY package-lock.json /usr/src/cache/package-lock.json
+RUN npm ci
 
-ARG SENTRY_RELEASE_VERSION
-ENV SENTRY_RELEASE_VERSION=${SENTRY_RELEASE_VERSION}
+ARG BASE_PATH=/
+ENV BASE_PATH=$BASE_PATH
 ARG SENTRY_URL
 ARG SENTRY_AUTH_TOKEN
 ARG SENTRY_ORG
 ARG SENTRY_PROJECT
-
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
 RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
-RUN if [ "${NODE_ENV}" != "development" ]; then npm run dist && npm install -g serve; fi
-
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
-RUN chmod +x /wait
-
-EXPOSE 5000
-CMD /wait && serve -s -l 5000 dist
+COPY . /usr/src/app
+RUN if [ "${NODE_ENV}" != "development" ]; then \
+      export CHECKSUM="$(find . ! -path */node_modules/* -type f -exec md5sum {} \; | sort -k 2 | printf "$(cat - )\\nBASE_PATH ${BASE_PATH}" | md5sum | awk '{ print $1 }')" && \
+      SENTRY_RELEASE_VERSION="${NODE_ENV}-${CHECKSUM}" npm run dist && \
+      npm install -g serve; \
+    fi

services/oodikone2-frontend/package.json

@@ -14,7 +14,7 @@
     "lint:css": "stylelint 'src/**/*.css'",
     "lint:js": "eslint server ./ --ext .jsx,.js",
     "lintfix": "eslint server ./ --ext .jsx,.js --fix",
-    "stats": "webpack --json --mode=production > stats.json",
+    "stats": "webpack --json --mode=production --profile --progress > stats.json",
     "test": "npm run lint",
     "test_docker": "docker exec -t frontend /bin/bash -c 'npm test'",
     "notest": "echo",

services/oodikone2-frontend/src/apiConnection/index.js

@@ -1,7 +1,7 @@
 import axios from 'axios'
 import * as Sentry from '@sentry/browser'
 import { getToken, setToken } from '../common'
-import { API_BASE_PATH, TOKEN_NAME, BASE_PATH, ERROR_STATUSES_TO_CAPTURE } from '../constants'
+import { API_BASE_PATH, TOKEN_NAME, BASE_PATH, ERROR_STATUSES_NOT_TO_CAPTURE } from '../constants'
 import { login as loginAction } from '../redux/auth'
 
 const isTestEnv = BASE_PATH === '/testing/'
@@ -148,7 +148,7 @@ export const handleRequest = store => next => async (action) => {
 const handleError = (err) => {
   const { response } = err
   if (response && response.status) {
-    if (ERROR_STATUSES_TO_CAPTURE.includes(response.status)) {
+    if (!ERROR_STATUSES_NOT_TO_CAPTURE.includes(response.status)) {
       Sentry.captureException(err)
     }
   }

services/oodikone2-frontend/src/components/AccessDenied/index.jsx

@@ -4,27 +4,37 @@ import { bool, func } from 'prop-types'
 import Highcharts from 'highcharts'
 import ReactHighchart from 'react-highcharts'
 import { connect } from 'react-redux'
+import { random } from 'lodash'
 import { logout as logoutAction } from '../../redux/auth'
 import { log, images } from '../../common'
 import MulticolorBarChart from '../MulticolorBarChart'
 
-const dummyData = [
-  { name: 'mluukkai', data: [48.0] },
-  { name: 'jakousa', data: [33.3] },
-  { name: 'totutotu', data: [46.2] },
-  { name: 'sasumaki', data: [59.7] },
-
-  { name: 'ikuisma', data: [48.0] },
-  { name: 'eero3', data: [33.3] },
-  { name: 'mitiaine', data: [46.2] },
-  { name: 'woltsu', data: [59.7] }
+const names = [
+  'mluukkai',
+  'jakousa',
+  'totutotu',
+  'sasumaki',
+  'ikuisma',
+  'eero3',
+  'mitiaine',
+  'rimi',
+  'esakemp',
+  'woltsu'
 ]
+const dummyData = names.map(name => ({
+  name,
+  data: [0, 1, 2, 3, 4, 5, 6, 7]
+    .reduce((acc, i) => {
+      acc[i] = (acc[i - 1] || 0) + (random(0, 3) === 0 ? 0 : random(10.0, 100.0))
+      return acc
+    }, [])
+}))
 
 const AccessDenied = ({ notEnabled, logout }) => {
   const header = notEnabled ? 'Welcome to Oodikone!' : 'Something broke'
-  const subheader = notEnabled ? `You're currently not allowed to enter 
+  const subheader = notEnabled ? `You're currently not allowed to enter
   but you will get an email when you're authorized`
-    : `If this was not intended try refreshing your browser window, 
+    : `If this was not intended try refreshing your browser window,
     pressing log out or contacting grp-toska@helsinki.fi`
 
   const [easterEgg, setEasterEgg] = useState(false)
@@ -66,34 +76,11 @@ const AccessDenied = ({ notEnabled, logout }) => {
               }
             },
 
-            series: [{
-              name: 'mluukkai',
-              data: [18937, 29057, 33213, 40949, 55880, 60421, 77543, 87691]
-            }, {
-              name: 'jakousa',
-              data: [15411, 17960, 34037, 58382, 56367, 63103, 79570, 83898]
-            }, {
-              name: 'totutotu',
-              data: [12482, 26592, 32348, 69576, 78155, 80379, 83568, 83165]
-            }, {
-              name: 'sasumaki',
-              data: [17358, 24823, 36578, 47617, 59341, 68391, 72326, 96022]
-            }, {
-              name: 'ikuisma',
-              data: [8536, 21650, 35013, 45562, 65750, 68431, 74402, 83202]
-            },
-            {
-              name: 'eero3',
-              data: [18855, 24929, 38722, 45049, 51706, 68569, 66225, 72269]
-            },
-            {
-              name: 'mitiaine',
-              data: [500, 1000, 20000, 56000, 59000, 65425, 69800, 80000]
-            }]
+            series: dummyData
           }}
         />
 
-        <MulticolorBarChart chartTitle="Your students' future" chartData={dummyData} />
+        <MulticolorBarChart chartTitle="Your students' future" chartData={dummyData.map(e => ({ name: e.name, data: [Math.max(...e.data)] }))} />
       </Container>
       <Dimmer
         active

services/oodikone2-frontend/src/components/CheckStudentList/index.jsx

@@ -18,7 +18,7 @@ class CheckStudentList extends Component {
 
   renderResults() {
     return (
-      <Modal trigger={<Button color="green" onClick={() => this.checkStudents(this.state.input)}>check students</Button>}>
+      <Modal trigger={<Button color="green" disabled={this.state.input.length === 0} onClick={() => this.checkStudents(this.state.input)}>check students</Button>}>
         <Modal.Content>
           <Form>
             <h2> Results </h2>

services/oodikone2-frontend/src/components/CourseStatistics/CourseTable/index.jsx

@@ -16,10 +16,9 @@ const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, ma
     </Table.Row>
   )
 
-  const toCourseRow = (course, i) => (
-    // kinda hax solution for the problem, after fetching course course is array of selected course code letters
-    <Table.Row style={{ cursor: 'pointer' }} key={`${course.code ? course.code : course[0]}-${i}`} onClick={() => (course.min_attainment_date || mandatory ? onSelectCourse(course) : null)} >
-      <Table.Cell key={`${course.code}-${i}`} width={10}>
+  const toCourseRow = course => (
+    <Table.Row style={{ cursor: 'pointer' }} key={course.code} onClick={() => (course.min_attainment_date || mandatory ? onSelectCourse(course) : null)} >
+      <Table.Cell width={10}>
         <div>{course.name}</div>
         <div>{getActiveYears(course)}</div>
       </Table.Cell>

services/oodikone2-frontend/src/components/CourseStatistics/ProgrammeDropdown/index.jsx

@@ -14,10 +14,12 @@ const ProgrammeDropdown = ({ options, label, name, onChange, onClear, value, ...
         ...rest
     }))}
     selection
+    multiple
     fluid
     label={label}
     name={name}
     onChange={onChange}
+    search
     value={value}
     {...props}
   />
@@ -28,7 +30,7 @@ ProgrammeDropdown.propTypes = {
   name: string.isRequired,
   onChange: func.isRequired,
   onClear: func,
-  value: oneOfType([string, number]),
+  value: arrayOf(string),
   options: arrayOf(shape({
     code: oneOfType([string, number]),
     count: oneOfType([string, number]),
@@ -39,7 +41,7 @@ ProgrammeDropdown.propTypes = {
 }
 
 ProgrammeDropdown.defaultProps = {
-  value: undefined,
+  value: [],
   onClear: undefined
 }
 

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/cumulative.jsx

@@ -1,14 +1,16 @@
 import React from 'react'
 import qs from 'query-string'
 import { Header, Icon } from 'semantic-ui-react'
+import { uniq } from 'lodash'
 import { shape, string, number, oneOfType, arrayOf } from 'prop-types'
 import SortableTable from '../../../../SortableTable'
 import { userIsAdmin } from '../../../../../common'
 
 const CumulativeTable = ({ stats, name, history }) => {
   const admin = userIsAdmin()
-  const showPopulation = (yearcode, coursecode, year) => {
-    const queryObject = { yearcode, coursecode, year }
+  const showPopulation = (yearcode, year) => {
+    const coursecodes = stats.map(s => s.coursecode)
+    const queryObject = { yearcode, coursecodes: JSON.stringify(uniq(coursecodes)), year }
     const searchString = qs.stringify(queryObject)
     history.push(`/coursepopulation?${searchString}`)
   }
@@ -17,6 +19,7 @@ const CumulativeTable = ({ stats, name, history }) => {
     <div>
       <Header as="h3" content={name} textAlign="center" />
       <SortableTable
+        defaultdescending
         getRowKey={s => s.code}
         tableProps={{ celled: true }}
         columns={[
@@ -24,7 +27,7 @@ const CumulativeTable = ({ stats, name, history }) => {
             key: 'TIME',
             title: 'Time',
             getRowVal: s => s.code,
-            getRowContent: s => (admin ? (<div>{s.name}<Icon name="level up alternate" onClick={() => showPopulation(s.code, s.coursecode, s.name)} /></div>) : s.name),
+            getRowContent: s => (admin ? (<div>{s.name}<Icon name="level up alternate" onClick={() => showPopulation(s.code, s.name, s)} /></div>) : s.name),
             cellProps: { width: 4 }
           },
           { key: 'PASSED', title: 'Passed', getRowVal: s => s.cumulative.categories.passed, cellProps: { width: 4 } },
@@ -35,7 +38,7 @@ const CumulativeTable = ({ stats, name, history }) => {
             getRowVal: s => s.cumulative.categories.passed /
               (s.cumulative.categories.failed + s.cumulative.categories.passed),
             getRowContent: stat => `${Number((100 * stat.cumulative.categories.passed) /
-              (stat.cumulative.categories.failed + stat.cumulative.categories.passed)).toFixed(2)} %`,
+              (stat.cumulative.categories.failed + stat.cumulative.categories.passed) || 0).toFixed(2)} %`,
             cellProps: { width: 4 }
           }
         ]}

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/grades.jsx

@@ -2,6 +2,7 @@ import React from 'react'
 import qs from 'query-string'
 import { arrayOf, number, oneOfType, shape, string } from 'prop-types'
 import { Header, Icon } from 'semantic-ui-react'
+import { uniq } from 'lodash'
 import SortableTable from '../../../../SortableTable'
 import { userIsAdmin } from '../../../../../common'
 import { getGradeSpread, getThesisGradeSpread, isThesisGrades, THESIS_GRADE_KEYS } from '../util'
@@ -50,14 +51,15 @@ const GradesTable = ({ stats, name, history }) => {
   const isGradeSeries = !isThesisGrades(grades)
   const admin = userIsAdmin()
 
-  const showPopulation = (yearcode, coursecode, year) => {
-    const queryObject = { yearcode, coursecode, year }
+  const showPopulation = (yearcode, year) => {
+    const coursecodes = stats.map(s => s.coursecode)
+    const queryObject = { yearcode, coursecodes: JSON.stringify(uniq(coursecodes)), year }
     const searchString = qs.stringify(queryObject)
     history.push(`/coursepopulation?${searchString}`)
   }
 
   const columns = [
-    getSortableColumn('TIME', 'Time', s => s.code, s => (admin ? (<div>{s.name}<Icon name="level up alternate" onClick={() => showPopulation(s.code, s.coursecode, s.name)} /></div>) : s.name)),
+    getSortableColumn('TIME', 'Time', s => s.code, s => (admin ? (<div>{s.name}<Icon name="level up alternate" onClick={() => showPopulation(s.code, s.name)} /></div>) : s.name)),
     getSortableColumn('ATTEMPTS', 'Attempts', s => s.attempts),
     ...getGradeColumns(isGradeSeries)
   ]
@@ -68,6 +70,7 @@ const GradesTable = ({ stats, name, history }) => {
     <div>
       <Header as="h3" content={name} textAlign="center" />
       <SortableTable
+        defaultdescending
         getRowKey={s => s.code}
         tableProps={{ celled: true, structured: true }}
         columns={columns}

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/student.jsx

@@ -1,6 +1,7 @@
 import React from 'react'
 import qs from 'query-string'
 import { Header, Icon } from 'semantic-ui-react'
+import { uniq } from 'lodash'
 import { shape, string, number, oneOfType, arrayOf } from 'prop-types'
 import SortableTable from '../../../../SortableTable'
 import { userIsAdmin } from '../../../../../common'
@@ -10,8 +11,9 @@ const formatPercentage = p => `${(p * 100).toFixed(2)} %`
 const StudentTable = ({ stats, name, history }) => {
   const admin = userIsAdmin()
 
-  const showPopulation = (yearcode, coursecode, year) => {
-    const queryObject = { yearcode, coursecode, year }
+  const showPopulation = (yearcode, year) => {
+    const coursecodes = stats.map(s => s.coursecode)
+    const queryObject = { yearcode, coursecodes: JSON.stringify(uniq(coursecodes)), year }
     const searchString = qs.stringify(queryObject)
     history.push(`/coursepopulation?${searchString}`)
   }
@@ -42,6 +44,7 @@ const StudentTable = ({ stats, name, history }) => {
     <div>
       <Header as="h3" content={name} textAlign="center" />
       <SortableTable
+        defaultdescending
         getRowKey={s => s.code}
         tableProps={{ celled: true, structured: true }}
         columns={[
@@ -49,7 +52,7 @@ const StudentTable = ({ stats, name, history }) => {
             key: 'TIME',
             title: 'Time',
             getRowVal: s => s.code,
-            getRowContent: s => (admin ? (<div>{s.name}<Icon name="level up alternate" onClick={() => showPopulation(s.code, s.coursecode, s.name)} /></div>) : s.name),
+            getRowContent: s => (admin ? (<div>{s.name}<Icon name="level up alternate" onClick={() => showPopulation(s.code, s.name)} /></div>) : s.name),
             headerProps: { rowSpan: 2, width: 3 }
           }, {
             key: 'TOTAL',

services/oodikone2-frontend/src/components/CourseStatistics/SearchForm/YearFilter/index.jsx

@@ -1,8 +1,8 @@
 import React, { Fragment } from 'react'
-import { arrayOf, bool, func, number, shape } from 'prop-types'
+import { arrayOf, func, number, shape } from 'prop-types'
 import { Form } from 'semantic-ui-react'
 
-const YearFilter = ({ years, fromYear, toYear, handleChange, separate, onToggleCheckbox, showCheckbox }) => (
+const YearFilter = ({ years, fromYear, toYear, handleChange }) => (
   <Fragment>
     <Form.Group widths="equal" inline>
       <Form.Dropdown
@@ -26,16 +26,6 @@ const YearFilter = ({ years, fromYear, toYear, handleChange, separate, onToggleC
         value={toYear}
       />
     </Form.Group>
-    {
-      showCheckbox && (
-        <Form.Checkbox
-          label="Separate statistics for Spring and Fall semesters"
-          name="separate"
-          onChange={onToggleCheckbox}
-          checked={separate}
-        />
-      )
-    }
   </Fragment>
 )
 
@@ -43,16 +33,12 @@ YearFilter.propTypes = {
   years: arrayOf(shape({})).isRequired,
   fromYear: number,
   toYear: number,
-  handleChange: func.isRequired,
-  onToggleCheckbox: func,
-  separate: bool.isRequired,
-  showCheckbox: bool.isRequired
+  handleChange: func.isRequired
 }
 
 YearFilter.defaultProps = {
   fromYear: undefined,
-  toYear: undefined,
-  onToggleCheckbox: () => null
+  toYear: undefined
 }
 
 export default YearFilter

services/oodikone2-frontend/src/components/CourseStatistics/SearchForm/index.jsx

@@ -3,8 +3,8 @@ import { Segment, Header, Form } from 'semantic-ui-react'
 import { connect } from 'react-redux'
 import { withRouter } from 'react-router-dom'
 import qs from 'query-string'
+import { sortBy } from 'lodash'
 import { func, arrayOf, shape, bool } from 'prop-types'
-import { getSemesters } from '../../../redux/semesters'
 import { clearCourses, findCoursesV2 } from '../../../redux/coursesearch'
 import { getCourseStats, clearCourseStats } from '../../../redux/coursestats'
 import AutoSubmitSearchInput from '../../AutoSubmitSearchInput'
@@ -12,15 +12,11 @@ import CourseTable from '../CourseTable'
 import { getCourseSearchResults } from '../../../selectors/courses'
 import { useSearchHistory } from '../../../common'
 import SearchHistory from '../../SearchHistory'
-import { getStartAndEndYearValues } from '../courseStatisticsUtils'
-import YearFilter from './YearFilter'
 
 const INITIAL = {
   courseName: '',
   courseCode: '',
   selectedCourses: {},
-  fromYear: undefined,
-  toYear: undefined,
   separate: false
 }
 
@@ -34,20 +30,16 @@ const SearchForm = (props) => {
     courseName,
     courseCode,
     selectedCourses,
-    fromYear,
-    toYear,
     separate
   } = state
 
   const parseQueryFromUrl = () => {
     const { location } = props
-    const { courseCodes, fromYear, toYear, separate, ...rest } = qs.parse(location.search)
+    const { courseCodes, separate, ...rest } = qs.parse(location.search)
     const query = {
       ...INITIAL,
       ...rest,
       courseCodes: JSON.parse(courseCodes),
-      fromYear: JSON.parse(fromYear),
-      toYear: JSON.parse(toYear),
       separate: JSON.parse(separate)
     }
     return query
@@ -55,13 +47,12 @@ const SearchForm = (props) => {
 
   const fetchStatisticsFromUrlParams = () => {
     const query = parseQueryFromUrl()
-    setState({ ...state, ...query, selectedCourses: query.courseCodes })
+    setState({ ...state, ...query })
     props.getCourseStats(query, props.onProgress)
   }
 
   useEffect(() => {
     const { location } = props
-    props.getSemesters()
     if (!location.search) {
       props.clearCourses()
       props.clearCourseStats()
@@ -78,29 +69,24 @@ const SearchForm = (props) => {
   const onSelectCourse = (course) => {
     course.selected = !course.selected
     const isSelected = !!selectedCourses[course.code]
-    const { fromYear: newFromYear, toYear: newToYear } = getStartAndEndYearValues(course, props.years)
 
     if (isSelected) {
       const { [course.code]: omit, ...rest } = selectedCourses
-      setState({ ...state, selectedCourses: rest })
+      setState({
+        ...state,
+        selectedCourses: rest
+      })
     } else {
       setState({
         ...state,
         selectedCourses: {
           ...selectedCourses,
           [course.code]: { ...course, selected: true }
-        },
-        fromYear: fromYear < newFromYear ? fromYear : newFromYear,
-        toYear: toYear > newToYear ? toYear : newToYear
+        }
       })
     }
   }
 
-  const onToggleCheckbox = (e, target) => {
-    const { name } = target
-    setState({ ...state, [name]: !state[name] })
-  }
-
   const pushQueryToUrl = (query) => {
     const { history } = props
     const { courseCodes, ...rest } = query
@@ -110,18 +96,14 @@ const SearchForm = (props) => {
   }
 
   const onSubmitFormClick = async () => {
+    const codes = sortBy(Object.keys(selectedCourses))
     const params = {
-      fromYear,
-      toYear,
-      courseCodes: Object.keys(selectedCourses),
+      courseCodes: codes,
       separate
     }
-    const { years } = props
-    const from = years.find(year => year.key === fromYear)
-    const to = years.find(year => year.key === toYear)
-    const searchHistoryText = params.courseCodes.map(code => `${selectedCourses[code].name} ${code}`)
+    const searchHistoryText = codes.map(code => `${selectedCourses[code].name} ${code}`)
     addItemToSearchHistory({
-      text: `${searchHistoryText.join(', ')} from: ${from.text} to: ${to.text}`,
+      text: searchHistoryText.join(', '),
       params
     })
     pushQueryToUrl(params)
@@ -141,15 +123,15 @@ const SearchForm = (props) => {
     return Promise.resolve()
   }
 
-  const handleChange = (e, target) => {
-    const { name, value } = target
-    setState({ ...state, [name]: value })
+  const onToggleCheckbox = (e, target) => {
+    const { name } = target
+    setState({ ...state, [name]: !state[name] })
   }
 
-  const { years, isLoading, matchingCourses } = props
+  const { isLoading, matchingCourses } = props
   const courses = matchingCourses.filter(c => !selectedCourses[c.code])
 
-  const disabled = (!fromYear || Object.keys(selectedCourses).length === 0) || isLoading
+  const disabled = isLoading || Object.keys(selectedCourses).length === 0
   const selected = Object.values(selectedCourses).map(course => ({ ...course, selected: true }))
   const noSelectedCourses = selected.length === 0
   const noQueryStrings = !courseName && !courseCode
@@ -193,14 +175,11 @@ const SearchForm = (props) => {
             />
             {!noSelectedCourses &&
               <Fragment>
-                <YearFilter
-                  fromYear={fromYear}
-                  toYear={toYear}
-                  years={years}
-                  separate={separate}
-                  handleChange={handleChange}
-                  onToggleCheckbox={onToggleCheckbox}
-                  showCheckbox
+                <Form.Checkbox
+                  label="Separate statistics for Spring and Fall semesters"
+                  name="separate"
+                  onChange={onToggleCheckbox}
+                  checked={separate}
                 />
                 <Form.Button
                   type="button"
@@ -239,12 +218,10 @@ SearchForm.defaultProps = {
 
 SearchForm.propTypes = {
   findCoursesV2: func.isRequired,
-  getSemesters: func.isRequired,
   getCourseStats: func.isRequired,
   clearCourses: func.isRequired,
   clearCourseStats: func.isRequired,
   matchingCourses: arrayOf(shape({})).isRequired,
-  years: arrayOf(shape({})).isRequired,
   isLoading: bool.isRequired,
   coursesLoading: bool.isRequired,
   history: shape({}).isRequired,
@@ -253,22 +230,15 @@ SearchForm.propTypes = {
 }
 
 const mapStateToProps = (state) => {
-  const { years = [] } = state.semesters.data
   const { pending: courseStatsPending } = state.courseStats
   return {
     matchingCourses: getCourseSearchResults(state),
-    years: Object.values(years).map(({ yearcode, yearname }) => ({
-      key: yearcode,
-      text: yearname,
-      value: yearcode
-    })).reverse(),
     isLoading: courseStatsPending,
     coursesLoading: state.courseSearch.pending
   }
 }
 
 export default withRouter(connect(mapStateToProps, {
-  getSemesters,
   getCourseStats,
   clearCourses,
   findCoursesV2,

services/oodikone2-frontend/src/components/CourseStatistics/SingleCourseStats/index.jsx

@@ -1,15 +1,18 @@
 import React, { Component } from 'react'
-import { Segment, Header, Form } from 'semantic-ui-react'
-import { shape, string, arrayOf, objectOf, oneOfType, number } from 'prop-types'
+import { Segment, Header, Form, Grid } from 'semantic-ui-react'
+import { shape, string, arrayOf, objectOf, oneOfType, number, func } from 'prop-types'
 import { connect } from 'react-redux'
 import { withRouter } from 'react-router-dom'
 import { getActiveLanguage } from 'react-localize-redux'
+import { difference, min, max } from 'lodash'
 import qs from 'query-string'
 import ResultTabs from '../ResultTabs'
+import { setSelectedCourse, clearSelectedCourse } from '../../../redux/singleCourseStats'
 import ProgrammeDropdown from '../ProgrammeDropdown'
 import selectors, { ALL } from '../../../selectors/courseStats'
 import YearFilter from '../SearchForm/YearFilter'
 import { getTextIn } from '../../../common'
+import { getSemesters } from '../../../redux/semesters'
 
 const countFilteredStudents = (stat, filter) => Object.entries(stat).reduce((acc, entry) => {
   const [category, students] = entry
@@ -20,71 +23,107 @@ const countFilteredStudents = (stat, filter) => Object.entries(stat).reduce((acc
 }, {})
 
 class SingleCourseStats extends Component {
-  state = {
-    primary: ALL.value,
-    comparison: null,
-    separate: null
+  constructor(props) {
+    super(props)
+
+    const yearcodes = props.stats.statistics.map(s => s.yearcode)
+    const fromYear = min(yearcodes)
+    const toYear = max(yearcodes)
+
+    this.state = {
+      primary: [ALL.value],
+      comparison: [],
+      fromYear,
+      toYear,
+      separate: null
+    }
   }
 
-  componentDidMount = () => {
-    const { location } = this.props
+  componentWillMount = () => {
+    const { setSelectedCourse, location, stats: { coursecode }, getSemesters, years, semesters } = this.props
+    if (years.length === 0 || semesters.length === 0) getSemesters()
     if (location.search) {
-      this.setState({ ...this.parseQueryFromUrl() })
+      const params = this.parseQueryFromUrl()
+      this.setState(params)
     }
+    setSelectedCourse(coursecode)
+  }
+
+  componentDidUpdate = (prevProps) => {
+    const { primary } = this.state
+    if (this.props.programmes.length !== prevProps.programmes.length) {
+      if (primary.every(c => !this.props.programmes.map(p => p.key).includes(c))) {
+        this.setState({ // eslint-disable-line
+          primary: [ALL.value]
+        })
+      }
+    }
+  }
+
+  componentWillUnmount = () => {
+    const { clearSelectedCourse } = this.props
+    clearSelectedCourse()
   }
 
   getProgrammeName = (progcode) => {
     if (progcode === ALL.value) {
       return 'All'
+    } else if (progcode === 'EXCLUDED') {
+      return 'Excluded'
     }
     const { activeLanguage } = this.props
     const { name } = this.props.stats.programmes[progcode]
     return getTextIn(name, activeLanguage)
   }
 
-  parseQueryFromUrl = () => {
-    const { location } = this.props
-    const { separate, fromYear, toYear } = qs.parse(location.search)
-    return {
-      separate: JSON.parse(separate),
-      fromYear: JSON.parse(fromYear),
-      fromYearInitial: JSON.parse(fromYear),
-      toYear: JSON.parse(toYear),
-      toYearInitial: JSON.parse(toYear)
-    }
-  }
+  setExcludedToComparison = () => this.setState({
+    comparison: this.state.primary.includes(ALL.value) ?
+      [] :
+      ['EXCLUDED']
+  })
+
+  getExcluded = () => (
+    this.state.primary.includes(ALL.value) ?
+      [] :
+      difference(
+        this.props.programmes.map(p => p.value).filter(v => v !== ALL.value),
+        this.state.primary
+      )
+  )
+
+  belongsToAtLeastOneProgramme = (codes) => {
+    if (codes.includes(ALL.value)) return () => true
 
-  belongsToProgramme = (code) => {
-    if (code === ALL.value) {
-      return () => true
-    }
     const { programmes } = this.props.stats
-    const programme = programmes[code]
-    if (!programme) {
-      return () => false
-    }
-    const numberset = new Set(programme.students)
+    const studentNumbers = []
+    codes.forEach((code) => {
+      if (programmes[code]) {
+        studentNumbers.push(...programmes[code].students)
+      }
+    })
+
+    const numberset = new Set(studentNumbers)
+    if (!numberset.size) return () => false
     return studentnumber => numberset.has(studentnumber)
   }
 
   validProgCode = (code) => {
     const { programmes } = this.props.stats
-    return programmes[code] || (code === ALL.value)
+    return programmes[code] || (code === ALL.value || (code === 'EXCLUDED'))
   }
 
-  filteredYearsAndSemesters = (useInitialValues = false) => {
-    const { years, semesters } = this.props
-    const { fromYearInitial, fromYear, toYearInitial, toYear } = this.state
-    if (!fromYearInitial || !toYearInitial) {
+  filteredYearsAndSemesters = () => {
+    const { years, semesters, stats } = this.props
+    const yearcodes = stats.statistics.map(s => s.yearcode)
+    const from = min(yearcodes)
+    const to = max(yearcodes)
+    if (from == null || to == null) {
       return {
         filteredYears: years,
         filteredSemesters: semesters
       }
     }
-    const timeFilter = ({ value }) => (
-      value >= (useInitialValues ? fromYearInitial : fromYear) &&
-      value <= (useInitialValues ? toYearInitial : toYear)
-    )
+    const timeFilter = ({ value }) => value >= from && value <= to
     return {
       filteredYears: years.filter(timeFilter),
       filteredSemesters: semesters.filter(timeFilter)
@@ -92,17 +131,20 @@ class SingleCourseStats extends Component {
   }
 
   isStatInYearRange = ({ name }) => {
-    const { separate } = this.state
-    const { filteredYears, filteredSemesters } = this.filteredYearsAndSemesters()
+    const { separate, fromYear, toYear } = this.state
+    const { years, semesters } = this.props
+    const timeFilter = ({ value }) => value >= fromYear && value <= toYear
+    const filteredSemesters = semesters.filter(timeFilter)
+    const filteredYears = years.filter(timeFilter)
     return separate ?
       filteredSemesters.find(year => year.texts.includes(name)) :
       filteredYears.find(year => year.text === name)
   }
 
-  statsForProgramme = (progcode) => {
+  statsForProgrammes = (progCodes, name) => {
     const { statistics } = this.props.stats
-    const filter = this.belongsToProgramme(progcode)
-    const progstats = statistics.map(({ code, name, students: allstudents, attempts, coursecode }) => {
+    const filter = this.belongsToAtLeastOneProgramme(progCodes)
+    const progStats = statistics.filter(this.isStatInYearRange).map(({ code, name, students: allstudents, attempts, coursecode }) => {
       const cumulative = {
         grades: countFilteredStudents(attempts.grades, filter),
         categories: countFilteredStudents(attempts.classes, filter)
@@ -112,71 +154,100 @@ class SingleCourseStats extends Component {
         categories: countFilteredStudents(allstudents.classes, filter)
       }
       return { code, name, cumulative, students, coursecode }
-    }).filter(this.isStatInYearRange)
+    })
     return {
-      code: progcode,
-      name: this.getProgrammeName(progcode),
-      stats: progstats
+      codes: progCodes,
+      name,
+      stats: progStats
     }
   }
 
+  handleSelect = (e, { name, value }) => {
+    let selected = [...value].filter(v => v !== ALL.value)
+
+    if (name === 'primary') {
+      this.setState({ comparison: this.state.comparison.filter(p => p !== 'EXCLUDED') })
+    }
+
+    if ((!this.state[name].includes(ALL.value) && value.includes(ALL.value)) || (name === 'primary' && value.length === 0)) {
+      selected = [ALL.value]
+    }
+
+    this.setState({ [name]: selected })
+  }
+
   handleChange = (e, { name, value }) => {
     this.setState({ [name]: value })
   }
 
+  handleYearChange = (e, { name, value }) => {
+    if (name === 'fromYear') this.setState({ fromYear: value })
+    else this.setState({ toYear: value })
+  }
+
   filteredProgrammeStatistics = () => {
     const { primary, comparison } = this.state
-    const pstats = !this.validProgCode(primary) ? undefined : this.statsForProgramme(primary)
-    const cstats = !this.validProgCode(comparison)
-      ? undefined
-      : this.statsForProgramme(comparison)
+    const filter = p => this.validProgCode(p)
+
+    const excludedProgrammes = this.getExcluded()
+
+    const primaryProgrammes = primary
+    const comparisonProgrammes = comparison.filter(filter)
+    if (comparison.includes('EXCLUDED')) comparisonProgrammes.push(...excludedProgrammes)
+
+    const pstats = primaryProgrammes.length ? this.statsForProgrammes(
+      primaryProgrammes,
+      primaryProgrammes.length === 1 ? this.getProgrammeName(primaryProgrammes[0]) : 'Primary'
+    ) : undefined
+    const cstats = comparisonProgrammes.length ? this.statsForProgrammes(
+      comparisonProgrammes,
+      comparisonProgrammes.length === 1 ? this.getProgrammeName(comparisonProgrammes[0]) : 'Comparison'
+    ) : undefined
+
     return {
-      primary: pstats,
-      comparison: cstats
+      primary: pstats || undefined,
+      comparison: cstats || undefined
     }
   }
 
-  selectedProgrammes = () => {
-    const { primary: p, comparison: c } = this.state
-    const { programmes } = this.props.stats
-    const primary = !programmes[p] ? ALL.value : p
-    const comparison = (!programmes[c] && c !== ALL.value) ? undefined : c
-    return { primary, comparison }
+  parseQueryFromUrl = () => {
+    const { location } = this.props
+    const { separate } = qs.parse(location.search)
+    return {
+      separate: JSON.parse(separate)
+    }
+  }
+
+  clearComparison = () => this.setState({ comparison: [] })
+
+  comparisonProgrammes = () => {
+    const { primary, comparison } = this.state
+    const result = this.props.programmes.filter(({ key }) => key !== 'EXCLUDED')
+    const excludedProgrammes = this.getExcluded()
+
+    if (!primary.includes(ALL.value)) {
+      const excludedStudents = result
+        .filter(({ key }) => excludedProgrammes.includes(key) && key !== 'ALL')
+        .reduce((res, { students }) => [...res, ...students], [])
+      result.push({
+        key: 'EXCLUDED',
+        size: excludedStudents.length,
+        students: excludedStudents,
+        text: 'Excluded',
+        value: 'EXCLUDED'
+      })
+    }
+    return result.filter(({ key }) => !comparison.includes('EXCLUDED') || !excludedProgrammes.includes(key))
   }
 
   render() {
     const { programmes } = this.props
-    const { fromYear, toYear } = this.state
-    const { primary, comparison } = this.selectedProgrammes()
+    const { primary, comparison, fromYear, toYear } = this.state
     const statistics = this.filteredProgrammeStatistics()
-    const { filteredYears } = this.filteredYearsAndSemesters(true)
+    const { filteredYears } = this.filteredYearsAndSemesters()
 
     return (
       <div>
-        <Segment>
-          <Form>
-            <Header content="Filter statistics by study programme" as="h4" />
-            <Form.Group widths="equal" >
-              <ProgrammeDropdown
-                name="primary"
-                options={programmes}
-                label="Primary group"
-                placeholder="Select a study programme"
-                value={primary}
-                onChange={this.handleChange}
-              />
-              <ProgrammeDropdown
-                name="comparison"
-                options={programmes}
-                label="Comparison group"
-                placeholder="Optional"
-                value={comparison}
-                onChange={this.handleChange}
-                onClear={() => this.setState({ comparison: undefined })}
-              />
-            </Form.Group>
-          </Form>
-        </Segment>
         <Segment>
           <Form>
             <Header content="Filter statistics by time range" as="h4" />
@@ -184,12 +255,51 @@ class SingleCourseStats extends Component {
               years={filteredYears}
               fromYear={fromYear}
               toYear={toYear}
-              handleChange={this.handleChange}
-              showCheckbox={false}
-              separate={false}
+              handleChange={this.handleYearChange}
             />
           </Form>
         </Segment>
+        <Segment>
+          <Form>
+            <Header content="Filter statistics by study programmes" as="h4" />
+            <Grid>
+              <Grid.Column width={8}>
+                <ProgrammeDropdown
+                  name="primary"
+                  options={programmes}
+                  label="Primary group"
+                  placeholder="Select study programmes"
+                  value={primary}
+                  onChange={this.handleSelect}
+                />
+              </Grid.Column>
+              <Grid.Column width={8}>
+                <ProgrammeDropdown
+                  name="comparison"
+                  options={this.comparisonProgrammes()}
+                  label="Comparison group"
+                  placeholder="Optional"
+                  value={comparison}
+                  onChange={this.handleSelect}
+                />
+              </Grid.Column>
+              <Grid.Column width={8} />
+              <Grid.Column width={8}>
+                <Form.Group>
+                  <Form.Button
+                    content="Select excluded study programmes"
+                    onClick={this.setExcludedToComparison}
+                    disabled={primary.length === 1 && primary[0] === ALL.value}
+                  />
+                  <Form.Button
+                    content="Clear"
+                    onClick={this.clearComparison}
+                  />
+                </Form.Group>
+              </Grid.Column>
+            </Grid>
+          </Form>
+        </Segment>
         <ResultTabs
           primary={statistics.primary}
           comparison={statistics.comparison}
@@ -221,7 +331,10 @@ SingleCourseStats.propTypes = {
   years: arrayOf(shape({})).isRequired,
   semesters: arrayOf(shape({})).isRequired,
   location: shape({}).isRequired,
-  activeLanguage: string.isRequired
+  activeLanguage: string.isRequired,
+  setSelectedCourse: func.isRequired,
+  clearSelectedCourse: func.isRequired,
+  getSemesters: func.isRequired
 }
 
 const mapStateToProps = (state) => {
@@ -242,4 +355,10 @@ const mapStateToProps = (state) => {
   }
 }
 
-export default withRouter(connect(mapStateToProps)(SingleCourseStats))
+const mapDispatchToProps = {
+  setSelectedCourse,
+  clearSelectedCourse,
+  getSemesters
+}
+
+export default withRouter(connect(mapStateToProps, mapDispatchToProps)(SingleCourseStats))

services/oodikone2-frontend/src/components/CourseStatistics/SummaryTab/index.jsx

@@ -2,13 +2,19 @@ import React, { Component } from 'react'
 import { Form, Label, Segment, Header } from 'semantic-ui-react'
 import { connect } from 'react-redux'
 import { shape, arrayOf, func, oneOfType, number, string } from 'prop-types'
-import selectors from '../../../selectors/courseStats'
+import selectors, { ALL } from '../../../selectors/courseStats'
 import { fields, setValue } from '../../../redux/coursesSummaryForm'
 import CumulativeTable from '../CumulativeTable'
 import ProgrammeDropdown from '../ProgrammeDropdown'
 
 class SummaryTab extends Component {
-    handleChange = (e, { name, value }) => this.props.setValue(name, value)
+    handleChange = (e, { name, value }) => {
+      let selected = [...value].filter(v => v !== ALL.value)
+      if ((!this.props.form[fields.programmes].includes(ALL.value) && value.includes(ALL.value)) || value.length === 0) {
+        selected = [ALL.value]
+      }
+      this.props.setValue(name, selected)
+    }
 
     render() {
       const { statistics, programmes, queryInfo } = this.props
@@ -29,13 +35,13 @@ class SummaryTab extends Component {
         <div>
           <Segment>
             <Form>
-              <Header content="Filter statistics by study programme" as="h4" />
+              <Header content="Filter statistics by study programmes" as="h4" />
               <ProgrammeDropdown
                 options={programmes}
-                label="Study programme:"
-                name={fields.programme}
+                label="Study programmes:"
+                name={fields.programmes}
                 onChange={this.handleChange}
-                value={this.props.form[fields.programme]}
+                value={this.props.form[fields.programmes]}
               />
               <Form.Field>
                 <label>Timeframe:</label>
@@ -47,11 +53,11 @@ class SummaryTab extends Component {
               </Form.Field>
             </Form>
           </Segment>
-          <CumulativeTable
+          {<CumulativeTable
             categoryName="Course"
             onClickCourse={this.props.onClickCourse}
             data={data}
-          />
+          />}
         </div>
       )
     }
@@ -79,10 +85,10 @@ SummaryTab.propTypes = {
 
 const mapStateToProps = (state) => {
   const programmes = selectors.getAllStudyProgrammes(state)
-  const programme = state.courseSummaryForm[fields.programme]
+  const programmeCodes = state.courseSummaryForm[fields.programmes]
   return {
     form: state.courseSummaryForm,
-    statistics: selectors.summaryStatistics(state, { programmes, programme }),
+    statistics: selectors.summaryStatistics(state, { programmes, programmeCodes }),
     queryInfo: selectors.getQueryInfo(state),
     programmes
   }

services/oodikone2-frontend/src/components/CourseStatistics/courseStatisticsUtils.js

@@ -5,9 +5,6 @@ const MAX_YEAR = 2112
 
 const isSpring = date => moment(date).month() < 9
 const isPre2016Course = course => !Number.isNaN(Number(course.code.charAt(0)))
-const twoDigitYear = year => year.toString().substring(2, 4)
-const getSemesterText = (start, end) => `${start}-${twoDigitYear(end)}`
-const getYearDataText = (year, spring) => (spring ? getSemesterText(year - 1, year) : getSemesterText(year, year + 1))
 const getYearText = (year, spring) => (spring ? `Spring ${year}` : `Fall ${year}`)
 
 const getCourseYears = course => ({
@@ -35,35 +32,7 @@ const getActiveYears = (course) => {
   return `${startYearText} — ${endYearText}`
 }
 
-const getCourseSemesters = (course) => {
-  const { startYear, endYear } = getCourseYears(course)
-  return {
-    start: getYearDataText(startYear, isSpring(course.min_attainment_date)),
-    end: getYearDataText(endYear, isSpring(course.max_attainment_date))
-  }
-}
-
-const maximumYear = years => (
-  years.reduce((prev, curr) => {
-    if (prev.value > curr.value) return prev
-    return curr
-  })
-)
-
-const getStartAndEndYearValues = (course, years) => {
-  if (!course.min_attainment_date && !course.max_attainment_date) return { fromYear: null, toYear: null }
-  const { start, end } = getCourseSemesters(course)
-  const startYear = years.find(year => year.text === start)
-
-  const endYear = Number(start.substring(0, 4) === MAX_YEAR) ?
-    maximumYear(years) :
-    years.find(year => year.text === end)
-  const fromYear = startYear ? startYear.value : undefined
-  const toYear = endYear ? endYear.value : undefined
-  return { fromYear, toYear }
-}
-
 export {
   getActiveYears,
-  getStartAndEndYearValues
+  getYearText
 }

services/oodikone2-frontend/src/components/CourseStatistics/index.jsx

@@ -60,14 +60,16 @@ const CourseStatistics = (props) => {
             clearCourseStats()
           }
         },
-        render: () => <SearchForm />
+        render: () => null
       }
     ]
     return !singleCourseStats ? panes : panes.filter(p => p.menuItem !== MENU.SUM)
   }
 
-  const handleTabChange = (e, { activeIndex }) => {
-    setActiveIndex(activeIndex)
+  const handleTabChange = (e, { activeIndex, panes }) => {
+    if (panes[activeIndex].menuItem.key !== 'query') {
+      setActiveIndex(activeIndex)
+    }
   }
 
   const panes = getPanes()

services/oodikone2-frontend/src/components/CourseStudentCourses/index.jsx

@@ -0,0 +1,77 @@
+import React, { useEffect } from 'react'
+import { connect } from 'react-redux'
+import { func, shape, arrayOf, string, bool } from 'prop-types'
+import { Segment, Header, Popup } from 'semantic-ui-react'
+import { getTranslate } from 'react-localize-redux'
+
+import SegmentDimmer from '../SegmentDimmer'
+import PopulationCourseStats from '../PopulationCourseStats'
+import { refreshFilters } from '../../redux/populationFilters'
+import { getCustomPopulationCoursesByStudentnumbers } from '../../redux/populationCourses'
+
+const CourseStudentCourses = ({
+  refreshNeeded,
+  dispatchRefreshFilters,
+  translate,
+  courses,
+  pending,
+  selectedStudents,
+  getCustomPopulationCoursesByStudentnumbersDispatch,
+  query
+}) => {
+  const reloadCourses = () => {
+    dispatchRefreshFilters()
+    getCustomPopulationCoursesByStudentnumbersDispatch({ studentnumberlist: selectedStudents })
+  }
+
+
+  useEffect(() => {
+    if (refreshNeeded) {
+      reloadCourses()
+    }
+  }, [refreshNeeded])
+
+  return (
+    <Segment>
+      <Header size="medium" dividing >
+        <Popup
+          trigger={<Header.Content>{translate('populationCourses.header')}</Header.Content>}
+          content="Sort by clicking columns. Click course name to limit observed population to students who participated to the course."
+          wide
+          position="top left"
+        />
+      </Header>
+      <SegmentDimmer translate={translate} isLoading={pending} />
+      <PopulationCourseStats
+        courses={courses}
+        query={query}
+        pending={pending}
+        selectedStudents={selectedStudents}
+      />
+    </Segment>
+  )
+}
+
+CourseStudentCourses.propTypes = {
+  pending: bool.isRequired,
+  courses: shape([]).isRequired,
+  translate: func.isRequired,
+  selectedStudents: arrayOf(string).isRequired,
+  refreshNeeded: bool.isRequired,
+  getCustomPopulationCoursesByStudentnumbersDispatch: func.isRequired,
+  dispatchRefreshFilters: func.isRequired,
+  query: shape({}).isRequired
+}
+
+const mapStateToProps = ({ localize, populationFilters, populationCourses }) => ({
+  courses: populationCourses.data,
+  pending: populationCourses.pending,
+  translate: getTranslate(localize),
+  query: populationCourses.query,
+  refreshNeeded: populationFilters.refreshNeeded
+})
+
+export default connect(mapStateToProps, {
+  dispatchRefreshFilters: refreshFilters,
+  getCustomPopulationCoursesByStudentnumbersDispatch: getCustomPopulationCoursesByStudentnumbers
+})(CourseStudentCourses)

services/oodikone2-frontend/src/components/CourseStudents/index.jsx

@@ -6,32 +6,45 @@ import { getTranslate } from 'react-localize-redux'
 import { Segment, Header, Loader } from 'semantic-ui-react'
 import qs from 'query-string'
 import { intersection, difference } from 'lodash'
-import { getCoursePopulation, getCoursePopulationCourses } from '../../redux/coursePopulation'
+import { getCoursePopulation } from '../../redux/populations'
+import { getCoursePopulationCourses } from '../../redux/populationCourses'
 import { getSingleCourseStats } from '../../redux/singleCourseStats'
 import CreditAccumulationGraphHighCharts from '../CreditAccumulationGraphHighCharts'
 import PopulationStudents from '../PopulationStudents'
-import PopulationCourses from '../PopulationCourseStats'
+import CourseStudentCourses from '../CourseStudentCourses'
 import infoTooltips from '../../common/InfoToolTips'
 import InfoBox from '../InfoBox'
 import SegmentDimmer from '../SegmentDimmer'
 import CourseStudentsFilters from '../CourseStudentsFilters'
 
-const CourseStudents = ({ getCoursePopulationDispatch, getCoursePopulationCoursesDispatch, getSingleCourseStatsDispatch, studentData, courses, pending, history, translate, courseData, selectedStudents }) => {
+const CourseStudents = ({
+  getCoursePopulationDispatch,
+  getCoursePopulationCoursesDispatch,
+  getSingleCourseStatsDispatch,
+  studentData,
+  pending,
+  history,
+  translate,
+  courseData,
+  selectedStudents }) => {
   const parseQueryFromUrl = () => {
     const { location } = history
     const query = qs.parse(location.search)
     return query
   }
-  const [code, setCode] = useState('')
+  const [codes, setCodes] = useState([])
   const [headerYear, setYear] = useState('')
+  const [yearCode, setYearCode] = useState('')
   useEffect(() => {
     const query = parseQueryFromUrl()
-    getCoursePopulationDispatch({ coursecode: query.coursecode, yearcode: query.yearcode })
-    getCoursePopulationCoursesDispatch({ coursecode: query.coursecode, yearcode: query.yearcode })
-    getSingleCourseStatsDispatch({ fromYear: query.yearcode, toYear: query.yearcode, courseCodes: [query.coursecode], separate: false })
-    setCode(query.coursecode)
+    getCoursePopulationDispatch({ coursecodes: query.coursecodes, yearcode: query.yearcode })
+    getCoursePopulationCoursesDispatch({ coursecodes: JSON.parse(query.coursecodes), yearcode: query.yearcode })
+    getSingleCourseStatsDispatch({ fromYear: query.yearcode, toYear: query.yearcode, courseCodes: JSON.parse(query.coursecodes), separate: false })
+    setCodes(JSON.parse(query.coursecodes))
+    setYearCode(query.yearcode)
     setYear(query.year)
   }, [])
+
   const { CreditAccumulationGraph, CoursesOf } = infoTooltips.PopulationStatistics
   const header = courseData ? `${courseData.name} ${headerYear}` : null
 
@@ -40,7 +53,7 @@ const CourseStudents = ({ getCoursePopulationDispatch, getCoursePopulationCourse
       {studentData.students && !pending ? (
         <Segment className="contentSegment">
           <Header className="segmentTitle" size="large" textAlign="center">Population of course {header}</Header>
-          <CourseStudentsFilters samples={studentData.students} coursecode={code} />
+          <CourseStudentsFilters samples={studentData.students} coursecodes={codes} />
           <Segment>
             <Header size="medium" dividing>
               {translate('populationStatistics.graphSegmentHeader')} (for {selectedStudents.length} students)
@@ -60,10 +73,10 @@ const CourseStudents = ({ getCoursePopulationDispatch, getCoursePopulationCourse
               <InfoBox content={CoursesOf} />
             </Header>
             <SegmentDimmer translate={translate} isLoading={pending} />
-            <PopulationCourses
-              courses={courses}
-              pending={pending}
+            <CourseStudentCourses
               selectedStudents={selectedStudents}
+              codes={codes}
+              yearCode={yearCode}
             />
           </Segment>
           <PopulationStudents
@@ -81,7 +94,6 @@ CourseStudents.propTypes = {
   getCoursePopulationCoursesDispatch: func.isRequired,
   getSingleCourseStatsDispatch: func.isRequired,
   pending: bool.isRequired,
-  courses: shape([]).isRequired,
   studentData: shape({}).isRequired,
   history: shape({}).isRequired,
   translate: func.isRequired,
@@ -89,8 +101,8 @@ CourseStudents.propTypes = {
   selectedStudents: arrayOf(string).isRequired
 }
 
-const mapStateToProps = ({ coursePopulation, localize, singleCourseStats, populationFilters }) => {
-  const samples = coursePopulation.students.students ? coursePopulation.students.students : []
+const mapStateToProps = ({ localize, singleCourseStats, populationFilters, populations }) => {
+  const samples = populations.data.students ? populations.data.students : []
   let selectedStudents = samples.length > 0 ? samples.map(s => s.studentNumber) : []
   const { complemented } = populationFilters
 
@@ -109,14 +121,12 @@ const mapStateToProps = ({ coursePopulation, localize, singleCourseStats, popula
       selectedStudents = difference(samples.map(s => s.studentNumber), selectedStudents)
     }
   }
-
   return ({
-    studentData: coursePopulation.students,
-    courses: coursePopulation.courses,
-    pending: coursePopulation.pending,
+    studentData: populations.data,
+    pending: populations.pending,
     translate: getTranslate(localize),
-    query: coursePopulation.query,
-    courseData: singleCourseStats.stats,
+    query: populations.query,
+    courseData: singleCourseStats.stats || {},
     selectedStudents
   })
 }

services/oodikone2-frontend/src/components/CourseStudentsFilters/CourseCreditFilter.jsx

@@ -0,0 +1,81 @@
+import React, { useState } from 'react'
+import { connect } from 'react-redux'
+import { Segment, Icon, Button, Form, Input, Popup } from 'semantic-ui-react'
+import { func, shape, string, arrayOf } from 'prop-types'
+
+import { removePopulationFilter, setPopulationFilter } from '../../redux/populationFilters'
+import { courseCreditFilter } from '../../populationFilters'
+
+const CourseCreditFilter = ({ coursecodes, courseData, filter, setPopulationFilterAction, removePopulationFilterAction }) => {
+  const [credits, setCredits] = useState(0)
+  const handleFilter = () => {
+    setPopulationFilterAction(courseCreditFilter({ credits, coursecodes, coursename: courseData.name }))
+  }
+
+  const handleChange = (e, { value }) => {
+    setCredits(value)
+  }
+  const clearFilter = () => {
+    removePopulationFilterAction(filter.id)
+  }
+
+  if (filter.notSet) {
+    return (
+      <Segment>
+        <Form>
+          <Popup
+            trigger={<Icon style={{ float: 'right' }} name="info" />}
+          />
+          <Form.Group inline>
+            <Form.Field>
+              <label>Select students that </label>
+            </Form.Field>
+            <Form.Field>
+              <Input
+                type="number"
+                onChange={handleChange}
+                value={credits}
+              />
+            </Form.Field>
+            <Form.Field>
+              <label>credits from course {courseData.name}</label>
+            </Form.Field>
+            <Form.Field>
+              <Button
+                onClick={handleFilter}
+                disabled={credits === 0}
+              >
+                set filter
+              </Button>
+            </Form.Field>
+          </Form.Group>
+        </Form>
+      </Segment>
+    )
+  }
+  return (
+    <Segment>
+      Students that have {filter.params.credits} credits from course {filter.params.coursename}
+      <span style={{ float: 'right' }}>
+        <Icon name="remove" onClick={clearFilter} />
+      </span>
+    </Segment>
+  )
+}
+
+CourseCreditFilter.propTypes = {
+  setPopulationFilterAction: func.isRequired,
+  removePopulationFilterAction: func.isRequired,
+  filter: shape({}).isRequired,
+  courseData: shape({}).isRequired,
+  coursecodes: arrayOf(string).isRequired
+}
+
+const mapStateToProps = ({ singleCourseStats }) => ({
+  courseData: singleCourseStats.stats
+})
+
+export default connect(mapStateToProps, {
+  setPopulationFilterAction: setPopulationFilter,
+  removePopulationFilterAction: removePopulationFilter
+})(CourseCreditFilter)

services/oodikone2-frontend/src/components/CourseStudentsFilters/GradeFilter.jsx

@@ -1,15 +1,15 @@
 import React, { useState } from 'react'
 import { connect } from 'react-redux'
 import { Segment, Icon, Button, Form, Dropdown, Popup } from 'semantic-ui-react'
-import { func, shape, string } from 'prop-types'
+import { func, shape, string, arrayOf } from 'prop-types'
 
 import { removePopulationFilter, setPopulationFilter } from '../../redux/populationFilters'
 import { gradeFilter } from '../../populationFilters'
 
-const GradeFilter = ({ coursecode, courseData, filter, setPopulationFilterAction, removePopulationFilterAction }) => {
+const GradeFilter = ({ coursecodes, courseData, filter, setPopulationFilterAction, removePopulationFilterAction }) => {
   const [grade, setGrade] = useState(0)
   const handleFilter = () => {
-    setPopulationFilterAction(gradeFilter({ grade, coursecode, coursename: courseData[coursecode].name }))
+    setPopulationFilterAction(gradeFilter({ grade, coursecodes, coursename: courseData.name }))
   }
 
   const handleChange = (e, { value }) => {
@@ -42,7 +42,7 @@ const GradeFilter = ({ coursecode, courseData, filter, setPopulationFilterAction
               />
             </Form.Field>
             <Form.Field>
-              <label>from course {courseData[coursecode].name}</label>
+              <label>from course {courseData.name}</label>
             </Form.Field>
             <Form.Field>
               <Button
@@ -72,11 +72,11 @@ GradeFilter.propTypes = {
   removePopulationFilterAction: func.isRequired,
   filter: shape({}).isRequired,
   courseData: shape({}).isRequired,
-  coursecode: string.isRequired
+  coursecodes: arrayOf(string).isRequired
 }
 
-const mapStateToProps = ({ courseStats }) => ({
-  courseData: courseStats.data
+const mapStateToProps = ({ singleCourseStats }) => ({
+  courseData: singleCourseStats.stats
 })
 
 export default connect(mapStateToProps, {

services/oodikone2-frontend/src/components/CourseStudentsFilters/index.jsx

@@ -13,12 +13,14 @@ import InfoBox from '../InfoBox'
 import infotooltips from '../../common/InfoToolTips'
 import GradeFilter from './GradeFilter'
 import ProgrammeFilter from './ProgrammeFilter'
+import CourseCreditFilter from './CourseCreditFilter'
 import {
   clearPopulationFilters, setComplementFilter
 } from '../../redux/populationFilters'
 
 const componentFor = {
   GradeFilter,
+  CourseCreditFilter,
   ProgrammeFilter,
   CreditsAtLeast,
   CreditsLessThan,
@@ -27,7 +29,7 @@ const componentFor = {
   CourseParticipation
 }
 
-const CourseStudentsFilters = ({ samples, filters, clearPopulationFiltersDispatch, complemented, setComplementFilterDispatch, allStudyrights, coursecode }) => {
+const CourseStudentsFilters = ({ samples, filters, clearPopulationFiltersDispatch, complemented, setComplementFilterDispatch, allStudyrights, coursecodes }) => {
   const [visible, setVisible] = useState(false)
 
   const renderAddFilters = () => {
@@ -58,7 +60,7 @@ const CourseStudentsFilters = ({ samples, filters, clearPopulationFiltersDispatc
     return (
       <Segment>
         <Header>Add filters <InfoBox content={Add} /></Header>
-        {unsetFilters.map(filterName => React.createElement(componentFor[filterName], { filter: { notSet: true }, key: filterName, samples, allStudyrights, coursecode }))}
+        {unsetFilters.map(filterName => React.createElement(componentFor[filterName], { filter: { notSet: true }, key: filterName, samples, allStudyrights, coursecodes }))}
         <Button onClick={() => setVisible(false)}>cancel</Button>
       </Segment>
     )
@@ -74,7 +76,7 @@ const CourseStudentsFilters = ({ samples, filters, clearPopulationFiltersDispatc
     return (
       <Segment>
         <Header>Filters <InfoBox content={Filters} /></Header>
-        {filters.map(filter => React.createElement(componentFor[filter.type], { filter, key: filter.id, samples, allStudyrights, coursecode }))}
+        {filters.map(filter => React.createElement(componentFor[filter.type], { filter, key: filter.id, samples, allStudyrights, coursecodes }))}
         <Form>
           <Form.Group inline>
             <Form.Field>
@@ -107,14 +109,14 @@ CourseStudentsFilters.propTypes = {
   samples: arrayOf(shape([])).isRequired,
   clearPopulationFiltersDispatch: func.isRequired,
   setComplementFilterDispatch: func.isRequired,
-  coursecode: string.isRequired,
+  coursecodes: arrayOf(string).isRequired,
   allStudyrights: shape({}).isRequired
 }
 
-const mapStateToProps = ({ populationFilters, coursePopulation }) => ({
+const mapStateToProps = ({ populationFilters, populations }) => ({
   filters: populationFilters.filters,
   complemented: populationFilters.complemented,
-  allStudyrights: coursePopulation.students.studyrights
+  allStudyrights: populations.data.studyrights
 })
 
 export default connect(mapStateToProps, {

services/oodikone2-frontend/src/components/CreditAccumulationGraphHighCharts/index.jsx

@@ -116,7 +116,6 @@ class CreditAccumulationGraphHighCharts extends Component {
       const lastDate = moment(students.maxDate)
       const lastMonth = Math.ceil(this.getXAxisMonth(lastDate, started))
       lastCredits = lastMonth * (55 / 12)
-
       dataOfSelected.push({
         data: [
           [students.minDate, 0],

services/oodikone2-frontend/src/components/CustomPopulation/index.jsx

@@ -0,0 +1,115 @@
+import React, { useState, useEffect } from 'react'
+import { connect } from 'react-redux'
+import { Button, Modal, Form, TextArea, Segment, Header } from 'semantic-ui-react'
+import { getTranslate } from 'react-localize-redux'
+import { shape, func, arrayOf, bool } from 'prop-types'
+
+import { userIsAdmin } from '../../common'
+import { getCustomPopulation } from '../../redux/populations'
+import { getCustomPopulationCoursesByStudentnumbers } from '../../redux/populationCourses'
+import CreditAccumulationGraphHighCharts from '../CreditAccumulationGraphHighCharts'
+import PopulationStudents from '../PopulationStudents'
+import PopulationCourseStats from '../PopulationCourseStats'
+
+const CustomPopulation = ({ getCustomPopulationDispatch, getCustomPopulationCoursesByStudentnumbers, custompop, translate, courses, pending }) => {
+  const [admin, setAdmin] = useState(false)
+  const [modal, setModal] = useState(false)
+  const [input, setInput] = useState('')
+
+  useEffect(() => {
+    const isAdmin = userIsAdmin()
+    setAdmin(isAdmin)
+  }, [])
+
+  const onClicker = (e) => {
+    e.preventDefault()
+    const studentnumbers = input.match(/[0-9]+/g)
+    getCustomPopulationDispatch({ studentnumberlist: studentnumbers })
+    getCustomPopulationCoursesByStudentnumbers({ studentnumberlist: studentnumbers })
+    setModal(false)
+  }
+  const renderCustomPopulationSearch = () => (
+    <Modal
+      trigger={<Button size="small" onClick={() => setModal(true)}>Custom population</Button>}
+      open={modal}
+      onClose={() => setModal(false)}
+      size="small"
+    >
+      <Modal.Content>
+        <Form>
+          <h2> Custom population </h2>
+          <Form.Field>
+            <em> Insert studentnumbers you wish to use for population here </em>
+            <TextArea placeholder="011111111" onChange={e => setInput(e.target.value)} />
+          </Form.Field>
+        </Form>
+      </Modal.Content>
+      <Modal.Actions>
+        <Button
+          negative
+          onClick={() => setModal(false)}
+        >
+          Cancel
+        </Button>
+        <Button
+          positive
+          onClick={e => onClicker(e)}
+        >
+          Search population
+        </Button>
+      </Modal.Actions>
+    </Modal>
+  )
+
+  const renderCustomPopulation = () => {
+    const selectedStudents = custompop.map(student => student.studentNumber)
+    return (
+      <div>
+        <Segment>
+          <Header size="medium" dividing>
+            {translate('populationStatistics.graphSegmentHeader')} (for {selectedStudents.length} students)
+          </Header>
+          <CreditAccumulationGraphHighCharts students={custompop} selectedStudents={selectedStudents} translate={translate} />
+        </Segment>
+        <Segment>
+          <PopulationCourseStats
+            courses={courses}
+            pending={pending}
+            selectedStudents={selectedStudents}
+          />
+        </Segment>
+        <PopulationStudents
+          samples={custompop}
+          selectedStudents={selectedStudents}
+        />
+      </div>
+    )
+  }
+
+  if (!admin) return <div>you are not an admin, go away</div>
+
+  return (
+    <div>
+      {renderCustomPopulationSearch()}
+      {custompop.length > 0 ? (renderCustomPopulation()) : null}
+    </div>
+  )
+}
+
+CustomPopulation.propTypes = {
+  translate: func.isRequired,
+  custompop: arrayOf(shape({})).isRequired,
+  getCustomPopulationDispatch: func.isRequired,
+  getCustomPopulationCoursesByStudentnumbers: func.isRequired,
+  courses: shape({}).isRequired,
+  pending: bool.isRequired
+}
+
+const mapStateToProps = ({ populations, localize, populationCourses }) => ({
+  translate: getTranslate(localize),
+  custompop: populations.data.students || [],
+  courses: populationCourses.data,
+  pending: populationCourses.pending
+})
+
+export default connect(mapStateToProps, { getCustomPopulationDispatch: getCustomPopulation, getCustomPopulationCoursesByStudentnumbers })(CustomPopulation)

services/oodikone2-frontend/src/components/EnableUsers/index.jsx

@@ -1,6 +1,6 @@
 import React, { Component } from 'react'
 import { withRouter } from 'react-router-dom'
-import { Button, Radio, Icon, Header, Segment, Confirm, Loader, Label, Message } from 'semantic-ui-react'
+import { Button, Radio, Icon, Header, Segment, Confirm, Loader, Label } from 'semantic-ui-react'
 import { connect } from 'react-redux'
 import { func, shape, string, bool, arrayOf, number } from 'prop-types'
 import { getTranslate, getActiveLanguage } from 'react-localize-redux'
@@ -186,10 +186,9 @@ class EnableUsers extends Component {
     return (
       <div className="segmentContainer">
         <Header className="segmentTitle" size="large">
-          Enable or disable access to Oodikone
+          Oodikone users
         </Header>
-        <Radio toggle onClick={() => this.toggleEnabledOnly()} />
-        <Message>Showing {enabledOnly ? 'only enabled' : 'all'} users</Message>
+        <Radio label={`Showing ${enabledOnly ? 'only enabled' : 'all'} users`} toggle onClick={() => this.toggleEnabledOnly()} />
         <Segment loading={pending} className="contentSegment">
           {!userid ? this.renderUserSearchList() : this.renderUserPage(userid)}
         </Segment>

services/oodikone2-frontend/src/components/Faculty/FacultySelector/index.jsx

@@ -1,89 +1,62 @@
-import React, { useEffect, useState } from 'react'
-import { withRouter } from 'react-router-dom'
+import React, { useMemo } from 'react'
 import { connect } from 'react-redux'
 import { getActiveLanguage } from 'react-localize-redux'
-import { Segment, Form } from 'semantic-ui-react'
-import { string, func, arrayOf, shape } from 'prop-types'
-import { uniq } from 'lodash'
-import { getFaculties as getFacultiesAction, getFacultiesYearlyStats as getFacultiesYearlyStatsAction } from '../../../redux/faculties'
+import { string, arrayOf, shape, number, func } from 'prop-types'
+import {
+  calculateTotalPassedCourses,
+  calculateTotalFailedCourses
+} from '../facultyUtils'
 import { getTextIn } from '../../../common'
 import SortableTable from '../../SortableTable'
-import YearFilter from '../../CourseStatistics/SearchForm/YearFilter'
+import FacultyStatsGraph from '../FacultyStatsGraph'
 
-const FacultySelector = ({ language, getFaculties, getFacultiesYearlyStats, faculties, facultyYearlyStats }) => {
-  const [selectedFaculties, setSelectedFaculties] = useState([])
-  const [fromYear, setFromYear] = useState(-1)
-  const [toYear, setToYear] = useState(-1)
-  const [years, setYears] = useState([])
-
-  useEffect(() => {
-    getFaculties()
-    getFacultiesYearlyStats()
-  }, [])
-
-  const handleSelect = code => setSelectedFaculties(selectedFaculties.includes(code) ?
-    selectedFaculties.filter(c => c !== code) :
-    selectedFaculties.concat(code))
-
-  const getYearFilterData = () => {
-    const filterYears = uniq(facultyYearlyStats
-      .filter(f => selectedFaculties.includes(f.id))
-      .map(({ data }) => Object.keys(data).map(y => parseInt(y, 10)))
-      .reduce((acc, curr) => [...acc, ...curr], []))
-      .sort((a, b) => b - a)
-
-    return {
-      fromYear: filterYears[filterYears.length - 1],
-      toYear: filterYears[0],
-      years: filterYears.map(y => ({ key: y, text: y, value: y }))
+const FacultySelector = ({ language, faculties, facultyYearlyStats, fromYear, toYear, handleSelect }) => {
+  const calculateYearlyStatsForFaculty = (facultyCode) => {
+    const res = {}
+    const yearData = {
+      studentCredits: 0,
+      coursesPassed: 0,
+      coursesFailed: 0
     }
+
+    const faculty = facultyYearlyStats.find(f => f.id === facultyCode)
+    if (!faculty) return res
+    Object.values(faculty.data).forEach((programmeYearlyStats) => {
+      Object.entries(programmeYearlyStats).forEach(([year, stat]) => {
+        if (!res[year]) res[year] = { ...yearData }
+        res[year].studentCredits += stat.studentCredits
+        res[year].coursesPassed += stat.coursesPassed
+        res[year].coursesFailed += stat.coursesFailed
+      })
+    })
+    return res
   }
 
-  useEffect(() => {
-    const { fromYear: newFromYear, toYear: newToYear, years: newYears } = getYearFilterData()
-    if (fromYear < newFromYear || fromYear > newToYear) setFromYear(newFromYear)
-    if (toYear < newFromYear || toYear > newToYear) setToYear(newToYear)
-    setYears(newYears)
-  }, [selectedFaculties])
-
-  const handleYearChange = (e, target) => {
-    const { name, value } = target
-    if (name === 'fromYear') setFromYear(value)
-    else setToYear(value)
-  }
-
-  const calculateStatsFor = (facultyCode) => {
-    const yearlyStats = facultyYearlyStats.find(f => f.id === facultyCode)
-    const res = {
+  const calculateTotalStatsForFaculty = (facultyCode) => {
+    const initial = {
       totalStudentCredits: 0,
       totalCoursesPassed: 0,
       totalCoursesFailed: 0
     }
-    if (!yearlyStats) return res
-    Object.entries(yearlyStats.data)
+    return Object.entries(calculateYearlyStatsForFaculty(facultyCode))
       .filter(([year]) => year >= fromYear && year <= toYear)
-      .forEach(([, stat]) => {
-        const { studentCredits, coursesPassed, coursesFailed } = stat
-        res.totalStudentCredits += studentCredits
-        res.totalCoursesPassed += coursesPassed
-        res.totalCoursesFailed += coursesFailed
-      })
-    return res
+      .reduce((res, [, curr]) => ({
+        totalStudentCredits: res.totalStudentCredits + curr.studentCredits,
+        totalCoursesPassed: res.totalCoursesPassed + curr.coursesPassed,
+        totalCoursesFailed: res.totalCoursesFailed + curr.coursesFailed
+      }), { ...initial })
   }
 
-  const calculateTotalPassedCoursesFor = (facultyCode) => {
-    const { totalCoursesPassed, totalCoursesFailed } = calculateStatsFor(facultyCode)
-    const ratio = ((totalCoursesPassed / (totalCoursesPassed + totalCoursesFailed)) * 100)
-    return Number.isNaN(ratio) ? 0 : ratio
-  }
+  const totalStats = useMemo(() => faculties.reduce((res, { code }) => {
+    res[code] = calculateTotalStatsForFaculty(code)
+    return { ...res }
+  }, {}), [fromYear, toYear])
 
-  const calculateTotalFailedCoursesFor = (facultyCode) => {
-    const totalPassedRatio = calculateTotalPassedCoursesFor(facultyCode)
-    if (totalPassedRatio === 0) return 0
-    return 100 - totalPassedRatio
-  }
+  const graphData = useMemo(() => faculties.map(({ code, name }) => ({
+    name: getTextIn(name, language),
+    data: calculateYearlyStatsForFaculty(code)
+  })), [])
 
-  if (!faculties || !facultyYearlyStats) return null
   const headers = [
     {
       key: 'name',
@@ -94,84 +67,52 @@ const FacultySelector = ({ language, getFaculties, getFacultiesYearlyStats, facu
       key: 'code',
       title: 'code',
       getRowVal: faculty => faculty.code
-    }
-  ]
-
-  const selectedHeaders = [
-    ...headers,
+    },
     {
       key: 'students',
       title: 'Student credits',
-      getRowVal: ({ code }) => Math.round(calculateStatsFor(code).totalStudentCredits)
+      getRowVal: ({ code }) => Math.round(totalStats[code].totalStudentCredits)
     },
     {
       key: 'coursesPassed',
       title: 'Courses passed',
-      getRowVal: ({ code }) => calculateTotalPassedCoursesFor(code),
-      getRowContent: ({ code }) => `${calculateTotalPassedCoursesFor(code).toFixed(2)}%`
+      getRowVal: ({ code }) => calculateTotalPassedCourses(totalStats[code]),
+      getRowContent: ({ code }) => `${calculateTotalPassedCourses(totalStats[code]).toFixed(2)}%`
     },
     {
       key: 'coursesFailed',
       title: 'Courses failed',
-      getRowVal: ({ code }) => calculateTotalFailedCoursesFor(code),
-      getRowContent: ({ code }) => `${calculateTotalFailedCoursesFor(code).toFixed(2)}%`
+      getRowVal: ({ code }) => calculateTotalFailedCourses(totalStats[code]),
+      getRowContent: ({ code }) => `${calculateTotalFailedCourses(totalStats[code]).toFixed(2)}%`
     }
   ]
 
-  const selectedLength = selectedFaculties.length
   return (
-    <div>
-      { selectedLength !== faculties.length &&
-        <SortableTable
-          columns={headers}
-          getRowKey={faculty => faculty.code}
-          getRowProps={faculty => ({ onClick: () => handleSelect(faculty.code), style: { cursor: 'pointer' } })}
-          data={faculties.filter(({ code }) => !selectedFaculties.includes(code))}
-        />
-      }
-      { selectedLength !== 0 && (
-        <div>
-          <Segment>
-            <Form>
-              <YearFilter
-                fromYear={fromYear}
-                toYear={toYear}
-                years={years}
-                handleChange={handleYearChange}
-                showCheckbox={false}
-                separate={false}
-              />
-            </Form>
-          </Segment>
-          <SortableTable
-            columns={selectedHeaders}
-            getRowKey={faculty => faculty.code}
-            getRowProps={faculty => ({ onClick: () => handleSelect(faculty.code), style: { cursor: 'pointer' } })}
-            data={faculties.filter(({ code }) => selectedFaculties.includes(code))}
-          />
-        </div>
-      ) }
-    </div>
+    <React.Fragment>
+      <SortableTable
+        columns={headers}
+        getRowKey={faculty => faculty.code}
+        getRowProps={faculty => ({ onClick: () => handleSelect(faculty.code), style: { cursor: 'pointer' } })}
+        data={faculties}
+      />
+      <FacultyStatsGraph
+        data={graphData}
+      />
+    </React.Fragment>
   )
 }
 
 FacultySelector.propTypes = {
   language: string.isRequired,
-  getFaculties: func.isRequired,
-  getFacultiesYearlyStats: func.isRequired,
   faculties: arrayOf(shape({})).isRequired,
-  facultyYearlyStats: arrayOf(shape({})).isRequired
+  facultyYearlyStats: arrayOf(shape({})).isRequired,
+  fromYear: number.isRequired,
+  toYear: number.isRequired,
+  handleSelect: func.isRequired
 }
 
-const mapStateToProps = ({ faculties, localize }) => ({
-  faculties: faculties.data,
-  facultyYearlyStats: faculties.yearlyStats,
+const mapStateToProps = ({ localize }) => ({
   language: getActiveLanguage(localize).code
 })
 
-const mapDispatchToProps = {
-  getFaculties: getFacultiesAction,
-  getFacultiesYearlyStats: getFacultiesYearlyStatsAction
-}
-
-export default connect(mapStateToProps, mapDispatchToProps)(withRouter(FacultySelector))
+export default connect(mapStateToProps)(FacultySelector)

services/oodikone2-frontend/src/components/Faculty/FacultyStats/index.jsx

@@ -0,0 +1,93 @@
+import React, { useMemo } from 'react'
+import { connect } from 'react-redux'
+import { getActiveLanguage } from 'react-localize-redux'
+import { string, arrayOf, shape, number } from 'prop-types'
+import { Segment } from 'semantic-ui-react'
+import { getTextIn } from '../../../common'
+import {
+  calculateStatsForProgramme,
+  calculateTotalPassedCourses,
+  calculateTotalFailedCourses
+} from '../facultyUtils'
+import SortableTable from '../../SortableTable'
+import FacultyStatsGraph from '../FacultyStatsGraph'
+
+const FacultyStats = ({ facultyProgrammes, selectedFacultyProgrammesStats, language, fromYear, toYear }) => {
+  const totalStats = useMemo(() => Object.entries(selectedFacultyProgrammesStats).reduce((res, [code, stats]) => {
+    res[code] = calculateStatsForProgramme(stats, fromYear, toYear)
+    return { ...res }
+  }, {}), [fromYear, toYear])
+
+  const getNameOfProgramme = (code) => {
+    const foundProgramme = facultyProgrammes.find(p => p.code === code)
+    return foundProgramme ? getTextIn(foundProgramme.name, language) : code
+  }
+
+  const graphData = useMemo(() => Object.entries(selectedFacultyProgrammesStats).map(([code, data]) => ({ name: getNameOfProgramme(code), data })), [])
+
+  if (!Object.keys(selectedFacultyProgrammesStats).length) {
+    return (
+      <Segment textAlign="center">
+        No data
+      </Segment>
+    )
+  }
+
+  const headers = [
+    {
+      key: 'name',
+      title: 'name',
+      getRowVal: ({ code }) => getNameOfProgramme(code)
+    },
+    {
+      key: 'code',
+      title: 'code',
+      getRowVal: ({ code }) => code
+    },
+    {
+      key: 'students',
+      title: 'Student credits',
+      getRowVal: ({ code }) => Math.round(totalStats[code].totalStudentCredits)
+    },
+    {
+      key: 'coursesPassed',
+      title: 'Courses passed',
+      getRowVal: ({ code }) => calculateTotalPassedCourses(totalStats[code]),
+      getRowContent: ({ code }) => `${calculateTotalPassedCourses(totalStats[code]).toFixed(2)}%`
+    },
+    {
+      key: 'coursesFailed',
+      title: 'Courses failed',
+      getRowVal: ({ code }) => calculateTotalFailedCourses(totalStats[code]),
+      getRowContent: ({ code }) => `${calculateTotalFailedCourses(totalStats[code]).toFixed(2)}%`
+    }
+  ]
+
+  return (
+    <React.Fragment>
+      <SortableTable
+        columns={headers}
+        getRowKey={({ code }) => code}
+        data={Object.entries(selectedFacultyProgrammesStats).map(([code, stats]) => ({ code, stats }))}
+      />
+      <FacultyStatsGraph
+        data={graphData}
+      />
+    </React.Fragment>
+  )
+}
+
+FacultyStats.propTypes = {
+  facultyProgrammes: arrayOf(shape({})).isRequired,
+  selectedFacultyProgrammesStats: shape({}).isRequired,
+  language: string.isRequired,
+  fromYear: number.isRequired,
+  toYear: number.isRequired
+}
+
+const mapStateToProps = ({ faculties, localize }) => ({
+  facultyProgrammes: faculties.facultyProgrammes,
+  language: getActiveLanguage(localize).code
+})
+
+export default connect(mapStateToProps)(FacultyStats)

services/oodikone2-frontend/src/components/Faculty/FacultyStatsGraph/index.jsx

@@ -0,0 +1,68 @@
+import React, { useState, useMemo } from 'react'
+import { arrayOf, string, number, shape } from 'prop-types'
+import { Segment, Button } from 'semantic-ui-react'
+import Highcharts from 'highcharts/highstock'
+import boost from 'highcharts/modules/boost'
+import ReactHighstock from 'react-highcharts/ReactHighstock'
+
+boost(Highcharts)
+
+const FacultyStatsGraph = ({ data }) => {
+  const [mode, setMode] = useState('studentCredits')
+
+  const series = useMemo(() => data.map(({ name, data: entryData }) => ({
+    name,
+    data: Object.entries(entryData).map(([year, stats]) => ({
+      x: new Date(parseInt(year, 10), 0, 1, 0, 0, 0, 0).getTime(),
+      y: Math.round(mode === 'studentCredits' ? stats[mode] : ((stats[mode] / (stats.coursesPassed + stats.coursesFailed)) * 100))
+    }))
+  })), [data, mode])
+
+  const { min, max } = useMemo(() => series.reduce((res, { data }) => {
+    data.forEach(({ x }) => {
+      res.min = Math.min(res.min, x)
+      res.max = Math.max(res.max, x)
+    })
+    return res
+  }, { min: Infinity, max: -Infinity }), [series])
+
+  const options = {
+    findNearestPointBy: 'xy',
+    series,
+    xAxis: {
+      min,
+      max,
+      ordinal: false
+    }
+  }
+
+  const handleClick = (_, { name }) => setMode(name)
+
+  return (
+    <Segment>
+      <ReactHighstock
+        highcharts={Highcharts}
+        constructorType="stockChart"
+        config={options}
+      />
+      <Button active={mode === 'studentCredits'} onClick={handleClick} name="studentCredits">Total credits</Button>
+      <Button active={mode === 'coursesPassed'} onClick={handleClick} name="coursesPassed">% of courses passed</Button>
+      <Button active={mode === 'coursesFailed'} onClick={handleClick} name="coursesFailed">% of courses failed</Button>
+    </Segment>
+  )
+}
+
+FacultyStatsGraph.propTypes = {
+  data: arrayOf(shape({
+    name: string,
+    data: shape({
+      year: shape({
+        studentCredits: number,
+        coursesPassed: number,
+        coursesFailed: number
+      })
+    })
+  })).isRequired
+}
+
+export default FacultyStatsGraph

services/oodikone2-frontend/src/components/Faculty/facultyUtils.js

@@ -0,0 +1,29 @@
+export const calculateStatsForProgramme = (stats, fromYear, toYear) => {
+  const res = {
+    totalStudentCredits: 0,
+    totalCoursesPassed: 0,
+    totalCoursesFailed: 0
+  }
+
+  Object.entries(stats)
+    .filter(([year]) => year >= fromYear && year <= toYear)
+    .forEach(([, stat]) => {
+      const { studentCredits, coursesPassed, coursesFailed } = stat
+      res.totalStudentCredits += studentCredits
+      res.totalCoursesPassed += coursesPassed
+      res.totalCoursesFailed += coursesFailed
+    })
+
+  return res
+}
+
+export const calculateTotalPassedCourses = ({ totalCoursesPassed, totalCoursesFailed }) => {
+  const ratio = ((totalCoursesPassed / (totalCoursesPassed + totalCoursesFailed)) * 100)
+  return Number.isNaN(ratio) ? 0 : ratio
+}
+
+export const calculateTotalFailedCourses = ({ totalCoursesPassed, totalCoursesFailed }) => {
+  const totalPassedRatio = calculateTotalPassedCourses({ totalCoursesPassed, totalCoursesFailed })
+  if (totalPassedRatio === 0) return 0
+  return 100 - totalPassedRatio
+}

services/oodikone2-frontend/src/components/Faculty/index.jsx

@@ -1,17 +1,141 @@
-import React from 'react'
+import React, { useState, useEffect } from 'react'
+import { connect } from 'react-redux'
 import { withRouter } from 'react-router-dom'
-import { Header, Segment } from 'semantic-ui-react'
+import { getActiveLanguage } from 'react-localize-redux'
+import { Header, Segment, Form } from 'semantic-ui-react'
+import { uniq } from 'lodash'
+import { string, arrayOf, shape, func } from 'prop-types'
+import { getTextIn } from '../../common'
+import { getFaculties, getFacultiesYearlyStats, getFacultyProgrammes } from '../../redux/faculties'
+import YearFilter from '../CourseStatistics/SearchForm/YearFilter'
 import FacultySelector from './FacultySelector'
+import FacultyStats from './FacultyStats'
 
-const Faculty = () => (
-  <div className="segmentContainer">
-    <Header className="segmentTitle" size="large">
-      Faculties
-    </Header>
-    <Segment className="contentSegment" >
-      <FacultySelector />
-    </Segment>
-  </div>
-)
+const Faculty = ({ getFaculties, getFacultiesYearlyStats, getFacultyProgrammes, faculties, facultyYearlyStats, history, match, language }) => {
+  const [selectedFaculty, setSelectedFaculty] = useState(null)
+  const [fromYear, setFromYear] = useState(-1)
+  const [toYear, setToYear] = useState(-1)
+  const [years, setYears] = useState([])
 
-export default withRouter(Faculty)
+  const facultyCodes = faculties.map(({ code }) => code)
+  const selectedFacultyProgrammesStats = facultyYearlyStats.find(({ id }) => id === selectedFaculty)
+  const hasLoaded = (faculties.length > 0 && facultyYearlyStats.length > 0)
+
+  useEffect(() => {
+    if (selectedFaculty) {
+      history.push(`/faculties/${selectedFaculty}`)
+    }
+  }, [selectedFaculty])
+
+  useEffect(() => {
+    const { params: { facultyid } } = match
+    if (facultyid) setSelectedFaculty(facultyid)
+    else setSelectedFaculty(null)
+  }, [history.location.pathname])
+
+  useEffect(() => {
+    getFaculties()
+    getFacultiesYearlyStats()
+    getFacultyProgrammes()
+  }, [])
+
+  const getYearFilterData = () => {
+    const filterYears = uniq(facultyYearlyStats
+      .filter(f => facultyCodes.includes(f.id))
+      .map(({ data }) => Object.values(data).reduce((res, curr) => [...res, ...Object.keys(curr)], []))
+      .reduce((acc, curr) => [...acc, ...curr], []))
+      .sort((a, b) => b - a).map(year => parseInt(year, 10))
+
+    return {
+      fromYear: filterYears[filterYears.length - 1],
+      toYear: filterYears[0],
+      years: filterYears.map(y => ({ key: y, text: y, value: y }))
+    }
+  }
+
+  useEffect(() => {
+    if (faculties.length && facultyYearlyStats.length) {
+      const { fromYear: newFromYear, toYear: newToYear, years: newYears } = getYearFilterData()
+      if (fromYear < newFromYear || fromYear > newToYear) setFromYear(newFromYear)
+      if (toYear < newFromYear || toYear > newToYear) setToYear(newToYear)
+      setYears(newYears)
+    }
+  }, [faculties, facultyYearlyStats, selectedFaculty])
+
+  const handleYearChange = (e, target) => {
+    const { name, value } = target
+    if (name === 'fromYear') setFromYear(value)
+    else setToYear(value)
+  }
+
+  const getTitle = () => (
+    selectedFaculty && hasLoaded ?
+      getTextIn(faculties.find(({ code }) => code === selectedFaculty).name, language) :
+      'Faculties'
+  )
+
+  return (
+    <div className="segmentContainer">
+      <Header className="segmentTitle" size="large">
+        { hasLoaded && getTitle() }
+      </Header>
+      <Segment className="contentSegment" loading={!hasLoaded}>
+        { !hasLoaded && null }
+        { hasLoaded && (
+          <React.Fragment>
+            <Segment>
+              <Form>
+                <Header content="Filter by time range" as="h4" />
+                <YearFilter
+                  fromYear={fromYear}
+                  toYear={toYear}
+                  years={years}
+                  handleChange={handleYearChange}
+                />
+              </Form>
+            </Segment>
+            { !selectedFaculty ?
+              <FacultySelector
+                faculties={faculties}
+                facultyYearlyStats={facultyYearlyStats}
+                fromYear={fromYear}
+                toYear={toYear}
+                handleSelect={setSelectedFaculty}
+              /> :
+              <FacultyStats
+                selectedFacultyProgrammesStats={selectedFacultyProgrammesStats ? selectedFacultyProgrammesStats.data : {}}
+                fromYear={fromYear}
+                toYear={toYear}
+              />
+            }
+          </React.Fragment>
+        )}
+      </Segment>
+    </div>
+  )
+}
+
+Faculty.propTypes = {
+  getFaculties: func.isRequired,
+  getFacultiesYearlyStats: func.isRequired,
+  getFacultyProgrammes: func.isRequired,
+  faculties: arrayOf(shape({})).isRequired,
+  facultyYearlyStats: arrayOf(shape({})).isRequired,
+  history: shape({}).isRequired,
+  match: shape({}).isRequired,
+  language: string.isRequired
+}
+
+const mapStateToProps = ({ faculties, localize }) => ({
+  faculties: faculties.data,
+  facultyYearlyStats: faculties.yearlyStats,
+  language: getActiveLanguage(localize).code
+})
+
+const mapDispatchToProps = {
+  getFaculties,
+  getFacultiesYearlyStats,
+  getFacultyProgrammes
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(withRouter(Faculty))

services/oodikone2-frontend/src/components/Feedback/index.jsx

@@ -23,7 +23,7 @@ const Feedback = ({ sendFeedback, success }) => {
       setConfirm(false)
     }
   }, [success])
-
+  // just a dummy commit
   const open = () => setConfirm(true)
 
   const close = () => setConfirm(false)

services/oodikone2-frontend/src/components/OodiLearn/StudentInfo.jsx

@@ -1,35 +1,28 @@
-import React, { Component } from 'react'
+import React from 'react'
 import { Segment, Card, Button, Divider } from 'semantic-ui-react'
 import { shape, string, func } from 'prop-types'
 import ProfileSpiderGraph from './ProfileSpiderGraph'
 
-class SearchResult extends Component {
-    state={}
-
-    render() {
-      const { student, goBack } = this.props
-      return (
-        <Segment basic>
-          <Button
-            icon="arrow circle left"
-            basic
-            content="Back"
-            size="small"
-            onClick={goBack}
-          />
-          <Divider />
-          <Card
-            fluid
-            header="Opiskelijan Nimi" // {student.name}
-            meta={student.studentnumber}
-          />
-          <Segment>
-            <ProfileSpiderGraph profile={student.profile} />
-          </Segment>
-        </Segment>
-      )
-    }
-}
+const SearchResult = ({ student, goBack }) => (
+  <Segment basic>
+    <Button
+      icon="arrow circle left"
+      basic
+      content="Back"
+      size="small"
+      onClick={goBack}
+    />
+    <Divider />
+    <Card
+      fluid
+      header="Opiskelijan Nimi" // {student.name}
+      meta={student.studentnumber}
+    />
+    <Segment>
+      <ProfileSpiderGraph profile={student.profile} />
+    </Segment>
+  </Segment>
+)
 
 SearchResult.propTypes = {
   student: shape({

services/oodikone2-frontend/src/components/PopulationCourseStats/PassingSemesters/CourseRow.jsx

@@ -68,7 +68,7 @@ const CourseRow = ({ statistics, cumulative, onCourseNameClickFn, isActiveCourse
 
   return (
     <Table.Row key={course.code} active={isActive}>
-      <Table.Cell onClick={() => onCourseNameClickFn(statistics)} className="clickableCell">
+      <Table.Cell onClick={() => onCourseNameClickFn(course.code)} className="clickableCell">
         {getTextIn(course.name, activeLanguage)}
       </Table.Cell>
       <Table.Cell>

services/oodikone2-frontend/src/components/PopulationCourseStats/index.jsx

@@ -86,6 +86,7 @@ class PopulationCourseStats extends Component {
     }).isRequired,
     translate: func.isRequired,
     setPopulationFilter: func.isRequired,
+    populationCourses: shape({ data: shape({ coursestatistics: arrayOf(shape({ course: shape({ code: string, name: shape({}) }) })) }) }).isRequired,
     selectedCourses: arrayOf(object).isRequired,
     removePopulationFilterOfCourse: func.isRequired,
     history: shape({}).isRequired,
@@ -99,14 +100,14 @@ class PopulationCourseStats extends Component {
 
   static getDerivedStateFromProps(props, state) {
     if (state && props.courses) {
-      const studentAmountLimit = state.studentAmountLimit > props.selectedStudents.length
-        ? Math.round(props.selectedStudents.length / 2) : state.studentAmountLimit
-
+      const studentAmountLimit = state.selectedStudentsLength !== props.selectedStudents.length ?
+        Math.round(props.selectedStudents.length * 0.3) : state.studentAmountLimit
       return {
         ...state,
         courseStatistics: PopulationCourseStats.updateCourseStatisticsCriteria(props, state),
         initialSortReady: true,
-        studentAmountLimit
+        studentAmountLimit,
+        selectedStudentsLength: props.selectedStudents.length
       }
     }
 
@@ -146,7 +147,8 @@ class PopulationCourseStats extends Component {
     reversed: true,
     studentAmountLimit: Math.round(this.props.selectedStudents.length * 0.3),
     codeFilter: '',
-    activeView: null
+    activeView: null,
+    selectedStudentsLength: 0
   }
 
   onCodeFilterChange = (e) => {
@@ -200,12 +202,15 @@ class PopulationCourseStats extends Component {
     clearCourseStatsfn()
   }
 
-  onCourseNameCellClick = (courseStats) => {
-    if (!this.isActiveCourse(courseStats.course)) {
-      const params = { course: courseStats, field: 'all' }
-      this.props.setPopulationFilter(courseParticipation(params))
-    } else {
-      this.props.removePopulationFilterOfCourse(courseStats.course)
+  onCourseNameCellClick = (code) => {
+    const courseStatistic = this.props.populationCourses.data.coursestatistics.find(cs => cs.course.code === code)
+    if (courseStatistic) {
+      if (!this.isActiveCourse(courseStatistic.course)) {
+        const params = { course: courseStatistic, field: 'all' }
+        this.props.setPopulationFilter(courseParticipation(params))
+      } else {
+        this.props.removePopulationFilterOfCourse(courseStatistic.course)
+      }
     }
   }
 
@@ -301,7 +306,7 @@ class PopulationCourseStats extends Component {
       return (
         <Table.Row active={this.isActiveCourse(course)}>
           <Table.Cell
-            onClick={() => this.onCourseNameCellClick(courseStats)}
+            onClick={() => this.onCourseNameCellClick(code)}
             content={getTextIn(name, language)}
             className="clickableCell"
           />
@@ -406,11 +411,11 @@ class PopulationCourseStats extends Component {
       return ((
         <Table.Row key={code} active={this.isActiveCourse(course)}>
           <Table.Cell
-            onClick={() => this.onCourseNameCellClick(courseStats)}
+            onClick={() => this.onCourseNameCellClick(code)}
             content={getTextIn(name, language)}
             className="clickableCell"
           />
-          {history.location.pathname === '/coursepopulation' ? <Table.Cell className="iconCell" /> : (<Table.Cell
+          {['/coursepopulation', '/custompopulation'].includes(history.location.pathname) ? <Table.Cell className="iconCell" /> : (<Table.Cell
             icon="level up alternate"
             onClick={() => this.onGoToCourseStatisticsClick(code)}
             className="iconCell"
@@ -458,7 +463,7 @@ class PopulationCourseStats extends Component {
               value={studentAmountLimit}
               onChange={this.onStudentAmountLimitChange}
             />
-            {history.location.pathname === '/coursepopulation' ? null : (
+            {['/coursepopulation', '/custompopulation'].includes(history.location.pathname) ? null : (
               <Button
                 active={this.state.activeView === 'passingSemester'}
                 floated="right"
@@ -498,9 +503,10 @@ const mapStateToProps = (state) => {
   return {
     language: getActiveLanguage(state.localize).code,
     translate: getTranslate(state.localize),
-    query: state.populations.query ? state.populations.query : state.coursePopulation.query,
+    query: state.populations.query ? state.populations.query : {},
     years,
-    selectedCourses
+    selectedCourses,
+    populationCourses: state.populationCourses
   }
 }
 

services/oodikone2-frontend/src/components/PopulationCourses/index.jsx

@@ -9,22 +9,25 @@ import SegmentDimmer from '../SegmentDimmer'
 import PopulationCourseStats from '../PopulationCourseStats'
 import InfoBox from '../InfoBox'
 import infotooltips from '../../common/InfoToolTips'
-import { getPopulationCourses } from '../../redux/populationCourses'
+import { getPopulationSelectedStudentCourses } from '../../redux/populationSelectedStudentCourses'
 import { refreshFilters } from '../../redux/populationFilters'
 
 const PopulationCourses = ({
+  populationSelectedStudentCourses,
   populationCourses,
   refreshNeeded,
   dispatchRefreshFilters,
   selectedStudents,
   translate,
-  getPopulationCourses: gpc
+  getPopulationSelectedStudentCourses: gpc
 }) => {
+  const selectedPopulationCourses = populationSelectedStudentCourses.data ? populationSelectedStudentCourses : populationCourses
+
   const { CoursesOf } = infotooltips.PopulationStatistics
-  const { pending } = populationCourses
+  const { pending } = selectedPopulationCourses
   const reloadCourses = () => {
     dispatchRefreshFilters()
-    gpc({ ...populationCourses.query, uuid: uuidv4(), selectedStudents })
+    gpc({ ...selectedPopulationCourses.query, uuid: uuidv4(), selectedStudents })
   }
 
   useEffect(() => {
@@ -48,10 +51,10 @@ const PopulationCourses = ({
         </Header>
         <SegmentDimmer translate={translate} isLoading={pending} />
         <PopulationCourseStats
-          key={populationCourses.query.uuid}
-          courses={populationCourses.data}
-          query={populationCourses.query}
-          pending={populationCourses.pending}
+          key={selectedPopulationCourses.query.uuid}
+          courses={selectedPopulationCourses.data}
+          query={selectedPopulationCourses.query}
+          pending={pending}
           selectedStudents={selectedStudents}
         />
       </Segment>
@@ -60,20 +63,22 @@ const PopulationCourses = ({
 }
 
 PopulationCourses.propTypes = {
-  populationCourses: shape({}).isRequired,
+  populationSelectedStudentCourses: shape({ query: shape({}), data: shape({}), pending: bool }).isRequired,
+  populationCourses: shape({ query: shape({}), data: shape({}), pending: bool }).isRequired,
   refreshNeeded: bool.isRequired,
   translate: func.isRequired,
   selectedStudents: arrayOf(string).isRequired,
-  getPopulationCourses: func.isRequired,
+  getPopulationSelectedStudentCourses: func.isRequired,
   dispatchRefreshFilters: func.isRequired
 }
 
-const mapStateToProps = ({ populationCourses, localize, populationFilters }) => ({
+const mapStateToProps = ({ populationSelectedStudentCourses, populationCourses, localize, populationFilters }) => ({
   populationCourses,
+  populationSelectedStudentCourses,
   refreshNeeded: populationFilters.refreshNeeded,
   translate: getTranslate(localize)
 })
 
 export default connect(mapStateToProps, {
-  getPopulationCourses, dispatchRefreshFilters: refreshFilters
+  getPopulationSelectedStudentCourses, dispatchRefreshFilters: refreshFilters
 })(PopulationCourses)

services/oodikone2-frontend/src/components/PopulationFilters/CourseParticipationNTimes.jsx

@@ -12,7 +12,7 @@ class CourseParticipationNTimes extends Component {
     filter: shape({}).isRequired,
     removePopulationFilter: func.isRequired,
     setPopulationFilter: func.isRequired,
-    populationCourses: shape({}).isRequired
+    selectedPopulationCourses: shape({}).isRequired
   }
 
   state = {
@@ -24,8 +24,8 @@ class CourseParticipationNTimes extends Component {
   }
 
   studyRights = () => {
-    const { populationCourses } = this.props
-    return populationCourses ? populationCourses.query.studyRights : []
+    const { selectedPopulationCourses } = this.props
+    return selectedPopulationCourses ? selectedPopulationCourses.query.studyRights : []
   }
 
   handleLimit = () => {
@@ -98,8 +98,13 @@ class CourseParticipationNTimes extends Component {
   }
 }
 
-const mapStateToProps = ({ populationCourses }) => ({
+const mapStateToProps = ({
+  populationSelectedStudentCourses,
   populationCourses
+}) => ({
+  selectedPopulationCourses: populationSelectedStudentCourses.data
+    ? populationSelectedStudentCourses
+    : populationCourses
 })
 
 export default connect(mapStateToProps, {

services/oodikone2-frontend/src/components/PopulationFilters/DisciplineTypes.jsx

@@ -144,11 +144,19 @@ DisciplineTypes.defaultProps = {
   courseTypes: {}
 }
 
-const mapStateToProps = ({ populationCourses }) => ({
-  courseTypes: populationCourses.data.coursetypes,
-  disciplines: populationCourses.data.disciplines,
-  courses: populationCourses.data.coursestatistics
-})
+const mapStateToProps = ({
+  populationSelectedStudentCourses,
+  populationCourses
+}) => {
+  const selectedPopulationCourses = populationSelectedStudentCourses.data
+    ? populationSelectedStudentCourses
+    : populationCourses
+  return {
+    courseTypes: selectedPopulationCourses.data.coursetypes,
+    disciplines: selectedPopulationCourses.data.disciplines,
+    courses: selectedPopulationCourses.data.coursestatistics
+  }
+}
 
 export default connect(
   mapStateToProps,

services/oodikone2-frontend/src/components/PopulationFilters/TransferFilter.jsx

@@ -157,10 +157,7 @@ class TransferFilter extends Component {
   }
 }
 
-const mapStateToProps = ({ populationCourses, localize }) => ({
-  courseTypes: populationCourses.data.coursetypes,
-  disciplines: populationCourses.data.disciplines,
-  courses: populationCourses.data.coursestatistics,
+const mapStateToProps = ({ localize }) => ({
   activeLanguage: getActiveLanguage(localize).code
 })
 

services/oodikone2-frontend/src/components/PopulationFilters/index.jsx

@@ -73,8 +73,9 @@ class PopulationFilters extends Component {
     savePopulationFilters: func.isRequired,
     setPopulationFilter: func.isRequired,
     studyRights: shape({ programme: string, degree: string, studyTrack: string }).isRequired,
-    populationFilters: object.isRequired, //eslint-disable-line
-    populationCourses: object.isRequired //eslint-disable-line
+    populationFilters: shape({}).isRequired,
+    populationSelectedStudentCourses: shape({}).isRequired,
+    populationCourses: shape({}).isRequired
   }
 
   state = {
@@ -101,7 +102,8 @@ class PopulationFilters extends Component {
   initialFilterLoading = async () => {
     const untilCoursesLoaded = () => {
       const poll = (resolve) => {
-        const { data, pending } = this.props.populationCourses
+        const selectedPopulationCourses = this.props.populationSelectedStudentCourses.data ? this.props.populationSelectedStudentCourses : this.props.populationCourses
+        const { data, pending } = selectedPopulationCourses
         if (data && !pending) {
           resolve()
         } else {
@@ -163,13 +165,14 @@ class PopulationFilters extends Component {
   }
 
   updateFilterList(filtersToCreate) {
+    const selectedPopulationCourses = this.props.populationSelectedStudentCourses.data ? this.props.populationSelectedStudentCourses : this.props.populationCourses
     // sorry for the uglyness but it kinda works (I think)
     const regenerateFilterFunctions = filters =>    /* eslint-disable */
       filters.map(f => f.type === 'Preset' ?
         getFilterFunction(f.type, { ...f, filters: regenerateFilterFunctions(f.filters) },
-          this.props.populationCourses.data)
+        selectedPopulationCourses.data)
         :
-        getFilterFunction(f.type, f.params, this.props.populationCourses.data))
+        getFilterFunction(f.type, f.params, selectedPopulationCourses.data))
 
     if (filtersToCreate) {
       const newFilters = filtersToCreate.map(newFilter =>
@@ -184,8 +187,9 @@ class PopulationFilters extends Component {
 
 
   renderAddFilters(allStudyRights) {
-    const { extents, transfers, populationCourses } = this.props
+    const { extents, transfers, populationSelectedStudentCourses, populationCourses } = this.props
     const { Add } = infotooltips.PopulationStatistics.Filters
+    const selectedPopulationCourses = populationSelectedStudentCourses.data ? populationSelectedStudentCourses : populationCourses
     const allFilters = union(Object.keys(componentFor).filter(f =>
       !(Object.keys(advancedFilters).includes(f) && !this.state.advancedUser)).map(f =>
         String(f)), this.state.presetFilters.map(f => f.id).filter(f => this.state.advancedUser))
@@ -203,10 +207,10 @@ class PopulationFilters extends Component {
       return (
         <Segment>
           <Header>Add filters <InfoBox content={Add} /></Header>
-          <Loader active={populationCourses.pending} inline="centered" />
+          <Loader active={selectedPopulationCourses.pending} inline="centered" />
           <Button
             onClick={() => this.setState({ visible: true })}
-            disabled={populationCourses.pending}
+            disabled={selectedPopulationCourses.pending}
           >
             add
           </Button>
@@ -358,8 +362,10 @@ const mapStateToProps = ({
   localize,
   graphSpinner,
   populations,
+  populationSelectedStudentCourses,
   populationCourses
 }) => ({
+  populationSelectedStudentCourses,
   populationCourses,
   populationFilters,
   filters: populationFilters.filters,

services/oodikone2-frontend/src/components/PopulationSearchForm/index.jsx

@@ -12,6 +12,7 @@ import moment from 'moment'
 
 import { getPopulationStatistics } from '../../redux/populations'
 import { getPopulationCourses } from '../../redux/populationCourses'
+import { getPopulationSelectedStudentCourses } from '../../redux/populationSelectedStudentCourses'
 import { getPopulationFilters, setPopulationFilter, clearPopulationFilters } from '../../redux/populationFilters'
 import { getMandatoryCourses } from '../../redux/populationMandatoryCourses'
 import { getSemesters } from '../../redux/semesters'
@@ -117,7 +118,7 @@ const PopulationSearchForm = (props) => {
     const formattedQueryParams = formatQueryParamsToArrays(query, ['semesters', 'studentStatuses'])
     const queryCodes = Object.values(query.studyRights).filter(e => e != null)
     const uuid = uuidv4()
-    const request = { ...formattedQueryParams, studyRights: queryCodes, uuid }
+    const request = { ...formattedQueryParams, studyRights: queryCodes, uuid, tag: selectedTag }
     setState({ isLoading: true })
     props.setLoading()
 
@@ -132,8 +133,6 @@ const PopulationSearchForm = (props) => {
     if (success) {
       if (queryCodes[0] === 'KH50_001') {
         props.setPopulationFilter(transferTo(false))
-      } else {
-        props.clearPopulationFilters()
       }
       setState({ isLoading: false })
       // not a good solution FIX
@@ -755,7 +754,6 @@ PopulationSearchForm.propTypes = {
   getPopulationCourses: func.isRequired,
   getMandatoryCourses: func.isRequired,
   getPopulationFilters: func.isRequired,
-  clearPopulationFilters: func.isRequired,
   setPopulationFilter: func.isRequired,
   queries: shape({}).isRequired,
   studyProgrammes: shape({}), //eslint-disable-line
@@ -792,6 +790,7 @@ const mapStateToProps = ({ semesters, settings, populations, populationDegreesAn
 export default withRouter(connect(mapStateToProps, {
   getPopulationStatistics,
   getPopulationCourses,
+  getPopulationSelectedStudentCourses,
   getPopulationFilters,
   getMandatoryCourses,
   setPopulationFilter,

services/oodikone2-frontend/src/components/PopulationSearchHistory/index.jsx

@@ -5,7 +5,6 @@ import { getTranslate } from 'react-localize-redux'
 
 import PopulationQueryCard from '../PopulationQueryCard'
 import { removePopulation, updatePopulationStudents } from '../../redux/populations'
-import { clearPopulationCourses } from '../../redux/populationCourses'
 import { clearPopulationFilters } from '../../redux/populationFilters'
 
 import './populationSearchHistory.css'
@@ -77,7 +76,6 @@ const mapStateToProps = ({ populations, populationDegreesAndProgrammes, localize
 const mapDispatchToProps = dispatch => ({
   removePopulation: (uuid) => {
     dispatch(removePopulation(uuid))
-    dispatch(clearPopulationCourses())
     dispatch(clearPopulationFilters())
   },
   updatePopulationStudents: students => dispatch(updatePopulationStudents(students))

services/oodikone2-frontend/src/components/PopulationStudents/index.jsx

@@ -154,6 +154,24 @@ class PopulationStudents extends Component {
       return codes.join(', ')
     }
 
+    const tags = (tags) => {
+      const studentTags = tags.map(t => t.tag.tagname)
+      return studentTags.join(', ')
+    }
+
+    const mainProgramme = (studyrights) => {
+      const studyprogrammes = []
+      studyrights.forEach((sr) => {
+        const studyrightElements = sr.studyrightElements.filter(srE => srE.element_detail.type === 20)
+        if (studyrightElements.length > 0) {
+          const newestStudyrightElement = studyrightElements.sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0]
+          studyprogrammes.push({ name: sr.highlevelname, startdate: newestStudyrightElement.startdate })
+        }
+      })
+      const programme = studyprogrammes.sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0]
+      return programme.name
+    }
+
     const studytrack = (studyrights) => {
       const { queryStudyrights } = this.props
       let startdate = '1900-01-01'
@@ -193,29 +211,32 @@ class PopulationStudents extends Component {
         key: 'icon',
         getRowVal: s => (<Icon name="level up alternate" onClick={() => pushToHistoryFn(s.studentNumber)} />),
         cellProps: { collapsing: true, className: 'iconCell' }
-      },
-      {
+      }
+    )
+    if (!['/coursepopulation', '/custompopulation'].includes(history.location.pathname)) {
+      columns.push({
         key: 'credits since start',
         title: 'credits since start',
         getRowVal: (s) => {
           const credits = getStudentTotalCredits(s)
           return roundToTwo(credits)
         }
-      },
-      {
-        key: 'all credits',
-        title: 'all credits',
-        getRowVal: s => s.credits
-      }
-    )
-    if (history.location.pathname !== '/coursepopulation') {
+      })
+    }
+    columns.push({
+      key: 'all credits',
+      title: 'all credits',
+      getRowVal: s => s.credits
+    })
+
+    if (!['/coursepopulation', '/custompopulation'].includes(history.location.pathname)) {
       columns.push({
         key: 'transferred from',
         title: 'transferred from',
         getRowVal: s => (s.transferredStudyright ? transferFrom(s) : '')
       })
     }
-    if (containsStudyTracks && history.location.pathname !== '/coursepopulation') {
+    if (containsStudyTracks && !['/coursepopulation', '/custompopulation'].includes(history.location.pathname)) {
       columns.push({
         key: 'studytrack',
         title: 'studytrack',
@@ -223,7 +244,7 @@ class PopulationStudents extends Component {
       })
     }
 
-    if (admin && history.location.pathname !== '/coursepopulation') {
+    if (admin && !['/coursepopulation', '/custompopulation'].includes(history.location.pathname)) {
       columns.push(
         {
           key: 'priority',
@@ -235,8 +256,29 @@ class PopulationStudents extends Component {
           title: 'extent',
           getRowVal: s => extentCodes(s.studyrights)
         },
+        {
+          key: 'tags',
+          title: 'tags',
+          getRowVal: s => tags(s.tags)
+        }
       )
     }
+
+    if (['/coursepopulation', '/custompopulation'].includes(history.location.pathname)) {
+      columns.push(
+        {
+          key: 'programme',
+          title: 'studyprogramme',
+          getRowVal: s => mainProgramme(s.studyrights)
+        },
+        {
+          key: 'startyear',
+          title: 'start year',
+          getRowVal: s => reformatDate(s.started, 'YYYY')
+        }
+      )
+    }
+
     if (admin) {
       columns.push({
         key: 'updatedAt',
@@ -534,7 +576,7 @@ class PopulationStudents extends Component {
       return workbook
     }
     const filteredPanes = (panesToFilter) => {
-      if (history.location.pathname === '/coursepopulation') {
+      if (['/coursepopulation', '/custompopulation'].includes(history.location.pathname)) {
         return panesToFilter.slice(0, 1)
       }
       if (!this.state.admin) {

services/oodikone2-frontend/src/components/ProgressBar/index.jsx

@@ -43,7 +43,6 @@ const ProgressBar = ({ progress, fixed }) => {
       <Progress
         percent={progress}
         disabled={progress === 100}
-        progress
         color="blue"
         className={classNames.join(' ')}
       />

services/oodikone2-frontend/src/components/ProgressBar/progressBar.css

@@ -1,6 +1,7 @@
 .progressBar,
 .progressBarHidden {
   margin: 10px 0 10px 0 !important;
+  z-index: 100;
 }
 
 .progressBarHidden {

services/oodikone2-frontend/src/components/Routes/index.jsx

@@ -15,6 +15,7 @@ const OodiLearn = React.lazy(() => import('../OodiLearn'))
 const Feedback = React.lazy(() => import('../Feedback'))
 const Faculty = React.lazy(() => import('../Faculty'))
 const CourseStudents = React.lazy(() => import('../CourseStudents'))
+const CustomPopulation = React.lazy(() => import('../CustomPopulation'))
 const Updater = React.lazy(() => import('../Updater'))
 
 const routes = {
@@ -27,7 +28,8 @@ const routes = {
   sandbox: '/sandbox',
   oodilearn: '/oodilearn',
   feedback: '/feedback',
-  coursepopulation: '/coursepopulation'
+  coursepopulation: '/coursepopulation',
+  custompopulation: '/custompopulation'
 }
 
 const Routes = () => (
@@ -47,6 +49,7 @@ const Routes = () => (
       <Route exact path={routes.oodilearn} component={OodiLearn} />
       <Route exact path={routes.feedback} component={Feedback} />
       <Route exact path={routes.coursepopulation} component={CourseStudents} />
+      <Route exact path={routes.custompopulation} component={CustomPopulation} />
       <Route exact path="/updater" component={Updater} />
     </Switch>
   </Suspense>

services/oodikone2-frontend/src/components/StudyProgramme/TagModal/index.jsx

@@ -0,0 +1,114 @@
+import React, { useState, useEffect } from 'react'
+import { withRouter } from 'react-router-dom'
+import { connect } from 'react-redux'
+import { arrayOf, string, bool, func, shape } from 'prop-types'
+import { Button, Modal, Form, TextArea, Dropdown, Message } from 'semantic-ui-react'
+
+import {
+  createMultipleStudentTagAction
+} from '../../../redux/tagstudent'
+
+const TagModal = ({ tags, studytrack, createMultipleStudentTag, pending, success, error }) => {
+  const [modalOpen, setModalOpen] = useState(false)
+  const [input, setInput] = useState('')
+  const [selectedValue, setSelected] = useState('')
+
+  useEffect(() => {
+    if (!pending) {
+      if (success) {
+        setModalOpen(false)
+        setSelected('')
+        setInput('')
+      }
+    }
+  }, [pending])
+
+  const handleClick = (event) => {
+    event.preventDefault()
+    const studentnumbers = input.match(/[^\s,]+/g)
+    createMultipleStudentTag(studentnumbers.map(sn => ({
+      tag_id: selectedValue,
+      studentnumber: sn
+    })), studytrack)
+  }
+
+  const handleChange = (event, { value }) => {
+    event.preventDefault()
+    setSelected(value)
+  }
+
+  const createdOptions = tags.map(tag => ({ key: tag.tag_id, text: tag.tagname, value: tag.tag_id }))
+
+  return (
+    <Modal
+      trigger={<Button size="small" onClick={() => setModalOpen(true)}>Add tags to students</Button>}
+      open={modalOpen}
+      onClose={() => setModalOpen(false)}
+      size="small"
+      closeOnEscape={false}
+    >
+      <Modal.Content>
+        <Form>
+          <h2> Add tags to students </h2>
+          <Message hidden={!error} content={error} negative />
+          <Form.Field>
+            <em> Select a tag </em>
+            <Dropdown
+              placeholder="Tag"
+              search
+              selection
+              selectOnBlur={false}
+              selectOnNavigation={false}
+              options={createdOptions}
+              onChange={handleChange}
+              value={selectedValue}
+            />
+            <em> Insert studentnumbers you wish to add tags to </em>
+            <TextArea
+              placeholder="011111111"
+              onChange={e => setInput(e.target.value)}
+              value={input}
+            />
+          </Form.Field>
+        </Form>
+      </Modal.Content>
+      <Modal.Actions>
+        <Button
+          negative
+          onClick={() => setModalOpen(false)}
+        >Cancel
+        </Button>
+        <Button
+          positive
+          onClick={event => handleClick(event)}
+          disabled={pending || selectedValue.length === 0 || !input.match(/[^\s,]+/g)}
+        >
+          Add tags
+        </Button>
+      </Modal.Actions>
+    </Modal>
+  )
+}
+
+TagModal.defaultProps = {
+  error: null
+}
+
+TagModal.propTypes = {
+  createMultipleStudentTag: func.isRequired,
+  tags: arrayOf(shape({ tag_id: string, tagname: string, studytrack: string })).isRequired,
+  studytrack: string.isRequired,
+  pending: bool.isRequired,
+  success: bool.isRequired,
+  error: string
+}
+
+const mapStateToProps = ({ tagstudent }) => ({
+  pending: tagstudent.pending,
+  success: tagstudent.success,
+  error: tagstudent.error
+})
+
+export default withRouter(connect(mapStateToProps, {
+  createMultipleStudentTag: createMultipleStudentTagAction
+})(TagModal))

services/oodikone2-frontend/src/components/StudyProgramme/Tags/index.jsx

@@ -1,13 +1,15 @@
 import React, { useState, useEffect } from 'react'
 import { withRouter } from 'react-router-dom'
 import { connect } from 'react-redux'
-import { Button, Input, List, Label, Icon, Header } from 'semantic-ui-react'
+import { Button, List, Label, Icon, Header, Confirm, Form } from 'semantic-ui-react'
 import { arrayOf, string, shape, func } from 'prop-types'
 
+import TagModal from '../TagModal'
 import { getTagsByStudytrackAction, createTagAction, deleteTagAction } from '../../../redux/tags'
 
 const Tags = ({ createTag, deleteTag, getTagsByStudytrack, tags, studyprogramme }) => {
   const [tagname, setTagname] = useState('')
+  const [confirm, setConfirm] = useState(null)
 
   useEffect(() => {
     getTagsByStudytrack(studyprogramme)
@@ -16,12 +18,13 @@ const Tags = ({ createTag, deleteTag, getTagsByStudytrack, tags, studyprogramme
   const handleDeleteTag = (event, tag) => {
     event.preventDefault()
     deleteTag(tag)
+    setConfirm(null)
   }
 
   const handleSubmit = (event) => {
     event.preventDefault()
     const newTag = {
-      tagname,
+      tagname: tagname.trim(),
       studytrack: studyprogramme
     }
     createTag(newTag)
@@ -36,16 +39,28 @@ const Tags = ({ createTag, deleteTag, getTagsByStudytrack, tags, studyprogramme
     <List.Item key={tag.tag_id}>
       <List.Content>
         <Label>
-          {tag.tagname} <Icon name="delete" link onClick={event => handleDeleteTag(event, tag)} />
+          {tag.tagname} <Icon name="delete" link onClick={() => setConfirm(tag.tag_id)} />
         </Label>
+        <Confirm
+          open={confirm === tag.tag_id}
+          onCancel={() => setConfirm(null)}
+          onConfirm={event => handleDeleteTag(event, tag)}
+          content={`Are you sure you want to delete tag "${tag.tagname}"? If you press confirm you will delete it from all students that have it. You and other users won't be able to use this tag again.`}
+          cancelButton="Cancel"
+          confirmButton="Confirm"
+        />
       </List.Content>
     </List.Item >
   ))
 
   return (
     <List>
-      <Input onChange={handleChange} value={tagname} />
-      <Button onClick={handleSubmit}>add new tag</Button>
+      <Form>
+        <Form.Group>
+          <Form.Input onChange={handleChange} value={tagname} action={<Button disabled={!tagname.trim() || tags.find(t => t.tagname === tagname.trim())} onClick={handleSubmit}>add new tag</Button>} />
+          <TagModal tags={tags} studytrack={studyprogramme} />
+        </Form.Group>
+      </Form>
       <Header size="medium">Study programme tags</Header>
       {rows}
     </List>
@@ -64,4 +79,8 @@ Tags.propTypes = {
   studyprogramme: string.isRequired
 }
 
-export default withRouter(connect(mapStateToProps, { createTag: createTagAction, deleteTag: deleteTagAction, getTagsByStudytrack: getTagsByStudytrackAction })(Tags))
+export default withRouter(connect(mapStateToProps, {
+  createTag: createTagAction,
+  deleteTag: deleteTagAction,
+  getTagsByStudytrack: getTagsByStudytrackAction
+})(Tags))

services/oodikone2-frontend/src/components/StudyProgramme/ThroughputTable/index.jsx

@@ -31,10 +31,15 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
   }
   const data = throughput && throughput.data ? throughput.data.filter(year => year.credits.length > 0) : []
   const genders = data.length > 0 ? uniq(flatten(data.map(year => Object.keys(year.genders)))) : []
-  const countries = data.length > 0 && throughput.totals.countries ?
-    uniq(flatten(data.map(year => Object.keys(year.countries)))).sort() : []
   const renderGenders = genders.length > 0
-  const renderCountries = countries.length > 0
+
+  const calculateTotalNationalities = () => (
+    data.length > 0 ?
+      Object.values(throughput.totals.nationalities).reduce((res, curr) => res + curr, 0) :
+      0
+  )
+
+  const renderRatioOfFinns = calculateTotalNationalities() > 0
   let thesisTypes = []
   if (thesis) {
     thesisTypes = thesis.map(t => t.thesisType)
@@ -43,6 +48,27 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
     callApi('/v2/studyprogrammes/throughput/recalculate', 'get', null, { code: studyprogramme })
       .then(() => { dispatchGetThroughput(studyprogramme) })
   }
+
+  const renderStudentsHeader = () => {
+    let colSpan = 1
+    let rowSpan = 1
+
+    if (renderGenders) colSpan += genders.length
+    if (renderRatioOfFinns) colSpan += 1
+    if (!renderGenders && !renderRatioOfFinns) rowSpan += 1
+
+    return <Table.HeaderCell colSpan={colSpan} rowSpan={rowSpan}>Students</Table.HeaderCell>
+  }
+
+  const ratioOfFinnsIn = (year) => {
+    const total = Object.values(year.nationalities).reduce((res, curr) => res + curr, 0)
+    return (
+      <Table.Cell>
+        {`${year.nationalities.Finland || 0} (${Math.floor((year.nationalities.Finland / total) * 100) || 0}%)`}
+      </Table.Cell>
+    )
+  }
+
   return (
     <React.Fragment>
       <Header>
@@ -76,17 +102,11 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
           <Table.Row>
             <Table.HeaderCell rowSpan="2">Year</Table.HeaderCell>
             {
-              renderGenders ?
-                <Table.HeaderCell colSpan={genders.length + 1}>Students</Table.HeaderCell> :
-                <Table.HeaderCell rowSpan="2">Students</Table.HeaderCell>
+              renderStudentsHeader()
             }
             <Table.HeaderCell colSpan={GRADUATED_FEATURE_TOGGLED_ON ? '3' : '1'}>Graduated</Table.HeaderCell>
 
             <Table.HeaderCell rowSpan="2">Transferred to this program</Table.HeaderCell>
-            {
-              renderCountries ?
-                <Table.HeaderCell colSpan={countries.length}>Countries</Table.HeaderCell> : null
-            }
             <Table.HeaderCell colSpan="5">Credits</Table.HeaderCell>
             {(thesisTypes.includes('BACHELOR') ||
               thesisTypes.includes('MASTER')) && (
@@ -97,8 +117,9 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
           </Table.Row>
 
           <Table.Row>
-            {renderGenders ? <Table.HeaderCell content="Total" /> : null}
+            {renderGenders || renderRatioOfFinns ? <Table.HeaderCell content="Total" /> : null}
             {genders.map(gender => <Table.HeaderCell key={gender} content={gender} />)}
+            {renderRatioOfFinns ? <Table.HeaderCell content="Finnish" /> : null}
             <Table.HeaderCell >Graduated overall</Table.HeaderCell>
             {GRADUATED_FEATURE_TOGGLED_ON &&
               <Fragment>
@@ -106,7 +127,6 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                 <Table.HeaderCell >Graduation median time</Table.HeaderCell>
               </Fragment>
             }
-            {renderCountries ? countries.map(country => <Table.HeaderCell key={country} content={country} />) : null}
             <Table.HeaderCell content="≥ 30" />
             <Table.HeaderCell content="≥ 60" />
             <Table.HeaderCell content="≥ 90" />
@@ -137,6 +157,7 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                     {`${year.genders[gender] || 0} (${Math.floor((year.genders[gender] / year.credits.length) * 100) || 0}%)`}
                   </Table.Cell>
                 ))}
+                { renderRatioOfFinns && ratioOfFinnsIn(year) }
                 <Table.Cell>{year.graduated}</Table.Cell>
                 {GRADUATED_FEATURE_TOGGLED_ON &&
                   <Fragment>
@@ -145,11 +166,6 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                   </Fragment>
                 }
                 <Table.Cell>{year.transferred}</Table.Cell>
-                {renderCountries ? countries.map(country => (
-                  <Table.Cell key={`${year.year} country:${country}`}>
-                    {year.countries[country] || 0}
-                  </Table.Cell>
-                )) : null}
                 {Object.keys(year.creditValues).map(creditKey => (
                   <Table.Cell key={`${year.year} credit:${creditKey}`}>{year.creditValues[creditKey]}
                   </Table.Cell>
@@ -173,6 +189,13 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                   {`${throughput.totals.genders[genderKey]} (${Math.floor((throughput.totals.genders[genderKey] / throughput.totals.students) * 100)}%)`}
                 </Table.HeaderCell>
               ))}
+              {
+                renderRatioOfFinns ?
+                  <Table.HeaderCell>
+                    {`${throughput.totals.nationalities.Finland || 0} (${Math.floor((throughput.totals.nationalities.Finland / calculateTotalNationalities()) * 100) || 0}%)`}
+                  </Table.HeaderCell> :
+                  null
+              }
               <Table.HeaderCell>{throughput.totals.graduated}</Table.HeaderCell>
               {GRADUATED_FEATURE_TOGGLED_ON &&
                 <Fragment>
@@ -181,11 +204,6 @@ const ThroughputTable = ({ history, throughput, thesis, loading, error, studypro
                 </Fragment>
               }
               <Table.HeaderCell>{throughput.totals.transferred}</Table.HeaderCell>
-              {renderCountries ? Object.keys(throughput.totals.countries).map(countryKey => (
-                <Table.HeaderCell key={`${countryKey}total`}>
-                  {throughput.totals.countries[countryKey]}
-                </Table.HeaderCell>
-              )) : null}
               {Object.keys(throughput.totals.credits).map(creditKey => (
                 <Table.HeaderCell key={`${creditKey}total`}>{throughput.totals.credits[creditKey]}
                 </Table.HeaderCell>

services/oodikone2-frontend/src/components/TagPopulation/index.jsx

@@ -5,20 +5,17 @@ import { Button, Dropdown, Checkbox, List } from 'semantic-ui-react'
 import { arrayOf, string, shape, func, bool } from 'prop-types'
 
 import {
-  createStudentTagAction,
-  getStudentTagsByStudytrackAction
+  createMultipleStudentTagAction
 } from '../../redux/tagstudent'
 
 const TagPopulation = ({
   allChecker,
   handleAllCheck,
   falsifyChecks,
-  createStudentTag,
+  createMultipleStudentTag,
   tags,
   checkedStudents,
-  getStudentTagsByStudytrack,
-  studytrack,
-  created }) => {
+  studytrack }) => {
   const [options, setOptions] = useState([])
   const [selectedValue, setSelected] = useState('')
 
@@ -27,32 +24,30 @@ const TagPopulation = ({
     setOptions(createdOptions)
   }, [])
 
-  useEffect(() => {
-    if (created) {
-      getStudentTagsByStudytrack(studytrack)
-    }
-  }, [created])
-
   const handleChange = (event, { value }) => {
     event.preventDefault()
     setSelected(value)
   }
 
-  const handleSubmit = (event) => {
+  const handleSubmit = async (event) => {
     event.preventDefault()
-    falsifyChecks()
+    const tagList = []
     checkedStudents.forEach((student) => {
       if (student.checked) {
         const tag = {
           tag_id: selectedValue,
           studentnumber: student.studentnumber
         }
-        createStudentTag(tag)
-        setSelected('')
+        tagList.push(tag)
       }
     })
+    setSelected('')
+    falsifyChecks()
+    await createMultipleStudentTag(tagList, studytrack)
   }
 
+  const checkCount = checkedStudents.reduce((n, student) => n + (student.checked), 0)
+
   return (
     <List horizontal>
       <List.Item>
@@ -71,18 +66,16 @@ const TagPopulation = ({
           value={selectedValue}
         />
       </List.Item>
-      <Button onClick={handleSubmit}>add tag to multiple students</Button>
+      <Button onClick={handleSubmit} disabled={selectedValue === '' || checkCount === 0}>add tag to multiple students</Button>
     </List>
   )
 }
 
 TagPopulation.propTypes = {
-  createStudentTag: func.isRequired,
-  getStudentTagsByStudytrack: func.isRequired,
+  createMultipleStudentTag: func.isRequired,
   checkedStudents: arrayOf(shape({ studentnumber: string, checked: bool })).isRequired,
   tags: arrayOf(shape({ tag_id: string, tagname: string, studytrack: string })).isRequired,
   studytrack: string.isRequired,
-  created: bool.isRequired,
   falsifyChecks: func.isRequired,
   allChecker: bool.isRequired,
   handleAllCheck: func.isRequired
@@ -92,4 +85,6 @@ const mapStateToProps = ({ tagstudent }) => ({
   created: tagstudent.created
 })
 
-export default withRouter(connect(mapStateToProps, { createStudentTag: createStudentTagAction, getStudentTagsByStudytrack: getStudentTagsByStudytrackAction })(TagPopulation))
+export default withRouter(connect(mapStateToProps, {
+  createMultipleStudentTag: createMultipleStudentTagAction
+})(TagPopulation))

services/oodikone2-frontend/src/components/TagStudent/index.jsx

@@ -6,14 +6,12 @@ import { arrayOf, string, shape, func, bool } from 'prop-types'
 
 import {
   createStudentTagAction,
-  deleteStudentTagAction,
-  getStudentTagsByStudytrackAction
+  deleteStudentTagAction
 } from '../../redux/tagstudent'
 
 const TagStudent = ({
   createStudentTag,
   deleteStudentTag,
-  getStudentTagsByStudytrack,
   studentnumber,
   studentstags,
   tags,
@@ -57,15 +55,13 @@ const TagStudent = ({
       tag_id: value,
       studentnumber
     }
-    await createStudentTag(tag)
-    getStudentTagsByStudytrack(studytrack)
+    await createStudentTag(tag, studytrack)
   }
 
   const deleteTag = async (event, tag) => {
     event.preventDefault()
     const removableTag = studentsTagIds.find(t => t.tag_id === tag.tag_id)
-    await deleteStudentTag(removableTag.id)
-    getStudentTagsByStudytrack(studytrack)
+    await deleteStudentTag(removableTag.tag_id, studentnumber, studytrack)
   }
 
   const studentsTags = allTags
@@ -107,7 +103,6 @@ const TagStudent = ({
 TagStudent.propTypes = {
   createStudentTag: func.isRequired,
   deleteStudentTag: func.isRequired,
-  getStudentTagsByStudytrack: func.isRequired,
   studentnumber: string.isRequired,
   studentstags: arrayOf(shape({ tag: shape({ tagname: string, tag_id: string }), id: string })).isRequired,
   tags: arrayOf(shape({ tag_id: string, tagname: string, studytrack: string })).isRequired,
@@ -123,6 +118,5 @@ const mapStateToProps = ({ tagstudent }) => ({
 
 export default withRouter(connect(mapStateToProps, {
   createStudentTag: createStudentTagAction,
-  deleteStudentTag: deleteStudentTagAction,
-  getStudentTagsByStudytrack: getStudentTagsByStudytrackAction
+  deleteStudentTag: deleteStudentTagAction
 })(TagStudent))

services/oodikone2-frontend/src/components/UserPage/AccessGroups.jsx

@@ -12,7 +12,7 @@ class AccessGroups extends Component {
   }
 
   componentDidMount() {
-    this.props.getAccessGroups()
+    if (this.props.groups.length === 0) this.props.getAccessGroups()
   }
 
   componentDidUpdate(prevState) {
@@ -88,7 +88,7 @@ AccessGroups.propTypes = {
 }
 
 const mapStateToProps = ({ accessGroups, users }, { user }) => {
-  const { data = [], pending } = accessGroups
+  const { data, pending } = accessGroups
   const { accessgroupError: saveError, accessgroupPending: savePending } = users
   const groups = data.map(group => ({
     key: group.id,

services/oodikone2-frontend/src/components/UserPage/index.jsx

@@ -6,7 +6,8 @@ import { sortBy } from 'lodash'
 import { withRouter } from 'react-router-dom'
 import { string, number, shape, bool, arrayOf, func, object } from 'prop-types'
 import { getTextIn, getRolesWithoutRefreshToken } from '../../common'
-import { removeUserUnits, getAccessGroups, setFaculties } from '../../redux/users'
+import { removeUserUnits, setFaculties } from '../../redux/users'
+import { getAccessGroups } from '../../redux/accessGroups'
 import { getFaculties } from '../../redux/faculties'
 import { getDegreesAndProgrammesUnfiltered } from '../../redux/populationDegreesAndProgrammesUnfiltered'
 import { login } from '../../redux/auth'
@@ -31,13 +32,13 @@ class UserPage extends Component {
   }
 
   async componentDidMount() {
-    const { associations, pending, getElementDetails } = this.props
-    getElementDetails()
+    const { associations, pending, getElementDetails, elementdetails, accessGroups, faculties } = this.props
+    if (elementdetails.length === 0) getElementDetails()
+    if (accessGroups.data.length === 0) await this.props.getAccessGroups()
+    if (faculties.length === 0) await this.props.getFaculties()
     if (Object.keys(associations).length === 0 && !pending) {
       this.props.getDegreesAndProgrammesUnfiltered()
-      await this.props.getAccessGroups()
     }
-    await this.props.getFaculties()
   }
 
   getDisabledUnits = (units, enabled) => {
@@ -94,7 +95,7 @@ class UserPage extends Component {
     elementdetailcodes.sort()
     return (
       <List divided>
-        {elementdetails && elementdetails.length > 0 && elementdetailcodes.map(({ elementDetailCode: code }) => {
+        {elementdetails.length > 0 && elementdetailcodes.map(({ elementDetailCode: code }) => {
           const element = elementdetails.find(e => e.code === code)
           return (
             <List.Item key={code}>
@@ -176,6 +177,7 @@ class UserPage extends Component {
                 <Dropdown
                   placeholder="Select faculties"
                   fluid
+                  selection
                   multiple
                   search
                   value={this.props.user.faculty.map(f => f.faculty_code)}
@@ -238,7 +240,7 @@ const mapStateToProps = state => ({
   associations: state.populationDegreesAndProgrammesUnfiltered.data,
   pending: !!state.populationDegreesAndProgrammesUnfiltered.pending,
   elementdetails: state.elementdetails.data,
-  accessGroups: state.users.accessGroupsData || [],
+  accessGroups: state.accessGroups,
   isAdmin: getRolesWithoutRefreshToken().includes('admin')
 })
 

services/oodikone2-frontend/src/components/WelcomePage/index.jsx

@@ -1,5 +1,6 @@
 import React from 'react'
 import { Container, Header, Image, Divider } from 'semantic-ui-react'
+import moment from 'moment'
 import { images } from '../../common'
 
 const WelcomePage = () => (
@@ -47,6 +48,7 @@ const WelcomePage = () => (
         Contact team Oodikone by email: grp-toska@helsinki.fi
       </p>
       <Divider section />
+      <p>Site updated at: {moment(process.env.BUILT_AT).toDate().toLocaleString()}</p>
     </Container>
     <Image src={images.toskaLogo} size="medium" centered style={{ bottom: 0 }} />
   </div>

services/oodikone2-frontend/src/constants/index.js

@@ -209,7 +209,5 @@ export const PRIORITYCODE_TEXTS = {
   30: 'Graduated'
 }
 
-export const ERROR_STATUSES_TO_CAPTURE = [
-  400,
-  500
+export const ERROR_STATUSES_NOT_TO_CAPTURE = [
 ]

services/oodikone2-frontend/src/populationFilters/index.js

@@ -271,22 +271,39 @@ export const tagFilter = (params) => {
 }
 
 export const gradeFilter = (params) => {
-  const { coursecode, grade, coursename } = params
+  const { coursecodes, grade, coursename } = params
   return ({
     id: uuidv4(),
     type: 'GradeFilter',
     params: {
-      coursecode,
+      coursecodes,
       grade,
       coursename
     },
     filter: (student) => {
-      const course = student.courses.find(c => c.course.code === coursecode)
+      const course = student.courses.find(c => coursecodes.includes(c.course.code))
       return Number(course.grade) === grade
     }
   })
 }
 
+export const courseCreditFilter = (params) => {
+  const { coursecodes, credits, coursename } = params
+  return ({
+    id: uuidv4(),
+    type: 'GradeFilter',
+    params: {
+      coursecodes,
+      credits,
+      coursename
+    },
+    filter: (student) => {
+      const course = student.courses.find(c => coursecodes.includes(c.course.code))
+      return Number(course.credits) === Number(credits)
+    }
+  })
+}
+
 export const programmeFilter = (params) => {
   const { programme, programmeName } = params
   return ({
@@ -336,8 +353,8 @@ export const getFilterFunction = (type, params, populationCourses) => {
     case 'CourseParticipation':
       return courseParticipation({
         field: params.field,
-        course: populationCourses.coursestatistics.filter(c =>
-          c.course.code === params.course.course.code)[0]
+        course: populationCourses.coursestatistics.find(c =>
+          c.course.code === params.course.course.code)
       })
     default:
       return typeList[type](params)

services/oodikone2-frontend/src/redux/coursePopulation.js

@@ -1,66 +0,0 @@
-import { callController } from '../apiConnection'
-
-export const getCoursePopulation = ({ coursecode, yearcode }) => {
-  const route = '/v3/populationstatisticsbycourse'
-  const prefix = 'GET_STUDENTS_OF_COURSE_'
-  const params = { coursecode, yearcode }
-  const query = { coursecode, yearcode }
-
-  return callController(route, prefix, null, 'get', query, params)
-}
-
-export const getCoursePopulationCourses = ({ coursecode, yearcode }) => {
-  const route = '/v2/populationstatistics/coursesbycoursecode'
-  const prefix = 'GET_COURSES_OF_COURSE_POP_'
-  const params = { coursecode, yearcode }
-  const query = { coursecode, yearcode }
-  const body = { coursecode, yearcode }
-
-  return callController(route, prefix, body, 'post', query, params)
-}
-
-const reducer = (state = { students: {}, courses: {}, pending: false, query: {} }, action) => {
-  switch (action.type) {
-    case 'GET_STUDENTS_OF_COURSE_ATTEMPT':
-      return {
-        ...state,
-        pending: true
-      }
-
-    case 'GET_STUDENTS_OF_COURSE_FAILURE':
-      return {
-        ...state,
-        pending: false
-      }
-
-    case 'GET_STUDENTS_OF_COURSE_SUCCESS':
-      return {
-        ...state,
-        pending: false,
-        students: action.response
-      }
-    case 'GET_COURSES_OF_COURSE_POP_ATTEMPT':
-      return {
-        ...state,
-        pending: true
-      }
-
-    case 'GET_COURSES_OF_COURSE_POP_FAILURE':
-      return {
-        ...state,
-        pending: false
-      }
-
-    case 'GET_COURSES_OF_COURSE_POP_SUCCESS':
-      return {
-        ...state,
-        pending: false,
-        courses: action.response,
-        query: action.query
-      }
-    default:
-      return state
-  }
-}
-
-export default reducer

services/oodikone2-frontend/src/redux/coursesSummaryForm.js

@@ -4,10 +4,10 @@ import { ALL } from '../selectors/courseStats'
 const prefix = 'COURSE_SUMMARY_FORM'
 
 export const fields = {
-  programme: 'programme'
+  programmes: 'programmes'
 }
 
-const reducer = formreducer(prefix, { [fields.programme]: ALL.value })
+const reducer = formreducer(prefix, { [fields.programmes]: [ALL.value] })
 
 export const { setValue, clear } = reducer
 

services/oodikone2-frontend/src/redux/coursestats.js

@@ -7,12 +7,10 @@ const { reset } = actions(prefix)
 
 export const clearCourseStats = reset
 
-export const getCourseStats = ({ fromYear, toYear, courseCodes, separate }, onProgress) => {
+export const getCourseStats = ({ courseCodes, separate }, onProgress) => {
   const route = '/v3/courseyearlystats'
   const params = {
     codes: courseCodes,
-    startyearcode: fromYear,
-    endyearcode: toYear,
     separate
   }
   return callController(route, prefix, [], 'get', params, params, onProgress)

services/oodikone2-frontend/src/redux/elementdetails.js

@@ -18,7 +18,7 @@ const reducer = (state = { data: [] }, action) => {
       return {
         pending: false,
         error: true,
-        data: action.response
+        data: []
       }
     case 'GET_ELEMENTDETAILS_SUCCESS':
       return {