Destructure props on function declaration

Bump axios from 0.21.1 to 0.21.2 in /services/frontend

.env.template

@@ -1,4 +1,3 @@
 # Populate and change this to .env
 IMPORTER_DB_API_TOKEN=
-IMPORTER_DB_API_URL=
 PATE_TOKEN=

.eslintrc.json

@@ -8,17 +8,10 @@
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "eslint:recommended",
-    "plugin:prettier/recommended",
-    "plugin:cypress/recommended"
-  ],
-  "plugins": [
-    "prettier",
-    "cypress"
-  ],
+  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "plugins": ["prettier", "cypress"],
   "rules": {
-    "no-console": "off",
+    "no-console": "error",
     "react/no-unescaped-entities": "off",
     "no-async-promise-executor": "off",
     "prettier/prettier": "error"

.github/workflows/release-oodikone.yaml

@@ -87,7 +87,7 @@ jobs:
           environment: production
           set_commits: 'skip'
           sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
+          url_prefix: '~/static/js'
           version: ${{ steps.extract_version.outputs.VERSION }}
 
   build_and_release_userservice:

.github/workflows/test-oodikone.yaml

@@ -545,7 +545,7 @@ jobs:
           environment: staging
           set_commits: 'skip'
           sourcemaps: './build/static/js'
-          url_prefix: '~/oodikone/static/js'
+          url_prefix: '~/static/js'
           version: ${{ github.sha }}
       - name: 'Push to docker hub'
         run: |

README.md

@@ -126,6 +126,10 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 You should always install the dependencies **inside** the container to have the application **inside** the container access them. This might be the case when someone else installs a new library and you only pull the changes in package.json. Use `docker exec -it <service> sh` to get inside the container and run `npm ci` to install modules.
 
+### I get the "You are not authorized to use Oodikone" -message when I try to work with anon-data
+
+Make sure you have your VPN on
+
 ### Everything is broken, can't get oodikone running, data is not there etc.
 
 Just do clean install by launching cli with `npm cli` and running option 1: _Set up oodikone from scratch_.
@@ -135,3 +139,7 @@ Just do clean install by launching cli with `npm cli` and running option 1: _Set
 Toska of course.
 
 University of Helsinki
+
+## Tietosuoja / data protection
+
+See [tietosuoja.md](tietosuoja.md).

cli.sh

@@ -44,20 +44,22 @@ will take care of setting up Oodikone for you. See README for more details."
   msg ""
 }
 
-# Define custom shell prompt for the interactive select loop
-PS3="Please enter your choice: "
+set_custom_select_prompt() {
+  PS3="Please enter your choice: "
 
-options=(
-  "Set up oodikone from scratch."
-  "Reset all real data."
-  "Reset sis-importer data."
-  "Quit."
-)
+  options=(
+    "Set up oodikone from scratch."
+    "Reset all real data."
+    "Reset single database."
+    "Quit."
+  )
+}
 
-# Run scripts before showing interactive prompt
+# Run scripts and set the prompt
 show_welcome
 init_dirs
 get_username
+set_custom_select_prompt
 
 while true; do
   select opt in "${options[@]}"; do
@@ -66,8 +68,10 @@ while true; do
         set_up_oodikone_from_scratch;;
       "Reset all real data.")
         reset_all_real_data;;
-      "Reset sis-importer data.")
-        reset_sis_importer_data;;
+      "Reset single database.")
+        reset_single_database
+        set_custom_select_prompt # Set prompt back to initial values
+        ;;
       "Quit.")
         break 2;;
       *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

cypress/integration/Filters.js

@@ -89,7 +89,7 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('ageFilter-header', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(43)
+      checkFilteringResult(42)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })
@@ -120,7 +120,8 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Admission type filter works', () => {
+  // Return when we have +2020 population with admission types in data
+  it.skip('Admission type filter works', () => {
     runTestStepWithPreAndPostParts('admissionTypeFilter-header', () => {
       cy.selectFromDropdown('admissionTypeFilter-dropdown', 0)
       checkFilteringResult(5)
@@ -189,7 +190,7 @@ describe('Course Statistics', () => {
     runTestStepWithPreAndPostParts('ageFilter-header', () => {
       cy.cs('ageFilter-min').type('20')
       cy.cs('ageFilter-max').type('40')
-      checkFilteringResult(33)
+      checkFilteringResult(32)
       cy.cs('ageFilter-min').find('input').clear()
       cy.cs('ageFilter-max').find('input').clear()
     })

cypress/integration/Population_statistics.js

@@ -1,5 +1,8 @@
 /// <reference types="Cypress" />
 
+const moment = require('moment')
+const _ = require('lodash')
+
 const setPopStatsUntil = (until, includeSettings = []) => {
   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
   includeSettings.forEach(setting => {
@@ -13,6 +16,8 @@ const setPopStatsUntil = (until, includeSettings = []) => {
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
+  const pathToCSMaster2019 =
+    '/populations?months=27&semesters=FALL&semesters=SPRING&studyRights=%7B%22programme%22%3A%22MH50_009%22%7D&tag&year=2019'
   describe('when using basic user', () => {
     beforeEach(() => {
       cy.init('/populations')
@@ -59,6 +64,7 @@ describe('Population Statistics tests', () => {
 
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
+      cy.cs('filtered-students')
       setPopStatsUntil('toukokuu 2020')
 
       cy.get('.card').within(() => {
@@ -69,7 +75,8 @@ describe('Population Statistics tests', () => {
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
-      cy.contains('Courses of population').click({ force: true })
+      cy.cs('filtered-students')
+      cy.contains('Courses of population').click()
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -123,18 +130,62 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit accumulation (for 170 students)')
     })
 
-    it('Credit Statistics, Statistics pane works', () => {
+    it('Credit Statistics, Credits Gained tab works', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
+
+      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
+
+      const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
+      const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
+
+      cy.get('.credits-gained-table').should('contain', '(n=149)')
+
+      for (const [start, end] of ranges) {
+        let value = 'credits'
+
+        if (start !== null) {
+          value = `${start} ≤ ${value}`
+        }
+
+        if (end !== null) {
+          value = `${value} < ${end}`
+        }
+
+        cy.get('.credits-gained-table').should('contain', value)
+      }
+
+      cy.get("[data-cy='credits-gained-table-body'] td:nth-child(3)").then($els => {
+        const sum = [...$els].map($el => parseInt($el.innerText)).reduce((a, b) => a + b, 0)
+
+        expect(sum).to.equal(149)
+      })
+
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
+    })
+
+    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+      cy.visit(pathToCSMaster2019)
+      cy.cs('filtered-students')
+      cy.contains('Credit statistics')
+      cy.get("[data-cy='credit-statistics']").click()
+      cy.get('.credits-gained-divider').click()
+      cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Muu']").should('exist')
+      cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('exist')
+    })
+
+    it('Credit Statistics, Statistics pane works', () => {
+      cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('Credit statistics').click({ force: true })
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
       cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 149 Students')
       cy.get("[data-cy='credit-stats-mean']").should('contain', '123.89')
       cy.get("[data-cy='credit-stats-stdev']").should('contain', '61.32')
       cy.get("[data-cy='credit-stats-min']").should('contain', '11')
-      cy.get("[data-cy='credit-stats-q1']").should('contain', '70')
-      cy.get("[data-cy='credit-stats-q2']").should('contain', '129')
-      cy.get("[data-cy='credit-stats-q3']").should('contain', '168')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
     })
   })

cypress/integration/Studyprogramme_overview.js

@@ -79,7 +79,7 @@ describe('Studyprogramme overview', () => {
     let originalProductivityCalculatedText
     const doRecalculation = () => {
       // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(2000)
+      cy.wait(5000)
       cy.get('.attached').contains('Admin').click()
       cy.contains('recalculate productivity').click()
       cy.contains('recalculate throughput').click()

cypress/integration/Users.js

@@ -4,16 +4,16 @@ describe('Users tests', () => {
   describe('Using user with just grp-oodikone-user, no other rights', () => {
     it('shows only frontpage, trends feedback', () => {
       cy.init('', 'norights') // login with norights user
-      cy.get('.navBar').contains('Trends').should('exist')
-      cy.get('.navBar').contains('Give feedback').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Give feedback').should('exist')
 
-      cy.get('.navBar').contains('Study programme').should('not.exist')
-      cy.get('.navBar').contains('Student statistics').should('not.exist')
-      cy.get('.navBar').contains('Course statistics').should('not.exist')
-      cy.get('.navBar').contains('Teachers').should('not.exist')
-      cy.get('.navBar').contains('Users').should('not.exist')
-      cy.get('.navBar').contains('Faculty').should('not.exist')
-      cy.get('.navBar').contains('Updater').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Teachers').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Faculty').should('not.exist')
+      cy.get('[data-cy="navBar"]').contains('Updater').should('not.exist')
     })
   })
 
@@ -29,12 +29,12 @@ describe('Users tests', () => {
     })
 
     it('should see more stuff than others', () => {
-      cy.get('.navBar').contains('Study programme').should('exist')
-      cy.get('.navBar').contains('Student statistics').should('exist')
-      cy.get('.navBar').contains('Course statistics').should('exist')
-      // TODO: admin should probably see this as well, since can access this page: cy.get('.navBar').contains('Teachers').should('exist')
-      cy.get('.navBar').contains('Users').should('exist')
-      cy.get('.navBar').contains('Trends').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Study programme').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Student statistics').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Course statistics').should('exist')
+      // TODO: admin should probably see this as well, since can access this page: cy.get('[data-cy="navBar"]').contains('Teachers').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Users').should('exist')
+      cy.get('[data-cy="navBar"]').contains('Trends').should('exist')
     })
 
     it("mocking normal user shows only the mocked user's programmes", () => {

package.json

@@ -6,6 +6,7 @@
     "concurrently": "concurrently",
     "eslint": "eslint",
     "stylelint": "stylelint",
+    "prettier": "prettier",
     "cli": "./cli.sh",
     "oodikone": "./run.sh oodikone anon up --detach",
     "oodikone:real": "./run.sh oodikone real up --detach",
@@ -48,5 +49,9 @@
     "stylelint-config-prettier": "^8.0.2",
     "stylelint-config-standard": "^22.0.0",
     "stylelint-prettier": "^1.2.0"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21",
+    "moment": "^2.29.1"
   }
 }

run.sh

@@ -64,8 +64,7 @@ parse_profiles() {
   return 0
 }
 
-# Set which docker-compose files to use based on version. In ci, profiles are also
-# emptied, since they're passed from github actions.
+# Set which docker-compose files to use based on version.
 parse_env() {
   env=""
   if [[ "$version" == "real" ]]; then

scripts/functions_for_setup.sh

@@ -20,10 +20,10 @@ USER_DB_NAME="user-db"
 DATABASES=("$KONE_DB_NAME" "$SIS_DB_NAME" "$SIS_IMPORTER_DB_NAME" "$USER_DB_NAME")
 
 ## Urls should be in same order as databases as both are iterated through by indexes.
-KONE_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-kone-pg.sqz"
+KONE_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-kone-db.sqz"
 SIS_DB_REAL_DUMP_URL="svm-96.cs.helsinki.fi:/home/updater_user/backups/latest-sis.sqz"
-SIS_IMPORTER_DB_REAL_DUMP_URL="importer:/home/importer_user/importer-db/backup/importer-db.sqz"
-USER_DB_REAL_DUMP_URL="oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/latest-user-pg.sqz"
+SIS_IMPORTER_DB_REAL_DUMP_URL="svm-92.cs.helsinki.fi:/home/importer_user/importer-db/backup/importer-db.sqz"
+USER_DB_REAL_DUMP_URL="svm-101.cs.helsinki.fi:/home/oodikone_user/backups/latest-user-db.sqz"
 REAL_DUMP_URLS=("$KONE_DB_REAL_DUMP_URL" "$SIS_DB_REAL_DUMP_URL" "$SIS_IMPORTER_DB_REAL_DUMP_URL" "$USER_DB_REAL_DUMP_URL")
 
 # Source utility functions
@@ -110,12 +110,54 @@ reset_all_real_data() {
   reset_databases "${DATABASES[@]}"
 }
 
-reset_sis_importer_data() {
-  infomsg "Downloading sis-importer-db dump"
-  local database=$SIS_IMPORTER_DB_NAME
-  local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
-  download_real_dump $database $url
-  reset_databases $database
+reset_single_database() {
+  # Define custom shell prompt for the nested interactive select loop
+  PS3="Please enter which database to reset: "
+
+  options=(
+    "kone-db"
+    "sis-db"
+    "sis-importer-db"
+    "user-db"
+    "Return to main menu."
+  )
+
+  local database=""
+  local url=""
+
+  while true; do
+    select opt in "${options[@]}"; do
+      case $opt in
+        "kone-db")
+          local database=$KONE_DB_NAME
+          local url=$KONE_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-db")
+          local database=$SIS_DB_NAME
+          local url=$SIS_DB_REAL_DUMP_URL
+          break 2;;
+        "sis-importer-db")
+          local database=$SIS_IMPORTER_DB_NAME
+          local url=$SIS_IMPORTER_DB_REAL_DUMP_URL
+          break 2;;
+        "user-db")
+          local database=$USER_DB_NAME
+          local url=$USER_DB_REAL_DUMP_URL
+          break 2;;
+        "Return to main menu.")
+          break 2;;
+        *) msg "${RED}Invalid option:${NOFORMAT} $REPLY
+  ";;
+      esac
+      break
+    done
+  done
+
+  if [ -n "$database" ]; then
+    infomsg "Downloading $database dump"
+    download_real_dump $database $url
+    reset_databases $database
+  fi
 }
 
 set_up_oodikone_from_scratch() {
@@ -152,10 +194,10 @@ linting and formatting"
   npm run prepare
 
   infomsg "Pulling images"
-  "$PROJECT_ROOT"/run.sh oodikone anon pull
+  "$PROJECT_ROOT"/run.sh both anon pull
 
   infomsg "Building images"
-  "$PROJECT_ROOT"/run.sh oodikone anon build
+  "$PROJECT_ROOT"/run.sh both anon build
 
   successmsg "Setup ready, oodikone can be started! See README for more info."
 }

services/backend/index.js

@@ -1,5 +1,6 @@
 require('./src/app')
+const logger = require('./src/util/logger')
 
 process.on('unhandledRejection', reason => {
-  console.log(reason)
+  logger.error({ message: 'unhandledRejection: ', meta: reason })
 })

services/backend/package.json

@@ -15,11 +15,12 @@
     "@sentry/tracing": "^6.11.0",
     "async": "^3.2.1",
     "async-lock": "^1.3.0",
-    "axios": "^0.21.1",
+    "axios": "^0.21.2",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
     "express": "^4.17.1",
+    "express-async-errors": "^3.1.1",
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
@@ -32,9 +33,7 @@
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0",
-    "winston-transport": "^4.4.0"
+    "winston": "^3.3.3"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/app.js

@@ -1,3 +1,4 @@
+require('express-async-errors')
 const express = require('express')
 const cors = require('cors')
 const Sentry = require('@sentry/node')
@@ -6,17 +7,19 @@ const routes = require('./routes')
 const { startCron } = require('./events')
 const { initializeDatabaseConnection, dbConnections } = require('./database/connection')
 const initializeSentry = require('./util/sentry')
+const errorMiddleware = require('./middleware/errormiddleware')
+const logger = require('./util/logger')
 
 initializeDatabaseConnection()
   .then(() => {
     dbConnections.connect()
 
     dbConnections.on('connect', () => {
-      console.log('Connected to sis db successfully')
+      logger.info('Connected to sis db successfully')
     })
 
     dbConnections.on('error', () => {
-      console.log('Failed connecting to sis db')
+      logger.error('Failed to connect to sis db!')
     })
 
     const app = express()
@@ -31,21 +34,23 @@ initializeDatabaseConnection()
 
     routes(app, baseUrl)
 
-    app.get('*', async (req, res) => {
+    app.get('*', async (_, res) => {
       const results = { error: 'unknown endpoint' }
       res.status(404).json(results)
     })
 
     app.use(Sentry.Handlers.errorHandler())
 
-    const server = app.listen(backendPort, () => console.log(`Backend listening on port ${backendPort}!`))
+    app.use(errorMiddleware)
+
+    const server = app.listen(backendPort, () => logger.info(`Backend listening on port ${backendPort}!`))
     process.on('SIGTERM', () => {
       server.close(() => {
-        console.log('Process terminated')
+        logger.info('Process terminated')
       })
     })
   })
   .catch(e => {
     process.exitCode = 1
-    console.log(e)
+    logger.error(e)
   })

services/backend/src/conf-backend.js

@@ -18,7 +18,6 @@ const pateToken = process.env.PATE_API_TOKEN || ''
 
 // Importer client
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
-const importerUrl = process.env.IMPORTER_DB_API_URL || ''
 
 // Networking: Urls & ports
 const baseUrl = isDev ? '/api' : ''
@@ -89,5 +88,4 @@ module.exports = {
   frontUrl,
   backendPort,
   importerToken,
-  importerUrl,
 }

services/backend/src/database/connection.js

@@ -3,6 +3,8 @@ const EventEmitter = require('events')
 const Umzug = require('umzug')
 const conf = require('../conf-backend')
 const { SIS_DB_URL } = process.env
+const logger = require('../util/logger')
+
 class DbConnection extends EventEmitter {
   constructor() {
     super()
@@ -31,8 +33,7 @@ class DbConnection extends EventEmitter {
         this.emit('error', e)
         return
       }
-      console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
-      console.log(e)
+      logger.error({ message: `Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`, meta: e })
       setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
     }
   }
@@ -54,10 +55,9 @@ const initializeDatabaseConnection = async () => {
       break
     } catch (e) {
       if (i === waitSeconds) {
-        console.log(`Could not connect to database in ${waitSeconds} seconds`)
+        logger.error(`Could not connect to database in ${waitSeconds} seconds`)
         throw e
       }
-      console.log('.')
       await sleep(1000)
     }
   }
@@ -69,7 +69,6 @@ const initializeDatabaseConnection = async () => {
         tableName: 'migrations',
         schema: conf.DB_SCHEMA_KONE,
       },
-      logging: console.log,
       migrations: {
         params: [sequelizeKone.getQueryInterface(), Sequelize],
         path: `${process.cwd()}/src/database/migrations_kone`,
@@ -78,27 +77,16 @@ const initializeDatabaseConnection = async () => {
       },
     })
     const migrations = await migrator.up()
-
-    console.log('Kone Migrations up to date', migrations)
+    logger.info({ message: 'Kone Migrations up to date', meta: migrations })
   } catch (e) {
-    console.log('Kone Migration error')
+    logger.error({ message: 'Kone Migration error: ', meta: e })
     throw e
   }
 }
 
-const forceSyncDatabase = async () => {
-  try {
-    await sequelizeKone.getQueryInterface().createSchema(conf.DB_SCHEMA_KONE)
-  } catch (e) {
-    //console.log(e)
-  }
-  await sequelizeKone.sync({ force: true })
-}
-
 const dbConnections = new DbConnection()
 module.exports = {
   dbConnections,
   sequelizeKone,
   initializeDatabaseConnection,
-  forceSyncDatabase,
 }

services/backend/src/database/migrations_kone/20211029_00_add_study_guidance_group_tags.js

@@ -0,0 +1,34 @@
+const { BIGINT, STRING, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable(
+      'study_guidance_group_tags',
+      {
+        id: {
+          primaryKey: true,
+          type: BIGINT,
+          autoIncrement: true,
+        },
+        study_guidance_group_id: {
+          type: STRING,
+          unique: true,
+        },
+        study_programme: {
+          type: STRING,
+        },
+        year: {
+          type: STRING,
+        },
+        created_at: {
+          type: DATE,
+        },
+        updated_at: {
+          type: DATE,
+        },
+      },
+      { underscored: true, timestamps: true }
+    )
+  },
+  down: async () => {},
+}

services/backend/src/events.js

@@ -9,209 +9,168 @@ const {
   getStartYears,
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
-const { getAllProgrammes, nonGraduatedStudentsOfElementDetail } = require('./services/studyrights')
+const { getAllProgrammes } = require('./services/studyrights')
 const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const {
-  setProductivity,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-  patchNonGraduatedStudents,
-} = require('./services/analyticsService')
-const { isNewHYStudyProgramme } = require('./util')
+const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
+const { getCurrentSemester } = require('./services/semesters')
+const logger = require('./util/logger')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
 const refreshStudyrightAssociations = async () => {
-  try {
-    console.log('Refreshing studyright associations...')
-    await refreshAssociationsInRedis()
-  } catch (e) {
-    console.error(e)
-  }
+  await refreshAssociationsInRedis()
+  logger.info('Studyright associations refreshed!')
 }
 
 const refreshOverview = async () => {
-  try {
-    console.log('Refreshing throughput and productivity for programmes...')
-    const codes = (await getAllProgrammes()).map(p => p.code)
-    let ready = 0
-    for (const code of codes) {
-      let programmeStatsSince = new Date('2017-07-31')
-      if (code.includes('MH') || code.includes('KH')) {
-        programmeStatsSince = new Date('2017-07-31')
-      } else {
-        programmeStatsSince = new Date('2000-07-31')
-      }
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-        const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-        await setThroughput(data)
-      } catch (e) {
-        try {
-          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-      }
-      try {
-        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-        const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-        await setProductivity(data)
-      } catch (e) {
-        try {
-          await patchProductivity({
-            [code]: { status: 'RECALCULATION ERRORED' },
-          })
-        } catch (e) {
-          console.error(e)
-        }
-        console.error(e)
-        console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-      }
-      ready += 1
-      console.log(`${ready}/${codes.length} programmes done`)
+  logger.info('Refreshing throughput and productivity for programmes...')
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
+  let ready = 0
+  for (const code of codes) {
+    let programmeStatsSince = new Date('2017-07-31')
+    if (code.includes('MH') || code.includes('KH')) {
+      programmeStatsSince = new Date('2017-07-31')
+    } else {
+      programmeStatsSince = new Date('2000-07-31')
     }
-  } catch (e) {
-    console.error(e)
+    try {
+      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
+      await setThroughput(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
+      try {
+        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
+      } catch (e) {
+        logger.error({ message: 'failed to update throughtput status to error', meta: e })
+      }
+    }
+    try {
+      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
+      await setProductivity(data)
+    } catch (e) {
+      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
+      try {
+        await patchProductivity({
+          [code]: { status: 'RECALCULATION ERRORED' },
+        })
+      } catch (e) {
+        logger.error({ message: 'failed to update productivity status to error', meta: e })
+      }
+    }
+    ready += 1
+    logger.info(`${ready}/${codes.length} programmes done`)
   }
+  logger.info('Throughput and productivity for programmes refreshed')
 }
 
 const refreshTeacherLeaderboard = async () => {
-  try {
-    const startyearcode = new Date().getFullYear() - 1950
-    const endyearcode = startyearcode + 1
-    console.log('Refreshing teacher leaderboard...')
-    await findAndSaveTeachers(startyearcode, endyearcode)
-  } catch (e) {
-    console.log(e)
-  }
-}
-
-const refreshNonGraduatedStudentsOfOldProgrammes = async () => {
-  try {
-    const oldProgrammeCodes = (await getAllProgrammes()).map(p => p.code).filter(c => !isNewHYStudyProgramme(c))
-    let i = 0
-    console.log('Refreshing non-graduated students of old programmes...')
-    await Promise.all(
-      oldProgrammeCodes.map(
-        c =>
-          new Promise(async res => {
-            try {
-              const [nonGraduatedStudents, studentnumbers] = await nonGraduatedStudentsOfElementDetail(c)
-              await patchNonGraduatedStudents({ [c]: { formattedData: nonGraduatedStudents, studentnumbers } })
-              console.log(`${++i}/${oldProgrammeCodes.length}`)
-            } catch (e) {
-              console.log(`Failed refreshing non-graduated students of programme ${c}!`)
-            }
-            res()
-          })
-      )
-    )
-  } catch (e) {
-    console.log(e)
-  }
+  // refresh this and previous year
+  const currentSemestersYearCode = (await getCurrentSemester()).getDataValue('yearcode')
+  await findAndSaveTeachers(currentSemestersYearCode - 1, currentSemestersYearCode)
 }
 
 const refreshProtoCtoRedis = async () => {
-  try {
-    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
-    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
-    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
-    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
-    console.log('Refreshing CDS ProtoC')
-    await refreshProtoC(defaultQuery)
-    await refreshProtoC(onlyOld)
-    await refreshProtoC(onlyEnr)
-    await refreshProtoC(bothToggles)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS ProtoC')
+  const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
+  const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
+  const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
+  const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
+  await refreshProtoC(defaultQuery)
+  await refreshProtoC(onlyOld)
+  await refreshProtoC(onlyEnr)
+  await refreshProtoC(bothToggles)
+  logger.info('Refreshing CDS ProtoC doned')
 }
 
 const refreshStatusToRedis = async () => {
-  try {
-    const unixMillis = moment().valueOf()
-    const date = new Date(Number(unixMillis))
+  const unixMillis = moment().valueOf()
+  const date = new Date(Number(unixMillis))
 
-    date.setHours(23, 59, 59, 999)
-    const showByYearOff = 'false'
-    const showByYear = 'true'
-    console.log('Refreshing CDS Status')
-    await refreshStatus(date.getTime(), showByYearOff)
-    await refreshStatus(date.getTime(), showByYear)
+  date.setHours(23, 59, 59, 999)
+  const showByYearOff = 'false'
+  const showByYear = 'true'
+  logger.info('Refreshing CDS Status')
+  await refreshStatus(date.getTime(), showByYearOff)
+  await refreshStatus(date.getTime(), showByYear)
+  logger.info('Refreshing CDS Status doned')
 
-    console.log('Refreshing CDS Graduated')
-    await refreshStatusGraduated(date.getTime(), showByYearOff)
-    await refreshStatusGraduated(date.getTime(), showByYear)
-  } catch (e) {
-    console.log(e)
-  }
+  logger.info('Refreshing CDS Graduated')
+  await refreshStatusGraduated(date.getTime(), showByYearOff)
+  await refreshStatusGraduated(date.getTime(), showByYear)
+  logger.info('Refreshing CDS graduated doned')
 }
 
 const refreshUberToRedis = async () => {
-  try {
-    const years = await getStartYears()
-    const mappedYears = years.map(({ studystartdate }) => studystartdate)
-    mappedYears.forEach(async year => {
-      console.log('Refreshing CDS Uber data for date', year)
-      const defaultQuery = { include_old_attainments: 'false', start_date: year }
-      const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+  const years = (await getStartYears()).map(({ studystartdate }) => studystartdate)
+  for (const year of years) {
+    const formattedYear = new Date(year).getFullYear()
+    logger.info(`Refreshing CDS uber year ${formattedYear}`)
+    const defaultQuery = { include_old_attainments: 'false', start_date: year }
+    const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+    try {
       await refreshUber(defaultQuery)
       await refreshUber(oldAttainmentsQuery)
-    })
-  } catch (e) {
-    console.log(e)
+      logger.info(`Refreshing CDS uber year ${formattedYear} doned`)
+    } catch (e) {
+      logger.error({ message: `Error when refreshing CDS uber year ${formattedYear}`, meta: e })
+    }
   }
+  logger.info(`Refreshing CDS Uber data doned`)
 }
 
 const refreshProtoCProgrammeToRedis = async () => {
-  try {
-    const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
-    codes.forEach(async code => {
-      console.log('refreshing code', code)
-      const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
-      const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
-      const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
-      const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
-      console.log('Refreshing CDS ProtoCProgramme for code ', code)
-      await refreshProtoCProgramme(defaultQuery)
-      await refreshProtoCProgramme(onlyOld)
-      await refreshProtoCProgramme(onlyEnr)
-      await refreshProtoCProgramme(bothToggles)
-    })
-  } catch (e) {
-    console.log(e)
+  const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
+  for (const code of codes) {
+    logger.info(`Refreshing ProtoCProgramme code ${code}`)
+    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
+    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
+    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
+    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
+    const queries = [defaultQuery, onlyOld, onlyEnr, bothToggles]
+    for (const query of queries) {
+      try {
+        await refreshProtoCProgramme(query)
+      } catch (e) {
+        logger.error({
+          message: `Error when refreshing ProtocProgramme code ${code}, query ${Object.keys({ query })[0]}`,
+          meta: e,
+        }) // object.keys is hax to print query variable name
+      }
+    }
+    logger.info(`Refreshing ProtoCProgramme code ${code} doned`)
   }
+  logger.info(`Refreshing protocprogramme doned`)
 }
 
 const refreshStatistics = async () => {
-  console.log('Starting refreshing statistics')
-  await refreshStudyrightAssociations()
-  await refreshOverview()
-  await refreshNonGraduatedStudentsOfOldProgrammes()
-  await refreshTeacherLeaderboard()
-  console.log('Statistics refreshed!')
+  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  logger.info('Refreshing statistics')
+  for (const func of statfuncs) {
+    await func()
+  }
+  logger.info('Statistics refreshed!')
 }
 
 const refreshTrends = async () => {
-  console.log('Starting refreshing trends')
-  await refreshProtoCtoRedis()
-  await refreshStatusToRedis()
-  await refreshUberToRedis()
-  await refreshProtoCProgrammeToRedis()
-  console.log('Trends refreshed!')
+  const trendfuncs = [refreshProtoCtoRedis, refreshStatusToRedis, refreshUberToRedis, refreshProtoCProgrammeToRedis]
+  logger.info('Refreshing trends')
+  for (const func of trendfuncs) {
+    await func()
+  }
+  logger.info('Trends refreshed!')
 }
 
 const startCron = () => {
   if (isProduction) {
-    schedule('0 6 * * *', async () => {
-      await refreshStatistics()
-      await refreshTrends()
+    // refresh 3am every day
+    schedule('0 3 * * *', async () => {
+      for (const func of [refreshStatistics, refreshTrends]) {
+        await func()
+      }
     })
   }
 }
@@ -219,4 +178,5 @@ const startCron = () => {
 module.exports = {
   startCron,
   refreshStatistics,
+  refreshTrends,
 }

services/backend/src/middleware/auth.js

@@ -5,7 +5,7 @@ const { getUserDataFor } = require('../services/userService')
 const { ACCESS_TOKEN_HEADER_KEY } = require('../conf-backend')
 const { hasRequiredGroup, parseHyGroups } = require('../util/utils')
 
-const TOKEN_VERSION = 1 // When token structure changes, increment in userservice, backend and frontend
+const TOKEN_VERSION = 1.1 // When token structure changes, increment in userservice, backend and frontend
 
 const checkAuth = async (req, res, next) => {
   const token = req.headers[ACCESS_TOKEN_HEADER_KEY]

services/backend/src/middleware/errormiddleware.js

@@ -0,0 +1,19 @@
+const Sentry = require('@sentry/node')
+const { ApplicationError } = require('../util/customErrors')
+const logger = require('../util/logger')
+
+const errorHandler = (error, req, res, next) => {
+  logger.error(`${error.message} ${error.name} ${error.stack}`)
+
+  Sentry.captureException(error)
+
+  if (res.headersSent) {
+    return next(error)
+  }
+
+  const normalizedError = error instanceof ApplicationError ? error : new ApplicationError(error.message)
+
+  return res.status(normalizedError.status).json(normalizedError)
+}
+
+module.exports = errorHandler

services/backend/src/models/index.js

@@ -30,6 +30,9 @@ Student.hasMany(SemesterEnrollment, { foreignKey: 'studentnumber', sourceKey: 's
 SemesterEnrollment.belongsTo(Semester, { foreignKey: 'semestercomposite', targetKey: 'composite' })
 Semester.hasMany(SemesterEnrollment, { foreignKey: 'semestercomposite', sourceKey: 'composite' })
 
+ProgrammeModule.belongsTo(Organization, { foreignKey: 'organization_id' })
+Organization.hasMany(ProgrammeModule, { foreignKey: 'organization_id' })
+
 Credit.notUnnecessary = credit => {
   return credit.credits > 0 && credit.credits <= 12
 }
@@ -96,6 +99,8 @@ ProgrammeModule.belongsToMany(ProgrammeModule, {
   foreignKey: 'parent_id',
 })
 
+Organization.hasMany(Organization, { foreignKey: 'parent_id', as: 'children' })
+
 module.exports = {
   Organization,
   Course,

services/backend/src/models/models_kone.js

@@ -135,6 +135,28 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
   },
 })
 
+const StudyGuidanceGroupTag = sequelizeKone.define(
+  'study_guidance_group_tags',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    studyGuidanceGroupId: {
+      type: Sequelize.STRING,
+      unique: true,
+    },
+    studyProgramme: {
+      type: Sequelize.STRING,
+    },
+    year: {
+      type: Sequelize.STRING,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 MandatoryCourse.belongsTo(MandatoryCourseLabels, {
   foreignKey: 'label',
   sourceKey: 'id',
@@ -157,4 +179,5 @@ module.exports = {
   CustomPopulationSearch,
   sequelizeKone,
   ExcludedCourse,
+  StudyGuidanceGroupTag,
 }

services/backend/src/models/organization.js

@@ -19,6 +19,9 @@ Organization.init(
     name: {
       type: JSONB,
     },
+    parent_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/programmeModule.js

@@ -18,6 +18,16 @@ ProgrammeModule.init(
     type: {
       type: STRING,
     },
+    studyLevel: {
+      type: STRING,
+    },
+    organization_id: {
+      type: STRING,
+      references: {
+        model: 'organization',
+        key: 'id',
+      },
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/routes.js

@@ -57,7 +57,7 @@ module.exports = (app, url) => {
   app.use(`${url}/faculties`, faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
-  app.use(`${url}/users`, auth.roles(['users']), users)
+  app.use(`${url}/users`, auth.roles(['admin']), users)
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/mandatory_courses`, mandatoryCourses)
   app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)

services/backend/src/routes/courses.js

@@ -2,7 +2,6 @@ const router = require('express').Router()
 const crypto = require('crypto')
 const Course = require('../services/courses')
 const { validateParamLength } = require('../util')
-const logger = require('../util/logger')
 
 router.get('/v2/coursesmulti', async (req, res) => {
   let results = { courses: [] }
@@ -17,35 +16,29 @@ router.get('/v2/coursesmulti', async (req, res) => {
 })
 
 router.get('/v3/courseyearlystats', async (req, res) => {
-  try {
-    const { rights, roles } = req
+  const { rights, roles } = req
 
-    const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
+  const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
 
-    // If user has rights to see at least one programme, then they are allowed
-    // to see all of them
-    if (!allowedRoles && rights.length < 1) {
-      return res.status(403).json({ error: 'No programmes so no access to course stats' })
-    }
+  // If user has rights to see at least one programme, then they are allowed
+  // to see all of them
+  if (!allowedRoles && rights.length < 1) {
+    return res.status(403).json({ error: 'No programmes so no access to course stats' })
+  }
 
-    const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
+  const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
 
-    const separate = !sep ? false : JSON.parse(sep)
-    const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
-    if (!codes) {
-      res.status(422).send('Missing required query parameters')
-    } else {
-      // Studentnumbers should be obfuscated to all other users except admins
-      // and users with rights to any specific study programmes
-      const anonymize = allowedRoles !== 'admin' && rights.length < 1
-      const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
-      const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
-      res.json(results)
-    }
-  } catch (e) {
-    logger.error(e.message)
-    console.log(e)
-    res.status(500).send('Something went wrong with handling the request.')
+  const separate = !sep ? false : JSON.parse(sep)
+  const unifyOpenUniCourses = !unify ? false : JSON.parse(unify)
+  if (!codes) {
+    res.status(422).send('Missing required query parameters')
+  } else {
+    // Studentnumbers should be obfuscated to all other users except admins
+    // and users with rights to any specific study programmes
+    const anonymize = allowedRoles !== 'admin' && rights.length < 1
+    const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
+    const results = await Course.courseYearlyStats(codes, separate, unifyOpenUniCourses, anonymizationSalt)
+    res.json(results)
   }
 })
 

services/backend/src/routes/customPopulationSearch.js

@@ -7,76 +7,56 @@ const {
 } = require('../services/customPopulationSearch')
 
 router.get('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-    } = req
-    const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
-    res.json(customPopulationSearches)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const {
+    decodedToken: { id },
+  } = req
+  const customPopulationSearches = await getCustomPopulationSearchesByUser(id)
+  res.json(customPopulationSearches)
 })
 
 router.post('/', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id },
-      body: { name, students },
-    } = req
+  const {
+    decodedToken: { id },
+    body: { name, students },
+  } = req
 
-    if (!name) return res.status(400).json({ error: 'name missing' })
-    if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!name) return res.status(400).json({ error: 'name missing' })
+  if (students && !Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
 
-    const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
-    res.json(customPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  const customPopulationSearch = await createCustomPopulationSearch(name, id, students || [])
+  res.json(customPopulationSearch)
 })
 
 router.put('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-      body: { students },
-    } = req
-    const { id } = req.params
+  const {
+    decodedToken: { id: userId },
+    body: { students },
+  } = req
+  const { id } = req.params
 
-    if (!id) return res.status(400).json({ error: 'id missing' })
-    if (!students) return res.status(400).json({ error: 'students missing' })
-    if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!students) return res.status(400).json({ error: 'students missing' })
+  if (!Array.isArray(students)) res.status(400).json({ error: 'students must be of type array' })
 
-    const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
-    if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
+  const updatedPopulationSearch = await updateCustomPopulationSearch(userId, id, students)
+  if (!updatedPopulationSearch) return res.status(404).json({ error: 'custom population search not found' })
 
-    res.json(updatedPopulationSearch)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(updatedPopulationSearch)
 })
 
 router.delete('/:id', async (req, res) => {
-  try {
-    const {
-      decodedToken: { id: userId },
-    } = req
-    const { id } = req.params
+  const {
+    decodedToken: { id: userId },
+  } = req
+  const { id } = req.params
 
-    if (!id) return res.status(400).json({ error: 'id missing' })
+  if (!id) return res.status(400).json({ error: 'id missing' })
 
-    const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
+  const deletedSuccessfully = (await deleteCustomPopulationSearch(userId, id)) > 0
 
-    if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
+  if (!deletedSuccessfully) return res.status(404).json({ error: 'custom population search not found' })
 
-    res.json(id)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
+  res.json(id)
 })
 
 module.exports = router

services/backend/src/routes/elementdetails.js

@@ -1,5 +1,4 @@
 const router = require('express').Router()
-const { getStudentsUserCanAccess } = require('../services/userService')
 const MandatoryCourses = require('../services/mandatoryCourses')
 const {
   productivityStatsForStudytrack,
@@ -7,7 +6,6 @@ const {
   optionData,
 } = require('../services/studyprogramme')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-
 const {
   getProductivity,
   setProductivity,
@@ -15,28 +13,20 @@ const {
   setThroughput,
   patchProductivity,
   patchThroughput,
-  getNonGraduatedStudents,
 } = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
+const logger = require('../util/logger')
 
 const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
-  try {
-    const elementdetails = await getAllElementDetails()
-    res.json(elementdetails)
-  } catch (e) {
-    res.status(500).json(e)
-  }
+  const elementdetails = await getAllElementDetails()
+  res.json(elementdetails)
 })
 
 router.get('/studyprogrammes', async (req, res) => {
-  try {
-    const studyrights = await getAllProgrammes()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
+  const studyrights = await getAllProgrammes()
+  res.json(studyrights)
 })
 
 router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
@@ -55,34 +45,6 @@ router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   res.json(data)
 })
 
-router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
-  try {
-    const {
-      roles,
-      decodedToken,
-      params: { id },
-    } = req
-    if (!id) return res.status(400).json({ error: 'programme id missing' })
-
-    const nonGraduatedStudents = await getNonGraduatedStudents(id)
-    if (!nonGraduatedStudents) return res.status(200).json({})
-    const {
-      data: { formattedData, studentnumbers },
-    } = nonGraduatedStudents
-
-    const filteredData = { ...formattedData }
-    const studentsUserCanAccess = await getStudentsUserCanAccess(studentnumbers, roles, decodedToken.userId)
-    Object.keys(filteredData).forEach(year => {
-      filteredData[year] = filteredData[year].filter(({ studentNumber }) => studentsUserCanAccess.has(studentNumber))
-    })
-
-    res.json(filteredData)
-  } catch (e) {
-    console.error('e', e)
-    res.status(500).json({ error: 'error' })
-  }
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -99,14 +61,14 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
     try {
       data = await getProductivity(code)
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} productivity`)
     }
     if (!data) {
       try {
         const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
         data = await setProductivity(stats)
       } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       }
     }
     return res.json(data)
@@ -118,14 +80,8 @@ router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
 router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
   const code = req.query.code
 
-  console.log('Productivity stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()
 
   try {
     if (code.includes('MH') || code.includes('KH')) {
@@ -141,13 +97,13 @@ router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
         [code]: { status: 'RECALCULATION ERRORED' },
       })
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       return
     }
-    console.error(e)
-    console.log(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
+
+    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
   }
-  console.log(`Productivity stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
 })
 
 router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
@@ -159,14 +115,14 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
     try {
       data = await getThroughput(code)
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to get code ${code} throughput`)
     }
     if (!data) {
       try {
         const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
         data = await setThroughput(result)
       } catch (e) {
-        console.error(e)
+        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       }
     }
     return res.json({ ...data, thesis: await thesisPromise })
@@ -178,14 +134,8 @@ router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
 router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
   const code = req.query.code
 
-  console.log('Throughput stats recalculation starting')
-  try {
-    await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-    res.status(200).end()
-  } catch (e) {
-    console.error(e)
-    return res.status(500).end()
-  }
+  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+  res.status(200).end()
 
   try {
     if (code.includes('MH') || code.includes('KH')) {
@@ -199,13 +149,12 @@ router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
     try {
       await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
     } catch (e) {
-      console.error(e)
+      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
       return
     }
-    console.error(e)
-    console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
+    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
   }
-  console.log(`Throughput stats recalculation for studyprogramme ${code} done`)
+  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
 })
 
 router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {

services/backend/src/routes/feedback.js

@@ -1,22 +1,27 @@
 const router = require('express').Router()
-const userService = require('../services/userService')
+const { getUserDataFor } = require('../services/userService')
 const { sendFeedbackToToska } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 router.post('/email', async (req, res) => {
   const { content } = req.body
-  const { uid } = req.headers
-  const { email, full_name } = await userService.byUsername(uid)
+  const {
+    decodedToken: { userId },
+  } = req
+
+  const { email, full_name } = await getUserDataFor(userId)
 
   const result = await sendFeedbackToToska({
     feedbackContent: content,
-    userId: uid,
+    userId,
     userEmail: email,
     userFullName: full_name,
   })
   if (result.error) {
     return res.status(400).json(result).end()
   }
-  console.log('Message sent succesfully!')
+  logger.info(`${userId} succesfully sent some feedback mail to toska`)
+
   res.status(200).json('success').end()
 })
 

services/backend/src/routes/login.js

@@ -2,55 +2,46 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 const { parseHyGroups } = require('../util/utils')
 const { sendNotificationAboutNewUser } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 router.post('/login', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const full_name = req.headers.displayname || ''
-      const mail = req.headers.mail || ''
-      const hyGroups = parseHyGroups(req.headers['hygroupcn'])
-      const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
-      const hyPersonSisuId = req.headers.hypersonsisuid || ''
-      console.log('headers', JSON.stringify(req.headers, null, 2))
-      console.log(uid, 'trying to login, referring to userservice.')
-      let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
-      if (isNew) {
-        const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
-        if (result.error) {
-          return res.status(400).json(result).end()
-        }
+  const uid = req.headers['uid']
+  if (req.headers['shib-session-id'] && uid) {
+    const full_name = req.headers.displayname || ''
+    const mail = req.headers.mail || ''
+    const hyGroups = parseHyGroups(req.headers['hygroupcn'])
+    const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
+    const hyPersonSisuId = req.headers.hypersonsisuid || ''
+    logger.info(`${uid} trying to login, referring to userservice`)
+    let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
+    if (isNew) {
+      const result = await sendNotificationAboutNewUser({ userId: uid, userFullName: full_name })
+      if (result.error) {
+        return res.status(400).json(result).end()
       }
-      res.status(200).json({ token })
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid: ${req.headers.uid}
-        session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
     }
-  } catch (err) {
-    console.log(err)
-    res.status(401).json({ message: 'problem with login', err })
+    res.status(200).json({ token })
+  } else {
+    res
+      .status(401)
+      .json({
+        message: `Not enough headers login, uid: ${req.headers.uid}
+        session-id ${req.headers['shib-session-id']}`,
+      })
+      .end()
   }
 })
 
 router.delete('/logout', async (req, res) => {
-  try {
-    const logoutUrl = req.headers.shib_logout_url
-    const { returnUrl } = req.body
-    if (logoutUrl) {
-      return res
-        .status(200)
-        .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
-        .end()
-    }
-    res.status(200).send({ logoutUrl: returnUrl }).end()
-  } catch (err) {
-    res.status(500).json({ message: 'Error with logout', err })
+  const logoutUrl = req.headers.shib_logout_url
+  const { returnUrl } = req.body
+  if (logoutUrl) {
+    return res
+      .status(200)
+      .send({ logoutUrl: `${logoutUrl}?return=${returnUrl}` })
+      .end()
   }
+  res.status(200).send({ logoutUrl: returnUrl }).end()
 })
 
 module.exports = router

services/backend/src/routes/mandatorycourselabels.js

@@ -1,5 +1,6 @@
 const router = require('express').Router()
 const MandatoryCourseLabels = require('../services/mandatoryCourseLabels')
+const logger = require('../util/logger')
 
 router.get('/:programme/labels', async (req, res) => {
   const { programme } = req.params
@@ -8,7 +9,7 @@ router.get('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -25,7 +26,7 @@ router.delete('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -42,7 +43,7 @@ router.post('/:programme/move', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {
@@ -59,7 +60,7 @@ router.post('/:programme/labels', async (req, res) => {
       const labels = await MandatoryCourseLabels.labelsByStudyprogramme(programme)
       res.status(200).json(labels)
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       res.status(400).json(err.message)
     }
   } else {

services/backend/src/routes/mandatorycourses.js

@@ -5,47 +5,32 @@ router.delete('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.remove(programme, course)
-    res.status(200).json(course)
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.remove(programme, course)
+  res.status(200).json(course)
 })
 
 router.post('/:programme', async (req, res) => {
   const { programme } = req.params
   const { course } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.create(programme, course)
-    const newCourse = await MandatoryCourses.find(programme, course)
-    if (!newCourse) return res.status(400).end()
-    res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.create(programme, course)
+  const newCourse = await MandatoryCourses.find(programme, course)
+  if (!newCourse) return res.status(400).end()
+  res.status(201).json({ name: newCourse.course.name, code: newCourse.course_code, label: newCourse.label })
 })
 
 router.post('/:programme/label/:course', async (req, res) => {
   const { programme, course } = req.params
   const { label } = req.body
   if (!programme || !course) return res.status(400).end()
-  try {
-    await MandatoryCourses.updateLabel(programme, course, label)
-    const updatedCourse = await MandatoryCourses.find(programme, course)
-    if (!updatedCourse) return res.status(400).end()
-    res.status(200).json({
-      name: updatedCourse.course.name,
-      code: updatedCourse.course_code,
-      label: updatedCourse.mandatory_course_label,
-    })
-  } catch (err) {
-    console.error(err)
-    res.status(500).json(err.message)
-  }
+  await MandatoryCourses.updateLabel(programme, course, label)
+  const updatedCourse = await MandatoryCourses.find(programme, course)
+  if (!updatedCourse) return res.status(400).end()
+  res.status(200).json({
+    name: updatedCourse.course.name,
+    code: updatedCourse.course_code,
+    label: updatedCourse.mandatory_course_label,
+  })
 })
 
 module.exports = router

services/backend/src/routes/population.js

@@ -8,6 +8,8 @@ const UserService = require('../services/userService')
 const TagService = require('../services/tags')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
+const { getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
+const { logger } = require('../util/logger')
 
 const filterPersonalTags = (population, userId) => {
   return {
@@ -21,265 +23,238 @@ const filterPersonalTags = (population, userId) => {
 
 // POST instead of GET because of too long params and "sensitive" data
 router.post('/v2/populationstatistics/courses', async (req, res) => {
-  try {
-    if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
-      Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
-      res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+  if (!req.body.year || !req.body.semesters || !req.body.studyRights) {
+    Sentry.captureException(new Error('The body should have a year, semester and study rights defined'))
+    res.status(400).json({ error: 'The body should have a year, semester and study rights defined' })
+    return
+  }
+  if (!Array.isArray(req.body.studyRights)) {
+    // studyRights should always be an array
+    req.body.studyRights = [req.body.studyRights]
+  }
+
+  if (req.body.months == null) {
+    req.body.months = 12
+  }
+
+  if (req.body.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multicoursestatPromises = Promise.all(
+      req.body.years.map(year => {
+        if (req.body.selectedStudentsByYear) {
+          req.body.selectedStudents = req.body.selectedStudentsByYear[year]
+        }
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const query = { ...req.body, year, months: newMonths }
+        const coursestatistics = Population.bottlenecksOf(query)
+        return coursestatistics
+      })
+    )
+    const multicoursestats = await multicoursestatPromises
+    const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
+    if (result.error) {
+      res.status(400).json(result)
       return
     }
-    if (!Array.isArray(req.body.studyRights)) {
-      // studyRights should always be an array
-      req.body.studyRights = [req.body.studyRights]
+
+    res.json(result)
+  } else {
+    const result = await Population.bottlenecksOf(req.body)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).json(result)
+      return
     }
 
-    if (req.body.months == null) {
-      req.body.months = 12
-    }
-
-    if (req.body.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multicoursestatPromises = Promise.all(
-        req.body.years.map(year => {
-          if (req.body.selectedStudentsByYear) {
-            req.body.selectedStudents = req.body.selectedStudentsByYear[year]
-          }
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const query = { ...req.body, year, months: newMonths }
-          const coursestatistics = Population.bottlenecksOf(query)
-          return coursestatistics
-        })
-      )
-      const multicoursestats = await multicoursestatPromises
-      const result = StatMergeService.populationCourseStatsMerger(multicoursestats)
-      console.log('result: ', result)
-      if (result.error) {
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    } else {
-      console.log('bottleneck väylä')
-      const result = await Population.bottlenecksOf(req.body)
-
-      if (result.error) {
-        Sentry.captureException(new Error(result.error))
-        res.status(400).json(result)
-        return
-      }
-
-      res.json(result)
-    }
-  } catch (e) {
-    console.log(e)
-    Sentry.captureException(e)
-    res.status(500).json({ error: e })
+    res.json(result)
   }
 })
 
 router.post('/v2/populationstatistics/coursesbycoursecode', async (req, res) => {
-  try {
-    const { from, to, coursecodes } = req.body
-    if (!from || !to || !coursecodes) {
-      return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
-    }
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
-    if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
-      return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { from, to, coursecodes } = req.body
+  if (!from || !to || !coursecodes) {
+    return res.status(400).json({ error: 'The body should have a yearcode and coursecode defined' })
   }
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(coursecodes)
+  if (Math.abs(to - from + 1) > maxYearsToCreatePopulationFrom) {
+    return res.status(400).json({ error: `Max years to create population from is ${maxYearsToCreatePopulationFrom}` })
+  }
+  let studentnumberlist
+  const studentnumbers = await Student.findByCourseAndSemesters(coursecodes, from, to)
+  const {
+    decodedToken: { userId },
+    roles,
+  } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })
 
 router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
-  try {
-    const { tag } = req.body
-    if (!tag) {
-      res.status(400).json({ error: 'The body should have a tag defined' })
-      return
-    }
-    let studentnumberlist
-    const studentnumbers = await Student.findByTag(tag)
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = studentnumbers
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
-    }
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-        tag,
-      },
-      studentnumberlist
-    )
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  const { tag } = req.body
+  if (!tag) {
+    res.status(400).json({ error: 'The body should have a tag defined' })
+    return
   }
+  let studentnumberlist
+  const studentnumbers = await Student.findByTag(tag)
+  const {
+    decodedToken: { userId },
+    roles,
+  } = req
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = studentnumbers
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
+  }
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+      tag,
+    },
+    studentnumberlist
+  )
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(result)
 })
 
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
-  try {
-    if (!req.body.studentnumberlist) {
-      res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
-      return
-    }
-    let studentnumberlist
-    const {
-      decodedToken: { userId },
-      roles,
-    } = req
-
-    if (roles && roles.includes('admin')) {
-      studentnumberlist = req.body.studentnumberlist
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
-    }
-
-    const result = await Population.bottlenecksOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      studentnumberlist
-    )
-    if (result.error) {
-      res.status(400).json(result)
-      return
-    }
-
-    res.json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: err })
+  if (!req.body.studentnumberlist) {
+    res.status(400).json({ error: 'The body should have a studentnumberlist defined' })
+    return
   }
+  let studentnumberlist
+  const {
+    decodedToken: { userId, sisPersonId },
+    roles,
+  } = req
+
+  if (roles && roles.includes('admin')) {
+    studentnumberlist = req.body.studentnumberlist
+  } else if (req.body.usingStudyGuidanceGroups) {
+    const studentsUserCanAccess = await getAllStudentsUserHasInGroups(sisPersonId)
+    studentnumberlist = req.body.studentnumberlist.filter(s => studentsUserCanAccess.has(s))
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentnumberlist = await Student.filterStudentnumbersByAccessrights(req.body.studentnumberlist, codes)
+  }
+
+  const result = await Population.bottlenecksOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    studentnumberlist
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).json(result)
+    return
+  }
+
+  res.json(result)
 })
 
 router.get('/v3/populationstatistics', async (req, res) => {
   const { year, semesters, studyRights: studyRightsJSON } = req.query
   const { decodedToken } = req
 
+  if (!year || !semesters || !studyRightsJSON) {
+    res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
+    return
+  }
+  let studyRights = null
   try {
-    if (!year || !semesters || !studyRightsJSON) {
-      res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
-      return
-    }
-    let studyRights = null
-    try {
-      studyRights = JSON.parse(studyRightsJSON)
-      const { rights, roles } = req
+    studyRights = JSON.parse(studyRightsJSON)
+    const { rights, roles } = req
 
-      if (!roles || !roles.includes('admin')) {
-        if (!rights.includes(studyRights.programme)) {
-          res.status(403).json([])
-          return
-        }
-      }
-    } catch (e) {
-      console.error(e)
-      res.status(400).json({ error: 'The query had invalid studyRights' })
-      return
-    }
-
-    if (req.query.months == null) {
-      req.query.months = 12
-    }
-
-    if (req.query.years) {
-      const upperYearBound = new Date().getFullYear() + 1
-      const multipopulationstudentPromises = Promise.all(
-        req.query.years.map(year => {
-          const newMonths = (upperYearBound - Number(year)) * 12
-          const populationStudents = Population.optimizedStatisticsOf({
-            ...req.query,
-            studyRights,
-            year,
-            months: newMonths,
-          })
-          return populationStudents
-        })
-      )
-      const multipopulationstudents = await multipopulationstudentPromises
-
-      const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
-
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
+    if (!roles || !roles.includes('admin')) {
+      if (!rights.includes(studyRights.programme)) {
+        res.status(403).json([])
         return
       }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
-    } else {
-      const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
-      if (result.error) {
-        console.log(result.error)
-        res.status(400).end()
-        return
-      }
-
-      console.log(`request completed ${new Date()}`)
-      res.json(filterPersonalTags(result, decodedToken.id))
     }
   } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+    res.status(400).json({ error: 'The query had invalid studyRights' })
+    return
+  }
+
+  if (req.query.months == null) {
+    req.query.months = 12
+  }
+
+  if (req.query.years) {
+    const upperYearBound = new Date().getFullYear() + 1
+    const multipopulationstudentPromises = Promise.all(
+      req.query.years.map(year => {
+        const newMonths = (upperYearBound - Number(year)) * 12
+        const populationStudents = Population.optimizedStatisticsOf({
+          ...req.query,
+          studyRights,
+          year,
+          months: newMonths,
+        })
+        return populationStudents
+      })
+    )
+    const multipopulationstudents = await multipopulationstudentPromises
+
+    const result = StatMergeService.populationStudentsMerger(multipopulationstudents)
+
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, decodedToken.id))
+  } else {
+    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+    if (result.error) {
+      Sentry.captureException(new Error(result.error))
+      res.status(400).end()
+      return
+    }
+
+    res.json(filterPersonalTags(result, decodedToken.id))
   }
 })
 
@@ -305,34 +280,28 @@ router.get('/v3/populationstatisticsbytag', async (req, res) => {
     const codes = unitsUserCanAccess.map(unit => unit.id)
     studentnumberlist = await Student.filterStudentnumbersByAccessrights(studentnumbers, codes)
   }
-  try {
-    const studyRights = JSON.parse(studyRightsJSON)
-    const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
-    const yearDifference = Number(year) - Number(newStartYear)
-    const newMonths = Number(months) + 12 * yearDifference
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: newStartYear,
-        studyRights,
-        semesters,
-        months: newMonths,
-        tag: foundTag,
-      },
-      studentnumberlist
-    )
+  const studyRights = JSON.parse(studyRightsJSON)
+  const newStartYear = await Population.getEarliestYear(studentnumberlist, studyRights)
+  const yearDifference = Number(year) - Number(newStartYear)
+  const newMonths = Number(months) + 12 * yearDifference
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: newStartYear,
+      studyRights,
+      semesters,
+      months: newMonths,
+      tag: foundTag,
+    },
+    studentnumberlist
+  )
 
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
   }
+
+  res.json(filterPersonalTags(result, decodedToken.id))
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
@@ -354,126 +323,108 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 
   const semesters = ['FALL', 'SPRING']
   const studentnumbers = await Student.findByCourseAndSemesters(JSON.parse(coursecodes), from, to, separate)
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        // Useless, because studentnumbers are already filtered above by from & to.
-        // We should probably refactor this to avoid more confusement.
-        year: 1900,
-        studyRights: [],
-        semesters,
-        months: 10000,
-      },
-      studentnumbers
-    )
+  const result = await Population.optimizedStatisticsOf(
+    {
+      // Useless, because studentnumbers are already filtered above by from & to.
+      // We should probably refactor this to avoid more confusement.
+      year: 1900,
+      studyRights: [],
+      semesters,
+      months: 10000,
+    },
+    studentnumbers
+  )
 
-    let studentsUserCanAccess
-    if (roles && roles.includes('admin')) {
-      studentsUserCanAccess = new Set(studentnumbers)
-    } else {
-      const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
-      const codes = unitsUserCanAccess.map(unit => unit.id)
-      studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
-    }
-
-    const randomHash = crypto.randomBytes(12).toString('hex')
-    const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
-      courses,
-      studyrights,
-      gender_code,
-      studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
-      firstnames: '',
-      lastname: '',
-      name: '',
-      email: '',
-      started: null,
-      credits: '',
-      tags: [],
-      obfuscated: true,
-    })
-
-    result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
-
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-
-    console.log(`request completed ${new Date()}`)
-    res.json(filterPersonalTags(result, decodedToken.id))
-  } catch (e) {
-    console.log(e)
-    res.status(500).json({ error: e })
+  let studentsUserCanAccess
+  if (roles && roles.includes('admin')) {
+    studentsUserCanAccess = new Set(studentnumbers)
+  } else {
+    const unitsUserCanAccess = await UserService.getUnitsFromElementDetails(decodedToken.userId)
+    const codes = unitsUserCanAccess.map(unit => unit.id)
+    studentsUserCanAccess = new Set(await Student.filterStudentnumbersByAccessrights(studentnumbers, codes))
   }
+
+  const randomHash = crypto.randomBytes(12).toString('hex')
+  const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
+    courses,
+    studyrights,
+    gender_code,
+    studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
+    firstnames: '',
+    lastname: '',
+    name: '',
+    email: '',
+    started: null,
+    credits: '',
+    tags: [],
+    obfuscated: true,
+  })
+
+  result.students = result.students.map(s => (studentsUserCanAccess.has(s.studentNumber) ? s : obfuscateStudent(s)))
+
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    res.status(400).end()
+    return
+  }
+
+  res.json(filterPersonalTags(result, decodedToken.id))
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist } = req.body
+  const { studentnumberlist, usingStudyGuidanceGroups } = req.body
   const { roles, decodedToken } = req
 
-  const studentsUserCanAccess = await UserService.getStudentsUserCanAccess(
-    studentnumberlist,
-    roles,
-    decodedToken.userId
-  )
+  if (usingStudyGuidanceGroups) {
+    if (!decodedToken.sisPersonId) {
+      logger.error(
+        `User ${decodedToken.userId} tried to get person groups but personId was ${decodedToken.sisPersonId} in header`
+      )
+      return res.status(400).json({ error: 'Not possible to get groups without personId' })
+    }
+  }
+
+  const studentsUserCanAccess = usingStudyGuidanceGroups
+    ? await getAllStudentsUserHasInGroups(decodedToken.sisPersonId)
+    : await UserService.getStudentsUserCanAccess(studentnumberlist, roles, decodedToken.userId)
   const filteredStudentNumbers = studentnumberlist.filter(s => studentsUserCanAccess.has(s))
 
-  try {
-    const result = await Population.optimizedStatisticsOf(
-      {
-        year: 1900,
-        studyRights: [],
-        semesters: ['FALL', 'SPRING'],
-        months: 10000,
-      },
-      filteredStudentNumbers
-    )
-    if (result.error) {
-      console.log(result.error)
-      res.status(400).end()
-      return
-    }
-    res.status(200).json(filterPersonalTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).end()
+  const result = await Population.optimizedStatisticsOf(
+    {
+      year: 1900,
+      studyRights: [],
+      semesters: ['FALL', 'SPRING'],
+      months: 10000,
+    },
+    filteredStudentNumbers
+  )
+  if (result.error) {
+    Sentry.captureException(new Error(result.error))
+    return
   }
+  res.status(200).json(filterPersonalTags(result, decodedToken.id))
 })
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
-  try {
-    const { rights, roles } = req
-    if (roles && roles.includes('admin')) {
-      const studyrights = await StudyrightService.getAssociations()
-      res.json(studyrights)
-    } else {
-      const studyrights = await StudyrightService.getFilteredAssociations(rights)
-      res.json(studyrights)
-    }
-  } catch (err) {
-    res.status(500).json(err)
+  const { rights, roles } = req
+  if (roles && roles.includes('admin')) {
+    const studyrights = await StudyrightService.getAssociations()
+    res.json(studyrights)
+  } else {
+    const studyrights = await StudyrightService.getFilteredAssociations(rights)
+    res.json(studyrights)
   }
 })
 
 router.get('/v3/populationstatistics/studyprogrammes/unfiltered', async (req, res) => {
-  try {
-    const studyrights = await StudyrightService.getAssociations()
-    res.json(studyrights)
-  } catch (err) {
-    res.status(500).json(err)
-  }
+  const studyrights = await StudyrightService.getAssociations()
+  res.json(studyrights)
 })
 
 router.get('/v3/populationstatistics/maxYearsToCreatePopulationFrom', async (req, res) => {
-  try {
-    const { courseCodes } = req.query
-    const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
-    return res.json(maxYearsToCreatePopulationFrom)
-  } catch (err) {
-    console.log('err', err)
-    res.status(500).json(err)
-  }
+  const { courseCodes } = req.query
+  const maxYearsToCreatePopulationFrom = await CourseService.maxYearsToCreatePopulationFrom(JSON.parse(courseCodes))
+  return res.json(maxYearsToCreatePopulationFrom)
 })
 
 router.use('*', (req, res, next) => next())

services/backend/src/routes/programmeModules.js

@@ -8,17 +8,20 @@ const {
 
 router.get('/v3/programme_modules/:code', async (req, res) => {
   const { code } = req.params
-  const module = await byProgrammeCode(code)
-  if (!module) {
-    res.status(400).end()
-    return
-  }
-  res.json(module)
+  const result = await byProgrammeCode(code)
+  res.json(result)
 })
 
 router.get('/v3/programme_modules/:code/modules', async (req, res) => {
   const { code } = req.params
   const result = await modulesByProgrammeCode(code)
+  res.json(result)
+})
+
+router.delete('/v3/programme_modules', async (req, res) => {
+  const { programmecode, ids } = req.body
+  await removeExcludedCourses(ids)
+  const result = await byProgrammeCode(programmecode)
   if (!result) {
     res.status(400).end()
     return
@@ -26,36 +29,15 @@ router.get('/v3/programme_modules/:code/modules', async (req, res) => {
   res.json(result)
 })
 
-router.delete('/v3/programme_modules', async (req, res) => {
-  const { programmecode, ids } = req.body
-  try {
-    await removeExcludedCourses(ids)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
-  }
-})
-
 router.post('/v3/programme_modules/:programmecode/', async (req, res) => {
   const { programmecode, coursecodes } = req.body
-  try {
-    await addExcludedCourses(programmecode, coursecodes)
-    const result = await byProgrammeCode(programmecode)
-    if (!result) {
-      res.status(400).end()
-      return
-    }
-    res.json(result)
-  } catch (e) {
-    console.log(e)
-    res.json(500).json(e)
+  await addExcludedCourses(programmecode, coursecodes)
+  const result = await byProgrammeCode(programmecode)
+  if (!result) {
+    res.status(400).end()
+    return
   }
+  res.json(result)
 })
 
 module.exports = router

services/backend/src/routes/students.js

@@ -2,7 +2,6 @@ const router = require('express').Router()
 const Student = require('../services/students')
 const userService = require('../services/userService')
 const Unit = require('../services/units')
-const { mismatchedStudents } = require('../services/student_credit_total_mismatches')
 
 const filterStudentTags = (student, userId) => {
   return {
@@ -11,13 +10,6 @@ const filterStudentTags = (student, userId) => {
   }
 }
 
-const creditTotalMismatch = student => {
-  return {
-    ...student,
-    mismatch: mismatchedStudents.has(student.studentNumber),
-  }
-}
-
 router.get('/students', async (req, res) => {
   const {
     roles,
@@ -58,10 +50,7 @@ router.get('/students/:id', async (req, res) => {
     const results = await Student.withId(studentId)
     return results.error
       ? res.status(400).json({ error: 'error finding student' }).end()
-      : res
-          .status(200)
-          .json(creditTotalMismatch(filterStudentTags(results, decodedToken.id)))
-          .end()
+      : res.status(200).json(filterStudentTags(results, decodedToken.id)).end()
   }
 
   const uid = req.decodedToken.userId

services/backend/src/routes/studyGuidanceGroups.js

@@ -1,21 +1,39 @@
 const router = require('express').Router()
-const { getImporterClient } = require('../util/importerClient')
+const { logger } = require('../util/logger')
+const { getAllGroupsAndStudents, changeGroupTags } = require('../services/studyGuidanceGroups')
 
 router.get('/', async (req, res) => {
-  const personId = req.headers.hypersonsisuid
-  if (!personId) {
-    const errorMessage = 'Not possible to get groups without personId header'
-    console.log(errorMessage)
-    return res.status(400).json({ error: errorMessage })
+  const {
+    decodedToken: { uid, sisPersonId },
+  } = req
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to get person groups but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to get groups without personId' })
   }
-  const importerClient = getImporterClient()
-  try {
-    const { data } = await importerClient.get(`/person-groups/person/${personId}`)
-    return res.json(Object.values(data))
-  } catch (e) {
-    console.log(e)
-    return res.status(500).json(e)
+  return res.json(await getAllGroupsAndStudents(sisPersonId))
+})
+
+router.put('/:id/tags', async (req, res) => {
+  const {
+    decodedToken: { uid, sisPersonId },
+    body: tags,
+    params: { id: groupId },
+  } = req
+
+  if (!sisPersonId) {
+    logger.error(`User ${uid} tried to change study guidance group tags but personId was ${sisPersonId} in header`)
+    return res.status(400).json({ error: 'Not possible to change groups without personId' })
   }
+
+  const groupsUserHasAccessTo = (await getAllGroupsAndStudents(sisPersonId)).map(group => group.id)
+
+  if (!groupsUserHasAccessTo.includes(groupId)) {
+    logger.error(
+      `User ${uid} tried to change study guidance group tags, but didn't have permission for the group ${groupId}`
+    )
+    return res.status(400).json({ error: 'Access denied' })
+  }
+  return res.status(200).json(await changeGroupTags({ groupId, tags }))
 })
 
 module.exports = router

services/backend/src/routes/superlogin.js

@@ -2,24 +2,19 @@ const router = require('express').Router()
 const userService = require('../services/userService')
 
 router.post('/:uid', async (req, res) => {
-  try {
-    const uid = req.headers['uid']
-    if (req.headers['shib-session-id'] && uid) {
-      const asUser = req.params.uid
-      const token = await userService.superlogin(uid, asUser)
-      res.status(200).json(token)
-    } else {
-      res
-        .status(401)
-        .json({
-          message: `Not enough headers login, uid:
+  const uid = req.headers['uid']
+  if (req.headers['shib-session-id'] && uid) {
+    const asUser = req.params.uid
+    const token = await userService.superlogin(uid, asUser)
+    res.status(200).json(token)
+  } else {
+    res
+      .status(401)
+      .json({
+        message: `Not enough headers login, uid:
         ${req.headers.uid} session-id ${req.headers['shib-session-id']}`,
-        })
-        .end()
-    }
-  } catch (err) {
-    console.log(err.message)
-    res.status(500).json({ message: 'problem with login' })
+      })
+      .end()
   }
 })
 

services/backend/src/routes/tags.js

@@ -13,187 +13,132 @@ const filterRelevantStudentTags = (studentTags, userId) => {
 }
 
 router.get('/tags', async (req, res) => {
-  try {
-    const tags = await Tags.findTags()
-    res.status(200).json(tags)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const tags = await Tags.findTags()
+  res.status(200).json(tags)
 })
 
 router.get('/tags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
+  const { studytrack } = req.params
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const tags = await Tags.findTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
 })
 
 router.post('/tags', async (req, res) => {
-  try {
-    const {
-      tag: { studytrack, tagname, year, personal_user_id },
-    } = req.body
-    const { rights, roles, decodedToken } = req
+  const {
+    tag: { studytrack, tagname, year, personal_user_id },
+  } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
-    const tags = await Tags.findTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantTags(tags, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err.message)
-  }
+  await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
+  const tags = await Tags.findTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantTags(tags, decodedToken.id))
 })
 
 router.delete('/tags', async (req, res) => {
-  try {
-    const { tag } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (
-      !rights.includes(tag.studytrack) &&
-      !(roles && roles.includes('admin')) &&
-      !(tag.personal_user_id === decodedToken.id)
-    )
-      return res.status(403).end()
+  if (
+    !rights.includes(tag.studytrack) &&
+    !(roles && roles.includes('admin')) &&
+    !(tag.personal_user_id === decodedToken.id)
+  )
+    return res.status(403).end()
 
-    await Tags.deleteTag(tag)
-    const t = await Tags.findTagsByStudytrack(tag.studytrack)
-    res.status(200).json(filterRelevantTags(t, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await Tags.deleteTag(tag)
+  const t = await Tags.findTagsByStudytrack(tag.studytrack)
+  res.status(200).json(filterRelevantTags(t, decodedToken.id))
 })
 
 router.get('/studenttags', async (req, res) => {
-  try {
-    const result = await TagStudent.getStudentTags()
-    res.status(200).json(result)
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const result = await TagStudent.getStudentTags()
+  res.status(200).json(result)
 })
 
 router.get('/studenttags/:studytrack', async (req, res) => {
-  try {
-    const { studytrack } = req.params
-    const { rights, roles, decodedToken } = req
+  const { studytrack } = req.params
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.get('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { studentnumber } = req.params
-    const { decodedToken } = req
-    const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  const { studentnumber } = req.params
+  const { decodedToken } = req
+  const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.post('/studenttags/:studentnumber', async (req, res) => {
-  try {
-    const { tag, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    await TagStudent.createStudentTag(tag)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.createStudentTag(tag)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.post('/studenttags', async (req, res) => {
-  try {
-    const { tags, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tags, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const existingtags = await Tags.findTagsByStudytrack(studytrack)
-    const existingTagids = existingtags.map(t => t.tag_id)
-    const tagids = [...new Set(tags.map(t => t.tag_id))]
-    if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
+  const existingtags = await Tags.findTagsByStudytrack(studytrack)
+  const existingTagids = existingtags.map(t => t.tag_id)
+  const tagids = [...new Set(tags.map(t => t.tag_id))]
+  if (!tagids.find(t => existingTagids.includes(t))) return res.status(400).json({ error: 'The tag does not exist' })
 
-    const studentnumbers = tags.map(t => t.studentnumber)
-    const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
-    const missingStudents = difference(studentnumbers, students)
-    if (missingStudents.length !== 0)
-      return res
-        .status(400)
-        .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
+  const studentnumbers = tags.map(t => t.studentnumber)
+  const students = await Students.filterStudentnumbersByAccessrights(studentnumbers, [studytrack])
+  const missingStudents = difference(studentnumbers, students)
+  if (missingStudents.length !== 0)
+    return res
+      .status(400)
+      .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
 
-    await TagStudent.createMultipleStudentTags(tags)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.createMultipleStudentTags(tags)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.delete('/studenttags/delete_one', async (req, res) => {
-  try {
-    const { tag_id, studentnumber, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tag_id, studentnumber, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tag_id])
+  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
-    await TagStudent.deleteStudentTag(studentnumber, tag_id)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(500).json(err)
-  }
+  await TagStudent.deleteStudentTag(studentnumber, tag_id)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.delete('/studenttags/delete_many', async (req, res) => {
-  try {
-    const { tagId, studentnumbers, studytrack } = req.body
-    const { rights, roles, decodedToken } = req
+  const { tagId, studentnumbers, studytrack } = req.body
+  const { rights, roles, decodedToken } = req
 
-    if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
+  if (!rights.includes(studytrack) && !(roles && roles.includes('admin'))) return res.status(403).end()
 
-    const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
-    if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
+  const tags = await Tags.findTagsFromStudytrackById(studytrack, [tagId])
+  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
-    await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-    const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-    res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
-  } catch (err) {
-    console.log(err)
-    res.status(400).json(err)
-  }
+  await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
+  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
+  res.status(200).json(filterRelevantStudentTags(result, decodedToken.id))
 })
 
 router.use('*', (req, res, next) => next())

services/backend/src/routes/teachers.js

@@ -1,5 +1,6 @@
 const teachers = require('../services/teachers')
 const topteachers = require('../services/topteachers')
+const { isFaculty, providersOfFaculty } = require('../services/organisations')
 const router = require('express').Router()
 const { mapToProviders } = require('../util/utils')
 
@@ -24,6 +25,7 @@ router.get('/top', async (req, res) => {
   if (!yearcode) {
     return res.status(422).send('Missing required yearcode query param')
   }
+
   const result = await topteachers.getTeacherStats(category, yearcode)
   res.json(result)
 })
@@ -51,7 +53,19 @@ router.get('/stats', async (req, res) => {
   if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')
   }
-  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd || semesterStart + 1)
+
+  // combine provider list that may include programmes and faculties
+  const parsedProviders = [
+    ...new Set(
+      await providers.reduce(
+        async (acc, curr) =>
+          isFaculty(curr) ? [...(await acc), ...(await providersOfFaculty(curr))] : [...(await acc), curr],
+        []
+      )
+    ),
+  ]
+
+  const result = await teachers.yearlyStatistics(parsedProviders, semesterStart, semesterEnd || semesterStart + 1)
   res.json(result)
 })
 

services/backend/src/routes/updater.js

@@ -8,116 +8,63 @@ const {
   abort,
   updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatistics } = require('../events')
+const { refreshStatistics, refreshTrends } = require('../events')
 
 router.get('/update/v2/meta', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISMetadata(req)
-    if (response) {
-      res.status(200).json('Update SIS meta scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISMetadata(req)
+  if (response) {
+    res.status(200).json('Update SIS meta scheduled')
   }
 })
 
 router.get('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISStudents()
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISStudents()
+  if (response) {
+    res.status(200).json('Update SIS students scheduled')
   }
 })
 
 router.post('/update/v2/students', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateStudentsByStudentNumber(req.body)
-    if (response) {
-      res.status(200).json('Update SIS students scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateStudentsByStudentNumber(req.body)
+  if (response) {
+    res.status(200).json('Update SIS students scheduled')
   }
 })
 
 router.post('/update/v2/courses', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateCoursesByCourseCode(req.body)
-    if (response) {
-      res.status(200).json('Update SIS courses scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateCoursesByCourseCode(req.body)
+  if (response) {
+    res.status(200).json('Update SIS courses scheduled')
   }
 })
 
 router.get('/update/v2/programmes', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISProgrammes()
-    if (response) {
-      res.status(200).json('Update SIS programmes scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISProgrammes()
+  if (response) {
+    res.status(200).json('Update SIS programmes scheduled')
   }
 })
 
 router.get('/refresh_redis_cache', async (req, res) => {
-  const { roles } = req
-  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
-
-  try {
-    const response = await updateSISRedisCache()
-    if (response) {
-      res.status(200).json('Refreshing SIS redis cache scheduled')
-    }
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
+  const response = await updateSISRedisCache()
+  if (response) {
+    res.status(200).json('Refreshing SIS redis cache scheduled')
   }
 })
 
 router.post('/refresh_statistic_v2', async (req, res) => {
-  try {
-    refreshStatistics()
-    res.status(200).json('Refreshing sis statistics')
-  } catch (err) {
-    console.error(err)
-    res.status(500).json({ error: 'error' })
-  }
+  await refreshStatistics()
+  res.status(200).json('Refreshing sis statistics')
+})
+
+router.post('/refresh_trends', async (req, res) => {
+  await refreshTrends()
+  res.status(200).json('Refreshing trends')
 })
 
 router.get('/abort', async (req, res) => {
-  try {
-    await abort()
-    res.status(200).json()
-  } catch (err) {
-    console.log(err)
-    res.status(500).json({ error: 'error' })
-  }
+  await abort()
+  res.status(200).json()
 })
 
 module.exports = router

services/backend/src/routes/users.js

@@ -3,6 +3,7 @@ const userService = require('../services/userService')
 const blacklist = require('../services/blacklist')
 const { userDataCache } = require('../services/cache')
 const { sendNotificationAboutAccessToUser, previewNotificationAboutAccessToUser } = require('../services/mailservice')
+const logger = require('../util/logger')
 
 const addUserToBlacklist = async (user, decodedToken) => {
   if (user) {
@@ -24,9 +25,11 @@ router.get('/access_groups', async (req, res) => {
 
 router.post('/modifyaccess', async (req, res) => {
   try {
-    const { uid } = req.body
+    const { username } = req.body
     const result = await userService.modifyAccess(req.body)
-    const user = await userService.byId(uid)
+    const user = {
+      username,
+    }
     await addUserToBlacklist(user, req.decodedToken)
     res.status(200).json(result)
   } catch (e) {
@@ -49,7 +52,7 @@ router.post('/email', async (req, res) => {
   if (result.error) {
     return res.status(400).json(result).end()
   }
-  console.log('Message sent successfully')
+  logger.info('Succesfully sent message about oodikone access to user')
   res.status(200).end()
 })
 

services/backend/src/services/analyticsService.js

@@ -3,7 +3,6 @@ const moment = require('moment')
 
 const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
 const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForNonGraduatedStudents = id => `NONGRADUATEDSTUDENTS_${id}`
 
 const getProductivity = async id => {
   const redisKey = createRedisKeyForProductivity(id)
@@ -88,32 +87,6 @@ const patchThroughput = async data => {
   }
 }
 
-const patchNonGraduatedStudents = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForNonGraduatedStudents(id)
-  const dataFromRedisRaw = await redisClient.getAsync(redisKey)
-  const dataFromRedis = dataFromRedisRaw ? JSON.parse(dataFromRedisRaw) : { data: null }
-  const patchedData = {
-    data: {
-      ...dataFromRedis.data,
-      ...dataToPatch,
-    },
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getNonGraduatedStudents = async id => {
-  const redisKey = createRedisKeyForNonGraduatedStudents(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return JSON.parse(dataFromRedis)
-}
-
 module.exports = {
   getProductivity,
   setProductivity,
@@ -121,6 +94,4 @@ module.exports = {
   getThroughput,
   setThroughput,
   patchThroughput,
-  patchNonGraduatedStudents,
-  getNonGraduatedStudents,
 }

services/backend/src/services/blacklist.js

@@ -1,10 +1,11 @@
 const { redisClient } = require('./redis')
 const moment = require('moment')
+const logger = require('../util/logger')
 
 const MAKE_REDIS_HASH_KEY = id => `BLACKLIST ${id}`
 
 const addUserToBlacklist = async userId => {
-  console.log('ADDED BLACKLIST', userId)
+  logger.info(`user ${userId} added to token blacklist`)
   await redisClient.setAsync(MAKE_REDIS_HASH_KEY(userId), moment().toISOString(false))
 }
 

services/backend/src/services/course_stats_counter.js

@@ -16,6 +16,7 @@ class CourseStatsCounter {
       name,
       disciplines: {},
       coursetypes: {},
+      substitutions: [],
     }
     this.students = {
       all: {},
@@ -142,6 +143,10 @@ class CourseStatsCounter {
     this.course.coursetypes[typecode] = name
   }
 
+  addCourseSubstitutions(subs) {
+    this.course.substitutions = subs
+  }
+
   addCourseDiscipline(id, name) {
     this.course.disciplines[id] = name
   }

services/backend/src/services/courses.js

@@ -172,7 +172,7 @@ const allCodeAltenatives = async code => {
   const temp = courses
     .map(c => c.code)
     .map(c => {
-      if (c.match(/^A/)) return [c, 4] // open university codes come last
+      if (c.match(/^AY?/)) return [c, 4] // open university codes come last
       if (c.match(/^\d/)) return [c, 2] // old numeric codes come second
       if (c.match(/^[A-Za-z]/)) return [c, 1] // new letter based codes come first
       return [c, 3] // unknown, comes before open uni?
@@ -368,39 +368,9 @@ const byNameAndOrCodeLike = async (name, code) => {
   })
 
   const courses = rawCourses
-    .map(
-      ({
-        id,
-        code,
-        name,
-        latest_instance_date,
-        is_study_module,
-        coursetypecode,
-        startdate,
-        max_attainment_date,
-        min_attainment_date,
-        createdAt,
-        updatedAt,
-        substitutions,
-        organizations,
-      }) => {
-        return {
-          id,
-          code,
-          name,
-          latest_instance_date,
-          is_study_module,
-          coursetypecode,
-          startdate,
-          max_attainment_date,
-          min_attainment_date,
-          createdAt,
-          updatedAt,
-          substitutions,
-          organizations: organizations.map(o => o.id),
-        }
-      }
-    )
+    .map(course => {
+      return { ...course.dataValues, organizations: course.organizations.map(o => o.id) }
+    })
     .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
 
   let substitutionGroupIndex = 0

services/backend/src/services/mailservice.js

@@ -1,5 +1,6 @@
 const axios = require('axios')
 const { isProduction, pateToken } = require('../conf-backend')
+const logger = require('../util/logger')
 
 const pateClient = axios.create({
   baseURL: 'https://pate.toska.cs.helsinki.fi',
@@ -21,13 +22,13 @@ const baseSettings = {
 const sendEmail = async (options = {}) => {
   if (!isProduction) {
     const errorMessage = 'Skipped sending email in non-production environment'
-    console.log(errorMessage, options)
+    logger.error({ message: errorMessage, meta: options })
     return { error: errorMessage }
   }
 
   if (!pateToken) {
     const errorMessage = 'Skipped sending email since pate token was not available'
-    console.log(errorMessage, options)
+    logger.error({ message: errorMessage, meta: options })
     return { error: errorMessage }
   }
 

services/backend/src/services/organisations.js

@@ -1,6 +1,155 @@
 const Sequelize = require('sequelize')
-const { Organization } = require('../models')
+const { Organization, ProgrammeModule } = require('../models')
 const Op = Sequelize.Op
+const { dbConnections } = require('../database/connection')
+
+// temp refactoring faculty <-> userservice mappings away from userservice
+const facultiesAndProgrammesInUserService = [
+  ['H10', 'MH10_001'],
+  ['H10', 'MH40_011'],
+  ['H10', 'KH10_001'],
+  ['H20', 'KH20_001'],
+  ['H20', 'MH20_001'],
+  ['H20', 'MH20_002'],
+  ['H30', 'KH30_001'],
+  ['H30', 'KH30_002'],
+  ['H30', 'MH30_002'],
+  ['H30', 'MH30_003'],
+  ['H30', 'MH30_004'],
+  ['H30', 'MH30_005'],
+  ['H30', 'MH30_001'],
+  ['H40', 'KH40_001'],
+  ['H40', 'KH40_002'],
+  ['H40', 'KH40_003'],
+  ['H40', 'KH40_004'],
+  ['H40', 'KH40_005'],
+  ['H40', 'KH40_006'],
+  ['H40', 'MH40_001'],
+  ['H40', 'MH40_002'],
+  ['H40', 'MH40_003'],
+  ['H40', 'MH40_004'],
+  ['H40', 'MH40_005'],
+  ['H40', 'MH40_006'],
+  ['H40', 'MH40_007'],
+  ['H40', 'MH40_008'],
+  ['H40', 'MH40_009'],
+  ['H40', 'MH40_010'],
+  ['H40', 'MH40_011'],
+  ['H40', 'MH40_012'],
+  ['H40', 'MH40_013'],
+  ['H40', 'MH40_014'],
+  ['H40', 'MH40_015'],
+  ['H40', 'MH70_001'],
+  ['H40', 'MH70_006'],
+  ['H50', 'KH50_001'],
+  ['H50', 'KH50_002'],
+  ['H50', 'KH50_003'],
+  ['H50', 'KH50_004'],
+  ['H50', 'KH50_005'],
+  ['H50', 'KH50_006'],
+  ['H50', 'KH50_007'],
+  ['H50', 'MH50_001'],
+  ['H50', 'MH50_002'],
+  ['H50', 'MH50_003'],
+  ['H50', 'MH50_004'],
+  ['H50', 'MH50_005'],
+  ['H50', 'MH50_006'],
+  ['H50', 'MH50_007'],
+  ['H50', 'MH50_008'],
+  ['H50', 'MH50_009'],
+  ['H50', 'MH50_010'],
+  ['H50', 'MH50_011'],
+  ['H50', 'MH50_012'],
+  ['H50', 'MH50_013'],
+  ['H55', 'KH55_001'],
+  ['H55', 'MH55_001'],
+  ['H57', 'KH57_001'],
+  ['H57', 'KH57_002'],
+  ['H57', 'KH57_003'],
+  ['H57', 'MH50_002'],
+  ['H57', 'MH50_013'],
+  ['H57', 'MH57_001'],
+  ['H57', 'MH57_002'],
+  ['H57', 'MH57_003'],
+  ['H57', 'MH57_004'],
+  ['H57', 'MH57_005'],
+  ['H57', 'MH80_007'],
+  ['H60', 'KH60_001'],
+  ['H60', 'MH60_001'],
+  ['H70', 'KH40_001'],
+  ['H70', 'KH50_001'],
+  ['H70', 'KH70_001'],
+  ['H70', 'KH70_002'],
+  ['H70', 'KH70_003'],
+  ['H70', 'KH70_004'],
+  ['H70', 'MH50_001'],
+  ['H70', 'MH57_005'],
+  ['H70', 'MH70_001'],
+  ['H70', 'MH70_002'],
+  ['H70', 'MH70_003'],
+  ['H70', 'MH70_004'],
+  ['H70', 'MH70_005'],
+  ['H70', 'MH70_006'],
+  ['H70', 'MH70_007'],
+  ['H70', 'MH70_008'],
+  ['H70', 'MH70_009'],
+  ['H70', 'MH70_010'],
+  ['H74', 'KH74_001'],
+  ['H80', 'KH57_002'],
+  ['H80', 'KH80_001'],
+  ['H80', 'KH80_002'],
+  ['H80', 'KH80_003'],
+  ['H80', 'KH80_004'],
+  ['H80', 'MH57_002'],
+  ['H80', 'MH57_003'],
+  ['H80', 'MH57_005'],
+  ['H80', 'MH80_001'],
+  ['H80', 'MH80_002'],
+  ['H80', 'MH80_003'],
+  ['H80', 'MH80_004'],
+  ['H80', 'MH80_005'],
+  ['H80', 'MH80_006'],
+  ['H80', 'MH80_007'],
+  ['H90', 'KH90_001'],
+  ['H30', '447000'],
+  ['H30', '447001'],
+  ['H30', '447002'],
+  ['H60', 'EDUK730'],
+  ['H60', 'K-MUUT-ERIL'],
+  ['H60', 'ED400'],
+  ['H60', 'ED200'],
+  ['H60', 'ED700'],
+  ['H60', 'ED100'],
+  ['H60', 'ED300'],
+  ['H60', 'ED600'],
+  ['H60', '8009'],
+  ['H60', '8118'],
+  ['H60', '8600'],
+  ['H60', 'DUK-PED'],
+  ['H60', '65'],
+  ['H60', '1102'],
+  ['H60', '20018'],
+  ['H60', '8601'],
+  ['H60', '9290'],
+  ['H60', '8172'],
+  ['H60', '83600'],
+  ['H60', 'DUM-PED'],
+  ['H60', 'ED500'],
+  ['H60', '83601'],
+  ['H60', '0130'],
+  ['H60', '20005-e'],
+  ['H60', '1040'],
+  ['H60', '8703'],
+  ['H60', '0005'],
+  ['H60', '7900'],
+  ['H60', '0152 '],
+  ['H90', 'MH90_001'],
+]
+
+const facultiesAndProgrammesForTrends = facultiesAndProgrammesInUserService.map(f => ({
+  faculty_code: f[0],
+  programme_code: f[1],
+}))
 
 // Have facultyfetching to work like it worked during oodi-db time
 const facultiesInOodi = [
@@ -33,6 +182,37 @@ const faculties = () => {
   })
 }
 
+const degreeProgrammeCodesOfFaculty = async facultyCode =>
+  (
+    await ProgrammeModule.findAll({
+      attributes: ['code'],
+      include: {
+        model: Organization,
+        where: {
+          code: facultyCode,
+        },
+      },
+    })
+  ).map(({ code }) => code)
+
+const providersOfFaculty = async facultyCode => {
+  const [result] = await dbConnections.sequelize.query(
+    `SELECT childOrg.code
+     FROM organization parentOrg
+     INNER JOIN organization childOrg 
+     ON childOrg.parent_id = parentOrg.id
+     WHERE parentOrg.code = ?`,
+    { replacements: [facultyCode] }
+  )
+  return result.map(({ code }) => code)
+}
+
+const isFaculty = facultyCode => facultiesInOodi.includes(facultyCode)
+
 module.exports = {
   faculties,
+  degreeProgrammeCodesOfFaculty,
+  isFaculty,
+  providersOfFaculty,
+  facultiesAndProgrammesForTrends,
 }

services/backend/src/services/populations.js

@@ -801,7 +801,7 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
 
 const findCourses = async (studentnumbers, beforeDate) => {
   const res = await Course.findAll({
-    attributes: ['code', 'name', 'coursetypecode'],
+    attributes: ['code', 'name', 'coursetypecode', 'substitutions'],
     include: [
       {
         required: true,
@@ -969,6 +969,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
     const coursestats = stats[maincourse.code]
 
     coursestats.addCourseType(course_type.coursetypecode, course_type.name)
+    coursestats.addCourseSubstitutions(course.substitutions)
     bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
 
     course.credits.forEach(credit => {

services/backend/src/services/programmeModules.js

@@ -1,13 +1,12 @@
 const Sequelize = require('sequelize')
 const { ExcludedCourse } = require('../models/models_kone')
 const { Op } = Sequelize
+const logger = require('../util/logger')
 
-const { dbConnections: sisConnections } = require('../database/connection')
-const { sequelizeKone } = require('../database/connection')
+const { dbConnections } = require('../database/connection')
 
 const recursivelyGetModuleAndChildren = async (code, type) => {
-  // TODO use only sis connection once sis is stable
-  const connection = sisConnections.established ? sisConnections.sequelize : sequelizeKone
+  const connection = dbConnections.sequelize
   try {
     const [result] = await connection.query(
       `WITH RECURSIVE children as (
@@ -23,15 +22,14 @@ const recursivelyGetModuleAndChildren = async (code, type) => {
     )
     return result
   } catch (e) {
-    console.error('Error when searching with code: ', code)
-    console.error('Details: ', e)
+    logger.error(`Error when searching modules and children with code: ${code}`)
     return []
   }
 }
 
 const byProgrammeCode = async code => {
   const result = await recursivelyGetModuleAndChildren(code, 'course')
-
+  if (!result.length) return []
   const excluded = await ExcludedCourse.findAll({
     where: {
       programme_code: {

services/backend/src/services/students.js

@@ -15,6 +15,7 @@ const {
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
+const logger = require('../util/logger')
 
 const createStudent = student => Student.create(student)
 
@@ -177,9 +178,10 @@ const formatStudent = ({
       },
       date: attainment_date,
       course_code: course.code,
-      passed: Credit.passed({ credittypecode }),
+      passed: Credit.passed({ credittypecode }) || Credit.improved({ credittypecode }),
       grade,
       credits,
+      credittypecode,
       isStudyModuleCredit: isStudyModule,
     }
   }
@@ -232,7 +234,7 @@ const withId = async id => {
     const result = await byId(id)
     return formatStudent(result)
   } catch (e) {
-    console.log(e)
+    logger.error({ message: 'error when fetching single student', meta: e })
     return {
       error: e,
     }
@@ -328,16 +330,6 @@ const filterStudentnumbersByAccessrights = async (studentnumbers, codes) => {
   return students.map(student => student.studentnumber)
 }
 
-const hasEnrolledForSemester = async (studentnumber, semestercode) => {
-  const enrollment = await SemesterEnrollment.findOne({
-    where: {
-      studentnumber,
-      semestercode,
-    },
-  })
-  return !!(enrollment && enrollment.enrollmenttype === 1)
-}
-
 module.exports = {
   withId,
   bySearchTerm,
@@ -348,5 +340,4 @@ module.exports = {
   findByCourseAndSemesters,
   findByTag,
   splitByEmptySpace,
-  hasEnrolledForSemester,
 }

services/backend/src/services/studyGuidanceGroups.js

@@ -0,0 +1,52 @@
+const { getImporterClient } = require('../util/importerClient')
+const importerClient = getImporterClient()
+const { StudyGuidanceGroupTag } = require('../models/models_kone')
+
+const getAllStudentsUserHasInGroups = async sisPersonId => {
+  if (!importerClient) {
+    return new Set()
+  }
+  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+  return new Set(
+    Object.values(data)
+      .map(group => group.members)
+      .flat()
+      .map(member => member.personStudentNumber)
+  )
+}
+
+const getAllGroupsAndStudents = async sisPersonId => {
+  if (!importerClient) {
+    return []
+  }
+  const { data } = await importerClient.get(`/person-groups/person/${sisPersonId}`)
+  const tagsByGroupId = (await StudyGuidanceGroupTag.findAll()).reduce((acc, curr) => {
+    const { studyProgramme, year } = curr
+    return { ...acc, [curr.studyGuidanceGroupId]: { studyProgramme, year } }
+  }, {})
+  return Object.values(data).map(group => ({ ...group, tags: tagsByGroupId[group.id] }))
+}
+
+const changeGroupTags = async ({ groupId, tags }) => {
+  const { studyProgramme, year } = tags
+  const tagToUpdate = studyProgramme ? { studyProgramme } : { year }
+
+  const [result] = await StudyGuidanceGroupTag.upsert(
+    {
+      studyGuidanceGroupId: groupId,
+      ...tagToUpdate,
+    },
+    {
+      where: {
+        studyGuidanceGroupId: groupId,
+      },
+    }
+  )
+  return result
+}
+
+module.exports = {
+  getAllStudentsUserHasInGroups,
+  getAllGroupsAndStudents,
+  changeGroupTags,
+}

services/backend/src/services/studyprogramme.js

@@ -385,7 +385,9 @@ const graduationsFromClass = async (studentnumbers, studytrack) => {
 const genderCodeToValue = code => {
   if (code === '1') return 'Male'
   if (code === '2') return 'Female'
-  return 'Other'
+  if (code === '3') return 'Other'
+  if (code === '0') return 'Unknown'
+  return 'Unknown'
 }
 
 const gendersFromClass = async studentnumbers => {

services/backend/src/services/studyrights.js

@@ -2,16 +2,11 @@ const {
   dbConnections: { sequelize },
 } = require('../database/connection')
 const { Studyright, StudyrightElement, ElementDetail } = require('../models')
-const { getUserElementDetails } = require('./userService')
-const { hasEnrolledForSemester } = require('./students')
-const { getCurrentSemester } = require('./semesters')
 const moment = require('moment')
 const { redisClient } = require('../services/redis')
 const _ = require('lodash')
 const { Op, col, where, fn } = require('sequelize')
 
-const createStudyright = apiData => Studyright.create(apiData)
-
 const REDIS_KEY = 'STUDYRIGHT_ASSOCIATIONS_V2'
 
 const byStudent = studentNumber => {
@@ -24,97 +19,6 @@ const byStudent = studentNumber => {
   })
 }
 
-const getActiveStudyrightElementsOfCodeBeforeDate = async (code, date) =>
-  await StudyrightElement.findAll({
-    where: {
-      code,
-      enddate: {
-        [Op.gte]: date,
-      },
-    },
-  })
-
-const getActiveStudyrightsFromIdsBeforeDate = async (studyrightIds, date) =>
-  await Studyright.findAll({
-    where: {
-      graduated: 0,
-      studyrightid: {
-        [Op.in]: studyrightIds,
-      },
-      canceldate: null,
-      enddate: {
-        [Op.gte]: date,
-      },
-    },
-  })
-
-const nonGraduatedStudentsOfElementDetail = async code => {
-  const today = new Date()
-  const currentSemesterCode = (await getCurrentSemester()).semestercode
-
-  // Get studyrights of target code that haven't ended yet
-  const studyrightElements = await getActiveStudyrightElementsOfCodeBeforeDate(code, today)
-  const studentToDatesMap = {}
-
-  studyrightElements.forEach(({ startdate, enddate, studentnumber }) => {
-    studentToDatesMap[studentnumber] = {
-      startdate,
-      enddate,
-    }
-  })
-
-  const studyrights = await getActiveStudyrightsFromIdsBeforeDate(
-    studyrightElements.map(sE => sE.studyrightid),
-    today
-  )
-
-  // Filter out students that are in new master's programmes,
-  // see if student is currently enrolled and format the result
-  // to shape of { year1: [studentnumbers], ... }
-  const result = {}
-  const studentnumbers = new Set()
-  const studentsToBeFiltered = new Set()
-  await Promise.all(
-    studyrights.map(
-      ({ studentStudentnumber: student_studentnumber, studyrightid }) =>
-        new Promise(async res => {
-          const [enrolled, studentElementDetails] = await Promise.all([
-            await hasEnrolledForSemester(student_studentnumber, currentSemesterCode),
-            await StudyrightElement.findAll({
-              where: {
-                studyrightid,
-              },
-            }),
-          ])
-
-          // If student is in new master's programme,
-          // then don't include them in the result
-          if (studentElementDetails.find(e => e.code.match(/^M[A-Z]*[0-9]*_[0-9]*$/))) {
-            studentsToBeFiltered.add(student_studentnumber)
-            return res()
-          }
-
-          if (studentnumbers.has(student_studentnumber)) return res()
-          const year = moment(studentToDatesMap[student_studentnumber].startdate).tz('Europe/Helsinki').year()
-          if (!result[year]) result[year] = []
-          studentnumbers.add(student_studentnumber)
-          result[year].push({
-            studentNumber: student_studentnumber,
-            enrolled,
-          })
-          res()
-        })
-    )
-  )
-
-  // Filter out some special cases
-  Object.keys(result).forEach(year => {
-    result[year] = result[year].filter(({ studentNumber }) => !studentsToBeFiltered.has(studentNumber))
-  })
-
-  return [result, [...studentnumbers]]
-}
-
 const studentNumbersWithAllStudyRightElements = async (codes, startedAfter, startedBefore) => {
   const studyrights = await StudyrightElement.findAll({
     attributes: ['studentnumber'],
@@ -235,15 +139,6 @@ const getAllStudyrightElementsAndAssociations = async () => {
   return JSON.parse(studyright_elements)
 }
 
-const getStudyrightElementsAndAssociationsForUser = async username => {
-  const studyrightelementcodes = await getUserElementDetails(username)
-  if (studyrightelementcodes.length === 0) {
-    return []
-  }
-  const associations = await getAssociatedStudyrights(studyrightelementcodes)
-  return formatStudyrightElements(studyrightelementcodes, associations)
-}
-
 const getAllProgrammes = async () => {
   const elementDetails = ElementDetail.findAll({
     where: {
@@ -419,24 +314,14 @@ const getFilteredAssociations = async codes => {
   return associations
 }
 
-const getUserAssociations = async userid => {
-  const codes = await getUserElementDetails(userid)
-  const associations = await getFilteredAssociations(codes)
-  return associations
-}
-
 module.exports = {
   byStudent,
-  createStudyright,
   studentNumbersWithAllStudyRightElements,
   getAssociatedStudyrights,
   getAllStudyrightElementsAndAssociations,
-  getStudyrightElementsAndAssociationsForUser,
   getAssociations,
   getFilteredAssociations,
-  getUserAssociations,
   refreshAssociationsInRedis,
   getAllProgrammes,
   getAllElementDetails,
-  nonGraduatedStudentsOfElementDetail,
 }

services/backend/src/services/teachers.js

@@ -191,6 +191,11 @@ const activeTeachers = async (providers, semestercodeStart, semestercodeEnd) =>
         },
       ],
     },
+    where: {
+      id: {
+        [Op.notLike]: '%hy-hlo-org%',
+      },
+    },
   })
   return teachers.map(({ id }) => id)
 }

services/backend/src/services/tester.js

@@ -1,59 +0,0 @@
-const { bottlenecksOf, optimizedStatisticsOf } = require('./populations')
-
-const moment = require('moment')
-
-// eslint-disable-next-line no-unused-vars
-const run2 = async () => {
-  const o = {
-    semesters: ['FALL', 'SPRING'],
-    year: 2017, // 2015
-    studyRights: ['KH10_001'], // ['KH50_005'], // ['320001']
-    months: 16, // 41
-  }
-
-  const result = await bottlenecksOf(o) // eslint-disable-line
-
-  //console.log(JSON.stringify(result.coursestatistics[0].stats, null, 2))
-
-  //console.log(result.coursestatistics.length)
-
-  process.exit()
-}
-
-// eslint-disable-next-line no-unused-vars
-const run1 = async () => {
-  // eslint-disable-line
-  const o = {
-    semesters: ['FALL', 'SPRING'],
-    year: 2018, // 2015
-    studyRights: ['KH50_001'], // ['KH50_005'], // ['320001']
-    months: 41, // 41
-  }
-
-  const a = await optimizedStatisticsOf(o)
-
-  const start = moment('2018-07-31')
-
-  a.students.forEach(s1 => {
-    // eslint-disable-next-line no-constant-condition
-    if (true || s1.studentNumber === '014612688') {
-      const studyright = s1.studyrights.find(s =>
-        s.studyright_elements.map(d => d.element_detail.code).includes('KH50_001')
-      )
-
-      console.log(studyright.startdate, start.isAfter(moment(studyright.startdate)), s1.changedStudyright)
-
-      //console.log(JSON.stringify(studyright, null, 2))
-    }
-  })
-
-  process.exit()
-}
-
-const run3 = async () => {
-  const userStatistics = require('./userService')
-  const resp = await userStatistics.ping()
-  console.log(resp)
-}
-
-run3()

services/backend/src/services/topteachers.js

@@ -1,7 +1,8 @@
 const { redisClient } = require('../services/redis')
-const { getSemestersAndYears, getMaxYearcode } = require('./semesters')
+const { getSemestersAndYears, getCurrentSemester } = require('./semesters')
 const { Teacher, Semester, Credit, Course } = require('../models')
 const { Op } = require('sequelize')
+const logger = require('../util/logger')
 
 const category = (name, rediskey) => ({ name, rediskey })
 
@@ -128,11 +129,13 @@ const findTopTeachers = async yearcode => {
   }
 }
 
-const findAndSaveTeachers = async (startcode = 50, to) => {
-  const endcode = to ? to : await getMaxYearcode()
+const findAndSaveTeachers = async (startcode = 1, to) => {
+  const endcode = to || (await getCurrentSemester()).getDataValue('yearcode')
   for (let code = startcode; code <= endcode; code++) {
     await findAndSaveTopTeachers(code)
+    logger.info(`Teacher leaderboard for yearcode ${code} calculated`)
   }
+  logger.info(`Teacher leaderboard calculations done`)
 }
 
 const findAndSaveTopTeachers = async yearcode => {

services/backend/src/services/trends/index.js

@@ -14,7 +14,6 @@ const CURRENT_DATE = new Date()
 
 const withErr = handler => (req, res, next) =>
   handler(req, res, next).catch(e => {
-    console.error(e)
     res.status(500).json({ error: { message: e.message, stack: e.stack } })
   })
 

services/backend/src/services/trends/shared.js

@@ -1,15 +1,8 @@
-const axios = require('axios')
 const { redisClient } = require('../../services/redis')
-const { USERSERVICE_URL } = require('../../conf-backend')
 const {
   dbConnections: { sequelize },
 } = require('../../database/connection')
 
-const userServiceClient = axios.create({
-  baseURL: USERSERVICE_URL,
-  headers: { secret: process.env.USERSERVICE_SECRET },
-})
-
 const getRedisCDS = async REDIS_KEY => {
   const raw = await redisClient.getAsync(REDIS_KEY)
   return raw && JSON.parse(raw)
@@ -177,6 +170,5 @@ const getTargetStudentCounts = async ({ codes, includeOldAttainments, excludeNon
 module.exports = {
   getRedisCDS,
   saveToRedis,
-  userServiceClient,
   getTargetStudentCounts,
 }

services/backend/src/services/trends/statusAttainments.js

@@ -4,9 +4,10 @@ const {
 } = require('../../database/connection')
 const { mapToProviders } = require('../../util/utils')
 const { ElementDetail, Organization } = require('../../models')
-const { getRedisCDS, saveToRedis, userServiceClient } = require('./shared')
+const { getRedisCDS, saveToRedis } = require('./shared')
+const { facultiesAndProgrammesForTrends } = require('../../services/organisations')
 
-const REDIS_KEY_STATUS = 'STATUS_DATA_V2'
+const REDIS_KEY_STATUS = 'STATUS_DATA_V5'
 
 const getCurrentStudyYearStartDate = _.memoize(
   async unixMillis =>
@@ -29,6 +30,140 @@ const getCurrentYearStartDate = () => {
   return new Date(new Date().getFullYear(), 0, 1)
 }
 
+const getOrganizationStudents = async (start, end) => {
+  const rows = await sequelize.query(
+    `
+    SELECT
+      sre.code AS code,
+      ARRAY_AGG(DISTINCT(sr.student_studentnumber)) AS students
+    FROM studyright sr
+    INNER JOIN studyright_elements sre ON sre.studyrightid = sr.studyrightid
+    INNER JOIN element_details ed ON ed.code = sre.code
+    WHERE ed.type = 20 AND (:start, :end) OVERLAPS (sr.startdate, sr.enddate)
+    GROUP BY sre.code
+  `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { start, end },
+    }
+  )
+
+  const codes = mapToProviders(rows.map(({ code }) => code))
+
+  const orgIdRows = await sequelize.query(
+    `
+    SELECT o.code, o.id FROM organization o WHERE o.code IN (:codes)
+  `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { codes },
+    }
+  )
+
+  const orgIds = Object.fromEntries(orgIdRows.map(({ code, id }) => [code, id]))
+
+  return rows.reduce((acc, { code, students }) => {
+    const [orgCode] = mapToProviders([code])
+    const orgId = orgIds[orgCode]
+    acc[orgId] = students
+    return acc
+  }, {})
+}
+
+const getOrganizationCredits = async (start, end) => {
+  const rows = await sequelize.query(
+    `
+    SELECT
+      cp.organizationcode AS code,
+      -- ARRAY_AGG(DISTINCT(cr.student_studentnumber)) AS students,
+      SUM(cr.credits) AS credits
+    FROM credit cr
+    INNER JOIN course co ON co.code = cr.course_code
+    INNER JOIN course_providers cp ON cp.coursecode = co.id
+    WHERE
+      cr.attainment_date BETWEEN :start AND :end AND
+      cr.credittypecode IN (4, 9) AND
+      (cr."isStudyModule" = FALSE OR cr."isStudyModule" IS NULL)
+    GROUP BY cp.organizationcode
+  `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { start, end },
+    }
+  )
+
+  return rows.reduce((acc, { code, credits }) => {
+    acc[code] = credits
+    return acc
+  }, {})
+}
+
+const getOrganizationStatistics = async (organizations, start, end) => {
+  const orgCredits = await getOrganizationCredits(start, end)
+  const orgStudents = await getOrganizationStudents(start, end)
+
+  const organizationStats = {}
+
+  let stack_pointer = 0
+  const stack = ['hy-university-root-id']
+
+  while (stack_pointer < stack.length) {
+    const orgId = stack[stack_pointer]
+    stack_pointer += 1
+
+    organizations.filter(({ parent_id }) => parent_id === orgId).forEach(({ id }) => stack.push(id))
+  }
+
+  const updateOrgStats = (orgId, stats, isDirect) => {
+    if (organizationStats[orgId] === undefined) {
+      organizationStats[orgId] = {
+        direct: { credits: 0, students: new Set() },
+        aggregated: { credits: 0, students: new Set() },
+      }
+    }
+
+    const orgStats = organizationStats[orgId]
+
+    const statsRefs = isDirect ? [orgStats.direct, orgStats.aggregated] : [orgStats.aggregated]
+
+    statsRefs.forEach(statsRef => {
+      statsRef.credits += stats.credits
+      stats.students.forEach(s => statsRef.students.add(s))
+    })
+  }
+
+  while (stack.length > 0) {
+    const orgId = stack.pop()
+    const credits = orgCredits[orgId] ?? 0
+    const students = orgStudents[orgId] ?? []
+
+    const org = organizations.find(({ id }) => id === orgId)
+    const parentOrgId = org.parent_id
+
+    updateOrgStats(orgId, { credits, students }, true)
+
+    if (parentOrgId) {
+      updateOrgStats(parentOrgId, organizationStats[orgId].aggregated, false)
+    }
+  }
+
+  return Object.entries(organizationStats).reduce((acc, [id, { direct, aggregated }]) => {
+    acc[id] = {
+      id,
+      direct: {
+        students: direct.students.size,
+        credits: direct.credits,
+      },
+      aggregated: {
+        students: aggregated.students.size,
+        credits: aggregated.credits,
+      },
+    }
+
+    return acc
+  }, {})
+}
+
 const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'sum2') => {
   return sequelize.query(
     `
@@ -43,7 +178,7 @@ const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'su
     WHERE
       cr.attainment_date BETWEEN :a AND :b
       AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
-      AND cr.credittypecode IN (4, 9)
+      AND cr.credittypecode IN (4, 9)      
     GROUP BY co.id, o.code
     -- HAVING SUM(cr.credits) > 0
     `,
@@ -54,25 +189,40 @@ const getTotalCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'su
   )
 }
 
-const sumMerge = (a, b) => {
-  if (!a) return b
-  if (!b) return a
-  return a + b
+const getTotalOpenUniCreditsOfCoursesBetween = async (a, b, alias = 'sum', alias2 = 'sum2') => {
+  return sequelize.query(
+    `
+    SELECT SUM(cr.credits) AS ${alias}, COUNT(DISTINCT(cr.student_studentnumber)) AS ${alias2},
+    o.code AS organizationcode, co.code, co.id, co.name
+    FROM credit cr
+    INNER JOIN course co ON cr.course_code = co.code
+    INNER JOIN course_providers cp ON cp.coursecode = co.id
+    INNER JOIN organization o ON o.id = cp.organizationcode
+    AND cr.attainment_date BETWEEN :a AND :b
+    AND (cr."isStudyModule" = false OR cr."isStudyModule" IS NULL)
+    AND cr.credittypecode IN (4, 9) AND cp.organizationcode = 'hy-org-48645785'
+    AND cr.student_studentnumber NOT IN
+      (SELECT student_studentnumber FROM studyright
+      WHERE studyright.student_studentnumber = cr.student_studentnumber
+      AND cr.attainment_date BETWEEN studyright.startdate AND studyright.enddate)
+    GROUP BY co.id, o.code -- HAVING SUM(cr.credits) > 0
+    `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { a, b },
+    }
+  )
 }
 
-const mergele = (a, b) => {
-  if (!a) return _.clone(b)
-  if (!b) return _.clone(a)
-  return _.mergeWith(a, b, sumMerge)
-}
-
-const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum', alias2 = 'sum2') => {
+const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum', alias2 = 'sum2', avoin) => {
   return years.map(
     year =>
       new Promise(async res => {
         const diff = currentYear - year
         const { from, to } = getRange(diff)
-        const creditsByCourse = await getTotalCreditsOfCoursesBetween(from, to, alias, alias2)
+        const creditsByCourse = avoin
+          ? await getTotalOpenUniCreditsOfCoursesBetween(from, to, alias, alias2)
+          : await getTotalCreditsOfCoursesBetween(from, to, alias, alias2)
         res(
           creditsByCourse.map(c => {
             c['year'] = year
@@ -83,6 +233,39 @@ const makeYearlyCreditsPromises = (currentYear, years, getRange, alias = 'sum',
   )
 }
 
+const getOrgStats = (stats, orgId) =>
+  stats[orgId] ?? {
+    accumulated: {
+      aggregated: { students: 0, credits: 0 },
+      direct: { students: 0, credits: 0 },
+    },
+    total: {
+      aggregated: { students: 0, credits: 0 },
+      direct: { students: 0, credits: 0 },
+    },
+  }
+
+const getOrgsOfOpenUniReqularCounterparts = async codes => {
+  const rows = await sequelize.query(
+    `
+      SELECT co.id, co.code, cp.organizationcode, org.code AS organization
+      FROM course co, course_providers cp, organization org
+      WHERE co.code IN(:codes)
+      AND co.id = cp.coursecode
+      AND org.id = cp.organizationcode;
+    `,
+    {
+      type: sequelize.QueryTypes.SELECT,
+      replacements: { codes },
+    }
+  )
+
+  return rows.reduce((acc, { code, organization }) => {
+    acc[code] = organization
+    return acc
+  }, {})
+}
+
 const calculateStatusStatistics = async (unixMillis, showByYear) => {
   const YEAR_TO_MILLISECONDS = 31556952000
   /* Memoize parses booleans into strings... */
@@ -90,40 +273,135 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
   const startYear = startDate.getFullYear()
   const startTime = startDate.getTime()
   const yearRange = _.range(2017, startYear + 1)
+  const getRangeAcc = diff => ({
+    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
+    to:
+      showByYear === 'true'
+        ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
+        : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
+  })
+  const getRangeTotal = diff => ({
+    from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
+    to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
+  })
   const yearlyAccCreditsPromises = makeYearlyCreditsPromises(
     startYear,
     yearRange,
-    diff => ({
-      from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to:
-        showByYear === 'true'
-          ? new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS)
-          : new Date(unixMillis - diff * YEAR_TO_MILLISECONDS),
-    }),
+    getRangeAcc,
     'acc',
-    'students'
+    'students',
+    false
+  )
+  const yearlyOpenUniAccCreditsPromises = makeYearlyCreditsPromises(
+    startYear,
+    yearRange,
+    getRangeAcc,
+    'acc',
+    'students',
+    true
   )
-
   const yearlyTotalCreditsPromises = makeYearlyCreditsPromises(
     startYear,
     yearRange.slice(0, -1),
-    diff => ({
-      from: new Date(startTime - diff * YEAR_TO_MILLISECONDS),
-      to: new Date(startTime - (diff - 1) * YEAR_TO_MILLISECONDS),
-    }),
+    getRangeTotal,
     'total',
-    'students'
+    'students',
+    false
+  )
+
+  const yearlyOpenUniTotalCreditsPromises = makeYearlyCreditsPromises(
+    startYear,
+    yearRange.slice(0, -1),
+    getRangeTotal,
+    'total',
+    'students',
+    true
   )
 
   /* Gather all required data */
-  const [yearlyAccCredits, yearlyTotalCredits, elementDetails, faculties, { data: facultyProgrammes }] =
-    await Promise.all([
-      Promise.all(yearlyAccCreditsPromises),
-      Promise.all(yearlyTotalCreditsPromises),
-      ElementDetail.findAll(),
-      Organization.findAll(),
-      userServiceClient.get('/faculty_programmes'),
-    ])
+  const [
+    yearlyAccCredits,
+    yearlyTotalCredits,
+    yearlyOpenUniAccCredits,
+    yearlyOpenUniTotalCredits,
+    elementDetails,
+    faculties,
+  ] = await Promise.all([
+    Promise.all(yearlyAccCreditsPromises),
+    Promise.all(yearlyTotalCreditsPromises),
+    Promise.all(yearlyOpenUniAccCreditsPromises),
+    Promise.all(yearlyOpenUniTotalCreditsPromises),
+    ElementDetail.findAll(),
+    Organization.findAll(),
+  ])
+
+  const isOpenUniCourseCode = code => code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
+
+  const unifyOpenUniversity = code => {
+    const regexresult = isOpenUniCourseCode(code)
+    if (!regexresult) return code
+    return regexresult[1]
+  }
+
+  const fromAyToNormalCourses = [
+    ...new Set(_.flatten(yearlyOpenUniAccCredits).map(c => unifyOpenUniversity(c.code))),
+    ...new Set(_.flatten(yearlyOpenUniTotalCredits).map(c => unifyOpenUniversity(c.code))),
+  ]
+
+  const orgsOfOpenUniReqularCounterparts = await getOrgsOfOpenUniReqularCounterparts(fromAyToNormalCourses)
+
+  const facultyProgrammes = facultiesAndProgrammesForTrends
+
+  const yearlyOrgStatPromises = yearRange.map(async year => {
+    const accumulatedStats = await getOrganizationStatistics(
+      faculties,
+      new Date(startTime - (startYear - year) * YEAR_TO_MILLISECONDS),
+      showByYear === 'true'
+        ? new Date(startTime - (startYear - year - 1) * YEAR_TO_MILLISECONDS)
+        : new Date(unixMillis - (startYear - year) * YEAR_TO_MILLISECONDS)
+    )
+
+    const totalStats =
+      year !== startYear &&
+      (await getOrganizationStatistics(
+        faculties,
+        new Date(startTime - (startYear - year) * YEAR_TO_MILLISECONDS),
+        new Date(startTime - (startYear - year - 1) * YEAR_TO_MILLISECONDS)
+      ))
+
+    return [
+      year,
+      _.merge(
+        _.mapValues(totalStats, total => ({ total })),
+        _.mapValues(accumulatedStats, accumulated => ({ accumulated }))
+      ),
+    ]
+  })
+
+  const orgStats = await Promise.all(yearlyOrgStatPromises)
+
+  const yearlyOrgStats = Object.fromEntries(orgStats)
+  const [[, currentOrgStats], [, prevOrgStats]] = orgStats.reverse()
+
+  const defaultYearStats = _.chain(_.range(2017, startYear + 1))
+    .map(year => [year, { acc: 0, accStudents: 0 }])
+    .fromPairs()
+    .value()
+
+  const getOrgYearlyStats = orgId =>
+    _.chain(yearlyOrgStats)
+      .mapValues(stats => {
+        const orgStats = getOrgStats(stats, orgId)
+
+        return {
+          acc: orgStats.accumulated.aggregated.credits,
+          accStudents: orgStats.accumulated.aggregated.students,
+          total: orgStats.total?.aggregated?.credits,
+          totalStudents: orgStats.total?.aggregated?.students,
+        }
+      })
+      .defaults(defaultYearStats)
+      .value()
 
   /* Construct some helper maps */
   const facultyCodeToFaculty = faculties.reduce((res, curr) => {
@@ -137,6 +415,8 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     return res
   }, {})
 
+  programmeToFaculties['H906'] = ['H906']
+
   const providerToProgramme = elementDetails.reduce((res, curr) => {
     const [p] = mapToProviders([curr.code])
     res[p] = {
@@ -146,21 +426,66 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
     return res
   }, {})
 
+  providerToProgramme['H906'] = {
+    code: 'H906',
+    name: { en: 'Language Centre', fi: 'Kielikeskus', sv: 'Språkcentrum' },
+  }
+
+  providerToProgramme['H930'] = {
+    code: 'H930',
+    name: { en: 'Open University', fi: 'Avoin yliopisto', sv: 'Öppna universitetet' },
+  }
+
+  const organizationCodeToOrganization = faculties.reduce((acc, org) => {
+    acc[org.code] = org
+    return acc
+  }, {})
+
+  const sumMerge = (a, b) => {
+    if (!a) return b
+    if (!b) return a
+    return a + b
+  }
+
+  const mergele = (a, b) => {
+    if (!a) return _.clone(b)
+    if (!b) return _.clone(a)
+    return _.mergeWith(a, b, sumMerge)
+  }
+
+  const refactoredOpenUniCredits = [..._.flatten(yearlyOpenUniAccCredits), ..._.flatten(yearlyOpenUniTotalCredits)].map(
+    ele => {
+      const org = orgsOfOpenUniReqularCounterparts[unifyOpenUniversity(ele.code)]
+      return {
+        ...ele,
+        organizationcode: org ? org : 'H930',
+      }
+    }
+  )
+
   /* Calculate course level stats and group by providers */
   const coursesGroupedByProvider = Object.entries(
     _.groupBy([..._.flatten(yearlyAccCredits), ..._.flatten(yearlyTotalCredits)], 'organizationcode')
   ).reduce((acc, [organizationcode, courseCredits]) => {
     acc[organizationcode] = Object.entries(_.groupBy(courseCredits, 'code')).reduce(
       (acc, [courseCode, yearlyInstances]) => {
-        acc[courseCode] = { yearly: {}, name: yearlyInstances[0].name }
-        const array = [2020, 2019, 2018, 2017]
+        acc[courseCode] = {
+          yearly: {},
+          name: yearlyInstances[0].name,
+          type: 'course',
+          code: courseCode,
+        }
+
+        const array = _.range(2017, startYear + 1)
+
         array.forEach(year => {
           acc[courseCode]['yearly'][year] = {}
           acc[courseCode]['yearly'][year]['acc'] = 0
           acc[courseCode]['yearly'][year]['accStudents'] = 0
-          acc[courseCode]['yearly'][year]['total'] = 0
-          acc[courseCode]['yearly'][year]['totalStudents'] = 0
+          acc[courseCode]['yearly'][year]['total'] = undefined
+          acc[courseCode]['yearly'][year]['totalStudents'] = undefined
         })
+
         yearlyInstances.forEach(instance => {
           if (!acc[courseCode]['yearly'][instance.year]) acc[courseCode]['yearly'][instance.year] = {}
 
@@ -172,6 +497,52 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
             acc[courseCode]['yearly'][instance.year]['totalStudents'] = Number(instance.students)
           }
         })
+
+        acc[courseCode]['current'] = _.get(acc, [courseCode, 'yearly', startYear, 'acc']) || 0
+        acc[courseCode]['previous'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'acc']) || 0
+        acc[courseCode]['currentStudents'] = _.get(acc, [courseCode, 'yearly', startYear, 'accStudents']) || 0
+        acc[courseCode]['previousStudents'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'accStudents']) || 0
+        return acc
+      },
+      {}
+    )
+    return acc
+  }, {})
+
+  const openUniCoursesGroupedByProvider = Object.entries(
+    _.groupBy(refactoredOpenUniCredits, 'organizationcode')
+  ).reduce((acc, [organizationcode, courseCredits]) => {
+    acc[organizationcode] = Object.entries(_.groupBy(courseCredits, 'code')).reduce(
+      (acc, [courseCode, yearlyInstances]) => {
+        acc[courseCode] = {
+          yearly: {},
+          name: yearlyInstances[0].name,
+          type: 'course',
+          code: courseCode,
+        }
+
+        const array = _.range(2017, startYear + 1)
+
+        array.forEach(year => {
+          acc[courseCode]['yearly'][year] = {}
+          acc[courseCode]['yearly'][year]['acc'] = 0
+          acc[courseCode]['yearly'][year]['accStudents'] = 0
+          acc[courseCode]['yearly'][year]['total'] = undefined
+          acc[courseCode]['yearly'][year]['totalStudents'] = undefined
+        })
+
+        yearlyInstances.forEach(instance => {
+          if (!acc[courseCode]['yearly'][instance.year]) acc[courseCode]['yearly'][instance.year] = {}
+
+          if (instance.acc !== undefined) {
+            acc[courseCode]['yearly'][instance.year]['acc'] = instance.acc
+            acc[courseCode]['yearly'][instance.year]['accStudents'] = Number(instance.students)
+          } else {
+            acc[courseCode]['yearly'][instance.year]['total'] = instance.total
+            acc[courseCode]['yearly'][instance.year]['totalStudents'] = Number(instance.students)
+          }
+        })
+
         acc[courseCode]['current'] = _.get(acc, [courseCode, 'yearly', startYear, 'acc']) || 0
         acc[courseCode]['previous'] = _.get(acc, [courseCode, 'yearly', startYear - 1, 'acc']) || 0
         acc[courseCode]['currentStudents'] = _.get(acc, [courseCode, 'yearly', startYear, 'accStudents']) || 0
@@ -186,28 +557,136 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
   /* Map providers into proper programmes and calculate programme level stats */
   const groupedByProgramme = Object.entries(coursesGroupedByProvider).reduce((acc, [organizationcode, courses]) => {
     const programme = providerToProgramme[organizationcode]
-    const courseValues = Object.values(courses)
-    const yearlyValues = courseValues.map(c => c.yearly)
+
     if (programme && programme.code) {
+      if (programme.code === 'H906') {
+        const courseValues = Object.values(courses)
+        const yearlyValues = courseValues.map(c => c.yearly)
+
+        acc[programme.code] = {
+          type: 'programme',
+          code: programme.code,
+          name: programme.name,
+          drill: courses,
+          yearly: _.mergeWith({}, ...yearlyValues, mergele),
+          current: _.sumBy(courseValues, 'current') | undefined,
+          previous: _.sumBy(courseValues, 'previous') | undefined,
+          currentStudents: _.sumBy(courseValues, 'currentStudents') || 'undefined',
+          previousStudents: _.sumBy(courseValues, 'previousStudents') | 'undefined',
+        }
+
+        return acc
+      }
+
+      const orgId = organizationCodeToOrganization[organizationcode]?.id
+
+      const { students: currentStudents, credits: current } = getOrgStats(currentOrgStats, orgId).accumulated.aggregated
+      const { students: previousStudents, credits: previous } = getOrgStats(prevOrgStats, orgId).accumulated.aggregated
+
+      const yearly = getOrgYearlyStats(orgId)
+
       acc[programme.code] = {
+        type: 'programme',
+        code: programme.code,
         name: programme.name,
         drill: courses,
-        yearly: _.mergeWith({}, ...yearlyValues, mergele),
-        current: _.sumBy(courseValues, 'current'),
-        previous: _.sumBy(courseValues, 'previous'),
+        yearly,
+        current,
+        previous,
+        currentStudents: currentStudents,
+        previousStudents: previousStudents,
       }
     }
+
     return acc
   }, {})
 
+  const openUniGroupedByProgramme = Object.entries(openUniCoursesGroupedByProvider).reduce(
+    (acc, [organizationcode, courses]) => {
+      const programme = providerToProgramme[organizationcode]
+
+      if (programme && programme.code) {
+        const courseValues = Object.values(courses)
+        const yearlyValues = courseValues.map(c => c.yearly)
+
+        acc[programme.code] = {
+          type: 'programme',
+          code: programme.code,
+          name: programme.name,
+          drill: courses,
+          yearly: _.mergeWith({}, ...yearlyValues, mergele),
+          current: _.sumBy(courseValues, 'current') | undefined,
+          previous: _.sumBy(courseValues, 'previous') | undefined,
+          currentStudents: _.sumBy(courseValues, 'currentStudents') || 'undefined',
+          previousStudents: _.sumBy(courseValues, 'previousStudents') | 'undefined',
+        }
+
+        return acc
+      }
+
+      return acc
+    },
+    {}
+  )
+
   /* Group programmes into faculties and calculate faculty level stats */
   const groupedByFaculty = Object.entries(groupedByProgramme).reduce((acc, [programmeCode, programmeStats]) => {
     const facultyCodes = programmeToFaculties[programmeCode]
     if (!facultyCodes) return acc
     facultyCodes.forEach(facultyCode => {
       if (!facultyCode) return
+      if (facultyCode === 'H906' || facultyCode === 'H930') {
+        if (!acc[facultyCode]) {
+          acc[facultyCode] = {
+            type: 'faculty',
+            code: facultyCode,
+            drill: {},
+            name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
+            yearly: programmeStats.yearly,
+            current: programmeStats.current,
+            previous: programmeStats.previous,
+            currentStudents: programmeStats.currentStudents,
+            previousStudents: programmeStats.previousStudents,
+          }
+        }
+      }
+
+      if (!acc[facultyCode]) {
+        const orgId = organizationCodeToOrganization[facultyCode].id
+
+        const { students: currentStudents, credits: current } = getOrgStats(currentOrgStats, orgId).accumulated
+          .aggregated
+        const { students: previousStudents, credits: previous } = getOrgStats(prevOrgStats, orgId).accumulated
+          .aggregated
+
+        const yearly = getOrgYearlyStats(orgId)
+
+        acc[facultyCode] = {
+          type: 'faculty',
+          code: facultyCode,
+          drill: {},
+          name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
+          yearly,
+          current,
+          previous,
+          currentStudents,
+          previousStudents,
+        }
+      }
+
+      acc[facultyCode]['drill'][programmeCode] = programmeStats
+    })
+    return acc
+  }, {})
+
+  const openUniGroupedByFaculty = Object.entries(openUniGroupedByProgramme).reduce(
+    (acc, [programmeCode, programmeStats]) => {
+      const facultyCode = 'H930'
+
       if (!acc[facultyCode]) {
         acc[facultyCode] = {
+          type: 'faculty',
+          code: facultyCode,
           drill: {},
           name: facultyCodeToFaculty[facultyCode] ? facultyCodeToFaculty[facultyCode].name : null,
           yearly: {},
@@ -215,15 +694,16 @@ const calculateStatusStatistics = async (unixMillis, showByYear) => {
           previous: 0,
         }
       }
+
       acc[facultyCode]['drill'][programmeCode] = programmeStats
       acc[facultyCode]['yearly'] = _.mergeWith(acc[facultyCode]['yearly'], programmeStats.yearly, mergele)
       acc[facultyCode]['current'] += programmeStats.current
       acc[facultyCode]['previous'] += programmeStats.previous
-    })
-    return acc
-  }, {})
-
-  return groupedByFaculty
+      return acc
+    },
+    {}
+  )
+  return { ...groupedByFaculty, ...openUniGroupedByFaculty }
 }
 
 const getStatus = async (unixMillis, showByYear, doRefresh = false) => {

services/backend/src/services/userService.js

@@ -1,117 +1,67 @@
 const axios = require('axios')
+const _ = require('lodash')
 const { USERSERVICE_URL } = require('../conf-backend')
 const UnitService = require('./units')
 const elementDetailService = require('./elementdetails')
 const { filterStudentnumbersByAccessrights } = require('./students')
 const { userDataCache } = require('./cache')
+const { getImporterClient } = require('../util/importerClient')
+const logger = require('../util/logger')
+const { facultiesAndProgrammesForTrends } = require('../services/organisations')
+
 const client = axios.create({ baseURL: USERSERVICE_URL, headers: { secret: process.env.USERSERVICE_SECRET } })
 
-const ping = async () => {
-  const url = '/ping'
-  const response = await axios.get(url)
-  return response.data
+// Private helpers
+const enrichProgrammesFromFaculties = faculties =>
+  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
+
+const enrichWithProgrammes = toEnrich => {
+  if (Array.isArray(toEnrich)) {
+    return toEnrich.map(obj => enrichWithProgrammes(obj))
+  }
+  if (toEnrich.rights) {
+    return {
+      ...toEnrich,
+      rights: _.union(toEnrich.rights, enrichProgrammesFromFaculties(toEnrich.faculties)),
+    }
+  }
+
+  if (toEnrich.elementdetails) {
+    return {
+      ...toEnrich,
+      elementdetails: _.union(
+        toEnrich.elementdetails,
+        enrichProgrammesFromFaculties(toEnrich.faculty.map(f => f.faculty_code))
+      ),
+    }
+  }
+  return toEnrich
+}
+const checkStudyGuidanceGroupsAccess = async hyPersonSisuId => {
+  if (!hyPersonSisuId) {
+    logger.error('Not possible to get groups without personId header')
+    return false
+  }
+  const importerClient = getImporterClient()
+  if (!importerClient) return false
+  const { data } = await importerClient.get(`/person-groups/person/${hyPersonSisuId}`)
+  return data && Object.values(data).length > 0
 }
 
-const findAll = async () => {
-  const response = await client.get('/findall')
-  return response.data
-}
-
-const login = async (uid, full_name, hyGroups, affiliations, email, hyPersonSisuId) => {
-  const response = await client.post('/login', {
-    uid,
-    full_name,
-    hyGroups,
-    affiliations,
-    email,
-    hyPersonSisuId,
-  })
-  return response.data
-}
-
-const superlogin = async (uid, asUser) => {
-  const response = await client.post('/superlogin', {
-    uid,
-    asUser,
-  })
-  return response.data
-}
-
-const byUsername = async uid => {
-  const url = `/user/${uid}`
+const byUsernameData = async username => {
+  // OK, enriched
+  const url = `/user/${username}/user_data`
   const response = await client.get(url)
-  return response.data
+  return enrichWithProgrammes(response.data)
 }
 
-const byUsernameData = async uid => {
-  const url = `/user/${uid}/user_data`
-  const response = await client.get(url)
-  return response.data
-}
-
-const getUserElementDetails = async username => {
-  const url = `/user/elementdetails/${username}`
-  const response = await client.get(url)
-  const elementdetailcodes = response.data
-  return elementDetailService.byCodes(elementdetailcodes)
-}
-
-const byId = async id => {
-  const url = `/user/id/${id}`
-  const response = await client.get(url)
-  return response.data
-}
-
-const updateUser = async (uid, fields) => {
-  userDataCache.del(uid)
-  const url = `/user/${uid}`
-  const response = await client.put(url, fields)
-  return response.data
-}
-
-const getRolesFor = async user => {
-  console.log('roles for', user)
-  const url = `/get_roles/${user}`
-  const response = await client.get(url)
-  console.log(response.data)
-  return response.data
-}
-
-const enableElementDetails = async (uid, codes) => {
-  const response = await client.post('/add_rights', { uid, codes })
-  return response.data.user
-}
-
-const removeElementDetails = async (uid, codes) => {
-  const response = await client.post('/remove_rights', { uid, codes })
-  return response.data.user
-}
-
-const setFaculties = async (uid, faculties) => {
-  const response = await client.post('/set_faculties', { uid, faculties })
-  return response.data.user
-}
-
-const getUnitsFromElementDetails = async username => {
-  const elementDetails = await getUserElementDetails(username)
-  return elementDetails.map(element => UnitService.parseUnitFromElement(element))
-}
-
-const modifyAccess = async body => {
-  const response = await client.post('/modifyaccess', body)
-  return response.data
-}
-
-const getAccessGroups = async () => {
-  const response = await client.get('/access_groups')
-  return response.data
-}
-
-const getUserDataFor = async uid => {
-  let userData = userDataCache.get(uid)
+// Exported helpers
+const getUserDataFor = async username => {
+  // no need to enrich
+  let userData = userDataCache.get(username)
   if (!userData) {
-    userData = await byUsernameData(uid)
-    userDataCache.set(uid, userData)
+    userData = await byUsernameData(username)
+    userDataCache.set(username, userData)
   }
 
   return {
@@ -120,9 +70,21 @@ const getUserDataFor = async uid => {
   }
 }
 
+const getUnitsFromElementDetails = async username => {
+  // No need to enrich
+  let userData = userDataCache.get(username)
+  if (!userData) {
+    userData = await byUsernameData(username)
+    userDataCache.set(username, userData)
+  }
+  const elementDetails = await elementDetailService.byCodes(userData.rights)
+  return elementDetails.map(element => UnitService.parseUnitFromElement(element))
+}
+
 const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
+  // No need to enrich
   let studentsUserCanAccess
-  if (roles && roles.includes('admin')) {
+  if (roles?.includes('admin')) {
     studentsUserCanAccess = new Set(studentnumbers)
   } else {
     const unitsUserCanAccess = await getUnitsFromElementDetails(userId)
@@ -132,12 +94,52 @@ const getStudentsUserCanAccess = async (studentnumbers, roles, userId) => {
   return studentsUserCanAccess
 }
 
+// FOR ROUTES 1-to-1 mapping
+const modifyAccess = async body => enrichWithProgrammes((await client.post('/modifyaccess', body)).data)
+
+const getAccessGroups = async () => enrichWithProgrammes((await client.get('/access_groups')).data)
+
+const enableElementDetails = async (uid, codes) =>
+  enrichWithProgrammes((await client.post('/add_rights', { uid, codes })).data.user)
+
+const findAll = async () => enrichWithProgrammes((await client.get('/findall')).data)
+
+const setFaculties = async (uid, faculties) =>
+  enrichWithProgrammes((await client.post('/set_faculties', { uid, faculties })).data.user)
+
+const removeElementDetails = async (uid, codes) =>
+  enrichWithProgrammes((await client.post('/remove_rights', { uid, codes })).data.user)
+
+const updateUser = async (username, fields) => {
+  userDataCache.del(username)
+  return enrichWithProgrammes((await client.put(`/user/${username}`, fields)).data)
+}
+
+const login = async (uid, full_name, hyGroups, affiliations, email, hyPersonSisuId) => {
+  const hasStudyGuidanceGroupAccess = await checkStudyGuidanceGroupsAccess(hyPersonSisuId)
+  return (
+    await client.post('/login', {
+      uid,
+      full_name,
+      hyGroups,
+      affiliations,
+      email,
+      hyPersonSisuId,
+      hasStudyGuidanceGroupAccess,
+    })
+  ).data
+}
+
+const superlogin = async (uid, asUser) =>
+  (
+    await client.post('/superlogin', {
+      uid,
+      asUser,
+    })
+  ).data
+
 module.exports = {
-  ping,
-  byUsername,
   updateUser,
-  byId,
-  getUserElementDetails,
   login,
   superlogin,
   enableElementDetails,
@@ -146,7 +148,6 @@ module.exports = {
   modifyAccess,
   getAccessGroups,
   getUnitsFromElementDetails,
-  getRolesFor,
   setFaculties,
   getUserDataFor,
   getStudentsUserCanAccess,

services/backend/src/util/customErrors.js

@@ -0,0 +1,25 @@
+class ApplicationError extends Error {
+  constructor(message, status, extra) {
+    super()
+
+    Error.captureStackTrace(this, this.constructor)
+
+    this.name = this.constructor.name
+
+    this.message = message || 'Something went wrong. Please try again.'
+
+    this.status = status || 500
+
+    this.extra = extra || {}
+  }
+
+  toJSON() {
+    return {
+      error: this.message,
+    }
+  }
+}
+
+module.exports = {
+  ApplicationError,
+}

services/backend/src/util/importerClient.js

@@ -1,17 +1,17 @@
 const axios = require('axios')
-
-const { importerToken, importerUrl } = require('../conf-backend')
+const { importerToken } = require('../conf-backend')
+const logger = require('./logger')
 
 const importerClient = axios.create({
   headers: {
     token: importerToken,
   },
-  baseURL: importerUrl,
+  baseURL: 'https://importer.cs.helsinki.fi',
 })
 
 const getImporterClient = () => {
-  if (!importerToken || !importerUrl) {
-    console.log("Importer token or importer url not set, can't return client!")
+  if (!importerToken) {
+    logger.error("Importer token not set, can't return client!")
     return null
   }
   return importerClient

services/backend/src/util/index.js

@@ -1,8 +1,5 @@
 const validateParamLength = (param, minLength) => param && param.trim().length >= minLength
 
-const isNewHYStudyProgramme = code => !!(code && code.match(/^[A-Z]*[0-9]*_[0-9]*$/))
-
 module.exports = {
   validateParamLength,
-  isNewHYStudyProgramme,
 }

services/backend/src/util/logger.js

@@ -1,27 +1,43 @@
 const winston = require('winston')
-const Log2gelf = require('winston-log2gelf')
+const { isDev } = require('../conf-backend')
+const { combine, timestamp, printf, splat } = winston.format
 
-const transports = []
+let transports = []
+
+if (isDev) {
+  const devFormat = printf(
+    ({ level, message, timestamp, ...rest }) => `${timestamp} ${level}: ${message} ${JSON.stringify(rest)}`
+  )
 
-if (process.env.LOG_PORT && process.env.LOG_HOST) {
   transports.push(
-    new Log2gelf({
-      hostname: process.env.LOG_HOSTNAME || 'oodikone-backend',
-      host: process.env.LOG_HOST,
-      port: process.env.LOG_PORT,
-      protocol: process.env.LOG_PROTOCOL || 'https',
-      environment: 'production',
-      protocolOptions: {
-        path: process.env.LOG_PATH || '/gelf',
-      },
+    new winston.transports.Console({
+      level: 'debug',
+      format: combine(splat(), timestamp(), devFormat),
     })
   )
+} else {
+  const levels = {
+    error: 0,
+    warn: 1,
+    info: 2,
+    http: 3,
+    verbose: 4,
+    debug: 5,
+    silly: 6,
+  }
+
+  const prodFormat = winston.format.printf(({ level, ...rest }) =>
+    JSON.stringify({
+      level: levels[level],
+      ...rest,
+    })
+  )
+
+  transports.push(new winston.transports.Console({ format: prodFormat }))
 }
 
-transports.push(new winston.transports.File({ filename: 'debug.log' }))
-
-transports.push(new winston.transports.Console({ level: 'debug' }))
-
 const logger = winston.createLogger({ transports })
 
+logger.on('error', e => console.error('Logging failed! Reason: ', e)) // eslint-disable-line no-console
+
 module.exports = logger

services/frontend/.eslintrc.json

@@ -1,16 +1,14 @@
 {
   "root": true,
   "env": {
-    "browser": true
+    "browser": true,
+    "node": true
   },
   "parser": "@babel/eslint-parser",
   "parserOptions": {
     "requireConfigFile": false
   },
-  "extends": [
-    "airbnb",
-    "plugin:prettier/recommended"
-  ],
+  "extends": ["airbnb", "plugin:prettier/recommended"],
   "rules": {
     "react/no-did-mount-set-state": "off",
     "react/destructuring-assignment": "off",
@@ -31,13 +29,15 @@
     "react/static-property-placement": "off",
     "react/no-deprecated": "off",
     "react/no-unescaped-entities": "off",
+    "react/prop-types": "off",
     "react/prefer-stateless-function": "error",
-    "import/no-unresolved": "error",
+    "import/no-unresolved": "off",
     "react/jsx-no-bind": "error",
     "prefer-object-spread/prefer-object-spread": "error",
     "babel/no-invalid-this": "error",
     "react-hooks/rules-of-hooks": "error",
     "prettier/prettier": "error",
+    "no-console": "error",
     "no-param-reassign": [
       "error",
       {
@@ -45,11 +45,5 @@
       }
     ]
   },
-  "plugins": [
-    "react",
-    "import",
-    "prefer-object-spread",
-    "babel",
-    "react-hooks"
-  ]
+  "plugins": ["react", "import", "prefer-object-spread", "babel", "react-hooks"]
 }

services/frontend/jsconfig.json

@@ -0,0 +1,6 @@
+{
+  "compilerOptions": {
+    "baseUrl": "src"
+  },
+  "include": ["src"]
+}

services/frontend/package-lock.json

@@ -1912,6 +1912,24 @@
       "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.9.2.tgz",
       "integrity": "sha512-VZMYa7+fXHdwIq1TDhSXoVmSPEGM/aa+6Aiq3nVVJ9bXr24zScr+NlKFKC3iPljA7ho/GAZr+d2jOf5GIRC30Q=="
     },
+    "@reduxjs/toolkit": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.6.2.tgz",
+      "integrity": "sha512-HbfI/hOVrAcMGAYsMWxw3UJyIoAS9JTdwddsjlr5w3S50tXhWb+EMyhIw+IAvCVCLETkzdjgH91RjDSYZekVBA==",
+      "requires": {
+        "immer": "^9.0.6",
+        "redux": "^4.1.0",
+        "redux-thunk": "^2.3.0",
+        "reselect": "^4.0.0"
+      },
+      "dependencies": {
+        "immer": {
+          "version": "9.0.6",
+          "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.6.tgz",
+          "integrity": "sha512-G95ivKpy+EvVAnAab4fVa4YGYn24J1SpEktnJX7JJ45Bd7xqME/SCplFzYFmTbrkwZbQ4xJK1xMTUYBkN6pWsQ=="
+        }
+      }
+    },
     "@rollup/plugin-node-resolve": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz",
@@ -2403,9 +2421,9 @@
       "integrity": "sha512-L28j2FcJfSZOnL1WBjDYp2vUHCeIFlyYI/53EwD/rKUBQ7MtUUfbQWiyKJGpcnv4/WgrhWsFKrcPstcAt/J0tQ=="
     },
     "@types/react": {
-      "version": "17.0.14",
-      "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.14.tgz",
-      "integrity": "sha512-0WwKHUbWuQWOce61UexYuWTGuGY/8JvtUe/dtQ6lR4sZ3UiylHotJeWpf3ArP9+DSGUoLY3wbU59VyMrJps5VQ==",
+      "version": "17.0.34",
+      "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.34.tgz",
+      "integrity": "sha512-46FEGrMjc2+8XhHXILr+3+/sTe3OfzSPU9YGKILLrUYbQ1CLQC9Daqo1KzENGXAWwrFwiY0l4ZbF20gRvgpWTg==",
       "requires": {
         "@types/prop-types": "*",
         "@types/scheduler": "*",
@@ -2413,9 +2431,9 @@
       }
     },
     "@types/react-redux": {
-      "version": "7.1.18",
-      "resolved": "https://registry.npmjs.org/@types/react-redux/-/react-redux-7.1.18.tgz",
-      "integrity": "sha512-9iwAsPyJ9DLTRH+OFeIrm9cAbIj1i2ANL3sKQFATqnPWRbg+jEFXyZOKHiQK/N86pNRXbb4HRxAxo0SIX1XwzQ==",
+      "version": "7.1.20",
+      "resolved": "https://registry.npmjs.org/@types/react-redux/-/react-redux-7.1.20.tgz",
+      "integrity": "sha512-q42es4c8iIeTgcnB+yJgRTTzftv3eYYvCZOh1Ckn2eX/3o5TdsQYKUWpLoLuGlcY/p+VAhV9IOEZJcWk/vfkXw==",
       "requires": {
         "@types/hoist-non-react-statics": "^3.3.0",
         "@types/react": "*",
@@ -3142,11 +3160,11 @@
       "integrity": "sha512-99FZt8qS/xukgxU/8daV8WT7wAakqBzt6lF3XCweO6pwcf50/NgxxBj6ZC7/ejR+F4PWeFpkb9lAMH3y2quBXA=="
     },
     "axios": {
-      "version": "0.21.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
-      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
+      "version": "0.21.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz",
+      "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==",
       "requires": {
-        "follow-redirects": "^1.10.0"
+        "follow-redirects": "^1.14.0"
       }
     },
     "axobject-query": {
@@ -4979,9 +4997,9 @@
       }
     },
     "csstype": {
-      "version": "3.0.8",
-      "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.8.tgz",
-      "integrity": "sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw=="
+      "version": "3.0.9",
+      "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
+      "integrity": "sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw=="
     },
     "cyclist": {
       "version": "1.0.1",
@@ -6959,6 +6977,37 @@
         "mime-types": "^2.1.12"
       }
     },
+    "formik": {
+      "version": "2.2.9",
+      "resolved": "https://registry.npmjs.org/formik/-/formik-2.2.9.tgz",
+      "integrity": "sha512-LQLcISMmf1r5at4/gyJigGn0gOwFbeEAlji+N9InZF6LIMXnFNkO42sCI8Jt84YZggpD4cPWObAZaxpEFtSzNA==",
+      "requires": {
+        "deepmerge": "^2.1.1",
+        "hoist-non-react-statics": "^3.3.0",
+        "lodash": "^4.17.21",
+        "lodash-es": "^4.17.21",
+        "react-fast-compare": "^2.0.1",
+        "tiny-warning": "^1.0.2",
+        "tslib": "^1.10.0"
+      },
+      "dependencies": {
+        "deepmerge": {
+          "version": "2.2.1",
+          "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz",
+          "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA=="
+        },
+        "react-fast-compare": {
+          "version": "2.0.4",
+          "resolved": "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-2.0.4.tgz",
+          "integrity": "sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw=="
+        },
+        "tslib": {
+          "version": "1.14.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
+          "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
+        }
+      }
+    },
     "forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -8228,11 +8277,6 @@
         "istanbul-lib-report": "^3.0.0"
       }
     },
-    "jStat": {
-      "version": "1.8.6",
-      "resolved": "https://registry.npmjs.org/jStat/-/jStat-1.8.6.tgz",
-      "integrity": "sha512-Oh/ePZVSoFigme69pHTQudcGh64cpNH9Lz3hBZcRJWLrDqpw7JfuYU9F3dj9py3tBYmHz7og7ZT8hXTNbYq9Rw=="
-    },
     "jest": {
       "version": "26.6.0",
       "resolved": "https://registry.npmjs.org/jest/-/jest-26.6.0.tgz",
@@ -12520,11 +12564,6 @@
         "whatwg-fetch": "^3.4.1"
       }
     },
-    "react-context-devtool": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/react-context-devtool/-/react-context-devtool-2.0.3.tgz",
-      "integrity": "sha512-GVpDaQqRsVoofcbViVWelWTUHNjk4CAA81ZU+TVjEwZUTjhwOgKDM4t6yGfLBoEEG8PFYKbp9QqzyWw2BASe5Q=="
-    },
     "react-datetime": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/react-datetime/-/react-datetime-3.0.4.tgz",
@@ -12729,22 +12768,25 @@
       }
     },
     "react-redux": {
-      "version": "7.2.4",
-      "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-7.2.4.tgz",
-      "integrity": "sha512-hOQ5eOSkEJEXdpIKbnRyl04LhaWabkDPV+Ix97wqQX3T3d2NQ8DUblNXXtNMavc7DpswyQM6xfaN4HQDKNY2JA==",
+      "version": "7.2.6",
+      "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-7.2.6.tgz",
+      "integrity": "sha512-10RPdsz0UUrRL1NZE0ejTkucnclYSgXp5q+tB5SWx2qeG2ZJQJyymgAhwKy73yiL/13btfB6fPr+rgbMAaZIAQ==",
       "requires": {
-        "@babel/runtime": "^7.12.1",
-        "@types/react-redux": "^7.1.16",
+        "@babel/runtime": "^7.15.4",
+        "@types/react-redux": "^7.1.20",
         "hoist-non-react-statics": "^3.3.2",
         "loose-envify": "^1.4.0",
         "prop-types": "^15.7.2",
-        "react-is": "^16.13.1"
+        "react-is": "^17.0.2"
       },
       "dependencies": {
-        "react-is": {
-          "version": "16.13.1",
-          "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
-          "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
+        "@babel/runtime": {
+          "version": "7.16.0",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.0.tgz",
+          "integrity": "sha512-Nht8L0O8YCktmsDV6FqFue7vQLRx3Hb0B37lS5y0jDRqRxlBG4wIJHnf9/bgSE2UyipKFA01YtS+npRdTWBUyw==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
         }
       }
     },
@@ -13031,17 +13073,17 @@
       }
     },
     "redux": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/redux/-/redux-4.1.0.tgz",
-      "integrity": "sha512-uI2dQN43zqLWCt6B/BMGRMY6db7TTY4qeHHfGeKb3EOhmOKjU3KdWvNLJyqaHRksv/ErdNH7cFZWg9jXtewy4g==",
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/redux/-/redux-4.1.2.tgz",
+      "integrity": "sha512-SH8PglcebESbd/shgf6mii6EIoRM0zrQyjcuQ+ojmfxjTtE0z9Y8pa62iA/OJ58qjP6j27uyW4kUF4jl/jd6sw==",
       "requires": {
         "@babel/runtime": "^7.9.2"
       }
     },
     "redux-thunk": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.3.0.tgz",
-      "integrity": "sha512-km6dclyFnmcvxhAcrQV2AkZmPQjzPDjgVlQtR0EQjxZPyJ0BnMf3in1ryuR8A2qU0HldVRfxYXbFSKlI3N7Slw=="
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.0.tgz",
+      "integrity": "sha512-/y6ZKQNU/0u8Bm7ROLq9Pt/7lU93cT0IucYMrubo89ENjxPa7i8pqLKu6V4X7/TvYovQ6x01unTeyeZ9lgXiTA=="
     },
     "regenerate": {
       "version": "1.4.2",
@@ -13214,9 +13256,9 @@
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
     },
     "reselect": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.0.0.tgz",
-      "integrity": "sha512-qUgANli03jjAyGlnbYVAV5vvnOmJnODyABz51RdBN7M4WaVu8mecZWgyQNkG8Yqe3KRGRt0l4K4B3XVEULC4CA=="
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.2.tgz",
+      "integrity": "sha512-wg60ebcPOtxcptIUfrr7Jt3h4BR86cCW3R7y4qt65lnNb4yz4QgrXcbSioVsIOYguyz42+XTHIyJ5TEruzkFgQ=="
     },
     "resolve": {
       "version": "1.18.1",
@@ -14877,9 +14919,9 @@
       "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA=="
     },
     "tar": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz",
-      "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==",
+      "version": "6.1.11",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz",
+      "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==",
       "requires": {
         "chownr": "^2.0.0",
         "fs-minipass": "^2.0.0",
@@ -15101,9 +15143,9 @@
       "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA=="
     },
     "tmpl": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz",
-      "integrity": "sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE="
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz",
+      "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw=="
     },
     "to-arraybuffer": {
       "version": "1.0.1",

services/frontend/package.json

@@ -6,13 +6,14 @@
     "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
   },
   "dependencies": {
+    "@reduxjs/toolkit": "^1.6.2",
     "@sentry/browser": "^6.9.0",
     "@sentry/tracing": "^6.11.0",
-    "axios": "^0.21.1",
+    "axios": "^0.21.2",
+    "formik": "^2.2.9",
     "highcharts": "^9.1.2",
     "highcharts-custom-events": "^3.0.10",
     "history": "^4.10.1",
-    "jStat": "^1.8.6",
     "jwt-decode": "^3.1.2",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
@@ -22,18 +23,14 @@
     "qs": "^6.10.1",
     "query-string": "^6.14.1",
     "react": "^16.14.0",
-    "react-context-devtool": "^2.0.3",
     "react-datetime": "^3.0.4",
     "react-dom": "^16.14.0",
     "react-highcharts": "^16.1.0",
     "react-markdown": "^6.0.2",
-    "react-redux": "^7.2.4",
+    "react-redux": "^7.2.6",
     "react-router-dom": "^5.2.0",
     "react-scripts": "4.0.3",
     "react-scroll-to-component": "^1.0.2",
-    "redux": "^4.1.0",
-    "redux-thunk": "^2.3.0",
-    "reselect": "^4.0.0",
     "semantic-ui-css": "^2.4.1",
     "semantic-ui-react": "^2.0.3",
     "unfuck-spa-shibboleth-session": "^3.1.3",

services/frontend/src/CommonContext.jsx

@@ -1,5 +1,4 @@
 import React from 'react'
-import PropTypes from 'prop-types'
 import FilterContextProvider from './components/FilterTray/FilterContextProvider'
 import { LanguageProvider } from './components/LanguagePicker/useLanguage'
 
@@ -9,8 +8,4 @@ const CommonContext = ({ children }) => (
   </LanguageProvider>
 )
 
-CommonContext.propTypes = {
-  children: PropTypes.node.isRequired,
-}
-
 export default CommonContext

services/frontend/src/apiConnection/index.js

@@ -1,9 +1,11 @@
 import axios from 'axios'
 import * as Sentry from '@sentry/browser'
+import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react'
 import { getMocked, setMocking } from '../common'
 import { apiBasePath, isDev } from '../conf'
+import { loginPrefix, logoutPrefix } from '../redux/auth'
 
-const getHeaders = () => {
+export const getHeaders = () => {
   // Set up dev user for development environment, mimicking production admin user
   const devUserHeaders = {
     uid: 'mluukkai',
@@ -19,16 +21,16 @@ const getHeaders = () => {
 
 export const api = axios.create({ baseURL: apiBasePath, headers: getHeaders() })
 
-const types = {
-  attempt: prefix => `${prefix}ATTEMPT`,
-  failure: prefix => `${prefix}FAILURE`,
-  success: prefix => `${prefix}SUCCESS`,
+export const actionSuffixes = {
+  attempt: `ATTEMPT`,
+  failure: `FAILURE`,
+  success: `SUCCESS`,
 }
 
 export const actionTypes = prefix => ({
-  attempt: types.attempt(prefix),
-  failure: types.failure(prefix),
-  success: types.success(prefix),
+  attempt: `${prefix}${actionSuffixes.attempt}`,
+  failure: `${prefix}${actionSuffixes.failure}`,
+  success: `${prefix}${actionSuffixes.success}`,
 })
 
 export const logout = async () => {
@@ -69,7 +71,8 @@ export const callController = (route, prefix, data, method = 'get', query, param
     params,
     onProgress,
   }
-  return { type: `${prefix}ATTEMPT`, requestSettings }
+  const { attempt } = actionTypes(prefix)
+  return { type: attempt, requestSettings }
 }
 
 const handleError = (err, actionHistory = []) => {
@@ -90,20 +93,24 @@ export const handleRequest = store => next => async action => {
   const { requestSettings } = action
   if (requestSettings) {
     const { route, method, data, prefix, query, params, onProgress } = requestSettings
+    const { success, failure } = actionTypes(prefix)
     try {
       const res = await callApi(route, method, data, params, 0, onProgress)
-      store.dispatch({ type: `${prefix}SUCCESS`, response: res.data, query })
+      store.dispatch({ type: success, response: res.data, query })
     } catch (e) {
-      store.dispatch({ type: `${prefix}FAILURE`, response: e, query })
+      store.dispatch({ type: failure, response: e, query })
       handleError(e, store.getState().actionHistory)
     }
   }
 }
 
 export const handleAuth = store => next => async action => {
+  const loginTypes = actionTypes(loginPrefix)
+  const logoutTypes = actionTypes(logoutPrefix)
   next(action)
   const { type } = action
-  if (type === 'LOGIN_ATTEMPT') {
+  if (type === loginTypes.attempt) {
+    const { success, failure } = loginTypes
     try {
       let token
       const mocked = getMocked()
@@ -118,18 +125,39 @@ export const handleAuth = store => next => async action => {
         ...api.defaults.headers.common,
         'x-access-token': token,
       }
-      store.dispatch({ type: 'LOGIN_SUCCESS', token })
+      store.dispatch({ type: success, token })
     } catch (err) {
-      store.dispatch({ type: 'LOGIN_FAILURE' })
+      store.dispatch({ type: failure })
       handleError(err, store.getState().actionHistory)
     }
-  } else if (type === 'LOGOUT_ATTEMPT') {
+  } else if (type === loginTypes.attempt) {
+    const { success, failure } = logoutTypes
     try {
       await logout()
-      store.dispatch({ type: 'LOGOUT_SUCCESSFUL' })
+      store.dispatch({ type: success })
     } catch (err) {
-      store.dispatch({ type: 'LOGOUT_FAILURE' })
+      store.dispatch({ type: failure })
       handleError(err, store.getState().actionHistory)
     }
   }
 }
+
+// Redux-toolkit query based API
+// Api can be extended with enhanceEndpoints method
+// All tags used for invalidating cache must be defined here
+export const RTKApi = createApi({
+  reducerPath: 'api',
+  tagTypes: ['StudyGuidanceGroups'],
+  baseQuery: fetchBaseQuery({
+    baseUrl: apiBasePath,
+    prepareHeaders: (headers, { getState }) => {
+      // Get token from state and add it to headers
+      const token = getState().auth.encodedToken
+      if (token) headers.set('x-access-token', token)
+      // Add possible default headers
+      Object.entries(getHeaders()).forEach(([key, value]) => headers.set(key, value))
+      return headers
+    },
+  }),
+  endpoints: () => ({}),
+})

services/frontend/src/common/InfoToolTips/coolDataScience.js

@@ -1,63 +1,78 @@
 export default {
   protoC: `Graafi ottaa huomioon vain opiskelijat joiden opinto-oikeus on alkanut 1.8 joka vuodelta vuoden 2017 jälkeen. 
-    Opiskelijat jotka ovat vaihtaneet ohjelmaa tiedekunnan sisällä suodatetaan pois.
-    <br> <br>
-    Opiskelijoiden tavoiteaika lasketaan seuraavasti: **3 vuoden tahdissa** olevat opiskelijat 60op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet, 
-    **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
-    **peruutettu** jos optinto-oikeus merkattu perutuksi,
-    **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
-    <br> <br>
-    Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois. **Include old attainments** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
-    <br> <br>
-    Vakiona opiskelijat jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi) on sisällytetty laskuihin. **Include only at least once enrolled students** 
-    suodattaa pois opiskelijat jotka eivät ole ollenkaan ilmoittautunut
-    <br> <br>
-    Graafissa pystyy pisteitä klikkaamalla porautumaan ensin ohjelmatasolle ja edelleen opintosuuntatasolle.
-    `,
+  Opiskelijat jotka ovat vaihtaneet ohjelmaa tiedekunnan sisällä suodatetaan pois.
+    
+  **3 vuoden tahdissa** olevat opiskelijat 60op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet, 
+  
+  **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
+  
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
+
+  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+    
+  **Include only at least once enrolled students** suodattaa pois opiskelijat jotka eivät ole ollenkaan ilmoittautunut. 
+  Vakiona opiskelijat jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi)
+  on sisällytetty laskuihin.  
+  
+  **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
+  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois.
+  `,
   protoC2: `Graafi ottaa huomioon vain opiskelijat joiden opinto-oikeus on alkanut 1.8 joka vuodelta vuoden 2017 jälkeen. 
   Opiskelijat jotka ovat vaihtaneet ohjelmaa tiedekunnan sisällä suodatetaan pois.
-  <br> <br>
-  Opiskelijoiden tavoiteaika lasketaan seuraavasti: **3 vuoden tahdissa** olevat opiskelijat 60op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet, 
+  
+  **3 vuoden tahdissa** olevat opiskelijat 60op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet, 
+  
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tämä päivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
-  **peruutettu** jos optinto-oikeus merkattu perutuksi,
+  
   **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu).
-  <br> <br>
-  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois. **Include old attainments** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
-  <br> <br>
-  Vakiona opiskelijat jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi) on sisällytetty laskuihin. **Include only at least once enrolled students** 
-  suodattaa pois opiskelijat jotka eivät ole ollenkaan ilmoittautunut
-  <br> <br>
-  Graafissa pystyy palkkeja klikkaamalla porautumaan kunkin tiedekunnan ohjelmatasolle. Painamalla ohjelmatasolla jotakin palkkia uudelleen pääsee takaisin tiedekuntatasolle.
+
+  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+  
+  **Include only at least once enrolled students** suodattaa pois opiskelijat,
+  jotka eivät ole ollenkaan ilmoittautunut. Vakiona opiskelijat, jotka eivät ole ilmoittautunut millekkään lukuvuodelle (läsnä tai poissa olevaksi) on sisällytetty laskuihin.
+  
+  **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
+  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois. 
   `,
   protoG: `Graafi ottaa huomioon vain opiskelijat joiden opinto-oikeus on alkanut 1.8 joka vuodelta vuoden 2017 jälkeen. 
   Opiskelijat jotka ovat vaihtaneet ohjelmaa tiedekunnan sisällä suodatetaan pois.
-  <br> <br>
-  Opiskelijoiden tavoiteaika lasketaan seuraavasti: **3 vuoden tahdissa** olevat opiskelijat 60op × ((tarkastuspäivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet, 
+  
+  **3 vuoden tahdissa** olevat opiskelijat 60op × ((tarkastuspäivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet, 
+  
   **4 vuoden tahdissa** olevat opiskelijat 45op × ((tarkastuspäivä - aloituspäivä) / 365) ≤ opiskelijan opintopisteet,
-  **peruutettu** jos optinto-oikeus merkattu perutuksi,
-  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu). **Tarkastuspäivät** ovat 31.7, 30.11 ja 1.4 joka vuodelle.
-  <br> <br>
-  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois. **Include old attainments** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.`,
+
+  **ei tahdissa** = (kaikki opiskelijat - 3v - 4v - peruutettu). 
+  
+  **Tarkastuspäivät** ovat 31.7, 30.11 ja 1.4 joka vuodelle.
+
+  **peruutettu**, jos opinto-oikeus merkattu perutuksi
+
+  **Include attainments attained before the studyright start** laskee mukaan opiskelijan ennen opinto-oikeuden alkua saadut opintopisteet.
+  Vakiona opintopisteet ennen opinto-oikeutta suodatetaan pois.
+  `,
   status: `
-  **Tilannekuva** näyttää "reaaliaikaisesti" eri tiedekuntien, ohjelmien ja kurssien tuottamat opintopisteet nykyisen lukuvuoden aikana 
-  verrattuna edelliseen lukuvuoteen. **Nuoli** kuvaa muutosta, paljonko opintopisteitä on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
+  Tilannekuva näyttää "reaaliaikaisesti" eri tiedekuntien, ohjelmien ja kurssien tuottamat opintopisteet nykyisen lukuvuoden aikana 
+  verrattuna edelliseen lukuvuoteen. 
+  
+  Nuoli kuvaa muutosta, paljonko opintopisteitä on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
   suhteessa edellisen lukuvuoden alusta vastaavaan tarkastelupäivään viime vuonna. Vain suoritetut ja hyväksiluetut opintopisteet huomioidaan.
-  <br> <br>
-  Tilannekuvassa voi **porautua** kortteja klikkaamalla ensin ohjelmatasolle ja edelleen kurssitasolle. **Asetukset**-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. Valinta **Näytä edelliset vuodet** näyttää
-  vuosittaisen opintopiste-kertymän tähän päivään mennessä vuonna X sekä koko lukuvuoden X opintopisteet muodossa "\`kerääntymä vuonna X / koko lukuvuoden X opintopisteet\`". Valinta **Näytä vuositasolla** laskee kertymän kalenterivuoden alusta,
-  eikä lukuvuoden alusta. Kohdasta **Näytä päivänä** voi valita tarkastelupäivän, joka on oletusarvoisesti tämänhetkinen päivä.
-  <br> <br>
-  Avoimen yliopiston suorituksia ei vielä lasketa mukaan.
+  
+  Tilannekuvassa voi porautua kortteja klikkaamalla ensin ohjelmatasolle ja edelleen kurssitasolle. Asetukset-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. Mahdolliset valinnat ovat selitettyinä alla.
   `,
   statusGraduated: `
-  **Tilannekuva** näyttää "reaaliaikaisesti" eri tiedekuntien ja ohjelmien valmistumiset nykyisen lukuvuoden aikana 
-  verrattuna edelliseen lukuvuoteen. **Nuoli** kuvaa muutosta, paljonko valmistuneita on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
+  Tilannekuva näyttää "reaaliaikaisesti" eri tiedekuntien ja ohjelmien valmistumiset nykyisen lukuvuoden aikana 
+  verrattuna edelliseen lukuvuoteen. 
+  
+  Nuoli kuvaa muutosta, paljonko valmistuneita on kertynyt tämänhetkisen lukuvuoden alusta tarkastelupäivään mennessä 
   suhteessa edellisen lukuvuoden alusta vastaavaan tarkastelupäivään viime vuonna.
-  <br> <br>
-  Tilannekuvassa voi **porautua** kortteja klikkaamalla ohjelmatasolle. **Asetukset**-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. Valinta **Näytä edelliset vuodet** näyttää
-  vuosittaisen valmistumis-kertymän tähän päivään mennessä vuonna X sekä koko lukuvuoden X valmistuneet muodossa "\`kerääntymä vuonna X / koko lukuvuoden X valmistuneet\`". Valinta **Näytä vuositasolla** 
-  laskee kertymän kalenterivuoden alusta,
-  eikä lukuvuoden alusta. Kohdasta **Näytä päivänä** voi valita tarkastelupäivän, joka on oletusarvoisesti tämänhetkinen päivä.
-  <br> <br>
+  
+  Tilannekuvassa voi porautua kortteja klikkaamalla ohjelmatasolle. 
+  Asetukset-nappi avaa valikon, josta voi valita missä muodossa data halutaan nähdä. 
+  
+  Valinta "Näytä edelliset vuodet" näyttää vuosittaisen valmistumiskertymän tähän päivään mennessä vuonna X sekä koko lukuvuoden X valmistuneet muodossa "kerääntymä vuonna X / koko lukuvuoden X valmistuneet". 
+  
+  Valinta "Näytä vuositasolla" laskee kertymän kalenterivuoden alusta, eikä lukuvuoden alusta.
+  
+  Kohdasta "Näytä päivänä" voi valita tarkastelupäivän, joka on oletusarvoisesti tämänhetkinen päivä.
   `,
 }

services/frontend/src/common/InfoToolTips/coursePopulation.js

@@ -0,0 +1,12 @@
+export default {
+  Students: `Students
+
+  Harmaata _Show_-painiketta klikkaamalla saa listan kaikista populaation opiskelijoista. (Huom. suodattimet vaikuttavat tähän listaan!) Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
+  
+  - **Student Number:** Klikkaamalla opiskelijanumeron vieressä olevaa sinistä nuolta, siirtyy "student statistic" -näkymään valitun opiskelijan osalta.
+  - **All Credits:** listaa opiskelijan kaikki opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
+  - **Tags:** Opiskelijoille voi luoda tageja ryhmittelyä varten kohdassa _Study Program > Overview > haluttu koulutusohjelma_.
+  - **Study Programme:** Opiskelijan koulutusohjelma.
+  - **Start Year at Uni:** Aloitusvuosi yliopistolla.
+  `,
+}

services/frontend/src/common/InfoToolTips/index.jsx

@@ -2,10 +2,12 @@ import PopulationStatistics from './populationStatistics'
 import PopulationOverview from './populationOverview'
 import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
+import CoursePopulation from './coursePopulation'
 
 export default {
   PopulationStatistics,
   PopulationOverview,
   CoolDataScience,
   CourseStatistics,
+  CoursePopulation,
 }

services/frontend/src/common/InfoToolTips/populationStatistics.js

@@ -2,6 +2,101 @@ export default {
   Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella ilmoittautuneita, haluttu ajanjakso valitaan tästä.
 
   **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
+  QueryCard: `**Valitun koulutusohjelmavuosikurssin perustiedot.**
+  
+  **Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
+  
+  **Fall** = 1.8. - 31.12. ilmoittautuneet ja **Spring** = 1.1. - 31.7. ilmoittautuneet.
+  
+  **Showing XX months**: Dataa näytetään ohjelman vuosikurssin alkamispäivästä lähtien haluttuun kuukauteen asti (oletusarvoisesti "tähän päivään").
+  
+  **Updated** kertoo milloin data on päivitetty.`,
+  CreditAccumulation: `**Credit Accumulation**
+  
+  Opiskelijoiden kumulatiiviset opintopistekertymät opiskelijakohtaisina kuvaajina.
+  Kun hiiren vie kuvaajien päälle, saa näkyviin jokaista kuvaajaa vastaavan opiskelijanumeron.
+  Opiskelijanumeron yhteydessä olevaa ympyrää klikkaamalla siirtyy ”student statistics” puolelle ja näkee valitun opiskelijan opintotiedot.
+
+  X-akselin skaalaa voi säätää harmaista painikkeista ”1m”, ”3m”, ”6m”, ”YTD”, ”1y” ja ”ALL” tai kuvaajan alla olevan sinisen näkymän harmaita ”kahvoja” siirtämällä.
+  Y-akselin skaalaa voi säätää harmaista painikkeista ”small” ”medium” ja "large”`,
+  CreditStatistics: `
+  **Credits gained**
+
+  Taulukko kertoo opintopistekertymän valitulle opiskelijapopulaatiolle. Opintopisteluokat on suhteutettu tarkasteltavan populaation aloitusvuoteen. 
+  Ylin luokka kertoo siksi aina tavoiteajassa etenevien määrän ja suhteellisen osuuden. 
+  
+  Huom! nollasuorittajissa on mukana myös poissaolevaksi ilmoittautuneet, ellei heitä ole erikseen suodatettu populaatiosta pois.
+
+  Kategorioita voi myös itsessään käyttää suodattimina. Klikkaamalla rivin vasemmassa reunassa olevaa suodatinta jäävät näkyviin vain sen kategorian opiskelijat.
+  Suodatin aukeaa samalla ikkunan vasempaan laitaan, josta sitä voi käyttää vapaavalintaisilla arvoilla.
+  
+  **Credit statistics**
+
+  Taulukko kertoo kaikkien tarkasteluun valittujen opiskelijoiden opintopistemääristä tähän päivään mennessä. Statistiikka on suodatettu käytettyjen filttereiden mukaan.
+  Esimerkiksi, jos mukaan on valittu vain naisopiskelijat, statistiikat koskevat vain heitä. 
+  
+  + **Total** on kokonaisopintopistemäärä kyseiselle populaatiolle.
+  + **Mean** kuvaa opintopistemäärän opiskelijakohtaista keskiarvoa.
+  + **Keskihajonta** kuvaa vaihtelua opintopistemäärissä. Karkeasti, mitä suurempi keskihajonta, sitä enemmän opintopistemäärissä on vaihtelua opiskelijoiden kesken.
+  + **Min ja Max** kertovat pienimmän ja suurimman opintopistemäärän, jotka jollain opiskelijoilla kyseisessä populaatiossa on.
+
+  Mikäli valintatapatieto kyseiselle populaatiolle on saatavissa, näytetään myös samat statistiikat per valintatapa.  
+  `,
+  CoursesOfPopulation: `**Courses of population**
+
+  Tämä taulukko sisältää näkyvissä olevien opiskelijoiden kaikki kurssisuoritukset.
+  
+  Opiskelijat on mahdollista suodattaa kurssin perusteella käyttämällä kurssirivin vasemmassa laidassa olevaa suodatinpainiketta.
+  Kurssisuodatin aukeaa ikkunan vasempaan laitaan ja tarjoaa lisää vaihtoehtoja suodatukseen. Voit myös valita monta kurssia kerralla suodattimeen.
+  
+  Klikkaamalla kurssin nimen vieressä olevaa sinistä nuolta, siirryt ko. kurssin *Course Statistics* -näkymään.
+  
+  Taulukon tietoja voi suodattaa antamalla opiskelijoiden vähimmäismäärän kurssilla kenttään *Limit to courses where student number at least*.
+  Lisäksi kurssin nimi- ja koodisarakkeissa on hakukentät *("filter here")* suodattamista varten.
+  Suodattimet saa tyhjennettyä painamalla kentän vieressä olevaa ruksia.
+  
+  **Huom! Nämä suodattimet vaikuttavat vain tähän taulukkoon, eivät koko populatioon.**
+  
+  **Taulukot**
+  
+  - **pass/fail** - oletusnäkymä, joka sisältää kurssisuoritukset, hylätyt sekä yritykset. Taulukon tiedot voi järjestää sarakkeita (n, after retry, percentage, many times, per student, passed ja attempted) klikkaamalla.
+  - **grades** - arvosanajakauma.
+  - **when passed** - kurssisuorituksen ajankohta. Huom.! *Passed*-sarakkeen kokonaissumma ei välttämättä täsmää puolivuosittaisten lukumäärien summan kanssa johtuen tuplakirjauksista Oodissa.
+  `,
+  Search: `Tässä osiossa voi tarkastella koulutusohjelmakohtaisia populaatioita sisäänotto-lukuvuoden mukaisesti,
+  vuosikurssi kerrallaan. Uudet koulutusohjelmat ovat pääsääntöisesti alkaneet 1.8.2017,
+  joten oletusvalinnat näyttävät tiedot tästä päivämäärästä lähtien.
+
+  - **Class of:** lukuvuosi, jolloin opiskelija on ilmoittautunut ensimmäisen kerran (läsnä- tai poissaolevaksi) ohjelmaan. Ilmoittautumisen opinto-oikeus voi olla ensi- tai toissijainen.
+  - **Study programme:** haluttu koulutusohjelma.
+  - **Select tag (Optional):** itsetehty tai valmis tägi.
+  - **Study track (Optional):** opintosuunta.`,
+  Students: `**Students**
+
+  Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
+  
+  **General**
+  
+  Klikkaamalla opiskelijanumeron vieressä olevaa sinistä nuolta, siirtyy "student statistic" näkymään valitun opiskelijan osalta.
+  
+  - **credits since start of studyright:**  listaa opiskelijan suoritetut opintopisteet valitun tarkastelujakson / populaation ajalta.
+  - **all credits:** listaa opiskelijan kaikki opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
+  - **transferred from:** opiskelijan vanha koulutusohjelma, josta opiskelija on siirtynyt uuteen.
+  - **studytracks:** opintosuunta
+  
+  **Mandatory Courses**
+  
+  Tällä välilehdellä näkyy, mitä pakollisiksi määriteltyjä kursseja opiskelija on suorittanut.
+  Kurssit voi määrittää kohdassa "Study program" > "Overview" > haluttu koulutusohjelma. Scrollatessa kurssitaulukkoa alaspäin, kurssin nimen saa näkyviin viemällä kursorin halutun solun kohdalle.
+  
+  **Tags**
+  
+  Opiskelijoille voi luoda tag:eja ryhmittelyä varten kohdassa "Study Program" > "Overview" > haluttu koulutusohjelma.
+  
+  **Download**
+  
+  Klikkaamalla tätä harmaata kuvaketta, saa ladattua .xlsx tiedoston, joka sisältää kaikki "General" ja "Mandatory Courses" välilehtien tiedot.
+  `,
   Filters: {
     Add: `Harmaata ”add” painiketta klikkaamalla aukeaa filters- eli suodatinvalikko, 
     josta voi lisätä haluamiaan suodattimia, jotka rajaavat valittua populaatiota.`,

services/frontend/src/common/hooks.js

@@ -2,6 +2,8 @@ import React, { useCallback, useState, useEffect, useRef } from 'react'
 import { chunk, isEqual } from 'lodash'
 import qs from 'query-string'
 import { useSelector } from 'react-redux'
+import { useHistory } from 'react-router-dom'
+import { setMocking } from './index'
 
 import { SEARCH_HISTORY_VERSION } from '../constants'
 import TSA from './tsa'
@@ -276,3 +278,24 @@ export const useIsAdmin = () => {
   if (!roles) return false
   return roles.some(role => role.group_code === 'admin')
 }
+
+// From: https://www.joshwcomeau.com/snippets/react-hooks/use-toggle/
+export const useToggle = (initialValue = false) => {
+  const [value, setValue] = useState(initialValue)
+  const toggle = useCallback(() => {
+    setValue(v => !v)
+  }, [])
+  return [value, toggle]
+}
+
+export const useShowAsUser = () => {
+  const history = useHistory()
+
+  const showAsUser = uid => {
+    setMocking(uid)
+    history.push('/')
+    window.location.reload()
+  }
+
+  return showAsUser
+}

services/frontend/src/common/index.js

@@ -12,11 +12,6 @@ export const setMocking = userid =>
   userid ? localStorage.setItem(MOCK_USERID, userid) : localStorage.removeItem(MOCK_USERID)
 export const getMocked = () => localStorage.getItem(MOCK_USERID)
 
-export const SIS_WARNING_FLAG = 'SIS_WARNING_FLAG'
-export const setHideSisWarningFlag = booleanFlag =>
-  booleanFlag ? localStorage.setItem(SIS_WARNING_FLAG, booleanFlag) : localStorage.removeItem(SIS_WARNING_FLAG)
-export const getHideSisWarningFlag = () => localStorage.getItem(SIS_WARNING_FLAG)
-
 export const textAndDescriptionSearch = (dropDownOptions, param) =>
   filter(dropDownOptions, option =>
     option.text
@@ -228,3 +223,41 @@ export const validateInputLength = (input, minLength) => input && input.trim().l
 export const splitByEmptySpace = str => str.replace(/\s\s+/g, ' ').split(' ')
 
 export const isNewHYStudyProgramme = code => !!(code && code.match(/^[A-Z]*[0-9]*_[0-9]*$/))
+
+// These are the new Bachelor's programmes in Matlu, that have BH possibility
+export const bachelorHonoursProgrammes = [
+  'KH50_001',
+  'KH50_002',
+  'KH50_003',
+  'KH50_004',
+  'KH50_005',
+  'KH50_006',
+  'KH50_007',
+  'KH50_008',
+]
+
+// Basic modules from Sisu programme structures. All modules that can be found for the past years' structures
+// taken into account
+export const bachelorHonoursBasicModules = {
+  KH50_001: ['MAT110', 'MAT120', 'MAT130', 'MAT120', 'TKT1'],
+  KH50_002: ['FYS1100', 'FYS1200'],
+  KH50_003: ['KEK100'],
+  KH50_004: ['MAT110', 'FYS1100', 'FYS1200', 'KEK100'],
+  KH50_005: ['TKT1'],
+  KH50_006: ['GEOK_100'],
+  KH50_007: ['MAA-100'],
+  KH50_008: ['BSCH1000', 'BSCS1000', 'BSMA1000', 'BSMS1000', 'BSPH1000'],
+}
+
+// Similarly intermediate modules from Sisu programme structures. All modules that can be found for the past years' structures
+// taken into account
+export const bachelorHonoursIntermediateModules = {
+  KH50_001: ['mat217', 'MAT210', 'MAT220', 'MAT240', 'MAT213', 'MAT230', 'MAT218', 'MAT011'],
+  KH50_002: ['FYS2100', 'FYS2200', 'FYS2300', 'FYS2400', 'FYS2500', 'FYS2600'],
+  KH50_003: ['KEK200'],
+  KH50_004: ['MFK-M200', 'MFK-F200', 'MFK-K200'],
+  KH50_005: ['TKT2'],
+  KH50_006: ['GEOK_200'],
+  KH50_007: ['MAA-200'],
+  KH50_008: ['BSCH2000', 'BSCS2000', 'BSMA2000', 'BSMS2000', 'BSPH2000'],
+}

services/frontend/src/common/markdown/coursePopulation/students.info.md

@@ -1,9 +0,0 @@
-# Students
-
-Harmaata _Show_-painiketta klikkaamalla saa listan kaikista populaation opiskelijoista. (Huom. suodattimet vaikuttavat tähän listaan!) Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
-
-- **Student Number:** Klikkaamalla opiskelijanumeron vieressä olevaa sinistä nuolta, siirtyy "student statistic" -näkymään valitun opiskelijan osalta.
-- **All Credits:** listaa opiskelijan kaikki opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
-- **Tags:** Opiskelijoille voi luoda tageja ryhmittelyä varten kohdassa _Study Program > Overview > haluttu koulutusohjelma_.
-- **Study Programme:** Opiskelijan koulutusohjelma.
-- **Start Year at Uni:** Aloitusvuosi yliopistolla.

services/frontend/src/common/markdown/populationStatistics/coursesOfPopulation.info.md

@@ -1,15 +0,0 @@
-# Courses of population
-
-Tämä taulukko sisältää näkyvissä olevien opiskelijoiden kaikki kurssisuoritukset.
-
-Opiskelijat on mahdollista suodattaa kurssin perusteella käyttämällä kurssirivin vasemmassa laidassa olevaa suodatinpainiketta. Kurssisuodatin aukeaa ikkunan vasempaan laitaan ja tarjoaa lisää vaihtoehtoja suodatukseen. Voit myös valita monta kurssia kerralla suodattimeen.
-
-Klikkaamalla kurssin nimen vieressä olevaa sinistä nuolta, siirryt ko. kurssin *Course Statistics* -näkymään.
-
-Taulukon tietoja voi suodattaa antamalla opiskelijoiden vähimmäismäärän kurssilla kenttään *Limit to courses where student number at least*. Lisäksi kurssin nimi- ja koodisarakkeissa on hakukentät *("filter here")* suodattamista varten. Suodattimet saa tyhjennettyä painamalla kentän vieressä olevaa ruksia. *Huom! Nämä suodattimet vaikuttavat vain tähän taulukkoon, eivät koko populatioon.*
-
-## Taulukot:
-
-- **pass/fail** - oletusnäkymä, joka sisältää kurssisuoritukset, hylätyt sekä yritykset. Taulukon tiedot voi järjestää sarakkeita (n, after retry, percentage, many times, per student, passed ja attempted) klikkaamalla.
-- **grades** - arvosanajakauma.
-- **when passed** - kurssisuorituksen ajankohta. Huom.! *Passed*-sarakkeen kokonaissumma ei välttämättä täsmää puolivuosittaisten lukumäärien summan kanssa johtuen tuplakirjauksista Oodissa.

services/frontend/src/common/markdown/populationStatistics/creditAccumulation.info.md

@@ -1,5 +0,0 @@
-# Credit Accumulation
-
-Opiskelijoiden kumulatiiviset opintopistekertymät opiskelijakohtaisina kuvaajina. Kun hiiren vie kuvaajien päälle, saa näkyviin jokaista kuvaajaa vastaavan opiskelijanumeron. Opiskelijanumeron yhteydessä olevaa ympyrää klikkaamalla siirtyy ”student statistics” puolelle ja näkee valitun opiskelijan opintotiedot.
-
-X-akselin skaalaa voi säätää harmaista painikkeista ”1m”, ”3m”, ”6m”, ”YTD”, ”1y” ja ”ALL” tai kuvaajan alla olevan sinisen näkymän harmaita ”kahvoja” siirtämällä. Y-akselin skaalaa voi säätää harmaista painikkeista ”small” ”medium” ja "large”

services/frontend/src/common/markdown/populationStatistics/creditStatistics.info.md

@@ -1,5 +0,0 @@
-# Opintopistekertymä
-
-Taulukon rivit skaalautuvat valitun tarkastelujakson mukaan. Jos tarkastelujaksoksi valitsee 12 kuukautta, on ylin kategoria 55 opintopistettä tai yli, eli opiskelija etenee tavoiteajassa. Taulukosta näkee silmäyksellä tavoiteajassa etenevät (ylin kategoria) ja esim. nollasuorittajat. *Huom! nollasuorittajissa on mukana myös poissaolevaksi ilmoittautuneet, ellei heitä ole erikseen suodatettu populaatiosta pois.*
-
-Kategorioita voi myös itsessään käyttää suodattimina. Klikkaamalla rivin vasemmassa reunassa olevaa suodatinta jäävät näkyviin vain sen kategorian opiskelijat. Suodatin aukeaa samalla ikkunan vasempaan laitaan, josta sitä voi käyttää vapaavalintaisilla arvoilla.

services/frontend/src/common/markdown/populationStatistics/queryCard.info.md

@@ -1,12 +0,0 @@
-Valitun koulutusohjelmavuosikurssin perustiedot.
-<br>
-<br>
-**Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
-<br>
-**Fall** = 1.8. - 31.12. ilmoittautuneet ja **Spring** = 1.1. - 31.7. ilmoittautuneet.
-<br>
-**showing XX months**: Dataa näytetään ohjelman vuosikurssin alkamispäivästä lähtien haluttuun kuukauteen asti (oletusarvoisesti "tähän päivään").
-<br>
-**Updated** kertoo milloin data on päivitetty
-<br>
-Mustasta rastista voi sulkea valitun koulutusohjelmavuosikurssin.

services/frontend/src/common/markdown/populationStatistics/search.info.md

@@ -1,6 +0,0 @@
-Tässä osiossa voi tarkastella koulutusohjelmakohtaisia populaatioita sisäänotto-lukuvuoden mukaisesti, vuosikurssi kerrallaan. Uudet koulutusohjelmat ovat pääsääntöisesti alkaneet 1.8.2017 joten oletusvalinnat näyttävät tiedot tästä päivämäärästä lähtien.
-
-- **Class of:** lukuvuosi, jolloin opiskelija on ilmoittautunut ensimmäisen kerran (läsnä- tai poissaolevaksi) ohjelmaan. Ilmoittautumisen opinto-oikeus voi olla ensi- tai toissijainen.
-- **Study programme:** haluttu koulutusohjelma.
-- **Select tag (Optional):** itsetehty tai valmis tägi.
-- **Study track (Optional):** opintosuunta.

services/frontend/src/common/markdown/populationStatistics/students.info.md

@@ -1,24 +0,0 @@
-# Students
-
-Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
-
-## General
-
-Klikkaamalla opiskelijanumeron vieressä olevaa sinistä nuolta, siirtyy "student statistic" näkymään valitun opiskelijan osalta.
-
-- **credits since start of studyright:**  listaa opiskelijan suoritetut opintopisteet valitun tarkastelujakson / populaation ajalta.
-- **all credits:** listaa opiskelijan kaikki opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
-- **transferred from:** opiskelijan vanha koulutusohjelma, josta opiskelija on siirtynyt uuteen.
-- **studytracks:** opintosuunta
-
-## Mandatory Courses
-
-Tällä välilehdellä näkyy, mitä pakollisiksi määriteltyjä kursseja opiskelija on suorittanut. Kurssit voi määrittää kohdassa "Study program" > "Overview" > haluttu koulutusohjelma. Scrollatessa kurssitaulukkoa alaspäin, kurssin nimen saa näkyviin viemällä kursorin halutun solun kohdalle.
-
-## Tags
-
-Opiskelijoille voi luoda tag:eja ryhmittelyä varten kohdassa "Study Program" > "Overview" > haluttu koulutusohjelma.
-
-## Download
-
-Klikkaamalla tätä harmaata kuvaketta, saa ladattua .xlsx tiedoston, joka sisältää kaikki "General" ja "Mandatory Courses" välilehtien tiedot.

services/frontend/src/components/AccessDenied/index.jsx

@@ -1,11 +1,10 @@
 import React, { useState, useEffect } from 'react'
 import { Transition, Dimmer, Header, Image, Container, Button } from 'semantic-ui-react'
-import { bool, func } from 'prop-types'
 import Highcharts from 'highcharts'
 import ReactHighchart from 'react-highcharts'
-import { connect } from 'react-redux'
+import { useDispatch } from 'react-redux'
 import { random } from 'lodash'
-import { logout as logoutAction } from '../../redux/auth'
+import { logout } from '../../redux/auth'
 import { images } from '../../common'
 import MulticolorBarChart from './MulticolorBarChart'
 
@@ -28,6 +27,7 @@ const names = [
   'otahontas',
   'saarasat',
   'popalmu',
+  'ollikehy',
 ]
 
 const dummyData = names.map(name => ({
@@ -38,7 +38,7 @@ const dummyData = names.map(name => ({
   }, []),
 }))
 
-const AccessDenied = ({ notEnabled, logout }) => {
+const AccessDenied = ({ notEnabled }) => {
   const header = notEnabled ? 'Welcome to Oodikone!' : 'Something broke'
   const subheader = notEnabled
     ? `You're currently not allowed to enter
@@ -47,6 +47,7 @@ const AccessDenied = ({ notEnabled, logout }) => {
     pressing log out or contacting grp-toska@helsinki.fi`
 
   const [easterEgg, setEasterEgg] = useState(false)
+  const dispatch = useDispatch()
 
   useEffect(() => {
     if (notEnabled) {
@@ -95,7 +96,7 @@ const AccessDenied = ({ notEnabled, logout }) => {
           <p>{header}</p>
           <Header.Subheader>{subheader}</Header.Subheader>
           <br />
-          <Button onClick={logout} color="pink">
+          <Button onClick={() => dispatch(logout())} color="pink">
             {' '}
             Log out{' '}
           </Button>
@@ -114,13 +115,4 @@ const AccessDenied = ({ notEnabled, logout }) => {
   )
 }
 
-AccessDenied.propTypes = {
-  notEnabled: bool.isRequired,
-  logout: func.isRequired,
-}
-
-const mapDispatchToProps = {
-  logout: logoutAction,
-}
-
-export default connect(null, mapDispatchToProps)(AccessDenied)
+export default AccessDenied

services/frontend/src/components/CoolDataScience/CoolDataScience.jsx

@@ -1,5 +1,4 @@
 import React from 'react'
-import PropTypes from 'prop-types'
 import { Segment } from 'semantic-ui-react'
 import { Switch, Route, Link, useRouteMatch } from 'react-router-dom'
 
@@ -32,20 +31,10 @@ const NavTile = ({ to, title, imageSrc }) => {
   )
 }
 
-NavTile.propTypes = {
-  to: PropTypes.string.isRequired,
-  title: PropTypes.string.isRequired,
-  imageSrc: PropTypes.string.isRequired,
-}
-
 const NavTiles = ({ children }) => {
   return <div className="cool-ds-nav-tiles">{children}</div>
 }
 
-NavTiles.propTypes = {
-  children: PropTypes.node.isRequired,
-}
-
 const baseURI = '/cool-data-science'
 
 const CoolDataScience = () => {

services/frontend/src/components/CoolDataScience/ProtoC.jsx

@@ -5,12 +5,12 @@ import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
 import { Segment, Loader, Dimmer, Checkbox, Button, Message, Icon } from 'semantic-ui-react'
 import _ from 'lodash'
-import ReactMarkdown from 'react-markdown'
 import HighchartsCustomEvents from 'highcharts-custom-events'
 
 import TSA from '../../common/tsa'
 import InfoToolTips from '../../common/InfoToolTips'
 import { getProtoC, getProtoCProgramme } from '../../redux/coolDataScience'
+import ReactMarkdown from 'react-markdown'
 
 HighchartsCustomEvents(Highcharts)
 
@@ -578,7 +578,7 @@ const ProtoC = ({
     })
 
   const SorterButtons = () => (
-    <div align="center" style={{ marginTop: '10px' }}>
+    <div align="center" style={{ marginTop: '15px' }}>
       <Button.Group>
         <Button style={{ cursor: 'default' }} active color="black">
           Sort by:
@@ -622,14 +622,11 @@ const ProtoC = ({
           checked={includeOldAttainments}
         />
       </div>
-      <Message>
-        <ReactMarkdown source={CoolDataScience.protoC} escapeHtml={false} />
-      </Message>
     </>
   )
 
   const DrilldownMessage = () => (
-    <Message color='blue'
+    <Message style={{ margin: '2em' }} color='blue'
       content="Graafissa pystyy pisteitä klikkaamalla porautumaan ensin ohjelmatasolle ja edelleen opintosuuntatasolle. 
       Uusi graafi avautuu nykyisen alle."
     />
@@ -675,6 +672,9 @@ const ProtoC = ({
         )}
       </Segment>
       <RenderBelowGraph />
+      <Message>
+        <ReactMarkdown children={CoolDataScience.protoC} escapeHtml={false} />
+      </Message>
     </Segment>
   )
 }

services/frontend/src/components/CoolDataScience/ProtoC2.jsx

@@ -466,7 +466,10 @@ const ProtoC = ({
             />
           </div>
           <Message>
-            <ReactMarkdown source={CoolDataScience.protoC2} escapeHtml={false} />
+            {
+              // eslint-disable-next-line react/no-children-prop
+              <ReactMarkdown children={CoolDataScience.protoC2} escapeHtml={false} />
+            }
           </Message>
         </Segment>
       </Segment>
@@ -529,7 +532,10 @@ const ProtoC = ({
           />
         </div>
         <Message>
-          <ReactMarkdown source={CoolDataScience.protoC2} escapeHtml={false} />
+          {
+            // eslint-disable-next-line react/no-children-prop
+            <ReactMarkdown children={CoolDataScience.protoC2} escapeHtml={false} />
+          }
         </Message>
       </Segment>
     </Segment>

services/frontend/src/components/CoolDataScience/ProtoG.jsx

@@ -291,7 +291,9 @@ const ProtoG = ({ uberdata, years, isLoading, loadingYears, getUberDispatch, get
           </Table>
         )}
         <Message>
-          <ReactMarkdown source={CoolDataScience.protoG} escapeHtml={false} />
+          {//eslint-disable-next-line react/no-children-prop
+            <ReactMarkdown children={CoolDataScience.protoC} escapeHtml={false} />
+          }
         </Message>
       </Segment>
     </Segment>

services/frontend/src/components/CoolDataScience/Status.jsx

@@ -1,12 +1,12 @@
-import React, { useState, useEffect } from 'react'
-import PropTypes, { shape, bool, func } from 'prop-types'
-import { connect } from 'react-redux'
-import { Segment, Loader, Dimmer, Icon, Accordion, Checkbox, Message, Form } from 'semantic-ui-react'
+import React, { useState, useEffect, useMemo, useRef } from 'react'
+import { useSelector, useDispatch } from 'react-redux'
+import { Divider, Segment, Button, Popup, Loader, Icon, Checkbox, Form, Breadcrumb } from 'semantic-ui-react'
 import _ from 'lodash'
 import moment from 'moment'
 import ReactMarkdown from 'react-markdown'
 import Datetime from 'react-datetime'
 
+import { useHistory, useLocation } from 'react-router-dom'
 import TSA from '../../common/tsa'
 import { getTextIn } from '../../common'
 import { useLocalStorage } from '../../common/hooks'
@@ -28,31 +28,128 @@ const mapValueToRange = (x, min1, max1, min2, max2) => {
   return ((x - min1) * (max2 - min2)) / (max1 - min1) + min2
 }
 
+const settingDefinitions = {
+  showByYear: {
+    label: 'Näytä kalenterivuosittain',
+    short: 'Näytä tilastot kalenterivuosittain lukuvuosien sijasta.',
+    long: `
+      Kun tämä valinta on käytössä, vuosittaiset ajanjaksot lasketaan kalenterivuoden alusta sen loppuun.
+      Muulloin vuosittaiset ajanjaksot lasketaan lukukauden alusta seuraavan lukukauden alkuun.
+    `,
+    defaultValue: false,
+  },
+
+  showYearlyValues: {
+    label: 'Näytä edelliset vuodet',
+    short: 'Näytä tilastot vuosittain, alkaen vuodesta 2017.',
+    long: `
+      Näyttää tilastot vuodesta 2017 eteenpäin.
+
+      Jokaiselta vuodelta näytetään kaksi tilastoa: ajanjakson kokonaistilasto ja ns. tähän mennessä "*kertynyt*" tilasto.
+      Kokonaistilasto vastaa nimensä mukaisesti koko ajanjaksoa, kun taas *kertynyt* tilasto kattaa ajanjakson vuoden 
+      tai lukuvuoden alusta "Näytä päivänä"-valintaa vastaavaan päivämäärään tuona vuotena. Luvut näytetään muodossa *<kerynyt>*/*<kokonais>*.
+      Kuluvalta vuodelta näytetään ainoastaan kertynyt tilasto.
+    `,
+    defaultValue: false,
+  },
+
+  showRelativeValues: {
+    label: 'Näytä suhteutettuna opiskelijoiden määrään',
+    short:
+      'Näyttää tilastot suhteutettuna opiskelijoiden määrään kyseisellä aikavälillä ja kyseisessä organisaatiossa.',
+    long: `
+      Näyttää tilastot suhteutettuna opiskelijoiden määrään kyseisellä aikavälillä ja kyseisessä organisaatiossa.
+      Opiskelijoiden määrä perustuu ajanjaksolla kyseisen organisaation alaisista kursseista suoritusmerkintöjä saaneiden opiskelijoiden määrään.
+      Luku siis sisältää muutkin kuin kyseiseen ohjelman tai osaston opinto-oikeuden omaavat opiskelijat.
+    `,
+    defaultValue: false,
+  },
+
+  showCountingFrom: {
+    label: 'Näytä päivänä',
+    short: 'Valitse päivä johon asti kertyneet tilastot näytetään.',
+    long: `
+      Tämä valinta määrittää päivämäärän, jota käyttäen kertyneet tilastot lasketaan.
+      Esimerkiksi "Näytä kalenterivuosittain" valinnan ollessa pois päältä,
+      lasketaan kertyneet tilastot (vrt. lukuvuosien kokonaistilastot) kunkin lukuvuoden alusta
+      tätä päivämäärää vastaavaan päivään kyseisenä lukuvuonna.
+    `,
+    defaultValue: null,
+  },
+
+  showStudentCounts: {
+    label: 'Näytä kurssien opiskelijamäärät',
+    short: 'Näyttää suoritettujen opintopisteiden sijasta opiskelijoiden määrät kurssitason näkymässä.',
+    long: `
+      Oletuksena kurssitason näkymässä näytetään organisaatio- ja ohjelmatason näkymien tapaan suoritettujen opintopisteiden
+      kokonaismäärä. Kun tämä valinta on käytössä, tämän sijasta näytettävät luvut vastaavat kurssin suorittaneiden 
+      *yksilöityjen* opiskelijoiden määrää. Kukin opiskelija lasketaan siis vain kerran tähän tilastoon.
+    `,
+    defaultValue: false,
+  },
+}
+
 const StatusContainer = ({
-  title,
-  current,
-  previous,
-  clickable,
+  stats,
   handleClick,
   min1,
   max1,
   showYearlyValues,
-  yearlyValues,
+  showRelativeValues,
   showByYear,
+  showStudentCounts,
 }) => {
-  const diff = Math.round(current - previous)
+  const title = getTextIn(stats.name)
+  const clickable = !!stats.drill
+
+  let current
+  let previous
+
+  if (!stats.drill) {
+    current = stats.currentStudents
+    previous = stats.previousStudents
+  } else {
+    current = stats.current
+    previous = stats.previous
+
+    if (showRelativeValues) {
+      current /= stats.currentStudents
+      previous /= stats.previousStudents
+    }
+  }
+
+  const diff = current - previous
   const p = getP(current, previous)
-  const change = Math.round((p - 1) * 1000) / 10
+  const change = p - 1
 
   const getColor = v => {
-    if (v > 2.5) return '#6ab04c'
-    if (v < -2.5) return '#ff7979'
+    if (v > 0.025) return '#6ab04c'
+    if (v < -0.025) return '#ff7979'
     return '#7B9FCF'
   }
 
-  const plussify = x => {
-    if (x > 0) return `+${x.toLocaleString('fi')}`
-    return x.toLocaleString('fi')
+  const plussify = (x, decimals = 0) =>
+    x.toLocaleString('fi', {
+      minimumFractionDigits: decimals,
+      maximumFractionDigits: decimals,
+      signDisplay: 'always',
+    })
+
+  const getDisplayValue = (value, denominator) => {
+    let displayValue
+
+    if (denominator === 0) {
+      displayValue = value === 0 ? 0 : Infinity
+    } else if (showRelativeValues) {
+      displayValue = value / denominator
+    } else {
+      displayValue = value
+    }
+
+    return displayValue.toLocaleString('fi', {
+      minimumFractionDigits: showRelativeValues ? 2 : 0,
+      maximumFractionDigits: showRelativeValues ? 2 : 0,
+    })
   }
 
   return (
@@ -80,239 +177,86 @@ const StatusContainer = ({
           overflowWrap: 'break-word',
         }}
       >
-        <span className="status-title">{title}</span>
+        <Popup position="bottom center" size="tiny" trigger={<span className="status-title">{title}</span>}>
+          {stats.code}
+        </Popup>
         <div style={{ margin: '10px 0' }}>
           <Icon
             style={{
               color: getColor(change),
-              transform: `rotateZ(${-mapValueToRange(change, min1, max1, -45, 45)}deg)`,
+              transform: `rotateZ(${-mapValueToRange(change * 100, min1, max1, -45, 45)}deg)`,
+              transitionProperty: 'transform',
+              transitionDuration: '200ms',
             }}
             size="huge"
             name="arrow right"
           />
         </div>
         <div>
-          <span style={{ fontSize: 20, fontWeight: 'bold', color: getColor(change) }}>{plussify(diff)}</span>
+          <span style={{ fontSize: 20, fontWeight: 'bold', color: getColor(change) }}>
+            {plussify(diff, showRelativeValues ? 2 : 0)}
+          </span>
+          <br />
+          <span style={{ fontWeight: 'bold', color: getColor(change) }}>({plussify(change * 100, 1)}%)</span>
           <br />
-          <span style={{ fontWeight: 'bold', color: getColor(change) }}>({plussify(change)}%)</span>
         </div>
       </div>
       {showYearlyValues && (
-        <div style={{ marginTop: '10px', textAlign: 'start' }}>
-          {_.orderBy(Object.entries(yearlyValues), ([y]) => y, ['desc']).map(
-            ([year, { acc, total, accStudents, totalStudents }]) => {
-              return (
-                <div style={{ margin: '5px 0' }} key={`${title}-${year}`}>
-                  <span>
-                    <b>
-                      {year}
-                      {!showByYear && `-${`${Number(year) + 1}`.slice(-2)}`}:
-                    </b>{' '}
-                    {/* render num of students instead of credits if no credits are given from the course */}
-                    {accStudents <= acc ? Math.round(acc).toLocaleString('fi') : accStudents}
-                    {/* if no total students add students after accStudents */}
-                    {acc <= accStudents && totalStudents < 1 && ' students'}
-                    {/* render total if there are credits or students for either */}
-                    {(total > 0 || totalStudents > 0) &&
-                      ` / ${
-                        // same logic as before to check if render num of students or credits
-                        totalStudents <= total && total > 0
-                          ? Math.round(total).toLocaleString('fi')
-                          : `${totalStudents} students`
-                      }`}
-                  </span>
-                </div>
-              )
-            }
-          )}
+        <div className={`years item-type-${stats.type}`}>
+          {_.orderBy(Object.entries(stats.yearly), ([y]) => y, ['desc']).map(([year, yearStats]) => {
+            return (
+              <span className="year-row">
+                <b className="year-label">
+                  {year}
+                  {!showByYear && `-${`${Number(year) + 1}`.slice(-2)}`}:
+                </b>
+                {stats.type === 'course' && showStudentCounts ? (
+                  <>
+                    <span className="year-value">{yearStats.accStudents}</span>
+                    {!!yearStats.totalStudents && (
+                      <>
+                        <span className="separator">{yearStats.totalStudents !== undefined && '/'}</span>
+                        <span className="year-value">{yearStats.totalStudents}</span>
+                      </>
+                    )}
+                    <span className="unit">students</span>
+                  </>
+                ) : (
+                  <>
+                    <span className="year-value accumulated">
+                      {getDisplayValue(yearStats.acc, yearStats.accStudents)}
+                    </span>
+                    <span className="separator">{yearStats.total !== undefined && '/'}</span>
+                    <span className="year-value total">
+                      {yearStats.total !== undefined && getDisplayValue(yearStats.total, yearStats.totalStudents)}
+                    </span>
+                  </>
+                )}
+              </span>
+            )
+          })}
         </div>
       )}
     </Segment>
   )
 }
 
-StatusContainer.propTypes = {
-  title: PropTypes.string.isRequired,
-  current: PropTypes.number.isRequired,
-  previous: PropTypes.number.isRequired,
-  clickable: PropTypes.bool.isRequired,
-  handleClick: PropTypes.func.isRequired,
-  min1: PropTypes.number.isRequired,
-  max1: PropTypes.number.isRequired,
-  showYearlyValues: PropTypes.bool.isRequired,
-  yearlyValues: PropTypes.shape({}).isRequired,
-  showByYear: PropTypes.bool.isRequired,
-}
+const StatusContent = ({ data, settings, onDrill }) => {
+  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts } = settings
 
-const VerticalLine = () => <div style={{ margin: '0 10px', fontSize: '20px' }}>|</div>
-
-const Status = ({ getStatusDispatch, data, loading }) => {
-  const DATE_FORMAT = 'DD.MM.YYYY'
-  const [showYearlyValues, setShowYearlyValues] = useLocalStorage('showYearlyValues', true)
-  const [showByYear, setShowByYear] = useLocalStorage('showByYear', false)
-  const [drillStack, setDrillStack] = useState([])
-  const [showSettings, setShowSettings] = useState(true)
-  const [selectedDate, setSelectedDate] = useState(moment())
-  const [codes, setCodes] = useState([])
-  const [showByStudents, setShowByStudents] = useState(false)
-  const { CoolDataScience } = InfoToolTips
-
-  const isValidDate = d => moment.isMoment(d) && moment().diff(d) > 0
-
-  useEffect(() => {
-    if (selectedDate && isValidDate(selectedDate)) {
-      getStatusDispatch({ date: selectedDate.valueOf(), showByYear, showByStudents })
-    }
-  }, [selectedDate, showByYear, showByStudents])
-
-  useEffect(() => {
-    if (codes.length > 0) {
-      const updatedDrillStack = codes.reduce((acc, code) => {
-        const drilled = data[code]
-        // check if the code is on first level of drilldown
-        if (drilled) acc.push(drilled.drill)
-        // if not on first level then use the previous object in array
-        // this might be source of bugs but its the best I could come up with
-        else if (acc.length > 0 && acc[0][code]) acc.push(acc[0][code].drill)
-        return acc
-      }, [])
-      setDrillStack(updatedDrillStack)
-    }
-  }, [data])
-
-  const handleShowYearlyValuesToggled = () => {
-    const yearlyValues = showYearlyValues
-    setShowYearlyValues(!showYearlyValues)
-    sendAnalytics(`S Show yearly values toggle ${!yearlyValues ? 'on' : 'off'}`, 'Status')
-  }
-
-  const handleShowByYearToggled = () => {
-    const byYear = showByYear
-    setShowByYear(!showByYear)
-    sendAnalytics(`S Show by year toggle ${!byYear ? 'on' : 'off'}`, 'Status')
-  }
-
-  const handleShowByStudentsToggled = () => {
-    const byStudents = showByStudents
-    setShowByStudents(!showByStudents)
-    sendAnalytics(`S Show by students toggle ${!byStudents ? 'on' : 'off'}`, 'Status')
-  }
-
-  const pushToDrillStack = (values, code) => {
-    const updatedCodes = [...codes].concat(code)
-    const updatedDrillStack = [...drillStack].concat(values)
-    setCodes(updatedCodes)
-    setDrillStack(updatedDrillStack)
-    sendAnalytics('S Drilldown clicked', 'Status')
-  }
-
-  const popFromDrillStack = () => {
-    const updatedDrillStack = _.dropRight([...drillStack], 1)
-    const updatedCodes = _.dropRight([...codes], 1)
-    setCodes(updatedCodes)
-    setDrillStack(updatedDrillStack)
-    sendAnalytics('S Drillup clicked', 'Status')
-  }
-
-  const renderSettings = () => {
-    return (
-      <Accordion style={{ padding: 0, paddingTop: '10px', flex: 1 }}>
-        <Accordion.Title style={{ padding: 0, cursor: 'default' }} active={showSettings}>
-          <span style={{ cursor: 'pointer' }} onClick={() => setShowSettings(!showSettings)}>
-            <Icon name="setting" />
-            <span>Asetukset</span>
-          </span>
-        </Accordion.Title>
-        <Accordion.Content style={{ padding: 0, marginTop: '10px' }} active={showSettings}>
-          <Segment style={{ display: 'flex', alignItems: 'center' }}>
-            <div
-              style={{
-                width: '10px',
-                height: '10px',
-                background: 'red',
-                transform: 'rotateY(0deg) rotate(45deg)',
-                position: 'absolute',
-                top: '-6px',
-                left: '35px',
-                border: '1px solid #dededf',
-                borderRight: 'none',
-                borderBottom: 'none',
-                backgroundColor: 'white',
-              }}
-            />
-            <Checkbox
-              style={{ fontSize: '0.9em', fontWeight: 'normal' }}
-              label="Näytä edelliset vuodet"
-              onChange={handleShowYearlyValuesToggled}
-              checked={showYearlyValues}
-            />
-            <VerticalLine />
-            <Checkbox
-              style={{ fontSize: '0.9em', fontWeight: 'normal' }}
-              label="Näytä kalenterivuosittain"
-              onChange={handleShowByYearToggled}
-              checked={showByYear}
-            />
-            <VerticalLine />
-            <Checkbox
-              style={{ fontSize: '0.9em', fontWeight: 'normal' }}
-              label="Näytä suhteessa opiskelijamäärään"
-              onChange={handleShowByStudentsToggled}
-              checked={showByStudents}
-            />
-            <VerticalLine />
-            <Form>
-              <Form.Field error={!isValidDate(selectedDate)} style={{ display: 'flex', alignItems: 'center' }}>
-                <span style={{ fontSize: '0.9em' }}>Näytä päivänä:</span>
-                <Datetime
-                  className="status-date-time-input"
-                  dateFormat={DATE_FORMAT}
-                  timeFormat={false}
-                  closeOnSelect
-                  value={selectedDate}
-                  locale="fi"
-                  isValidDate={isValidDate}
-                  onChange={setSelectedDate}
-                />
-              </Form.Field>
-            </Form>
-          </Segment>
-        </Accordion.Content>
-      </Accordion>
-    )
-  }
-
-  const DrilldownMessage = () => (
-    <Message
-      color="blue"
-      content="Klikkaamalla tiedekuntaa pystyt porautumaan koulutusohjelma tasolle ja ohjelmaa klikkaamalla pystyt porautumaan kurssitasolle.
-      Vasemmassa yläkulmassa olevaa nuolta klikkaamalla pääset edelliseen näkymään."
-    />
-  )
-
-  if (!data || loading)
-    return (
-      <Segment style={{ padding: '40px' }} textAlign="center">
-        <Dimmer inverted active />
-        <Loader active={loading} />
-      </Segment>
-    )
-
-  const orderedAbsDiffs = _.orderBy(
-    Object.values(_.last(drillStack) || data).map(({ current, previous }) => {
-      return Math.abs(Math.floor((getP(current, previous) - 1) * 1000) / 10)
+  const orderedAbsDiffs = _.chain(data)
+    .map(({ current, currentStudents, previous, previousStudents }) => {
+      const currentValue = showRelativeValues ? current / currentStudents : current
+      const previousValue = showRelativeValues ? previous / previousStudents : previous
+      return Math.abs(Math.floor((getP(currentValue, previousValue) - 1) * 1000) / 10)
     })
-  )
+    .orderBy()
+    .value()
+
   const medianDiff = orderedAbsDiffs[Math.round((orderedAbsDiffs.length - 1) / 2)]
+
   return (
     <>
-      <h2>Uusien ohjelmien tuottamat opintopisteet</h2>
-      <DrilldownMessage />
-      <div style={{ display: 'flex', marginBottom: '20px', marginRight: '40px' }}>
-        {drillStack.length > 0 && (
-          <Icon onClick={popFromDrillStack} style={{ fontSize: '40px', cursor: 'pointer' }} name="arrow left" />
-        )}
-      </div>
       <div
         style={{
           display: 'grid',
@@ -323,51 +267,389 @@ const Status = ({ getStatusDispatch, data, loading }) => {
         }}
       >
         {_.orderBy(
-          Object.entries(_.last(drillStack) || data),
-          ([, { current, previous, currentStudents, previousStudents }]) =>
+          data,
+          ({ current, previous, currentStudents, previousStudents }) =>
             getP(current || currentStudents, previous || previousStudents), // oh god<r
           ['desc']
-        ).map(([code, stats]) => {
-          const handleClick = () => pushToDrillStack(stats.drill, code)
-          // check if the course has credits or not (if credits is zero but there are students who have completed it)
-          const current =
-            !stats.drill && stats.current === 0 && stats.currentStudents > 1 ? stats.currentStudents : stats.current
-          const previous =
-            !stats.drill && stats.previous === 0 && stats.previousStudents > 1 ? stats.previousStudents : stats.previous
+        ).map(stats => {
+          const handleClick = () => onDrill(stats.code)
+
           return (
             <StatusContainer
-              key={code}
-              clickable={!!stats.drill}
+              key={stats.code}
+              stats={stats}
               handleClick={handleClick}
-              title={getTextIn(stats.name)}
-              current={current}
-              previous={previous}
+              showRelativeValues={showRelativeValues}
               showYearlyValues={showYearlyValues}
+              showStudentCounts={showStudentCounts}
               min1={-medianDiff * 2}
               max1={medianDiff * 2}
-              yearlyValues={stats.yearly}
               showByYear={showByYear}
             />
           )
         })}
       </div>
-      {renderSettings()}
-      <Message>
-        <ReactMarkdown source={CoolDataScience.status} escapeHtml={false} />
-      </Message>
     </>
   )
 }
 
-Status.propTypes = {
-  data: shape({}).isRequired,
-  loading: bool.isRequired,
-  getStatusDispatch: func.isRequired,
+const WithHelpTooltip = ({ children, tooltip, onOpenDetails, ...rest }) => {
+  const popupContext = useRef()
+  const [popupOpen, setPopupOpen] = useState(false)
+
+  const trigger = (
+    <div>
+      {children}
+      <div ref={popupContext} style={{ display: 'inline-block', paddingTop: '0.2em', cursor: 'help' }}>
+        <Icon
+          onClick={() => setPopupOpen(!popupOpen)}
+          ref={popupContext}
+          style={{ marginLeft: '0.3em', color: '#888' }}
+          name="question circle outline"
+        />
+      </div>
+    </div>
+  )
+
+  const popupProps = _.defaults(rest, {
+    position: 'right center',
+  })
+
+  return (
+    <>
+      <Popup
+        hoverable
+        size="tiny"
+        open={popupOpen}
+        onOpen={() => setPopupOpen(true)}
+        onClose={() => setPopupOpen(false)}
+        trigger={trigger}
+        context={popupContext}
+        {...popupProps}
+        on="hover"
+        mouseEnterDelay={1000}
+      >
+        <div>{tooltip}</div>
+        <span style={{ color: '#2185d0', cursor: 'pointer' }} onClick={onOpenDetails}>
+          Lue lisää...
+        </span>
+      </Popup>
+    </>
+  )
 }
 
-const mapStateToProps = ({ coolDataScience }) => ({
-  data: coolDataScience.data.status,
-  loading: coolDataScience.pending.status,
-})
+const isValidDate = d => moment().diff(moment(d)) > 0
 
-export default connect(mapStateToProps, { getStatusDispatch: getStatus })(Status)
+const StatusSettings = ({ onSettingsChange, settings, onOpenDetails }) => {
+  const { selectedDate } = settings
+  const DATE_FORMAT = 'DD.MM.YYYY'
+
+  const changeSetting = (property, value) => {
+    sendAnalytics(`S Set setting "${property}" to ${value}`, 'Status')
+
+    onSettingsChange({
+      ...settings,
+      [property]: value,
+    })
+  }
+
+  const itemStyles = {
+    margin: '0.5rem 1rem',
+    display: 'flex',
+    alignItems: 'center',
+  }
+
+  const createSettingToggle = key => (
+    <div style={itemStyles}>
+      <WithHelpTooltip tooltip={settingDefinitions[key].short} onOpenDetails={() => onOpenDetails(key)}>
+        <Checkbox
+          style={{ fontSize: '0.9em', fontWeight: 'normal' }}
+          label={settingDefinitions[key].label}
+          checked={settings[key]}
+          onChange={() => changeSetting(key, !settings[key])}
+        />
+      </WithHelpTooltip>
+    </div>
+  )
+
+  return (
+    <div style={{ display: 'flex', alignItems: 'stretch', padding: 0, flexDirection: 'column' }}>
+      {createSettingToggle('showYearlyValues')}
+      {createSettingToggle('showByYear')}
+      {createSettingToggle('showRelativeValues')}
+      {createSettingToggle('showStudentCounts')}
+      <div style={itemStyles}>
+        <Form>
+          <Form.Field
+            error={selectedDate !== null && !isValidDate(selectedDate)}
+            style={{ display: 'flex', alignItems: 'center' }}
+          >
+            <WithHelpTooltip
+              tooltip={settingDefinitions.showCountingFrom.short}
+              onOpenDetails={() => onOpenDetails('showCountingFrom')}
+            >
+              <span style={{ fontSize: '0.9em' }}>{settingDefinitions.showCountingFrom.label}</span>
+            </WithHelpTooltip>
+            <Datetime
+              className="status-date-time-input"
+              dateFormat={DATE_FORMAT}
+              timeFormat={false}
+              closeOnSelect
+              value={moment(selectedDate)}
+              locale="fi"
+              isValidDate={isValidDate}
+              inputProps={{ placeholder: 'Valise päivämäärä' }}
+              onChange={value => {
+                changeSetting('selectedDate', value.format(DATE_FORMAT) === moment().format(DATE_FORMAT) ? null : value)
+              }}
+            />
+          </Form.Field>
+        </Form>
+      </div>
+    </div>
+  )
+}
+
+const getDefaultSettings = () =>
+  _.chain(settingDefinitions)
+    .toPairs()
+    .map(([key, { defaultValue }]) => [key, defaultValue])
+    .fromPairs()
+    .value()
+
+const Status = () => {
+  const [explicitSettings, setSettings] = useLocalStorage('trendsStatusSettings', {})
+  const [usageDetailsOpen, setUsageDetailsOpen] = useState(false)
+  const moreDetailsRef = useRef(null)
+  const history = useHistory()
+  const location = useLocation()
+
+  const drillStack = location.state?.drillStack ?? []
+
+  const data = useSelector(state => state.coolDataScience.data.status)
+  const loading = useSelector(state => state.coolDataScience.pending.status)
+  const dispatch = useDispatch()
+
+  const settings = useMemo(() => _.defaults(explicitSettings, getDefaultSettings()), [explicitSettings])
+
+  const { CoolDataScience } = InfoToolTips
+
+  useEffect(() => {
+    const { selectedDate, showByYear } = settings
+
+    let date
+
+    if (!selectedDate) {
+      date = moment().valueOf()
+    } else if (isValidDate(selectedDate)) {
+      date = moment(selectedDate).valueOf()
+    } else {
+      return
+    }
+
+    dispatch(
+      getStatus({
+        date,
+        showByYear,
+      })
+    )
+  }, [settings.selectedDate, settings.showByYear])
+
+  const pushToDrillStack = code => {
+    history.push('/trends/status', {
+      drillStack: [...drillStack, code],
+    })
+
+    sendAnalytics('S Drilldown clicked', 'Status')
+  }
+
+  const breadcrumb = useMemo(() => {
+    const stack = _.chain(drillStack)
+      .reduce(
+        ([stack, data], code) => {
+          let content = code
+          let next = null
+
+          if (data) {
+            const entry = data[code]
+
+            if (entry) {
+              content = getTextIn(entry.name)
+              next = entry.drill
+            }
+          }
+
+          const item = {
+            key: code,
+            content,
+            link: !!next,
+            active: !next,
+            onClick: () => {
+              history.push('/trends/status', {
+                drillStack: [...stack.map(({ key }) => key).splice(1), code],
+              })
+            },
+          }
+
+          return [[...stack, item], next]
+        },
+        [[], data]
+      )
+      .value()[0]
+
+    const hy = {
+      key: '',
+      content: getTextIn({
+        fi: 'Helsingin Yliopisto',
+        en: 'University of Helsinki',
+        sv: 'Helsingfors universitet',
+      }),
+      link: true,
+      onClick: () =>
+        history.push('/trends/status', {
+          drillStack: [],
+        }),
+    }
+
+    return [hy, ...stack]
+  }, [data, drillStack])
+
+  const drilledData = useMemo(
+    () =>
+      _.chain(drillStack)
+        .reduce((data, code) => data && _.values(data).find(item => item.code === code)?.drill, data)
+        .defaultTo({})
+        .values()
+        .value(),
+    [data, drillStack]
+  )
+
+  const popFromDrillStack = () => {
+    const newDrillStack = [...drillStack]
+    newDrillStack.pop()
+    history.push('/trends/status', {
+      drillStack: newDrillStack,
+    })
+
+    sendAnalytics('S Drillup clicked', 'Status')
+  }
+
+  const [settingsOpen, setSettingsOpen] = useState(false)
+  const attentionSeekers = useRef({})
+
+  const scrollToAttentionSeeker = el => {
+    el.scrollIntoView({
+      block: 'nearest',
+      inline: 'nearest',
+      behavior: 'smooth',
+    })
+  }
+
+  const attentionSeekerRef = key => ref => {
+    if (ref !== null && attentionSeekers.current[key] === true) {
+      scrollToAttentionSeeker(ref)
+    }
+
+    attentionSeekers.current[key] = ref
+  }
+
+  return (
+    <>
+      <div style={{ display: 'flex', alignItems: 'center', marginBottom: '1rem' }} ref={moreDetailsRef}>
+        <h2 style={{ margin: 0, flexGrow: 1 }}>Koulutusohjelmien tuottamat opintopisteet</h2>
+        <Popup
+          trigger={
+            <Button>
+              <Icon name="settings" /> Asetukset
+            </Button>
+          }
+          position="bottom right"
+          on="click"
+          wide="very"
+          open={settingsOpen}
+          onOpen={() => setSettingsOpen(true)}
+          onClose={() => {
+            // Close the popup only after the event has had a chance to propagate.
+            setTimeout(() => setSettingsOpen(false), 0)
+          }}
+          style={{ padding: '0.25em 0em', maxHeight: '80vh' }}
+        >
+          <StatusSettings
+            settings={settings}
+            onSettingsChange={setSettings}
+            onOpenDetails={key => {
+              if (moreDetailsRef.current) {
+                moreDetailsRef.current.scrollIntoView({
+                  block: 'start',
+                  inline: 'end',
+                  behavior: 'smooth',
+                })
+              }
+
+              if (attentionSeekers.current[key]) {
+                scrollToAttentionSeeker(attentionSeekers.current[key])
+              } else {
+                attentionSeekers.current[key] = true
+              }
+
+              setUsageDetailsOpen(true)
+            }}
+          />
+        </Popup>
+        <Popup
+          trigger={
+            <Button>
+              <Icon name="question circle" /> Käyttöohje
+            </Button>
+          }
+          position="bottom right"
+          on="click"
+          wide="very"
+          open={usageDetailsOpen}
+          onOpen={() => setUsageDetailsOpen(true)}
+          onClose={() => setUsageDetailsOpen(false)}
+          style={{ padding: '0', paddingBottom: '0.5em', maxHeight: '80vh', overflowY: 'auto' }}
+        >
+          <div style={{ padding: '1em' }}>
+            {
+              // eslint-disable-next-line react/no-children-prop
+              <ReactMarkdown children={CoolDataScience.status} escapeHtml={false} />
+            }
+          </div>
+          {_.toPairs(settingDefinitions).map(([key, { label, long }]) => (
+            <div key={key}>
+              <Divider />
+              <div style={{ padding: '0 1em' }} ref={attentionSeekerRef(key)}>
+                <b>
+                  Valinta "<i>{label}</i>"
+                </b>
+                <div style={{ margin: '0.5em', fontSize: '0.9em' }}>
+                  <ReactMarkdown escapeHtml={false}>{long.replace(/(^|\n)[ \t]+/g, '\n')}</ReactMarkdown>
+                </div>
+              </div>
+            </div>
+          ))}
+        </Popup>
+      </div>
+
+      <div style={{ display: 'flex', marginBottom: '20px', marginRight: '40px', alignItems: 'center' }}>
+        <Breadcrumb icon="right angle" sections={breadcrumb} size="large" />
+        {drillStack.length > 0 && (
+          <Icon
+            onClick={popFromDrillStack}
+            style={{ cursor: 'pointer', color: 'rgba(0,0,0,.28)', marginLeft: '0.4em', marginTop: '-0.17em' }}
+            name="arrow left"
+          />
+        )}
+      </div>
+
+      {data && !loading ? (
+        <StatusContent settings={settings} onDrill={pushToDrillStack} data={drilledData} />
+      ) : (
+        <Loader active={loading} inline="centered" />
+      )}
+    </>
+  )
+}
+
+export default Status

services/frontend/src/components/CoolDataScience/StatusGraduated.jsx

@@ -309,7 +309,7 @@ const Status = ({ getStatusGraduatedDispatch, data, loading }) => {
   const medianDiff = orderedAbsDiffs[Math.round((orderedAbsDiffs.length - 1) / 2)]
   return (
     <>
-      <h2>Uusista ohjelmista valmistuneet</h2>
+      <h2>Koulutusohjelmista valmistuneet</h2>
       <DrilldownMessage />
       <div style={{ display: 'flex', marginBottom: '20px', marginRight: '40px' }}>
         {drillStack.length > 0 && (
@@ -351,7 +351,10 @@ const Status = ({ getStatusGraduatedDispatch, data, loading }) => {
       </div>
       {renderSettings()}
       <Message>
-        <ReactMarkdown source={CoolDataScience.statusGraduated} escapeHtml={false} />
+        {
+          // eslint-disable-next-line react/no-children-prop
+          <ReactMarkdown children={CoolDataScience.statusGraduated} escapeHtml={false} />
+        }
       </Message>
     </>
   )

services/frontend/src/components/CoolDataScience/StatusPanel.jsx

@@ -5,8 +5,22 @@ import Graduated from './StatusGraduated'
 
 const StatusPanel = () => {
   const panes = [
-    { menuItem: 'Attainments', render: () => <Status /> },
-    { menuItem: 'Graduations', render: () => <Graduated /> },
+    {
+      menuItem: 'Attainments',
+      render: () => (
+        <Tab.Pane>
+          <Status />
+        </Tab.Pane>
+      ),
+    },
+    {
+      menuItem: 'Graduations',
+      render: () => (
+        <Tab.Pane>
+          <Graduated />
+        </Tab.Pane>
+      ),
+    },
   ]
   return <Tab panes={panes} />
 }

services/frontend/src/components/CoolDataScience/status.css

@@ -10,6 +10,47 @@
   box-shadow: 0 3px 6px rgba(0, 0, 0, 0.16), 0 3px 6px rgba(0, 0, 0, 0.23);
 }
 
+.years {
+  margin-top: 10px;
+  text-align: start;
+  font-variant-numeric: tabular-nums;
+  display: grid;
+  grid-template-columns: repeat(4, fit-content(0));
+}
+
+.subtitle {
+  color: #888;
+}
+
+.unit {
+  margin-left: 0.3em;
+}
+
+.year-label {
+  white-space: nowrap;
+  padding-right: 0.5em;
+}
+
+.year-value {
+  text-align: right;
+  white-space: nowrap;
+}
+
+.year-row {
+  position: relative;
+  padding-left: 0.5em;
+  display: contents;
+}
+
+.year-row .unit {
+  grid-column: 5;
+}
+
+.year-row .separator {
+  color: #aaa;
+  margin: 0 0.3em;
+}
+
 .status-date-time-input {
   padding-right: 0;
   margin-left: 5px;

services/frontend/src/components/CoursePopulation/index.jsx

@@ -21,6 +21,7 @@ import infotooltips from '../../common/InfoToolTips'
 import InfoBox from '../InfoBox'
 import FilterTray from '../FilterTray'
 import useFilters from '../FilterTray/useFilters'
+import useProgrammeFilter from '../FilterTray/filters/Programmes/useProgrammeFilter'
 import { CoursePopulationFilters } from '../FilterTray/FilterSets'
 import useLanguage from '../LanguagePicker/useLanguage'
 import { queryParamsFromUrl } from '../../common/query'
@@ -39,6 +40,7 @@ const CoursePopulation = ({
 }) => {
   const { language } = useLanguage()
   const { setAllStudents, filteredStudents } = useFilters()
+  const { setCourseCodes } = useProgrammeFilter()
   const selectedStudents = filteredStudents.map(stu => stu.studentNumber)
 
   const [codes, setCodes] = useState([])
@@ -106,6 +108,7 @@ const CoursePopulation = ({
         separate,
       })
       setCodes(parsedCourseCodes)
+      setCourseCodes(parsedCourseCodes)
       if (years) {
         setYears(years)
       } else {
@@ -121,7 +124,6 @@ const CoursePopulation = ({
   }, [semesters])
 
   const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears}` : null
-  const coursecode = courseData ? courseData.alternatives : []
 
   const subHeader = codes.join(', ')
 
@@ -260,7 +262,7 @@ const CoursePopulation = ({
               filteredStudents={studentData.students.filter(stu => selectedStudents.includes(stu.studentNumber))}
               coursePopulation
               language={language}
-              coursecode={coursecode}
+              coursecode={codes}
             />
           </div>
         ),

services/frontend/src/components/CourseStatistics/SingleCourseStats/countTotalStats.js

@@ -13,7 +13,7 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
         if (!cgrades[grade]) cgrades[grade] = 0
         cgrades[grade] += curr.attempts.grades[grade]
 
-        if (['Eisa', 'Hyl.', '0', 'Luop'].includes(grade)) {
+        if (['EISA', 'Hyl.', 'HYL', '0', 'LUOP'].includes(grade)) {
           failed += curr.attempts.grades[grade]
         } else {
           passed += curr.attempts.grades[grade]