Merge pull request #3145 from UniversityOfHelsinkiCS/trunk

Goodbye faculties

.github/workflows/release-oodikone-staging.yaml

@@ -48,15 +48,16 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Set current date as env variable"
-        run: echo NOW::"$(date --iso-8601=seconds)" >> "$GITHUB_ENV"
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: "Build and push"
         uses: docker/build-push-action@v2
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
             SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ env.NOW }}
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
             type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
@@ -82,6 +83,7 @@ jobs:
           set_commits: 'skip'
           sourcemaps: './build/static/js'
           url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
 
   build_and_release_userservice:
     name: "Build and release userservice"

.github/workflows/release-oodikone.yaml

@@ -49,8 +49,9 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Set current date as env variable"
-        run: echo NOW::"$(date --iso-8601=seconds)" >> "$GITHUB_ENV"
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: "Extract release version"
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
@@ -63,7 +64,7 @@ jobs:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
-            BUILT_AT=${{ env.NOW }}
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
             type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
@@ -89,6 +90,7 @@ jobs:
           set_commits: 'skip'
           sourcemaps: './build/static/js'
           url_prefix: '~/oodikone/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
 
   build_and_release_userservice:
     name: "Build and release userservice"

cypress/integration/Course_statistics.js

@@ -119,6 +119,42 @@ describe('Course Statistics tests', () => {
       cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
     })
 
+    it('Searching course by name displays right courses, 10 credit courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
+      cy.contains('AYTKT20001')
+      cy.contains('TKT20001, 58131')
+      cy.contains('td', /^TKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get('.right').click()
+      cy.contains('No results')
+
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', /^AYTKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
+
+      cy.get('.right').click()
+      cy.contains('No results')
+      cy.get("input[name='unifyOpenUniCourses']").parent().click()
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+    })
+
     it('Can find course population', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')

cypress/integration/Filters.js

@@ -1,6 +1,6 @@
 /// <reference types="Cypress" />
 
-const baseUrl = Cypress.config().baseUrl
+// const baseUrl = Cypress.config().baseUrl
 
 const checkFilteringResult = studentCount => {
   cy.contains(`Students (${studentCount})`)
@@ -130,12 +130,9 @@ describe('Population Statistics', () => {
     })
   })
 
-  // TODO: this fails in CI, but not locally. Investigate and fix
+  // plz fix
   it.skip('Courses filter works', () => {
     // courses takes some time to load, wait for it to complete
-    cy.intercept({ path: '**/courses' }).as('coursesOfPopulation')
-    cy.wait('@coursesOfPopulation')
-    // moar waiting hack
     cy.wait(10000)
     runTestStepWithPreAndPostParts('courseFilter-header', () => {
       cy.cs('courseFilter-course-dropdown').click()

cypress/integration/Population_statistics.js

@@ -73,7 +73,7 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(3000) // a bit hacky way, wait until ui is ready
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
       cy.get('[data-cy=expand-TKT1]').click()
       cy.get('[data-cy=coursestats-link-TKT10002]').click()
       cy.wait('@coursePage')

docker-compose.ci.yml

@@ -29,8 +29,6 @@ services:
   frontend:
     <<: *defaults
     container_name: frontend
-    environment:
-      BACKEND_ADDR: backend
     image: toska/oodikone2-frontend:trunk
 
   kone-db:

docker-compose.yml

@@ -53,13 +53,6 @@ services:
       <<: *build_defaults
       context: services/frontend
     container_name: frontend
-    environment:
-      ADMINER_URL: http://localhost:5050/?pgsql=oodi-db&username=postgres
-      BACKEND_ADDR: backend
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=kone-db&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=sis-db&username=postgres
-      SIS_IMPORTER_ADMINER_URL: http://localhost:5050/?pgsql=sis-importer-db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user-db&username=postgres
     ports:
       - 3000:3000
     profiles:

old_oodikone_stuff/updater_api/package.json

@@ -1,34 +0,0 @@
-{
-  "name": "updater_api",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "axios": "^0.21.1",
-    "dotenv": "^7.0.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^5.1.1",
-    "moment": "^2.29.1",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_scheduler/package.json

@@ -1,35 +0,0 @@
-{
-  "name": "updater_scheduler",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node --max_old_space_size=4096 index.js",
-    "dev": "nodemon --max_old_space_size=4096 index.js",
-    "scheduleActiveStudents": "NATS_CLIENTID=manualActiveSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleActiveStudents()'",
-    "scheduleMeta": "NATS_CLIENTID=manualAllSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleMeta()'",
-    "getOldestTasks": "node --max_old_space_size=4096 -e 'require(\"./src/SchedulingStatistics.js\").getOldestTasks()'"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "cron": "^1.8.2",
-    "express": "^4.17.1",
-    "mongoose": "^5.12.3",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_writer/package.json

@@ -1,39 +0,0 @@
-{
-  "name": "updater_writer",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run jest",
-    "jest": "NODE_ENV=test jest --runInBand --verbose",
-    "test_docker": "docker exec -t updater_writer /bin/bash -c 'npm test'",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js",
-    "force_sync": "node -e \"require('./database/force_sync_database').sync()\"",
-    "dump": "node -e \"require('./database/dump_database').dumpDatabase()\""
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "dotenv": "^8.0.0",
-    "jest": "^24.8.0",
-    "lodash": "^4.17.19",
-    "node-nats-streaming": "^0.2.6",
-    "pg": "^8.6.0",
-    "sequelize": "^6.6.2",
-    "umzug": "^2.2.0",
-    "winston": "^3.2.1",
-    "winston-log2gelf": "^2.3.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.0.3",
-    "eslint": "^6.6.0",
-    "eslint-config-prettier": "^6.3.0",
-    "eslint-plugin-prettier": "^3.1.0",
-    "nodemon": "^1.19.4",
-    "prettier": "^1.18.2",
-    "prettier-eslint": "^9.0.0"
-  }
-}

package.json

@@ -19,7 +19,11 @@
     "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
     "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
     "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "lint:docker-compose": "docker-compose config -q && :",
+    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
+    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
+    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
   },
   "lint-staged": {
     "{services,updater}/**/*.{js,jsx}": "eslint --fix",
@@ -28,10 +32,10 @@
     ".github/workflows/*": "actionlint",
     "Dockerfile": "hadolint",
     "*.sh": "shellcheck -x",
-    "docker-compose.yml": "docker-compose config -q",
-    "docker-compose.real.yml": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q",
-    "docker-compose.ci.yml": "docker-compose -f docker-compose.ci.yml config -q",
-    "docker-compose.test.yml": "docker-compose -f docker-compose.test.yml config -q"
+    "docker-compose.yml": "npm run lint:docker-compose",
+    "docker-compose.real.yml": "npm run lint:docker-compose:real",
+    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
+    "docker-compose.test.yml": "npm run lint:docker-compose:test"
   },
   "repository": {
     "type": "git",

services/backend/Dockerfile

@@ -12,12 +12,12 @@ CMD ["npm", "run", "dev"]
 
 FROM base as production
 ENV NODE_ENV=production
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
-COPY --chown=node:node . /usr/src/app/
-USER node
 ARG SENTRY_ENVIRONMENT
 ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+USER node
 CMD ["npm", "start"]

services/backend/src/events.js

@@ -7,58 +7,35 @@ const {
   refreshUber,
   refreshProtoCProgramme,
   getStartYears,
-} = require('./services/coolDataScience')
-const { refreshAssociationsInRedis: refreshAssociationsInRedisV2 } = require('./servicesV2/studyrights')
+} = require('./servicesV2/trends')
+const { refreshAssociationsInRedis } = require('./servicesV2/studyrights')
+const { getAllProgrammes, nonGraduatedStudentsOfElementDetail } = require('./servicesV2/studyrights')
+const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./servicesV2/studyprogramme')
+const { findAndSaveTeachers } = require('./servicesV2/topteachers')
 const {
-  getAllProgrammes: getAllProgrammesV2,
-  nonGraduatedStudentsOfElementDetail: nonGraduatedStudentsOfElementDetailV2,
-} = require('./servicesV2/studyrights')
-
-const {
-  productivityStatsForStudytrack: productivityStatsForStudytrackV2,
-  throughputStatsForStudytrack: throughputStatsForStudytrackV2,
-} = require('./servicesV2/studyprogramme')
-
-const topteachersV2 = require('./servicesV2/topteachers')
-const { isNewHYStudyProgramme } = require('./util')
-
-const { calculateFacultyYearlyStats } = require('./services/faculties')
-const { patchFacultyYearlyStats } = require('./servicesV2/analyticsService')
-
-const {
-  setProductivity: setProductivityV2,
-  setThroughput: setThroughputV2,
-  patchProductivity: patchProductivityV2,
-  patchThroughput: patchThroughputV2,
-  patchNonGraduatedStudents: patchNonGraduatedStudentsV2,
+  setProductivity,
+  setThroughput,
+  patchProductivity,
+  patchThroughput,
+  patchNonGraduatedStudents,
 } = require('./servicesV2/analyticsService')
+const { isNewHYStudyProgramme } = require('./util')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
-// This is currently done with old data, update to use sis db
-const refreshFacultyYearlyStats = async () => {
-  try {
-    console.log('Refreshing faculty yearly stats...')
-    const data = await calculateFacultyYearlyStats()
-    await patchFacultyYearlyStats(data)
-  } catch (e) {
-    console.error(e)
-  }
-}
-
-const refreshStudyrightAssociationsV2 = async () => {
+const refreshStudyrightAssociations = async () => {
   try {
     console.log('Refreshing studyright associations...')
-    await refreshAssociationsInRedisV2()
+    await refreshAssociationsInRedis()
   } catch (e) {
     console.error(e)
   }
 }
 
-const refreshOverviewV2 = async () => {
+const refreshOverview = async () => {
   try {
     console.log('Refreshing throughput and productivity for programmes...')
-    const codes = (await getAllProgrammesV2()).map(p => p.code)
+    const codes = (await getAllProgrammes()).map(p => p.code)
     let ready = 0
     for (const code of codes) {
       let programmeStatsSince = new Date('2017-07-31')
@@ -68,12 +45,12 @@ const refreshOverviewV2 = async () => {
         programmeStatsSince = new Date('2000-07-31')
       }
       try {
-        await patchThroughputV2({ [code]: { status: 'RECALCULATING' } })
-        const data = await throughputStatsForStudytrackV2(code, programmeStatsSince.getFullYear())
-        await setThroughputV2(data)
+        await patchThroughput({ [code]: { status: 'RECALCULATING' } })
+        const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
+        await setThroughput(data)
       } catch (e) {
         try {
-          await patchThroughputV2({ [code]: { status: 'RECALCULATION ERRORED' } })
+          await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
         } catch (e) {
           console.error(e)
         }
@@ -81,12 +58,12 @@ const refreshOverviewV2 = async () => {
         console.log(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
       }
       try {
-        await patchProductivityV2({ [code]: { status: 'RECALCULATING' } })
-        const data = await productivityStatsForStudytrackV2(code, programmeStatsSince)
-        await setProductivityV2(data)
+        await patchProductivity({ [code]: { status: 'RECALCULATING' } })
+        const data = await productivityStatsForStudytrack(code, programmeStatsSince)
+        await setProductivity(data)
       } catch (e) {
         try {
-          await patchProductivityV2({
+          await patchProductivity({
             [code]: { status: 'RECALCULATION ERRORED' },
           })
         } catch (e) {
@@ -103,20 +80,20 @@ const refreshOverviewV2 = async () => {
   }
 }
 
-const refreshTeacherLeaderboardV2 = async () => {
+const refreshTeacherLeaderboard = async () => {
   try {
     const startyearcode = new Date().getFullYear() - 1950
     const endyearcode = startyearcode + 1
     console.log('Refreshing teacher leaderboard...')
-    await topteachersV2.findAndSaveTeachers(startyearcode, endyearcode)
+    await findAndSaveTeachers(startyearcode, endyearcode)
   } catch (e) {
     console.log(e)
   }
 }
 
-const refreshNonGraduatedStudentsOfOldProgrammesV2 = async () => {
+const refreshNonGraduatedStudentsOfOldProgrammes = async () => {
   try {
-    const oldProgrammeCodes = (await getAllProgrammesV2()).map(p => p.code).filter(c => !isNewHYStudyProgramme(c))
+    const oldProgrammeCodes = (await getAllProgrammes()).map(p => p.code).filter(c => !isNewHYStudyProgramme(c))
     let i = 0
     console.log('Refreshing non-graduated students of old programmes...')
     await Promise.all(
@@ -124,8 +101,8 @@ const refreshNonGraduatedStudentsOfOldProgrammesV2 = async () => {
         c =>
           new Promise(async res => {
             try {
-              const [nonGraduatedStudents, studentnumbers] = await nonGraduatedStudentsOfElementDetailV2(c)
-              await patchNonGraduatedStudentsV2({ [c]: { formattedData: nonGraduatedStudents, studentnumbers } })
+              const [nonGraduatedStudents, studentnumbers] = await nonGraduatedStudentsOfElementDetail(c)
+              await patchNonGraduatedStudents({ [c]: { formattedData: nonGraduatedStudents, studentnumbers } })
               console.log(`${++i}/${oldProgrammeCodes.length}`)
             } catch (e) {
               console.log(`Failed refreshing non-graduated students of programme ${c}!`)
@@ -193,8 +170,9 @@ const refreshUberToRedis = async () => {
 
 const refreshProtoCProgrammeToRedis = async () => {
   try {
-    const codes = (await getAllProgrammesV2()).map(p => p.code).filter(code => code.includes('KH'))
+    const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH'))
     codes.forEach(async code => {
+      console.log('refreshing code', code)
       const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
       const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
       const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
@@ -210,31 +188,30 @@ const refreshProtoCProgrammeToRedis = async () => {
   }
 }
 
-const refreshStatisticsV2 = async () => {
-  await refreshStudyrightAssociationsV2()
-  await refreshOverviewV2()
-  await refreshNonGraduatedStudentsOfOldProgrammesV2()
-  await refreshTeacherLeaderboardV2()
-  await refreshFacultyYearlyStats() // using old data
+const refreshStatistics = async () => {
+  await refreshStudyrightAssociations()
+  await refreshOverview()
+  await refreshNonGraduatedStudentsOfOldProgrammes()
+  await refreshTeacherLeaderboard()
 }
 
-const startCron = () => {
-  if (process.env.NODE_ENV === 'production') {
-    schedule('0 6 * * *', async () => {
-      await refreshStatisticsV2()
-    })
-  }
-}
-
-const refreshCDS = async () => {
+const refreshTrends = async () => {
   await refreshProtoCtoRedis()
   await refreshStatusToRedis()
   await refreshUberToRedis()
   await refreshProtoCProgrammeToRedis()
 }
 
+const startCron = () => {
+  if (process.env.NODE_ENV === 'production') {
+    schedule('0 6 * * *', async () => {
+      await refreshStatistics()
+      await refreshTrends()
+    })
+  }
+}
+
 module.exports = {
   startCron,
-  refreshStatisticsV2,
-  refreshCDS,
+  refreshStatistics,
 }

services/backend/src/routes.js

@@ -13,7 +13,6 @@ const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
-const facultyStats = require('./routes/facultyStats')
 const semesters = require('./routes/semesters')
 const courseGroups = require('./routes/courseGroups')
 const mandatoryCourses = require('./routes/mandatorycourses')
@@ -56,7 +55,6 @@ module.exports = (app, url) => {
   app.use(url, programmeModules)
   app.use(url, diff)
   app.use(`${url}/faculties`, faculties)
-  app.use(`${url}/facultystats`, auth.roles(['faculties']), facultyStats)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['users']), users)

services/backend/src/routes/faculties.js

@@ -1,9 +1,10 @@
 const router = require('express').Router()
-const { all } = require('../services/organisations')
+const { faculties } = require('../servicesV2/organisations')
 
 router.get('/', async (req, res) => {
-  const faculties = await all()
-  res.json(faculties)
+  const facultyList = await faculties()
+  console.log('faculties')
+  res.json(facultyList)
 })
 
 module.exports = router

services/backend/src/routes/facultyStats.js

@@ -1,34 +0,0 @@
-const router = require('express').Router()
-const { all } = require('../services/organisations')
-const { byType: getElementDetailsByType } = require('../services/elementdetails')
-const { getFacultyYearlyStats } = require('../servicesV2/analyticsService')
-
-router.get('/faculties', async (req, res) => {
-  const { roles, faculties } = req
-  const allFaculties = await all()
-  res.json(roles.includes('admin') ? allFaculties : allFaculties.filter(({ code }) => faculties.has(code)))
-})
-
-router.get('/yearlystats', async (req, res) => {
-  try {
-    const { roles, faculties } = req
-    const facultyYearlyStats = await getFacultyYearlyStats()
-    res.json(roles.includes('admin') ? facultyYearlyStats : facultyYearlyStats.filter(({ id }) => faculties.has(id)))
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-router.get('/programmes', async (req, res) => {
-  try {
-    const { roles } = req
-    const isAdmin = roles.includes('admin')
-    const facultyProgrammes = await getElementDetailsByType(20)
-    const userRightsSet = new Set(req.rights)
-    res.json(facultyProgrammes.filter(({ code }) => userRightsSet.has(code) || isAdmin))
-  } catch (e) {
-    res.status(400).json({ error: e })
-  }
-})
-
-module.exports = router

services/backend/src/routes/providers.js

@@ -1,5 +1,5 @@
 const router = require('express').Router()
-const { getAllProviders } = require('../services/providers')
+const { getAllProviders } = require('../servicesV2/providers')
 
 router.get('/providers', async (req, res) => {
   const providers = await getAllProviders()

services/backend/src/routes/updater.js

@@ -9,7 +9,7 @@ const {
   abort,
   updateCoursesByCourseCode,
 } = require('../services/sisUpdaterService')
-const { refreshStatisticsV2 } = require('../events')
+const { refreshStatistics } = require('../events')
 
 router.get('/update/v2/meta', async (req, res) => {
   const { roles } = req
@@ -117,7 +117,7 @@ router.get('/refresh_redis_cache', async (req, res) => {
 
 router.post('/refresh_statistic_v2', async (req, res) => {
   try {
-    refreshStatisticsV2()
+    refreshStatistics()
     res.status(200).json('Refreshing sis statistics')
   } catch (err) {
     console.error(err)

services/backend/src/services/faculties.js

@@ -1,91 +0,0 @@
-const axios = require('axios')
-const moment = require('moment')
-const eachLimit = require('async/eachLimit')
-const { USERSERVICE_URL, DB_MAX_CRON_CONNECTIONS } = require('../conf-backend')
-const { Credit, StudyrightElement } = require('../models')
-const { lruMemoize } = require('../util')
-
-const client = axios.create({ baseURL: USERSERVICE_URL, headers: { secret: process.env.USERSERVICE_SECRET } })
-
-const calculateFacultyYearlyStats = async () => {
-  const { data: facultyProgrammes } = await client.get('/faculty_programmes')
-
-  const getFilteredStudentCredits = lruMemoize(
-    studentNumber =>
-      Credit.findAll({
-        where: {
-          student_studentnumber: studentNumber,
-          credittypecode: [4, 10],
-          isStudyModule: false,
-        },
-      }),
-    args => args[0],
-    {
-      max: 10000,
-    }
-  )
-
-  const res = {}
-  const studentSets = {}
-
-  let amountDone = 0
-  // Don't try to make this fast with Promise.all without taking into consideration that
-  // we only have a limited amount of database connections
-  for (const { faculty_code, programme_code } of facultyProgrammes) {
-    if (!res[faculty_code]) {
-      res[faculty_code] = {}
-      studentSets[faculty_code] = {}
-    }
-    if (!res[faculty_code][programme_code]) {
-      res[faculty_code][programme_code] = {}
-      studentSets[faculty_code][programme_code] = {}
-    }
-
-    const facultyStudents = await StudyrightElement.findAll({
-      where: {
-        code: programme_code,
-      },
-      attributes: ['studentnumber'],
-    })
-
-    await eachLimit(facultyStudents, DB_MAX_CRON_CONNECTIONS, async student => {
-      const filteredStudentCredits = await getFilteredStudentCredits(student.studentnumber)
-
-      filteredStudentCredits.forEach(c => {
-        const attainmentYear = moment(c.attainment_date).year()
-        if (!res[faculty_code][attainmentYear] && !res[faculty_code][programme_code][attainmentYear]) {
-          const facultyYearStats = {}
-          facultyYearStats.studentCredits = 0
-          facultyYearStats.coursesPassed = 0
-          facultyYearStats.coursesFailed = 0
-          facultyYearStats.students = 0
-          facultyYearStats.studentArray = []
-          res[faculty_code][programme_code][attainmentYear] = facultyYearStats
-          studentSets[faculty_code][programme_code][attainmentYear] = new Set()
-        }
-        const facultyYearStats = res[faculty_code][programme_code][attainmentYear]
-        const studentSet = studentSets[faculty_code][programme_code][attainmentYear]
-        if (!studentSet.has(c.student_studentnumber)) {
-          facultyYearStats.students++
-          facultyYearStats.studentArray.push(c.student_studentnumber)
-          studentSet.add(c.student_studentnumber)
-        }
-        if (c.credittypecode === 4) {
-          facultyYearStats.studentCredits += c.credits
-          facultyYearStats.coursesPassed += 1
-        } else {
-          facultyYearStats.coursesFailed += 1
-        }
-      })
-    })
-
-    amountDone += 1
-    console.log(`Faculty programmes done ${amountDone}/${facultyProgrammes.length}`)
-  }
-
-  return res
-}
-
-module.exports = {
-  calculateFacultyYearlyStats,
-}

services/backend/src/services/providers.js

@@ -1,10 +0,0 @@
-const { Provider } = require('../models/index')
-
-const getAllProviders = async () =>
-  Provider.findAll({
-    attributes: ['providercode', 'name'],
-  })
-
-module.exports = {
-  getAllProviders,
-}

services/backend/src/servicesV2/analyticsService.js

@@ -88,21 +88,6 @@ const patchThroughput = async data => {
   }
 }
 
-const patchFacultyYearlyStats = async data => {
-  const redisKey = 'FACULTY_YEARLY_STATS'
-  const dataToPatch = Object.entries(data).reduce((acc, [id, data]) => [...acc, { id, data }], [])
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToPatch))
-  if (setOperationStatus !== 'OK') return null
-  return dataToPatch
-}
-
-const getFacultyYearlyStats = async () => {
-  const redisKey = 'FACULTY_YEARLY_STATS'
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return JSON.parse(dataFromRedis)
-}
-
 const patchNonGraduatedStudents = async data => {
   const [id, dataToPatch] = Object.entries(data)[0]
   const redisKey = createRedisKeyForNonGraduatedStudents(id)
@@ -135,8 +120,6 @@ module.exports = {
   getThroughput,
   setThroughput,
   patchThroughput,
-  patchFacultyYearlyStats,
-  getFacultyYearlyStats,
   patchNonGraduatedStudents,
   getNonGraduatedStudents,
 }

services/backend/src/servicesV2/courses.js

@@ -14,6 +14,7 @@ const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('../servicesV2/course_yearly_stats_counter')
 const _ = require('lodash')
+// const { CourseProvider } = require('../models')
 
 const byNameOrCode = (searchTerm, language) =>
   Course.findAll({
@@ -356,50 +357,75 @@ const codeLikeTerm = code =>
       }
 
 const byNameAndOrCodeLike = async (name, code) => {
-  let courses = await Course.findAll({
-    raw: true,
-    attributes: [
-      'id',
-      'name',
-      'code',
-      'latest_instance_date',
-      'startdate',
-      'enddate',
-      'max_attainment_date',
-      'min_attainment_date',
-      'substitutions',
-    ],
+  let rawCourses = await Course.findAll({
+    include: {
+      model: Organization,
+      required: true,
+    },
     where: {
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
     },
   })
 
+  const courses = rawCourses
+    .map(
+      ({
+        id,
+        code,
+        name,
+        latest_instance_date,
+        is_study_module,
+        coursetypecode,
+        startdate,
+        max_attainment_date,
+        min_attainment_date,
+        createdAt,
+        updatedAt,
+        substitutions,
+        organizations,
+      }) => {
+        return {
+          id,
+          code,
+          name,
+          latest_instance_date,
+          is_study_module,
+          coursetypecode,
+          startdate,
+          max_attainment_date,
+          min_attainment_date,
+          createdAt,
+          updatedAt,
+          substitutions,
+          organizations: organizations.map(o => o.id),
+        }
+      }
+    )
+    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
+
   let substitutionGroupIndex = 0
-  const subsGroups = {}
   const visited = []
-  const dfs = async courseId => {
-    if (visited.includes(courseId)) return
-    visited.push(courseId)
-    const course = courses.find(course => course.id === courseId)
 
-    if (!course) return
+  const organizeSubgroups = course => {
+    if (visited.includes(course.code)) return
 
-    subsGroups[course.code] = substitutionGroupIndex
-    course.subsId = substitutionGroupIndex
-
-    for (const courseId of course.substitutions) {
-      dfs(courseId)
-    }
+    let temp = courses.filter(c => (course.substitutions ? course.substitutions.includes(c.id) : false))
+    temp.unshift(course)
+    temp.forEach(cu => {
+      if (visited.includes(course.code)) return
+      visited.push(cu.id)
+      cu.subsId = substitutionGroupIndex
+    })
   }
 
-  for (const course of courses) {
+  courses.forEach(course => {
     if (!visited.includes(course.id)) {
       substitutionGroupIndex++
-      dfs(course.id)
+      organizeSubgroups(course)
     }
-  }
-
+  })
+  console.log(courses)
   return { courses }
 }
 

services/backend/src/servicesV2/organisations.js

@@ -0,0 +1,38 @@
+const Sequelize = require('sequelize')
+const { Organization } = require('../modelsV2')
+const Op = Sequelize.Op
+
+// Have facultyfetching to work like it worked during oodi-db time
+const facultiesInOodi = [
+  'H10',
+  'H20',
+  'H30',
+  'H40',
+  'H50',
+  'H55',
+  'H57',
+  'H60',
+  'H70',
+  'H74',
+  'H80',
+  'H90',
+  'H92',
+  'H930',
+  'H99',
+  'Y',
+  'Y01',
+]
+
+const faculties = () => {
+  return Organization.findAll({
+    where: {
+      code: {
+        [Op.in]: facultiesInOodi,
+      },
+    },
+  })
+}
+
+module.exports = {
+  faculties,
+}

services/backend/src/servicesV2/providers.js

@@ -0,0 +1,10 @@
+const { Organization } = require('../modelsV2/index')
+
+const getAllProviders = async () =>
+  Organization.findAll({
+    attributes: ['code', 'name'],
+  })
+
+module.exports = {
+  getAllProviders,
+}

services/backend/src/servicesV2/trends/protoC.js

@@ -102,10 +102,13 @@ const calculateProtoCProgramme = async query => {
     includeOldAttainments: query.include_old_attainments === 'true',
     excludeNonEnrolled: query.exclude_non_enrolled === 'true',
   })
+
   const programmeData = data.find(d => d.programmeType === 20)
   const studytrackData = data.filter(d => d.programmeType !== 20)
 
-  // mankel through studytracks
+  if (!programmeData) return {}
+
+  // mankel through studytracks if studyrightdata available
   const studytrackMankelid = _(studytrackData)
     // seems to return the numerical columns as strings, parse them first
     .map(programmeRow => ({
@@ -122,7 +125,7 @@ const calculateProtoCProgramme = async query => {
   const studytrackToBachelorProgrammes = Object.keys(associations.programmes).reduce((acc, curr) => {
     if (!curr.includes('KH')) return acc
     const studytracksForProgramme = studytrackMankelid.reduce((acc2, studytrackdata) => {
-      if (associations.programmes[curr].studytracks.includes(studytrackdata.programmeCode)) {
+      if (studytrackdata && associations.programmes[curr].studytracks.includes(studytrackdata.programmeCode)) {
         acc2.push({
           code: studytrackdata.programmeCode,
           name: studytrackdata.programmeName,

services/frontend/Dockerfile

@@ -3,23 +3,23 @@ FROM node:14.17-alpine3.13 as base
 WORKDIR /usr/src/app
 
 FROM base as development
-EXPOSE 3000
 COPY ./package* /usr/src/app/
 RUN npm ci
 COPY . /usr/src/app/
+EXPOSE 3000
 CMD ["npm", "start"]
 
 FROM base as production
-EXPOSE 5000
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
-COPY --chown=node:node . /usr/src/app/
-RUN npm run build && npm install -g serve@12.0.0 && chown -R node:node /usr/src/app/build
-USER node
 ARG SENTRY_ENVIRONMENT
 ENV REACT_APP_SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV REACT_APP_SENTRY_RELEASE=$SENTRY_RELEASE
 ARG BUILT_AT
 ENV REACT_APP_BUILT_AT=$BUILT_AT
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+RUN npm run build && npm install -g serve@12.0.0 && chown -R node:node /usr/src/app/build
+USER node
+EXPOSE 5000
 CMD ["serve", "--single", "--listen", "5000", "build"]

services/frontend/src/components/CourseStatistics/SearchForm/searchFormUtils.js

@@ -1,7 +1,8 @@
 import { orderBy } from 'lodash'
 
 // handle special case where the course's actual code starts with "A" so it is mistakenly taken as Open Uni course
-const isAvoin = code => !!code.match(/^[A][0-9]|^AY/)
+// const isAvoin = code => !!code.match(/^[A][0-9]|^AY/)
+const isAvoinTwo = course => course.organizations.some(c => c === 'hy-org-48645785')
 
 const sortAlternatives = alternatives => {
   const result = orderBy(
@@ -30,7 +31,7 @@ const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
   const mergedCourses = {}
   const sortedCourses = sortAlternatives(courses)
   sortedCourses.forEach(course => {
-    const groupId = isAvoin(course.code) && !unifyOpenUniCourses ? course.code : course.subsId
+    const groupId = isAvoinTwo(course) && !unifyOpenUniCourses ? course.code : course.subsId
 
     if (!(course.max_attainment_date && course.min_attainment_date)) {
       return

services/frontend/src/components/Faculty/FacultySelector/index.jsx

@@ -1,128 +0,0 @@
-import React, { useMemo } from 'react'
-import { arrayOf, shape, number, func } from 'prop-types'
-import { calculateTotalPassedCourses, calculateTotalFailedCourses } from '../facultyUtils'
-import { getTextIn } from '../../../common'
-import SortableTable from '../../SortableTable'
-import FacultyStatsGraph from '../FacultyStatsGraph'
-import useLanguage from '../../LanguagePicker/useLanguage'
-
-const FacultySelector = ({ faculties, facultyYearlyStats, fromYear, toYear, handleSelect }) => {
-  const { language } = useLanguage()
-  const calculateYearlyStatsForFaculty = facultyCode => {
-    const res = {}
-    const yearData = {
-      studentCredits: 0,
-      coursesPassed: 0,
-      coursesFailed: 0,
-      students: [],
-    }
-
-    const faculty = facultyYearlyStats.find(f => f.id === facultyCode)
-    if (!faculty) return res
-    Object.values(faculty.data).forEach(programmeYearlyStats => {
-      Object.entries(programmeYearlyStats).forEach(([year, stat]) => {
-        if (!res[year]) res[year] = { ...yearData }
-        res[year].studentCredits += stat.studentCredits
-        res[year].coursesPassed += stat.coursesPassed
-        res[year].coursesFailed += stat.coursesFailed
-        res[year].students = [...new Set(res[year].students.concat(stat.studentArray))]
-      })
-    })
-    return res
-  }
-
-  const calculateTotalStatsForFaculty = facultyCode => {
-    const initial = {
-      totalStudentCredits: 0,
-      totalCoursesPassed: 0,
-      totalCoursesFailed: 0,
-      totalStudents: [],
-    }
-    return Object.entries(calculateYearlyStatsForFaculty(facultyCode))
-      .filter(([year]) => year >= fromYear && year <= toYear)
-      .reduce(
-        (res, [, curr]) => ({
-          totalStudentCredits: res.totalStudentCredits + curr.studentCredits,
-          totalCoursesPassed: res.totalCoursesPassed + curr.coursesPassed,
-          totalCoursesFailed: res.totalCoursesFailed + curr.coursesFailed,
-          totalStudents: [...new Set(res.totalStudents.concat(curr.students))],
-        }),
-        { ...initial }
-      )
-  }
-
-  const totalStats = useMemo(
-    () =>
-      faculties.reduce((res, { code }) => {
-        res[code] = calculateTotalStatsForFaculty(code)
-        return { ...res }
-      }, {}),
-    [fromYear, toYear]
-  )
-
-  const graphData = useMemo(
-    () =>
-      faculties.map(({ code, name }) => ({
-        name: getTextIn(name, language),
-        data: calculateYearlyStatsForFaculty(code),
-      })),
-    []
-  )
-
-  const headers = [
-    {
-      key: 'name',
-      title: 'name',
-      getRowVal: faculty => getTextIn(faculty.name, language),
-    },
-    {
-      key: 'code',
-      title: 'code',
-      getRowVal: faculty => faculty.code,
-    },
-    {
-      key: 'studentCredits',
-      title: 'Student credits',
-      getRowVal: ({ code }) => Math.round(totalStats[code].totalStudentCredits),
-    },
-    {
-      key: 'coursesPassed',
-      title: 'Courses passed',
-      getRowVal: ({ code }) => calculateTotalPassedCourses(totalStats[code]),
-      getRowContent: ({ code }) => `${calculateTotalPassedCourses(totalStats[code]).toFixed(2)}%`,
-    },
-    {
-      key: 'coursesFailed',
-      title: 'Courses failed',
-      getRowVal: ({ code }) => calculateTotalFailedCourses(totalStats[code]),
-      getRowContent: ({ code }) => `${calculateTotalFailedCourses(totalStats[code]).toFixed(2)}%`,
-    },
-    {
-      key: 'students',
-      title: 'Students',
-      getRowVal: ({ code }) => totalStats[code].totalStudents.length,
-    },
-  ]
-
-  return (
-    <>
-      <SortableTable
-        columns={headers}
-        getRowKey={faculty => faculty.code}
-        getRowProps={faculty => ({ onClick: () => handleSelect(faculty.code), style: { cursor: 'pointer' } })}
-        data={faculties}
-      />
-      <FacultyStatsGraph data={graphData} />
-    </>
-  )
-}
-
-FacultySelector.propTypes = {
-  faculties: arrayOf(shape({})).isRequired,
-  facultyYearlyStats: arrayOf(shape({})).isRequired,
-  fromYear: number.isRequired,
-  toYear: number.isRequired,
-  handleSelect: func.isRequired,
-}
-
-export default FacultySelector

services/frontend/src/components/Faculty/FacultyStats/index.jsx

@@ -1,119 +0,0 @@
-import React, { useMemo } from 'react'
-import { connect } from 'react-redux'
-import { Link } from 'react-router-dom'
-import { string, arrayOf, shape, number } from 'prop-types'
-import { Segment, Icon, Header, Item } from 'semantic-ui-react'
-import { getTextIn } from '../../../common'
-import { calculateStatsForProgramme, calculateTotalPassedCourses, calculateTotalFailedCourses } from '../facultyUtils'
-import SortableTable from '../../SortableTable'
-import FacultyStatsGraph from '../FacultyStatsGraph'
-import useLanguage from '../../LanguagePicker/useLanguage'
-
-const ShowProgrammeOverView = ({ code }) => {
-  return (
-    <Item as={Link} title={`Overview of study programme ${code}`} to={`/study-programme/${code}`}>
-      <Icon name="level up alternate" />
-    </Item>
-  )
-}
-
-ShowProgrammeOverView.propTypes = {
-  code: string.isRequired,
-}
-
-const FacultyStats = ({ facultyProgrammes, selectedFacultyProgrammesStats, fromYear, toYear }) => {
-  const { language } = useLanguage()
-  const totalStats = useMemo(
-    () =>
-      Object.entries(selectedFacultyProgrammesStats).reduce((res, [code, stats]) => {
-        res[code] = calculateStatsForProgramme(stats, fromYear, toYear)
-        return { ...res }
-      }, {}),
-    [fromYear, toYear]
-  )
-
-  const getNameOfProgramme = code => {
-    const foundProgramme = facultyProgrammes.find(p => p.code === code)
-    return foundProgramme ? getTextIn(foundProgramme.name, language) : code
-  }
-
-  const graphData = useMemo(
-    () =>
-      Object.entries(selectedFacultyProgrammesStats).map(([code, data]) => ({ name: getNameOfProgramme(code), data })),
-    []
-  )
-
-  if (!Object.keys(selectedFacultyProgrammesStats).length) {
-    return <Segment textAlign="center">No data</Segment>
-  }
-
-  /* eslint-disable react/prop-types */
-  const headers = [
-    {
-      key: 'name',
-      title: 'name',
-      getRowVal: ({ code }) => code,
-      getRowContent: ({ code }) => (
-        <div>
-          {' '}
-          {getNameOfProgramme(code)} <ShowProgrammeOverView code={code} />
-        </div>
-      ),
-    },
-    {
-      key: 'code',
-      title: 'code',
-      getRowVal: ({ code }) => code,
-    },
-    {
-      key: 'studentCredits',
-      title: 'Student credits',
-      getRowVal: ({ code }) => Math.round(totalStats[code].totalStudentCredits),
-    },
-    {
-      key: 'coursesPassed',
-      title: 'Courses passed',
-      getRowVal: ({ code }) => calculateTotalPassedCourses(totalStats[code]),
-      getRowContent: ({ code }) => `${calculateTotalPassedCourses(totalStats[code]).toFixed(2)}%`,
-    },
-    {
-      key: 'coursesFailed',
-      title: 'Courses failed',
-      getRowVal: ({ code }) => calculateTotalFailedCourses(totalStats[code]),
-      getRowContent: ({ code }) => `${calculateTotalFailedCourses(totalStats[code]).toFixed(2)}%`,
-    },
-    {
-      key: 'students',
-      title: 'Students',
-      getRowVal: ({ code }) => totalStats[code].totalStudents.length,
-    },
-  ]
-  /* eslint-enable react/prop-types */
-
-  const data = Object.entries(selectedFacultyProgrammesStats).map(([code, stats]) => ({ code, stats }))
-  const bachelors = data.filter(programme => programme.code.includes('KH'))
-  const masters = data.filter(programme => programme.code.includes('MH'))
-
-  return (
-    <>
-      <Header>Bachelor degrees</Header>
-      <SortableTable columns={headers} getRowKey={({ code }) => code} data={bachelors} />
-      <Header>Masters degrees</Header>
-      <SortableTable columns={headers} getRowKey={({ code }) => code} data={masters} />
-      <FacultyStatsGraph data={graphData} />
-    </>
-  )
-}
-
-FacultyStats.propTypes = {
-  facultyProgrammes: arrayOf(shape({})).isRequired,
-  selectedFacultyProgrammesStats: shape({}).isRequired,
-  fromYear: number.isRequired,
-  toYear: number.isRequired,
-}
-
-const mapStateToProps = ({ faculties }) => ({
-  facultyProgrammes: faculties.facultyProgrammes,
-})
-
-export default connect(mapStateToProps)(FacultyStats)

services/frontend/src/components/Faculty/FacultyStatsGraph/index.jsx

@@ -1,88 +0,0 @@
-import React, { useState, useMemo } from 'react'
-import { arrayOf, string, number, shape } from 'prop-types'
-import { Segment, Button } from 'semantic-ui-react'
-import Highcharts from 'highcharts/highstock'
-import ReactHighstock from 'react-highcharts/ReactHighstock'
-
-const FacultyStatsGraph = ({ data }) => {
-  const [mode, setMode] = useState('studentCredits')
-  const defaultStartTimestamp = new Date(2000, 0, 1).getTime()
-
-  const series = useMemo(
-    () =>
-      data.map(({ name, data: entryData }) => ({
-        name,
-        data: Object.entries(entryData).map(([year, stats]) => ({
-          x: new Date(parseInt(year, 10), 0, 1, 0, 0, 0, 0).getTime(),
-          y: Math.round(
-            ['coursesPassed', 'coursesFailed'].includes(mode)
-              ? (stats[mode] / (stats.coursesPassed + stats.coursesFailed)) * 100
-              : stats[mode].length || stats[mode]
-          ),
-        })),
-      })),
-    [data, mode]
-  )
-
-  const { min, max } = useMemo(
-    () =>
-      series.reduce(
-        (res, { data }) => {
-          data.forEach(({ x }) => {
-            res.min = Math.min(res.min, x)
-            res.max = Math.max(res.max, x)
-          })
-          return res
-        },
-        { min: Infinity, max: -Infinity }
-      ),
-    [series]
-  )
-
-  const options = {
-    findNearestPointBy: 'xy',
-    series,
-    xAxis: {
-      min: Math.max(min, defaultStartTimestamp),
-      max,
-      ordinal: false,
-    },
-  }
-
-  const handleClick = (_, { name }) => setMode(name)
-
-  return (
-    <Segment>
-      <ReactHighstock highcharts={Highcharts} constructorType="stockChart" config={options} />
-      <Button active={mode === 'studentCredits'} onClick={handleClick} name="studentCredits">
-        Total credits
-      </Button>
-      <Button active={mode === 'coursesPassed'} onClick={handleClick} name="coursesPassed">
-        % of courses passed
-      </Button>
-      <Button active={mode === 'coursesFailed'} onClick={handleClick} name="coursesFailed">
-        % of courses failed
-      </Button>
-      <Button active={mode === 'students'} onClick={handleClick} name="students">
-        Amount of students
-      </Button>
-    </Segment>
-  )
-}
-
-FacultyStatsGraph.propTypes = {
-  data: arrayOf(
-    shape({
-      name: string,
-      data: shape({
-        year: shape({
-          studentCredits: number,
-          coursesPassed: number,
-          coursesFailed: number,
-        }),
-      }),
-    })
-  ).isRequired,
-}
-
-export default FacultyStatsGraph

services/frontend/src/components/Faculty/facultyUtils.js

@@ -1,30 +0,0 @@
-export const calculateStatsForProgramme = (stats, fromYear, toYear) => {
-  const res = {
-    totalStudentCredits: 0,
-    totalCoursesPassed: 0,
-    totalCoursesFailed: 0,
-    totalStudents: [],
-  }
-
-  Object.entries(stats)
-    .filter(([year]) => year >= fromYear && year <= toYear)
-    .forEach(([, stat]) => {
-      const { studentCredits, coursesPassed, coursesFailed, studentArray } = stat
-      res.totalStudentCredits += studentCredits
-      res.totalCoursesPassed += coursesPassed
-      res.totalCoursesFailed += coursesFailed
-      res.totalStudents = [...new Set(res.totalStudents.concat(studentArray))]
-    })
-  return res
-}
-
-export const calculateTotalPassedCourses = ({ totalCoursesPassed, totalCoursesFailed }) => {
-  const ratio = (totalCoursesPassed / (totalCoursesPassed + totalCoursesFailed)) * 100
-  return Number.isNaN(ratio) ? 0 : ratio
-}
-
-export const calculateTotalFailedCourses = ({ totalCoursesPassed, totalCoursesFailed }) => {
-  const totalPassedRatio = calculateTotalPassedCourses({ totalCoursesPassed, totalCoursesFailed })
-  if (totalPassedRatio === 0) return 0
-  return 100 - totalPassedRatio
-}

services/frontend/src/components/Faculty/index.jsx

@@ -1,169 +0,0 @@
-import React, { useState, useEffect } from 'react'
-import { connect } from 'react-redux'
-import { withRouter } from 'react-router-dom'
-import { Header, Segment, Form } from 'semantic-ui-react'
-import { uniq } from 'lodash'
-import { arrayOf, shape, func, bool } from 'prop-types'
-import { getTextIn } from '../../common'
-import { useTitle } from '../../common/hooks'
-import { getUserFaculties, getFacultiesYearlyStats, getFacultyProgrammes } from '../../redux/faculties'
-import YearFilter from '../CourseStatistics/SearchForm/YearFilter'
-import FacultySelector from './FacultySelector'
-import FacultyStats from './FacultyStats'
-import useLanguage from '../LanguagePicker/useLanguage'
-
-const Faculty = ({
-  getUserFaculties,
-  getFacultiesYearlyStats,
-  getFacultyProgrammes,
-  faculties,
-  pending,
-  error,
-  facultyYearlyStats,
-  history,
-  match,
-}) => {
-  const { language } = useLanguage()
-  const [selectedFaculty, setSelectedFaculty] = useState(null)
-  const [fromYear, setFromYear] = useState(-1)
-  const [toYear, setToYear] = useState(-1)
-  const [years, setYears] = useState([])
-  const [initialized, setInitialized] = useState(false)
-  useTitle('Faculties')
-
-  const facultyCodes = faculties.map(({ code }) => code)
-  const selectedFacultyProgrammesStats = facultyYearlyStats.find(({ id }) => id === selectedFaculty)
-  const hasLoaded = faculties.length > 0 && facultyYearlyStats.length > 0
-
-  useEffect(() => {
-    if (selectedFaculty) {
-      history.push(`/faculties/${selectedFaculty}`)
-    }
-  }, [selectedFaculty])
-
-  useEffect(() => {
-    const {
-      params: { facultyid },
-    } = match
-    if (facultyid) setSelectedFaculty(facultyid)
-    else setSelectedFaculty(null)
-  }, [history.location.pathname])
-
-  useEffect(() => {
-    getUserFaculties()
-    getFacultiesYearlyStats()
-    getFacultyProgrammes()
-    setInitialized(true)
-  }, [])
-
-  const getYearFilterData = () => {
-    const filterYears = uniq(
-      facultyYearlyStats
-        .filter(f => facultyCodes.includes(f.id))
-        .map(({ data }) => Object.values(data).reduce((res, curr) => [...res, ...Object.keys(curr)], []))
-        .reduce((acc, curr) => [...acc, ...curr], [])
-    )
-      .sort((a, b) => b - a)
-      .map(year => parseInt(year, 10))
-
-    return {
-      fromYear: filterYears[filterYears.length - 1],
-      toYear: filterYears[0],
-      years: filterYears.map(y => ({ key: y, text: y, value: y })),
-    }
-  }
-
-  useEffect(() => {
-    if (faculties.length && facultyYearlyStats.length) {
-      const { fromYear: newFromYear, toYear: newToYear, years: newYears } = getYearFilterData()
-      if (fromYear < newFromYear || fromYear > newToYear) setFromYear(Math.max(newFromYear, 2000))
-      if (toYear < newFromYear || toYear > newToYear) setToYear(newToYear)
-      setYears(newYears)
-    }
-  }, [faculties, facultyYearlyStats, selectedFaculty])
-
-  const handleYearChange = (e, target) => {
-    const { name, value } = target
-    if (name === 'fromYear') setFromYear(value)
-    else setToYear(value)
-  }
-
-  const getTitle = () =>
-    selectedFaculty && hasLoaded
-      ? getTextIn(faculties.find(({ code }) => code === selectedFaculty).name, language)
-      : 'Faculties'
-
-  if (!(faculties.length || pending) && initialized) {
-    return !error ? (
-      <Header textAlign="center" content="No access to any faculties!" as="h1" />
-    ) : (
-      <Header textAlign="center" content="Not authorized for faculties!" as="h1" />
-    )
-  }
-
-  return (
-    <div className="segmentContainer">
-      <Header className="segmentTitle" size="large">
-        {hasLoaded && getTitle()}
-      </Header>
-      <Segment className="contentSegment" loading={!hasLoaded}>
-        {!hasLoaded && null}
-        {hasLoaded && (
-          <>
-            <Segment>
-              <Form>
-                <Header content="Filter by time range" as="h4" />
-                <YearFilter fromYear={fromYear} toYear={toYear} years={years} handleChange={handleYearChange} />
-              </Form>
-            </Segment>
-            {!selectedFaculty ? (
-              <FacultySelector
-                faculties={faculties}
-                facultyYearlyStats={facultyYearlyStats}
-                fromYear={fromYear}
-                toYear={toYear}
-                handleSelect={setSelectedFaculty}
-              />
-            ) : (
-              <FacultyStats
-                selectedFacultyProgrammesStats={
-                  selectedFacultyProgrammesStats ? selectedFacultyProgrammesStats.data : {}
-                }
-                history={history}
-                fromYear={fromYear}
-                toYear={toYear}
-              />
-            )}
-          </>
-        )}
-      </Segment>
-    </div>
-  )
-}
-
-Faculty.propTypes = {
-  getUserFaculties: func.isRequired,
-  getFacultiesYearlyStats: func.isRequired,
-  getFacultyProgrammes: func.isRequired,
-  faculties: arrayOf(shape({})).isRequired,
-  facultyYearlyStats: arrayOf(shape({})).isRequired,
-  history: shape({}).isRequired,
-  match: shape({}).isRequired,
-  pending: bool.isRequired,
-  error: bool.isRequired,
-}
-
-const mapStateToProps = ({ faculties }) => ({
-  faculties: faculties.data,
-  facultyYearlyStats: faculties.yearlyStats,
-  pending: faculties.userFacultiesPending,
-  error: faculties.userFacultiesError,
-})
-
-const mapDispatchToProps = {
-  getUserFaculties,
-  getFacultiesYearlyStats,
-  getFacultyProgrammes,
-}
-
-export default connect(mapStateToProps, mapDispatchToProps)(withRouter(Faculty))

services/frontend/src/components/NavigationBar/index.jsx

@@ -25,7 +25,6 @@ const allNavigationItems = {
   teachers: { path: '/teachers', key: 'teachers', label: 'Teachers', reqRights: ['teachers'] },
   users: { path: '/users', key: 'users', label: 'Users', reqRights: ['users'] },
   trends: { path: '/trends', key: 'trends', label: 'Trends' },
-  faculty: { path: '/faculties', key: 'faculty', label: 'Faculty', reqRights: ['faculties'] },
   updater: { path: '/updater', key: 'updater', label: 'Updater', reqRights: ['dev', 'admin'] },
   sandbox: { path: '/sandbox', key: 'sandbox', label: 'Sandbox', reqRights: ['dev'] },
   feedback: { path: '/feedback', key: 'feedback', label: 'Give feedback' },

services/frontend/src/components/Routes/index.jsx

@@ -34,7 +34,6 @@ const StudyProgramme = React.lazy(() => retry(() => import('../StudyProgramme'))
 const Teachers = React.lazy(() => retry(() => import('../Teachers')))
 const Sandbox = React.lazy(() => retry(() => import('../Sandbox')))
 const Feedback = React.lazy(() => retry(() => import('../Feedback')))
-const Faculty = React.lazy(() => retry(() => import('../Faculty')))
 const CoursePopulation = React.lazy(() => retry(() => import('../CoursePopulation')))
 const CustomPopulation = React.lazy(() => retry(() => import('../CustomPopulation')))
 const Updater = React.lazy(() => retry(() => import('../Updater')))
@@ -45,7 +44,6 @@ const routes = {
   courseStatistics: '/coursestatistics',
   teachers: '/teachers/:teacherid?',
   users: '/users/:userid?',
-  faculty: '/faculties/:facultyid?',
   sandbox: '/sandbox',
   feedback: '/feedback',
   coursepopulation: '/coursepopulation',
@@ -63,7 +61,6 @@ const Routes = () => (
       <Route exact path="/study-programme/:studyProgrammeId/course-group/:courseGroupId" component={StudyProgramme} />
       <Route exact path={routes.students} component={StudentStatistics} />
       <Route exact path={routes.courseStatistics} component={CourseStatistics} />
-      <Route exact path={routes.faculty} component={Faculty} />
       <Route exact path={routes.users} component={EnableUsers} />
       <Route exact path={routes.teachers} component={Teachers} />
       <Route exact path={routes.sandbox} component={Sandbox} />

services/frontend/src/components/Teachers/TeacherStatistics/index.jsx

@@ -51,6 +51,9 @@ const TeacherStatistics = ({
     KH57_001
     500-K001
     --------
+    KH74_001
+    740-K001
+    --------
     KH80_003
     800-K003
     --------
@@ -58,13 +61,12 @@ const TeacherStatistics = ({
     */
   const mapToProviders = rights =>
     rights.map(r => {
+      const isNumber = str => !Number.isNaN(Number(str))
       if (r.includes('_')) {
-        let newPrefix = ''
-        let newSuffix = ''
-        const split = r.split('_')
-        newPrefix = `${split[0][2]}00`
-        newSuffix = `${split[0][0]}${split[1]}`
-        const providercode = `${newPrefix}-${newSuffix}`
+        const [left, right] = r.split('_')
+        const prefix = [...left].filter(isNumber).join('')
+        const suffix = `${left[0]}${right}`
+        const providercode = `${prefix}0-${suffix}`
         return providercode
       }
       return r
@@ -85,7 +87,7 @@ const TeacherStatistics = ({
 
   const userProviders = mapToProviders(rights)
   const invalidQueryParams = provs.length === 0 || !semesterStart
-  const providerOptions = isAdmin ? providers : providers.filter(p => userProviders.includes(p.value))
+  const providerOptions = isAdmin ? providers : providers.filter(p => userProviders.includes(p.code))
   const localizedProviderOptions = providerOptions.map(({ name, ...rest }) => ({
     ...rest,
     text: getTextIn(name, language),
@@ -184,11 +186,7 @@ const mapStateToProps = state => {
     },
   } = state
   const { semesters } = state.semesters.data
-  const providerOptions = providers.data.map(p => ({
-    key: p.providercode,
-    value: p.providercode,
-    name: p.name,
-  }))
+  const providerOptions = providers.data.map(prov => ({ key: prov.code, value: prov.code, name: prov.name }))
   const semesterOptions = !semesters
     ? []
     : Object.values(semesters)

services/frontend/src/conf.js

@@ -12,10 +12,10 @@ const databaseNames = ['kone-db', 'oodi-db', 'sis-db', 'sis-importer-db', 'user-
 const adminerUrls = databaseNames.map(db => ({ url: `${adminerBaseUrl}/?pgsql=${db}&username=postgres`, text: db }))
 
 // Base paths
-const basePath = process.env.PUBLIC_URL || ''
+const basePath = process.env.REACT_APP_PUBLIC_URL || ''
 const apiBasePath = `${basePath}/api`
 
-// Build at time
+// Update time for frontpage
 const builtAt = process.env.REACT_APP_BUILT_AT || ''
 
 module.exports = {

services/frontend/src/redux/faculties.js

@@ -6,24 +6,6 @@ export const getFaculties = () => {
   return callController(route, prefix)
 }
 
-export const getUserFaculties = () => {
-  const route = '/facultystats/faculties'
-  const prefix = 'GET_USER_FACULTIES_'
-  return callController(route, prefix)
-}
-
-export const getFacultiesYearlyStats = () => {
-  const route = '/facultystats/yearlystats'
-  const prefix = 'GET_FACULTIES_YEARLY_STATS_'
-  return callController(route, prefix)
-}
-
-export const getFacultyProgrammes = () => {
-  const route = '/facultystats/programmes'
-  const prefix = 'GET_FACULTY_PROGRAMMES_'
-  return callController(route, prefix)
-}
-
 const reducer = (
   state = {
     data: [],
@@ -56,57 +38,6 @@ const reducer = (
         pending: false,
         error: true,
       }
-    case 'GET_USER_FACULTIES_ATTEMPT':
-      return {
-        ...state,
-        userFacultiesPending: true,
-        userFacultiesError: false,
-      }
-    case 'GET_USER_FACULTIES_FAILURE':
-      return {
-        ...state,
-        userFacultiesPending: false,
-        userFacultiesError: true,
-      }
-    case 'GET_USER_FACULTIES_SUCCESS':
-      return {
-        ...state,
-        data: action.response,
-        userFacultiesPending: false,
-        userFacultiesError: false,
-      }
-    case 'GET_FACULTIES_YEARLY_STATS_FAILURE':
-      return {
-        ...state,
-        pending: false,
-        error: true,
-      }
-    case 'GET_FACULTIES_YEARLY_STATS_SUCCESS':
-      return {
-        ...state,
-        yearlyStats: action.response,
-        pending: false,
-        error: true,
-      }
-    case 'GET_FACULTY_PROGRAMMES_ATTEMPT':
-      return {
-        ...state,
-        pending: true,
-        error: false,
-      }
-    case 'GET_FACULTY_PROGRAMMES_FAILURE':
-      return {
-        ...state,
-        pending: false,
-        error: true,
-      }
-    case 'GET_FACULTY_PROGRAMMES_SUCCESS':
-      return {
-        ...state,
-        facultyProgrammes: action.response,
-        pending: false,
-        error: false,
-      }
     default:
       return state
   }

updater/sis-updater-worker/src/updater/mapper.js

@@ -220,7 +220,7 @@ const courseProviderMapper =
 
 const timify = t => new Date(t).getTime()
 
-const courseMapper = courseIdToAttainments => (groupedCourse, substitutions) => {
+const courseMapper = courseIdToAttainments => (groupedCourse, substitutions, organisation) => {
   const [groupId, courses] = groupedCourse
   const { code, name, study_level: coursetypecode } = courses[0]
 
@@ -263,6 +263,7 @@ const courseMapper = courseIdToAttainments => (groupedCourse, substitutions) =>
     enddate,
     is_study_module: false, // VALIDATE THIS PLS
     substitutions,
+    responsible_organisation: organisation,
   }
 }
 

updater/sis-updater-worker/src/updater/updateMeta.js

@@ -49,6 +49,14 @@ const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
         }, [])
       ),
     ]
+    // take responsible organisation from all course units
+    const organisation = [
+      ...new Set(
+        courses.reduce((acc, curr) => {
+          return [...acc, ...flatten(curr.organisations).map(({ organisationId }) => organisationId)]
+        }, [])
+      ),
+    ][0]
 
     // Take organisation for the newest course_unit of this groupid
     const coursesSortedByStartDate = sortBy(courses, 'validity_period.startDate')
@@ -59,8 +67,9 @@ const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
         .filter(({ roleUrn }) => roleUrn === 'urn:code:organisation-role:responsible-organisation')
         .map(mapCourseProvider)
     )
+    console.log('courseProviders: ', courseProviders)
 
-    return mapCourse(groupedCourse, substitutions)
+    return mapCourse(groupedCourse, substitutions, organisation)
   })
 
   await bulkCreate(Course, courses)

updater/sis-updater-worker/src/updater/updateStudents/index.js

@@ -13,12 +13,27 @@ const {
 const { selectFromByIds, selectFromSnapshotsByIds, bulkCreate, getCourseUnitsByCodes } = require('../../db')
 const { getEducation, getUniOrgId, loadMapsIfNeeded } = require('../shared')
 const { studentMapper, mapTeacher, creditMapper, semesterEnrollmentMapper, courseProviderMapper } = require('../mapper')
-
 const { dbConnections } = require('../../db/connection')
 const { isBaMa } = require('../../utils')
 const { updateStudyRights, updateStudyRightElements, updateElementDetails } = require('./studyRightUpdaters')
 const { getAttainmentsToBeExcluded } = require('./excludedPartialAttainments')
 
+// Accepted selection path is not available when degree programme doesn't have
+// studytrack or major subject. This is a known bug on SIS and has been reported
+// to funidata.
+// In these cases, degree programmes module group id must be fetched from education.
+const addSelectionPaths = studyrightSnapshots =>
+  studyrightSnapshots.map(snapshot =>
+    Object.keys(snapshot.accepted_selection_path).length === 0
+      ? {
+          ...snapshot,
+          accepted_selection_path: {
+            educationPhase1GroupId: getEducation(snapshot.education_id).structure.phase1.options[0].moduleGroupId,
+          },
+        }
+      : snapshot
+  )
+
 // Group snapshots by studyright id and find out when studyrights have begun
 const groupStudyrightSnapshots = studyrightSnapshots => {
   const snapshotsBystudyright = Object.entries(
@@ -45,12 +60,20 @@ const groupStudyrightSnapshots = studyrightSnapshots => {
 
     const groupedByPhases = groupBy(orderedSnapshots, byPhases)
 
+    // Get snapshot date time from oldest snapshot where date time is available. If no date
+    // times are available (e.g. all are null), take study start date of the oldest one
+    const parseSnapshotDateTime = snapshots => {
+      let oldestFirst = [...snapshots]
+      oldestFirst.reverse()
+      const firstWithSnapshotDateTime = oldestFirst.find(s => !!s.snapshot_date_time)
+      if (firstWithSnapshotDateTime) return firstWithSnapshotDateTime.snapshot_date_time
+      return oldestFirst[0].study_start_date
+    }
+
     const snapshotsWithRightDate = Object.keys(groupedByPhases).map(key => {
       const snapshots = groupedByPhases[key]
       const most_recent = snapshots[0]
-      const the_first = snapshots[snapshots.length - 1]
-      most_recent.first_snapshot_date_time = the_first.snapshot_date_time
-
+      most_recent.first_snapshot_date_time = parseSnapshotDateTime(snapshots)
       return most_recent
     })
 
@@ -132,7 +155,9 @@ const updateStudents = async personIds => {
     selectFromByIds('study_right_primalities', personIds, 'student_id'),
   ])
 
-  const groupedStudyRightSnapshots = groupStudyrightSnapshots(studyRightSnapshots)
+  const studyRightSnapshotsWithAddedSelectionPaths = addSelectionPaths(studyRightSnapshots)
+
+  const groupedStudyRightSnapshots = groupStudyrightSnapshots(studyRightSnapshotsWithAddedSelectionPaths)
 
   const personIdToStudentNumber = students.reduce((res, curr) => {
     res[curr.id] = curr.student_number
@@ -147,7 +172,9 @@ const updateStudents = async personIds => {
 
   const attainmentsToBeExluced = getAttainmentsToBeExcluded()
 
-  const mappedStudents = students.map(studentMapper(attainments, studyRightSnapshots, attainmentsToBeExluced))
+  const mappedStudents = students.map(
+    studentMapper(attainments, studyRightSnapshotsWithAddedSelectionPaths, attainmentsToBeExluced)
+  )
   await bulkCreate(Student, mappedStudents)
 
   const [moduleGroupIdToCode, formattedStudyRights] = await Promise.all([