Merge pull request #3144 from UniversityOfHelsinkiCS/trunk

Fix provider stuff in teachers and add regular trendsv2 updates

.github/workflows/release-oodikone-staging.yaml

@@ -48,15 +48,16 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Set current date as env variable"
-        run: echo NOW::"$(date --iso-8601=seconds)" >> "$GITHUB_ENV"
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: "Build and push"
         uses: docker/build-push-action@v2
         with:
           build-args: |
             SENTRY_RELEASE=${{ github.sha }}
             SENTRY_ENVIRONMENT=staging
-            BUILT_AT=${{ env.NOW }}
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
             type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
@@ -82,6 +83,7 @@ jobs:
           set_commits: 'skip'
           sourcemaps: './build/static/js'
           url_prefix: '~/oodikone/static/js'
+          version: ${{ github.sha }}
 
   build_and_release_userservice:
     name: "Build and release userservice"

.github/workflows/release-oodikone.yaml

@@ -49,8 +49,9 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: "Set current date as env variable"
-        run: echo NOW::"$(date --iso-8601=seconds)" >> "$GITHUB_ENV"
+      - name: "Get current time"
+        id: get_current_time
+        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
       - name: "Extract release version"
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
@@ -63,7 +64,7 @@ jobs:
           build-args: |
             SENTRY_RELEASE=${{ steps.extract_version.outputs.VERSION }}
             SENTRY_ENVIRONMENT=production
-            BUILT_AT=${{ env.NOW }}
+            BUILT_AT=${{ steps.get_current_time.outputs.NOW }}
           cache-from: |
             type=registry,ref=toska/oodikone2-frontend:trunk
             type=registry,ref=toska/oodikone2-frontend:staging
@@ -89,6 +90,7 @@ jobs:
           set_commits: 'skip'
           sourcemaps: './build/static/js'
           url_prefix: '~/oodikone/static/js'
+          version: ${{ steps.extract_version.outputs.VERSION }}
 
   build_and_release_userservice:
     name: "Build and release userservice"

cypress/integration/Course_statistics.js

@@ -119,6 +119,42 @@ describe('Course Statistics tests', () => {
       cy.contains('AYTKT10004 Avoin yo: Tietokantojen perusteet')
     })
 
+    it('Searching course by name displays right courses, 10 credit courses', () => {
+      cy.url().should('include', '/coursestatistics')
+      cy.contains('Search for courses')
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+
+      cy.contains('Tietorakenteet ja algoritmit')
+      cy.contains('Avoin yo: Tietorakenteet ja algoritmit')
+      cy.contains('AYTKT20001')
+      cy.contains('TKT20001, 58131')
+      cy.contains('td', /^TKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+
+      cy.contains('TKT20001, 58131 Tietorakenteet ja algoritmit')
+      cy.get('.right').click()
+      cy.contains('No results')
+
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', /^AYTKT20001/).click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('AYTKT20001 Avoin yo: Tietorakenteet ja algoritmit')
+
+      cy.get('.right').click()
+      cy.contains('No results')
+      cy.get("input[name='unifyOpenUniCourses']").parent().click()
+      cy.get("input[placeholder='Search by entering a course name']").type('tietorakenteet ja algoritmit')
+      cy.contains('td', 'TKT20001, 58131, AYTKT20001').click()
+
+      cy.contains('Fetch statistics').should('be.enabled').click()
+      cy.contains('Search for courses').should('not.exist')
+      cy.contains('TKT20001, 58131, AYTKT20001 Tietorakenteet ja algoritmit')
+    })
+
     it('Can find course population', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains('Search for courses')

cypress/integration/Filters.js

@@ -1,6 +1,6 @@
 /// <reference types="Cypress" />
 
-const baseUrl = Cypress.config().baseUrl
+// const baseUrl = Cypress.config().baseUrl
 
 const checkFilteringResult = studentCount => {
   cy.contains(`Students (${studentCount})`)
@@ -130,12 +130,9 @@ describe('Population Statistics', () => {
     })
   })
 
-  // TODO: this fails in CI, but not locally. Investigate and fix
+  // plz fix
   it.skip('Courses filter works', () => {
     // courses takes some time to load, wait for it to complete
-    cy.intercept({ path: '**/courses' }).as('coursesOfPopulation')
-    cy.wait('@coursesOfPopulation')
-    // moar waiting hack
     cy.wait(10000)
     runTestStepWithPreAndPostParts('courseFilter-header', () => {
       cy.cs('courseFilter-course-dropdown').click()

cypress/integration/Population_statistics.js

@@ -73,7 +73,7 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(3000) // a bit hacky way, wait until ui is ready
+      cy.wait(10000) // a bit hacky way, wait until ui is ready
       cy.get('[data-cy=expand-TKT1]').click()
       cy.get('[data-cy=coursestats-link-TKT10002]').click()
       cy.wait('@coursePage')

docker-compose.ci.yml

@@ -29,8 +29,6 @@ services:
   frontend:
     <<: *defaults
     container_name: frontend
-    environment:
-      BACKEND_ADDR: backend
     image: toska/oodikone2-frontend:trunk
 
   kone-db:

docker-compose.yml

@@ -53,13 +53,6 @@ services:
       <<: *build_defaults
       context: services/frontend
     container_name: frontend
-    environment:
-      ADMINER_URL: http://localhost:5050/?pgsql=oodi-db&username=postgres
-      BACKEND_ADDR: backend
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=kone-db&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=sis-db&username=postgres
-      SIS_IMPORTER_ADMINER_URL: http://localhost:5050/?pgsql=sis-importer-db&username=postgres
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user-db&username=postgres
     ports:
       - 3000:3000
     profiles:

old_oodikone_stuff/updater_api/package.json

@@ -1,34 +0,0 @@
-{
-  "name": "updater_api",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "axios": "^0.21.1",
-    "dotenv": "^7.0.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^5.1.1",
-    "moment": "^2.29.1",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_scheduler/package.json

@@ -1,35 +0,0 @@
-{
-  "name": "updater_scheduler",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node --max_old_space_size=4096 index.js",
-    "dev": "nodemon --max_old_space_size=4096 index.js",
-    "scheduleActiveStudents": "NATS_CLIENTID=manualActiveSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleActiveStudents()'",
-    "scheduleMeta": "NATS_CLIENTID=manualAllSchedule node --max_old_space_size=4096 -e 'require(\"./src/schedule_students.js\").scheduleMeta()'",
-    "getOldestTasks": "node --max_old_space_size=4096 -e 'require(\"./src/SchedulingStatistics.js\").getOldestTasks()'"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "cron": "^1.8.2",
-    "express": "^4.17.1",
-    "mongoose": "^5.12.3",
-    "node-nats-streaming": "^0.2.6",
-    "nodemon": "^1.19.4",
-    "winston": "^3.3.3",
-    "winston-log2gelf": "^2.4.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.4.0",
-    "prettier": "^1.19.1",
-    "prettier-eslint": "^9.0.2"
-  }
-}

old_oodikone_stuff/updater_writer/package.json

@@ -1,39 +0,0 @@
-{
-  "name": "updater_writer",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run jest",
-    "jest": "NODE_ENV=test jest --runInBand --verbose",
-    "test_docker": "docker exec -t updater_writer /bin/bash -c 'npm test'",
-    "lint": "eslint .",
-    "lintfix": "prettier --write **/*.{js,json} && eslint --fix .",
-    "start": "node index.js",
-    "dev": "nodemon index.js",
-    "force_sync": "node -e \"require('./database/force_sync_database').sync()\"",
-    "dump": "node -e \"require('./database/dump_database').dumpDatabase()\""
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "dotenv": "^8.0.0",
-    "jest": "^24.8.0",
-    "lodash": "^4.17.19",
-    "node-nats-streaming": "^0.2.6",
-    "pg": "^8.6.0",
-    "sequelize": "^6.6.2",
-    "umzug": "^2.2.0",
-    "winston": "^3.2.1",
-    "winston-log2gelf": "^2.3.0"
-  },
-  "devDependencies": {
-    "babel-eslint": "^10.0.3",
-    "eslint": "^6.6.0",
-    "eslint-config-prettier": "^6.3.0",
-    "eslint-plugin-prettier": "^3.1.0",
-    "nodemon": "^1.19.4",
-    "prettier": "^1.18.2",
-    "prettier-eslint": "^9.0.0"
-  }
-}

package.json

@@ -19,7 +19,11 @@
     "diff:courses": "docker exec backend node /usr/src/app/src/sisoodi_diff/courses/index.js",
     "diff:transfers": "docker exec backend node /usr/src/app/src/sisoodi_diff/transfers.js",
     "diff:trends": "docker exec backend node /usr/src/app/src/sisoodi_diff/trends.js",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "lint:docker-compose": "docker-compose config -q && :",
+    "lint:docker-compose:real": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q && :",
+    "lint:docker-compose:ci": "docker-compose -f docker-compose.ci.yml config -q && :",
+    "lint:docker-compose:test": "docker-compose -f docker-compose.test.yml config -q && :"
   },
   "lint-staged": {
     "{services,updater}/**/*.{js,jsx}": "eslint --fix",
@@ -28,10 +32,10 @@
     ".github/workflows/*": "actionlint",
     "Dockerfile": "hadolint",
     "*.sh": "shellcheck -x",
-    "docker-compose.yml": "docker-compose config -q",
-    "docker-compose.real.yml": "docker-compose -f docker-compose.yml -f docker-compose.real.yml config -q",
-    "docker-compose.ci.yml": "docker-compose -f docker-compose.ci.yml config -q",
-    "docker-compose.test.yml": "docker-compose -f docker-compose.test.yml config -q"
+    "docker-compose.yml": "npm run lint:docker-compose",
+    "docker-compose.real.yml": "npm run lint:docker-compose:real",
+    "docker-compose.ci.yml": "npm run lint:docker-compose:ci",
+    "docker-compose.test.yml": "npm run lint:docker-compose:test"
   },
   "repository": {
     "type": "git",

services/backend/Dockerfile

@@ -12,12 +12,12 @@ CMD ["npm", "run", "dev"]
 
 FROM base as production
 ENV NODE_ENV=production
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
-COPY --chown=node:node . /usr/src/app/
-USER node
 ARG SENTRY_ENVIRONMENT
 ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+USER node
 CMD ["npm", "start"]

services/backend/src/events.js

@@ -8,6 +8,16 @@ const {
   refreshProtoCProgramme,
   getStartYears,
 } = require('./services/coolDataScience')
+
+const {
+  refreshProtoC: refreshProtoCV2,
+  refreshStatus: refreshStatusV2,
+  refreshStatusGraduated: refreshStatusGraduatedV2,
+  refreshUber: refreshUberV2,
+  refreshProtoCProgramme: refreshProtoCProgrammeV2,
+  getStartYears: getStartYearsV2,
+} = require('./servicesV2/trends')
+
 const { refreshAssociationsInRedis: refreshAssociationsInRedisV2 } = require('./servicesV2/studyrights')
 const {
   getAllProgrammes: getAllProgrammesV2,
@@ -20,8 +30,8 @@ const {
 } = require('./servicesV2/studyprogramme')
 
 const topteachersV2 = require('./servicesV2/topteachers')
-const { isNewHYStudyProgramme } = require('./util')
 
+const { isNewHYStudyProgramme } = require('./util')
 const { calculateFacultyYearlyStats } = require('./services/faculties')
 const { patchFacultyYearlyStats } = require('./servicesV2/analyticsService')
 
@@ -210,6 +220,78 @@ const refreshProtoCProgrammeToRedis = async () => {
   }
 }
 
+const refreshProtoCtoRedisV2 = async () => {
+  try {
+    const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false' }
+    const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false' }
+    const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true' }
+    const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true' }
+    console.log('Refreshing CDS ProtoC')
+    await refreshProtoCV2(defaultQuery)
+    await refreshProtoCV2(onlyOld)
+    await refreshProtoCV2(onlyEnr)
+    await refreshProtoCV2(bothToggles)
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+const refreshStatusToRedisV2 = async () => {
+  try {
+    const unixMillis = moment().valueOf()
+    const date = new Date(Number(unixMillis))
+
+    date.setHours(23, 59, 59, 999)
+    const showByYearOff = 'false'
+    const showByYear = 'true'
+    console.log('Refreshing CDS Status')
+    await refreshStatusV2(date.getTime(), showByYearOff)
+    await refreshStatusV2(date.getTime(), showByYear)
+
+    console.log('Refreshing CDS Graduated')
+    await refreshStatusGraduatedV2(date.getTime(), showByYearOff)
+    await refreshStatusGraduatedV2(date.getTime(), showByYear)
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+const refreshUberToRedisV2 = async () => {
+  try {
+    const years = await getStartYearsV2()
+    const mappedYears = years.map(({ studystartdate }) => studystartdate)
+    mappedYears.forEach(async year => {
+      console.log('Refreshing CDS Uber data for date', year)
+      const defaultQuery = { include_old_attainments: 'false', start_date: year }
+      const oldAttainmentsQuery = { include_old_attainments: 'true', start_date: year }
+      await refreshUberV2(defaultQuery)
+      await refreshUberV2(oldAttainmentsQuery)
+    })
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+const refreshProtoCProgrammeToRedisV2 = async () => {
+  try {
+    const codes = (await getAllProgrammesV2()).map(p => p.code).filter(code => code.includes('KH'))
+    codes.forEach(async code => {
+      console.log('refreshing code', code)
+      const defaultQuery = { include_old_attainments: 'false', exclude_non_enrolled: 'false', code }
+      const onlyOld = { include_old_attainments: 'true', exclude_non_enrolled: 'false', code }
+      const onlyEnr = { include_old_attainments: 'false', exclude_non_enrolled: 'true', code }
+      const bothToggles = { include_old_attainments: 'true', exclude_non_enrolled: 'true', code }
+      console.log('Refreshing CDS ProtoCProgramme for code ', code)
+      await refreshProtoCProgrammeV2(defaultQuery)
+      await refreshProtoCProgrammeV2(onlyOld)
+      await refreshProtoCProgrammeV2(onlyEnr)
+      await refreshProtoCProgrammeV2(bothToggles)
+    })
+  } catch (e) {
+    console.log(e)
+  }
+}
+
 const refreshStatisticsV2 = async () => {
   await refreshStudyrightAssociationsV2()
   await refreshOverviewV2()
@@ -218,10 +300,18 @@ const refreshStatisticsV2 = async () => {
   await refreshFacultyYearlyStats() // using old data
 }
 
+const refreshTrendsV2 = async () => {
+  await refreshProtoCtoRedisV2()
+  await refreshStatusToRedisV2()
+  await refreshUberToRedisV2()
+  await refreshProtoCProgrammeToRedisV2()
+}
+
 const startCron = () => {
   if (process.env.NODE_ENV === 'production') {
     schedule('0 6 * * *', async () => {
       await refreshStatisticsV2()
+      await refreshTrendsV2()
     })
   }
 }

services/backend/src/routes/providers.js

@@ -1,5 +1,5 @@
 const router = require('express').Router()
-const { getAllProviders } = require('../services/providers')
+const { getAllProviders } = require('../servicesV2/providers')
 
 router.get('/providers', async (req, res) => {
   const providers = await getAllProviders()

services/backend/src/servicesV2/courses.js

@@ -14,6 +14,7 @@ const { parseCredit } = require('./parseCredits')
 const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('../servicesV2/course_yearly_stats_counter')
 const _ = require('lodash')
+// const { CourseProvider } = require('../models')
 
 const byNameOrCode = (searchTerm, language) =>
   Course.findAll({
@@ -356,50 +357,75 @@ const codeLikeTerm = code =>
       }
 
 const byNameAndOrCodeLike = async (name, code) => {
-  let courses = await Course.findAll({
-    raw: true,
-    attributes: [
-      'id',
-      'name',
-      'code',
-      'latest_instance_date',
-      'startdate',
-      'enddate',
-      'max_attainment_date',
-      'min_attainment_date',
-      'substitutions',
-    ],
+  let rawCourses = await Course.findAll({
+    include: {
+      model: Organization,
+      required: true,
+    },
     where: {
       ...nameLikeTerm(name),
       ...codeLikeTerm(code),
     },
   })
 
+  const courses = rawCourses
+    .map(
+      ({
+        id,
+        code,
+        name,
+        latest_instance_date,
+        is_study_module,
+        coursetypecode,
+        startdate,
+        max_attainment_date,
+        min_attainment_date,
+        createdAt,
+        updatedAt,
+        substitutions,
+        organizations,
+      }) => {
+        return {
+          id,
+          code,
+          name,
+          latest_instance_date,
+          is_study_module,
+          coursetypecode,
+          startdate,
+          max_attainment_date,
+          min_attainment_date,
+          createdAt,
+          updatedAt,
+          substitutions,
+          organizations: organizations.map(o => o.id),
+        }
+      }
+    )
+    .sort(a => (a.code.match(/^[A-Za-z]{3}[0-9]{1}/) ? -1 : 1))
+
   let substitutionGroupIndex = 0
-  const subsGroups = {}
   const visited = []
-  const dfs = async courseId => {
-    if (visited.includes(courseId)) return
-    visited.push(courseId)
-    const course = courses.find(course => course.id === courseId)
 
-    if (!course) return
+  const organizeSubgroups = course => {
+    if (visited.includes(course.code)) return
 
-    subsGroups[course.code] = substitutionGroupIndex
-    course.subsId = substitutionGroupIndex
-
-    for (const courseId of course.substitutions) {
-      dfs(courseId)
-    }
+    let temp = courses.filter(c => (course.substitutions ? course.substitutions.includes(c.id) : false))
+    temp.unshift(course)
+    temp.forEach(cu => {
+      if (visited.includes(course.code)) return
+      visited.push(cu.id)
+      cu.subsId = substitutionGroupIndex
+    })
   }
 
-  for (const course of courses) {
+  courses.forEach(course => {
     if (!visited.includes(course.id)) {
       substitutionGroupIndex++
-      dfs(course.id)
+      organizeSubgroups(course)
     }
-  }
-
+  })
+  console.log(courses)
   return { courses }
 }
 

services/backend/src/servicesV2/providers.js

@@ -0,0 +1,10 @@
+const { Organization } = require('../modelsV2/index')
+
+const getAllProviders = async () =>
+  Organization.findAll({
+    attributes: ['code', 'name'],
+  })
+
+module.exports = {
+  getAllProviders,
+}

services/backend/src/servicesV2/trends/protoC.js

@@ -102,10 +102,13 @@ const calculateProtoCProgramme = async query => {
     includeOldAttainments: query.include_old_attainments === 'true',
     excludeNonEnrolled: query.exclude_non_enrolled === 'true',
   })
+
   const programmeData = data.find(d => d.programmeType === 20)
   const studytrackData = data.filter(d => d.programmeType !== 20)
 
-  // mankel through studytracks
+  if (!programmeData) return {}
+
+  // mankel through studytracks if studyrightdata available
   const studytrackMankelid = _(studytrackData)
     // seems to return the numerical columns as strings, parse them first
     .map(programmeRow => ({
@@ -122,7 +125,7 @@ const calculateProtoCProgramme = async query => {
   const studytrackToBachelorProgrammes = Object.keys(associations.programmes).reduce((acc, curr) => {
     if (!curr.includes('KH')) return acc
     const studytracksForProgramme = studytrackMankelid.reduce((acc2, studytrackdata) => {
-      if (associations.programmes[curr].studytracks.includes(studytrackdata.programmeCode)) {
+      if (studytrackdata && associations.programmes[curr].studytracks.includes(studytrackdata.programmeCode)) {
         acc2.push({
           code: studytrackdata.programmeCode,
           name: studytrackdata.programmeName,

services/frontend/Dockerfile

@@ -3,23 +3,23 @@ FROM node:14.17-alpine3.13 as base
 WORKDIR /usr/src/app
 
 FROM base as development
-EXPOSE 3000
 COPY ./package* /usr/src/app/
 RUN npm ci
 COPY . /usr/src/app/
+EXPOSE 3000
 CMD ["npm", "start"]
 
 FROM base as production
-EXPOSE 5000
-COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
-COPY --chown=node:node . /usr/src/app/
-RUN npm run build && npm install -g serve@12.0.0 && chown -R node:node /usr/src/app/build
-USER node
 ARG SENTRY_ENVIRONMENT
 ENV REACT_APP_SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV REACT_APP_SENTRY_RELEASE=$SENTRY_RELEASE
 ARG BUILT_AT
 ENV REACT_APP_BUILT_AT=$BUILT_AT
+COPY --chown=node:node ./package* /usr/src/app/
+RUN npm ci --production
+COPY --chown=node:node . /usr/src/app/
+RUN npm run build && npm install -g serve@12.0.0 && chown -R node:node /usr/src/app/build
+USER node
+EXPOSE 5000
 CMD ["serve", "--single", "--listen", "5000", "build"]

services/frontend/src/components/CourseStatistics/SearchForm/searchFormUtils.js

@@ -1,7 +1,8 @@
 import { orderBy } from 'lodash'
 
 // handle special case where the course's actual code starts with "A" so it is mistakenly taken as Open Uni course
-const isAvoin = code => !!code.match(/^[A][0-9]|^AY/)
+// const isAvoin = code => !!code.match(/^[A][0-9]|^AY/)
+const isAvoinTwo = course => course.organizations.some(c => c === 'hy-org-48645785')
 
 const sortAlternatives = alternatives => {
   const result = orderBy(
@@ -30,7 +31,7 @@ const filterCourseSearchResults = (courses, unifyOpenUniCourses) => {
   const mergedCourses = {}
   const sortedCourses = sortAlternatives(courses)
   sortedCourses.forEach(course => {
-    const groupId = isAvoin(course.code) && !unifyOpenUniCourses ? course.code : course.subsId
+    const groupId = isAvoinTwo(course) && !unifyOpenUniCourses ? course.code : course.subsId
 
     if (!(course.max_attainment_date && course.min_attainment_date)) {
       return

services/frontend/src/components/Teachers/TeacherStatistics/index.jsx

@@ -51,6 +51,9 @@ const TeacherStatistics = ({
     KH57_001
     500-K001
     --------
+    KH74_001
+    740-K001
+    --------
     KH80_003
     800-K003
     --------
@@ -58,13 +61,12 @@ const TeacherStatistics = ({
     */
   const mapToProviders = rights =>
     rights.map(r => {
+      const isNumber = str => !Number.isNaN(Number(str))
       if (r.includes('_')) {
-        let newPrefix = ''
-        let newSuffix = ''
-        const split = r.split('_')
-        newPrefix = `${split[0][2]}00`
-        newSuffix = `${split[0][0]}${split[1]}`
-        const providercode = `${newPrefix}-${newSuffix}`
+        const [left, right] = r.split('_')
+        const prefix = [...left].filter(isNumber).join('')
+        const suffix = `${left[0]}${right}`
+        const providercode = `${prefix}0-${suffix}`
         return providercode
       }
       return r
@@ -85,7 +87,7 @@ const TeacherStatistics = ({
 
   const userProviders = mapToProviders(rights)
   const invalidQueryParams = provs.length === 0 || !semesterStart
-  const providerOptions = isAdmin ? providers : providers.filter(p => userProviders.includes(p.value))
+  const providerOptions = isAdmin ? providers : providers.filter(p => userProviders.includes(p.code))
   const localizedProviderOptions = providerOptions.map(({ name, ...rest }) => ({
     ...rest,
     text: getTextIn(name, language),
@@ -184,11 +186,7 @@ const mapStateToProps = state => {
     },
   } = state
   const { semesters } = state.semesters.data
-  const providerOptions = providers.data.map(p => ({
-    key: p.providercode,
-    value: p.providercode,
-    name: p.name,
-  }))
+  const providerOptions = providers.data.map(prov => ({ key: prov.code, value: prov.code, name: prov.name }))
   const semesterOptions = !semesters
     ? []
     : Object.values(semesters)

services/frontend/src/conf.js

@@ -12,10 +12,10 @@ const databaseNames = ['kone-db', 'oodi-db', 'sis-db', 'sis-importer-db', 'user-
 const adminerUrls = databaseNames.map(db => ({ url: `${adminerBaseUrl}/?pgsql=${db}&username=postgres`, text: db }))
 
 // Base paths
-const basePath = process.env.PUBLIC_URL || ''
+const basePath = process.env.REACT_APP_PUBLIC_URL || ''
 const apiBasePath = `${basePath}/api`
 
-// Build at time
+// Update time for frontpage
 const builtAt = process.env.REACT_APP_BUILT_AT || ''
 
 module.exports = {

updater/sis-updater-worker/src/updater/mapper.js

@@ -220,7 +220,7 @@ const courseProviderMapper =
 
 const timify = t => new Date(t).getTime()
 
-const courseMapper = courseIdToAttainments => (groupedCourse, substitutions) => {
+const courseMapper = courseIdToAttainments => (groupedCourse, substitutions, organisation) => {
   const [groupId, courses] = groupedCourse
   const { code, name, study_level: coursetypecode } = courses[0]
 
@@ -263,6 +263,7 @@ const courseMapper = courseIdToAttainments => (groupedCourse, substitutions) =>
     enddate,
     is_study_module: false, // VALIDATE THIS PLS
     substitutions,
+    responsible_organisation: organisation,
   }
 }
 

updater/sis-updater-worker/src/updater/updateMeta.js

@@ -49,6 +49,14 @@ const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
         }, [])
       ),
     ]
+    // take responsible organisation from all course units
+    const organisation = [
+      ...new Set(
+        courses.reduce((acc, curr) => {
+          return [...acc, ...flatten(curr.organisations).map(({ organisationId }) => organisationId)]
+        }, [])
+      ),
+    ][0]
 
     // Take organisation for the newest course_unit of this groupid
     const coursesSortedByStartDate = sortBy(courses, 'validity_period.startDate')
@@ -59,8 +67,9 @@ const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
         .filter(({ roleUrn }) => roleUrn === 'urn:code:organisation-role:responsible-organisation')
         .map(mapCourseProvider)
     )
+    console.log('courseProviders: ', courseProviders)
 
-    return mapCourse(groupedCourse, substitutions)
+    return mapCourse(groupedCourse, substitutions, organisation)
   })
 
   await bulkCreate(Course, courses)

updater/sis-updater-worker/src/updater/updateStudents/index.js

@@ -13,12 +13,27 @@ const {
 const { selectFromByIds, selectFromSnapshotsByIds, bulkCreate, getCourseUnitsByCodes } = require('../../db')
 const { getEducation, getUniOrgId, loadMapsIfNeeded } = require('../shared')
 const { studentMapper, mapTeacher, creditMapper, semesterEnrollmentMapper, courseProviderMapper } = require('../mapper')
-
 const { dbConnections } = require('../../db/connection')
 const { isBaMa } = require('../../utils')
 const { updateStudyRights, updateStudyRightElements, updateElementDetails } = require('./studyRightUpdaters')
 const { getAttainmentsToBeExcluded } = require('./excludedPartialAttainments')
 
+// Accepted selection path is not available when degree programme doesn't have
+// studytrack or major subject. This is a known bug on SIS and has been reported
+// to funidata.
+// In these cases, degree programmes module group id must be fetched from education.
+const addSelectionPaths = studyrightSnapshots =>
+  studyrightSnapshots.map(snapshot =>
+    Object.keys(snapshot.accepted_selection_path).length === 0
+      ? {
+          ...snapshot,
+          accepted_selection_path: {
+            educationPhase1GroupId: getEducation(snapshot.education_id).structure.phase1.options[0].moduleGroupId,
+          },
+        }
+      : snapshot
+  )
+
 // Group snapshots by studyright id and find out when studyrights have begun
 const groupStudyrightSnapshots = studyrightSnapshots => {
   const snapshotsBystudyright = Object.entries(
@@ -45,12 +60,20 @@ const groupStudyrightSnapshots = studyrightSnapshots => {
 
     const groupedByPhases = groupBy(orderedSnapshots, byPhases)
 
+    // Get snapshot date time from oldest snapshot where date time is available. If no date
+    // times are available (e.g. all are null), take study start date of the oldest one
+    const parseSnapshotDateTime = snapshots => {
+      let oldestFirst = [...snapshots]
+      oldestFirst.reverse()
+      const firstWithSnapshotDateTime = oldestFirst.find(s => !!s.snapshot_date_time)
+      if (firstWithSnapshotDateTime) return firstWithSnapshotDateTime.snapshot_date_time
+      return oldestFirst[0].study_start_date
+    }
+
     const snapshotsWithRightDate = Object.keys(groupedByPhases).map(key => {
       const snapshots = groupedByPhases[key]
       const most_recent = snapshots[0]
-      const the_first = snapshots[snapshots.length - 1]
-      most_recent.first_snapshot_date_time = the_first.snapshot_date_time
-
+      most_recent.first_snapshot_date_time = parseSnapshotDateTime(snapshots)
       return most_recent
     })
 
@@ -132,7 +155,9 @@ const updateStudents = async personIds => {
     selectFromByIds('study_right_primalities', personIds, 'student_id'),
   ])
 
-  const groupedStudyRightSnapshots = groupStudyrightSnapshots(studyRightSnapshots)
+  const studyRightSnapshotsWithAddedSelectionPaths = addSelectionPaths(studyRightSnapshots)
+
+  const groupedStudyRightSnapshots = groupStudyrightSnapshots(studyRightSnapshotsWithAddedSelectionPaths)
 
   const personIdToStudentNumber = students.reduce((res, curr) => {
     res[curr.id] = curr.student_number
@@ -147,7 +172,9 @@ const updateStudents = async personIds => {
 
   const attainmentsToBeExluced = getAttainmentsToBeExcluded()
 
-  const mappedStudents = students.map(studentMapper(attainments, studyRightSnapshots, attainmentsToBeExluced))
+  const mappedStudents = students.map(
+    studentMapper(attainments, studyRightSnapshotsWithAddedSelectionPaths, attainmentsToBeExluced)
+  )
   await bulkCreate(Student, mappedStudents)
 
   const [moduleGroupIdToCode, formattedStudyRights] = await Promise.all([