iritmaximus/oodikone
1
0
Fork 0
mirror of https://github.com/UniversityOfHelsinkiCS/oodikone.git synced 2026-10-12 19:03:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Updater] Rework course mapper

Browse Source 
Add groupId and validityPeriod, create entries for all courseUnits for a
group rather than just one
This commit is contained in:
joonas-a
2026-09-21 15:24:21 +03:00
committed by Aaro Koivukangas
parent ae4b0b3d52
commit b8e9d59217
4 changed files with 55 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
updater/sis-updater-worker/src/db/connection.js
+2
View File
@@ -22,7 +22,9 @@ class DbConnections extends EventEmitter {
RETRY_ATTEMPTS = 15
knexConnection = false
seqConnection = false
/** @type {import('knex').Knex | undefined}
knex = undefined
/** @type {import('sequelize').Sequelize | undefined} */
sequelize = undefined
constructor() {
updater/sis-updater-worker/src/updater/mapper.js
+19 -34
View File
@@ -236,44 +236,29 @@ export const courseProviderMapper =
shares,
})
const timify = date => new Date(date).getTime()
export const courseMapper = courseIdToAttainments => (groupId, course, substitution_groups) => {
const { id, code, name, study_level, course_unit_type, validity_period } = course
export const courseMapper =
courseIdToAttainments =>
([groupId, courses], substitutions, substitution_groups) => {
const { code, name, study_level: coursetypecode, course_unit_type } = courses[0]
const courseAttainments = courseIdToAttainments[id] ?? []
const { min_attainment_date, max_attainment_date } = courses.reduce(
(res, curr) => {
const courseAttainments = courseIdToAttainments[curr.id]
if (!courseAttainments || courseAttainments.length === 0) return res
let { min_attainment_date, max_attainment_date } = res
if (!min_attainment_date || timify(min_attainment_date) > timify(courseAttainments[0].attainment_date))
min_attainment_date = courseAttainments[0].attainment_date
if (
!max_attainment_date ||
timify(max_attainment_date) < timify(courseAttainments[courseAttainments.length - 1].attainment_date)
)
max_attainment_date = courseAttainments[courseAttainments.length - 1].attainment_date
// Attainments are sorted by attainment_date ASC
const min_attainment_date = courseAttainments[0]?.attainment_date ?? null
const max_attainment_date = courseAttainments[courseAttainments.length - 1]?.attainment_date ?? null
return { min_attainment_date, max_attainment_date }
},
{ min_attainment_date: null, max_attainment_date: null }
)
return {
id: groupId,
name,
code,
coursetypecode,
min_attainment_date,
max_attainment_date,
is_study_module: course_unit_type == null, // Only course units have a course_unit_type so if it's null, it must be a study module
substitutions,
substitution_groups,
course_unit_type,
}
return {
id,
groupId,
name,
code,
coursetypecode: study_level,
min_attainment_date,
max_attainment_date,
validity_period,
is_study_module: !course_unit_type, // Only course units have a course_unit_type. If null -> must be a study module
substitution_groups,
course_unit_type,
}
}
export const mapCourseType = studyLevel => ({
coursetypecode: studyLevel.id,
updater/sis-updater-worker/src/updater/updateMeta.js
+33 -108
View File
@@ -1,5 +1,4 @@
import { uniqBy, flatten, groupBy, memoize } from 'lodash-es'
import { Op } from 'sequelize'
import { uniqBy, groupBy } from 'lodash-es'
import { rootOrgId } from '../config.js'
import { bulkCreate, selectFromByIdsOrderBy } from '../db/index.js'
import {
@@ -23,122 +22,46 @@ export const updateOrganisations = async organisations => {
await bulkCreate(Organization, organisations)
}
// sort substitutions so that main code is first
const newLetterBasedCode = /^[A-Za-z]/ // new letter based codes come first
const oldNumericCode = /^\d/ // old numeric codes come second
const openUniCode = /^AY?(.+?)(?:en|fi|sv)?$/ // open university codes come last
const openUniCodeA = /A\d/ // open university with just A come last
const digi = /DIGI-A?(.+?)(?:en|fi|sv)?$/ // digi-a goes on top courses goes third
const bscsCode = /BSCS??/
const codeRegexes = [openUniCodeA, openUniCode, bscsCode, oldNumericCode, newLetterBasedCode, digi]
// Compile the above RegEx'es into one. This saves considerable amount of time
// compared to executing each one independently.
const codeRegex = RegExp(codeRegexes.map(r => `(${r.source})`).join('|'))
const getSubstitutionPriority = memoize(code => {
const match = codeRegex.exec(code)
if (!match) {
return 3 // if no hit, put before open uni courses
}
// Use the index of the first matched and captured subgroup as the priority
return match.splice(1).findIndex(x => x !== undefined)
})
const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
const courseProviders = []
const courses = Object.entries(groupIdToCourse).map(([groupId, courses]) => {
// Take substitutions from all course units
// NOTE: Modules DO NOT have substitutions fields.
const substitutionArrays = courses.flatMap(
course => course.substitutions?.map(subGroup => subGroup.map(sub => sub.courseUnitGroupId)) ?? []
)
const uniqueSubstitutionArrays = uniqBy(substitutionArrays, a => [...a].sort().join('|'))
const mappedCourses = []
// TODO: Replace this old implementation of substitutions
// Take substitutions from all course units
const substitutions = [
...new Set(
courses.reduce((acc, curr) => {
return [...acc, ...flatten(curr.substitutions).map(({ courseUnitGroupId }) => courseUnitGroupId)]
}, [])
),
]
const mapCourse = courseMapper(courseIdToAttainments)
const organisationsById = {}
for (const [groupId, courses] of Object.entries(groupIdToCourse)) {
for (const course of courses) {
/** @type string[][] @description nested arrays of courseUnit groupIds. Modules have no substitutions. */
const substitutionGroups = course.substitutions?.map(subGroup => subGroup.map(sub => sub.courseUnitGroupId)) ?? []
for (const { organisations, validity_period: courseValidityPeriod } of courses) {
if (!organisations) continue
for (const { share, organisationId, roleUrn, validityPeriod } of organisations) {
const { startDate, endDate } = validityPeriod ?? courseValidityPeriod ?? {}
const organisationsById = {}
const { organisations, validity_period: courseValidityPeriod } = course
organisationsById[organisationId] ??= { organisationId, roleUrn, shares: [] }
organisationsById[organisationId].shares.push({
share,
...(startDate && { startDate }),
...(endDate && { endDate }),
})
if (organisations) {
for (const { share, organisationId, roleUrn, validityPeriod: orgValidityPeriod } of organisations) {
if (roleUrn !== 'urn:code:organisation-role:responsible-organisation') continue
// OrgValidityPeriod is never defined in importer? Leaving it, as it doesn't matter because of the fallback.
const { startDate, endDate } = orgValidityPeriod ?? courseValidityPeriod ?? {}
organisationsById[organisationId] ??= { organisationId, roleUrn, shares: [] }
organisationsById[organisationId].shares.push({
share,
...(startDate && { startDate }),
...(endDate && { endDate }),
})
}
}
const mapCourseProvider = courseProviderMapper(groupId)
courseProviders.push(...Object.values(organisationsById).map(mapCourseProvider))
mappedCourses.push(mapCourse(groupId, course, substitutionGroups))
}
const mapCourseProvider = courseProviderMapper(groupId)
const organisations = Object.values(organisationsById)
.filter(({ roleUrn }) => roleUrn === 'urn:code:organisation-role:responsible-organisation')
.map(mapCourseProvider)
courseProviders.push(...organisations)
// TODO: Remove OLD substitutions when done
return courseMapper(courseIdToAttainments)([groupId, courses], substitutions, uniqueSubstitutionArrays)
})
// change substitutions ids to course codes and update
for (const course of courses) {
// TODO: OLD substitutions
const newSubstitutions = []
for (const sub of course.substitutions) {
const subs = await Course.findOne({
attributes: ['code'],
where: { id: sub },
})
if (subs) newSubstitutions.push(subs.dataValues.code)
}
course.substitutions = newSubstitutions
const substitutionSet = new Set()
for (const group of course.substitution_groups) {
for (const course of group) substitutionSet.add(course)
}
const idToCodePairs = (
await Course.findAll({
attributes: ['id', 'code'],
where: {
id: { [Op.in]: Array.from(substitutionSet) },
},
raw: true,
})
).map(({ id, code }) => [id, code])
const idToCodeMap = new Map(idToCodePairs)
course.substitution_groups = course.substitution_groups
.map(group => group.map(id => idToCodeMap.get(id)))
.filter(group => group.filter(Boolean).length > 0) // Filter out empty groups or groups with only undefined
course.mainCourseCode = [course.code, ...course.substitution_groups]
.sort((a, b) => getSubstitutionPriority(b) - getSubstitutionPriority(a))
.at(0)
// HACK: Fix updater explosion, delete this after substitutions are finished
if (Array.isArray(course.mainCourseCode)) course.mainCourseCode = course.mainCourseCode.at(0)
}
await bulkCreate(Course, courses)
// console.log('Creating:')
// console.dir(mappedCourses, { depth: null })
await bulkCreate(Course, mappedCourses)
await bulkCreate(
CourseProvider,
uniqBy(courseProviders, cP => `${cP.coursecode}-${cP.organizationcode}`),
@@ -149,6 +72,7 @@ const updateCourses = async (courseIdToAttainments, groupIdToCourse) => {
export const updateStudyModules = async studyModules => {
const organizationStudyModules = studyModules.filter(s => s.university_org_ids.includes(rootOrgId))
// Attainments are later assumed to be sorted
const attainments = await selectFromByIdsOrderBy(
'attainments',
organizationStudyModules.map(s => s.id),
@@ -163,6 +87,7 @@ export const updateStudyModules = async studyModules => {
}
export const updateCourseUnits = async courseUnits => {
// Attainments are later assumed to be sorted
const attainments = await selectFromByIdsOrderBy(
'attainments',
courseUnits.map(course => course.id),
updater/sis-updater-worker/src/updater/updateProgrammeModules/resolver.js
+1
View File
@@ -5,6 +5,7 @@ const unknownModule = mod => ({ name: mod.name, type: 'unknown' })
export class ModuleResolver {
constructor(knex) {
/** @type {import('knex').Knex} */
this.knex = knex
this.moduleCache = {}
this.orderTable = {}