usage and coursegroups byebye

cypress/support/commands.js

@@ -10,6 +10,7 @@ const adminUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member;employee;faculty',
   mail: 'grp-toska+mockadminuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-6666666',
 }
 
 const basicUserHeaders = {
@@ -19,6 +20,7 @@ const basicUserHeaders = {
   hygroupcn: 'grp-oodikone-users;grp-oodikone-basic-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mockbasicuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-1234567',
 }
 
 const onlycoursestatisticsUserHeaders = {
@@ -28,6 +30,7 @@ const onlycoursestatisticsUserHeaders = {
   hyGroupCn: 'grp-oodikone-basic-users',
   eduPersonAffiliation: 'member',
   mail: 'grp-toska+mockonlycoursestatisticsuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-7654321',
 }
 
 const norightsUserHeaders = {
@@ -37,6 +40,7 @@ const norightsUserHeaders = {
   hygroupcn: 'grp-oodikone-users',
   edupersonaffiliation: 'member',
   mail: 'grp-toska+mocknorightuser@helsinki.fi',
+  hypersonsisuid: 'hy-hlo-0000000',
 }
 
 const userHeaders = [adminUserHeaders, basicUserHeaders, onlycoursestatisticsUserHeaders, norightsUserHeaders]

services/backend/src/models/index.js

@@ -30,6 +30,9 @@ Student.hasMany(SemesterEnrollment, { foreignKey: 'studentnumber', sourceKey: 's
 SemesterEnrollment.belongsTo(Semester, { foreignKey: 'semestercomposite', targetKey: 'composite' })
 Semester.hasMany(SemesterEnrollment, { foreignKey: 'semestercomposite', sourceKey: 'composite' })
 
+ProgrammeModule.belongsTo(Organization, { foreignKey: 'organization_id' })
+Organization.hasMany(ProgrammeModule, { foreignKey: 'organization_id' })
+
 Credit.notUnnecessary = credit => {
   return credit.credits > 0 && credit.credits <= 12
 }
@@ -96,6 +99,8 @@ ProgrammeModule.belongsToMany(ProgrammeModule, {
   foreignKey: 'parent_id',
 })
 
+Organization.hasMany(Organization, { foreignKey: 'parent_id', as: 'children' })
+
 module.exports = {
   Organization,
   Course,

services/backend/src/models/organization.js

@@ -19,6 +19,9 @@ Organization.init(
     name: {
       type: JSONB,
     },
+    parent_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/programmeModule.js

@@ -18,6 +18,16 @@ ProgrammeModule.init(
     type: {
       type: STRING,
     },
+    studyLevel: {
+      type: STRING,
+    },
+    organization_id: {
+      type: STRING,
+      references: {
+        model: 'organization',
+        key: 'id',
+      },
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/routes/faculties.js

@@ -3,7 +3,6 @@ const { faculties } = require('../services/organisations')
 
 router.get('/', async (req, res) => {
   const facultyList = await faculties()
-  console.log('faculties')
   res.json(facultyList)
 })
 

services/backend/src/routes/login.js

@@ -12,7 +12,6 @@ router.post('/login', async (req, res) => {
       const hyGroups = parseHyGroups(req.headers['hygroupcn'])
       const affiliations = parseHyGroups(req.headers['edupersonaffiliation'])
       const hyPersonSisuId = req.headers.hypersonsisuid || ''
-
       console.log(uid, 'trying to login, referring to userservice.')
       let { token, isNew } = await userService.login(uid, full_name, hyGroups, affiliations, mail, hyPersonSisuId)
       if (isNew) {

services/backend/src/routes/teachers.js

@@ -1,5 +1,6 @@
 const teachers = require('../services/teachers')
 const topteachers = require('../services/topteachers')
+const { isFaculty, providersOfFaculty } = require('../services/organisations')
 const router = require('express').Router()
 const { mapToProviders } = require('../util/utils')
 
@@ -51,7 +52,19 @@ router.get('/stats', async (req, res) => {
   if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')
   }
-  const result = await teachers.yearlyStatistics(providers, semesterStart, semesterEnd || semesterStart + 1)
+
+  // combine provider list that may include programmes and faculties
+  const parsedProviders = [
+    ...new Set(
+      await providers.reduce(
+        async (acc, curr) =>
+          isFaculty(curr) ? [...(await acc), ...(await providersOfFaculty(curr))] : [...(await acc), curr],
+        []
+      )
+    ),
+  ]
+
+  const result = await teachers.yearlyStatistics(parsedProviders, semesterStart, semesterEnd || semesterStart + 1)
   res.json(result)
 })
 

services/backend/src/services/courses.js

@@ -424,7 +424,6 @@ const byNameAndOrCodeLike = async (name, code) => {
       organizeSubgroups(course)
     }
   })
-  console.log(courses)
   return { courses }
 }
 

services/backend/src/services/organisations.js

@@ -1,6 +1,7 @@
 const Sequelize = require('sequelize')
-const { Organization } = require('../models')
+const { Organization, ProgrammeModule } = require('../models')
 const Op = Sequelize.Op
+const { dbConnections } = require('../database/connection')
 
 // Have facultyfetching to work like it worked during oodi-db time
 const facultiesInOodi = [
@@ -33,6 +34,36 @@ const faculties = () => {
   })
 }
 
+const degreeProgrammeCodesOfFaculty = async facultyCode =>
+  (
+    await ProgrammeModule.findAll({
+      attributes: ['code'],
+      include: {
+        model: Organization,
+        where: {
+          code: facultyCode,
+        },
+      },
+    })
+  ).map(({ code }) => code)
+
+const providersOfFaculty = async facultyCode => {
+  const [result] = await dbConnections.sequelize.query(
+    `SELECT childOrg.code
+     FROM organization parentOrg
+     INNER JOIN organization childOrg 
+     ON childOrg.parent_id = parentOrg.id
+     WHERE parentOrg.code = ?`,
+    { replacements: [facultyCode] }
+  )
+  return result.map(({ code }) => code)
+}
+
+const isFaculty = facultyCode => facultiesInOodi.includes(facultyCode)
+
 module.exports = {
   faculties,
+  degreeProgrammeCodesOfFaculty,
+  isFaculty,
+  providersOfFaculty,
 }

services/backend/src/services/studyrights.js

@@ -402,7 +402,6 @@ const getAssociations = async (doRefresh = false) => {
 }
 
 const getFilteredAssociations = async codes => {
-  console.log(codes)
   const associations = await getAssociations()
   associations.programmes = _.pick(associations.programmes, codes)
 

services/backend/src/util/importerClient.js

@@ -10,8 +10,6 @@ const importerClient = axios.create({
 })
 
 const getImporterClient = () => {
-  console.log('importerToken', importerToken)
-  console.log('importerUrl', importerUrl)
   if (!importerToken || !importerUrl) {
     console.log("Importer token or importer url not set, can't return client!")
     return null

services/frontend/src/components/PopulationCourseStats/Students/index.jsx

@@ -55,18 +55,21 @@ const Students = ({ expandedGroups, toggleGroupExpansion, showNames }) => {
     })
 
     const allStudents = Array.from(studentSet)
-    return allStudents.map(student => {
-      let passed = 0
-      courseStatistics.forEach(course => {
-        if (course.students.passed[student]) {
-          passed++
-        }
+    return allStudents
+      .map(student => {
+        let passed = 0
+        courseStatistics.forEach(course => {
+          if (course.students.passed[student]) {
+            passed++
+          }
+        })
+
+        const found = filteredStudents?.find(s => s?.studentNumber === student)
+        const name = found ? `${found.lastname} ${found.firstnames}` : null
+
+        return { studentnumber: student, name, passed }
       })
-
-      const { firstnames, lastname } = filteredStudents.find(s => s.studentNumber === student)
-
-      return { studentnumber: student, name: `${lastname} ${firstnames}`, passed }
-    })
+      .filter(s => !!s.name)
   }, [courseStatistics])
 
   const maxPages = Math.floor(students.length / 10)

services/frontend/src/components/Teachers/TeacherStatistics/index.jsx

@@ -87,7 +87,7 @@ const TeacherStatistics = ({
 
   const userProviders = mapToProviders(rights)
   const invalidQueryParams = provs.length === 0 || !semesterStart
-  const providerOptions = isAdmin ? providers : providers.filter(p => userProviders.includes(p.code))
+  const providerOptions = isAdmin ? providers : providers.filter(p => userProviders.includes(p.value))
   const localizedProviderOptions = providerOptions.map(({ name, ...rest }) => ({
     ...rest,
     text: getTextIn(name, language),

services/frontend/src/components/UserPage/index.jsx

@@ -51,15 +51,21 @@ const UserPage = ({
     window.location.reload()
   }
 
-  const renderUnitList = (elementdetailcodes, elementdetails, user) => {
-    if (!elementdetailcodes) return null
+  const renderUnitList = (usersElementdetailCodes, elementdetails, user) => {
+    if (!usersElementdetailCodes) return null
     const nameInLanguage = element => getTextIn(element.name, language)
-    elementdetailcodes.sort()
+    const elementDetailCodesAvailable = new Set(elementdetails.map(({ code }) => code))
+    const usersElementdetailCodesAvailable = usersElementdetailCodes.filter(obj =>
+      elementDetailCodesAvailable.has(obj.elementDetailCode)
+    )
+    usersElementdetailCodesAvailable.sort()
     return (
       <List divided>
-        {elementdetails.length > 0 &&
-          elementdetailcodes.map(({ elementDetailCode: code }) => {
+        {usersElementdetailCodesAvailable.length > 0 &&
+          usersElementdetailCodesAvailable.map(({ elementDetailCode: code }) => {
+            console.log('elementDetailCode', code)
             const element = elementdetails.find(e => e.code === code)
+            console.log('element', element)
             return (
               <List.Item key={code}>
                 <List.Content floated="right">
@@ -74,7 +80,8 @@ const UserPage = ({
                   />
                 </List.Content>
                 <List.Content>
-                  {element.type === 30 ? <Icon name="minus" /> : null} {`${nameInLanguage(element)} (${code})`}
+                  {element && element.type === 30 ? <Icon name="minus" /> : null}{' '}
+                  {`${nameInLanguage(element)} (${code})`}
                 </List.Content>
               </List.Item>
             )

services/userservice/src/conf.js

@@ -9,6 +9,8 @@ const isStaging = NODE_ENV === 'staging'
 
 const courseStatisticsGroup = 'grp-oodikone-basic-users'
 
+const hyOneGroup = 'hy-one'
+
 const requiredGroup = isStaging
   ? ['grp-oodikone-staging-users', 'grp-oodikone-basic-staging-users']
   : ['grp-oodikone-users', 'grp-oodikone-basic-users']
@@ -22,4 +24,5 @@ module.exports = {
   sentryRelease,
   sentryEnvironment,
   runningInCI,
+  hyOneGroup,
 }

services/userservice/src/database/migrations/20210831_00_remove_usage_and_coursegroups.js

@@ -0,0 +1,10 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const Op = Sequelize.Op
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: { [Op.in]: ['usage', 'coursegroups'] },
+      },
+    ])
+  },
+}

services/userservice/src/services/users.js

@@ -16,7 +16,7 @@ const {
 const AccessService = require('./accessgroups')
 const AffiliationService = require('./affiliations')
 const HyGroupService = require('./hygroups')
-const { requiredGroup, courseStatisticsGroup, TOKEN_SECRET } = require('../conf')
+const { requiredGroup, courseStatisticsGroup, TOKEN_SECRET, hyOneGroup } = require('../conf')
 const Op = Sequelize.Op
 
 const TOKEN_VERSION = 1 // When token structure changes, increment in userservice, backend and frontend
@@ -112,6 +112,7 @@ const login = async (uid, full_name, hyGroups, affiliations, mail, hyPersonSisuI
   }
 
   await determineAccessToCourseStats(user, hyGroups)
+  await determineAccessToTeachersForOne(user, hyGroups)
 
   console.log('Generating token')
   const token = await generateToken(uid)
@@ -321,6 +322,14 @@ const determineAccessToCourseStats = async (user, hyGroups) => {
   }
 }
 
+const determineAccessToTeachersForOne = async (user, hyGroups) => {
+  const accessGroups = (user && user.accessgroup) || []
+  const alreadyAccess = accessGroups.some(({ group_code }) => group_code === 'teachers')
+  if (hyGroups.includes(hyOneGroup) && !alreadyAccess) {
+    await modifyRights(user.id, { teachers: true })
+  }
+}
+
 module.exports = {
   byUsername,
   byUsernameMinified,

updater/sis-updater-worker/src/db/migrations/20210902_00_add_organization_id_to_programme_module.js

@@ -0,0 +1,16 @@
+const { STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('programme_modules', 'organization_id', {
+      type: STRING,
+      references: {
+        model: 'organization',
+        key: 'id',
+      },
+    })
+  },
+  down: async queryInterface => {
+    await queryInterface.deleteColumn('programme_modules', 'organization_id')
+  },
+}

updater/sis-updater-worker/src/db/migrations/20210902_01_add_parent_org_to_organisations.js

@@ -0,0 +1,12 @@
+const { STRING } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.addColumn('organization', 'parent_id', {
+      type: STRING,
+    })
+  },
+  down: async queryInterface => {
+    await queryInterface.deleteColumn('organization', 'parent_id')
+  },
+}

updater/sis-updater-worker/src/db/models/index.js

@@ -29,6 +29,9 @@ Student.hasMany(SemesterEnrollment, { foreignKey: 'studentnumber', sourceKey: 's
 SemesterEnrollment.belongsTo(Semester, { foreignKey: 'semestercomposite', targetKey: 'composite' })
 Semester.hasMany(SemesterEnrollment, { foreignKey: 'semestercomposite', sourceKey: 'composite' })
 
+ProgrammeModule.belongsTo(Organization, { foreignKey: 'organization_id' })
+Organization.hasMany(ProgrammeModule, { foreignKey: 'organization_id' })
+
 Credit.notUnnecessary = credit => {
   return credit.credits > 0 && credit.credits <= 12
 }
@@ -96,6 +99,8 @@ ProgrammeModule.belongsToMany(ProgrammeModule, {
   foreignKey: 'childId',
 })
 
+Organization.hasMany(Organization, { foreignKey: 'parent_id', as: 'children' })
+
 module.exports = {
   Organization,
   Course,

updater/sis-updater-worker/src/db/models/organization.js

@@ -19,6 +19,9 @@ Organization.init(
     name: {
       type: JSONB,
     },
+    parent_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

updater/sis-updater-worker/src/db/models/programmeModule.js

@@ -24,6 +24,13 @@ ProgrammeModule.init(
     studyLevel: {
       type: STRING,
     },
+    organization_id: {
+      type: STRING,
+      references: {
+        model: 'organization',
+        key: 'id',
+      },
+    },
     createdAt: {
       type: DATE,
     },

updater/sis-updater-worker/src/updater/updateProgrammeModules.js

@@ -155,10 +155,6 @@ const recursiveWrite = async (module, parentId) => {
     type: module.type,
     studyLevel: module.study_level,
   }
-  if (module.type !== 'course') {
-    console.log('new module', newModule)
-    console.log('new modules orig details', module)
-  }
 
   let join = {
     composite: `${parentId}-${module.id}`,
@@ -183,12 +179,16 @@ const updateProgrammeModules = async (entityIds = []) => {
   const topModules = await selectFromByIds('modules', entityIds)
 
   for (const module of topModules) {
+    const responsibleOrg = module.organisations
+      ? module.organisations.find(o => o.roleUrn === 'urn:code:organisation-role:responsible-organisation')
+      : null
     const topModule = {
       id: module.group_id,
       code: module.code,
       name: module.name,
       type: 'module',
       studyLevel: module.study_level,
+      organization_id: responsibleOrg ? responsibleOrg.organisationId : null,
     }
     programmes[module.group_id] = topModule
     const submodule = await resolver(module.rule)

updater/sis-updater-worker/src/updater/updateStudents/index.js

@@ -11,37 +11,48 @@ const {
   CourseProvider,
 } = require('../../db/models')
 const { selectFromByIds, selectFromSnapshotsByIds, bulkCreate, getCourseUnitsByCodes } = require('../../db')
-const { getEducation, getUniOrgId, loadMapsIfNeeded } = require('../shared')
+const { getEducation, getUniOrgId, getEducationType, loadMapsIfNeeded } = require('../shared')
 const { studentMapper, mapTeacher, creditMapper, semesterEnrollmentMapper, courseProviderMapper } = require('../mapper')
 const { dbConnections } = require('../../db/connection')
 const { isBaMa } = require('../../utils')
 const { updateStudyRights, updateStudyRightElements, updateElementDetails } = require('./studyRightUpdaters')
 const { getAttainmentsToBeExcluded } = require('./excludedPartialAttainments')
 
+// Check if studyright is degree leading or not
+const studyRightHasDegreeEducation = studyRight => {
+  const education = getEducation(studyRight.education_id)
+  if (!education) return false
+  const educationType = getEducationType(education.education_type)
+  if (!educationType) return false
+  return !educationType.parent_id.startsWith('urn:code:education-type:non-degree-education')
+}
+
 // Accepted selection path is not available when degree programme doesn't have
 // studytrack or major subject. This is a known bug on SIS and has been reported
 // to funidata.
 // In these cases, degree programmes module group id must be fetched from education.
-const addSelectionPaths = studyrightSnapshots =>
-  studyrightSnapshots.map(snapshot =>
-    Object.keys(snapshot.accepted_selection_path).length === 0
-      ? {
-          ...snapshot,
-          accepted_selection_path: {
-            educationPhase1GroupId: getEducation(snapshot.education_id).structure.phase1.options[0].moduleGroupId,
-          },
-        }
-      : snapshot
-  )
+const addSelectionPathIfNeeded = snapshot =>
+  Object.keys(snapshot.accepted_selection_path).length === 0
+    ? {
+        ...snapshot,
+        accepted_selection_path: {
+          educationPhase1GroupId: getEducation(snapshot.education_id).structure.phase1.options[0].moduleGroupId,
+        },
+      }
+    : snapshot
+
+// Parse useful snapshots from the whole list and enrich snapshots when needed
+const parseStudyrightSnapshots = studyrightSnapshots =>
+  studyrightSnapshots.reduce((parsed, current) => {
+    if (current.document_state !== 'ACTIVE') return parsed
+    if (!studyRightHasDegreeEducation(current)) return parsed
+    parsed.push(addSelectionPathIfNeeded(current))
+    return parsed
+  }, [])
 
 // Group snapshots by studyright id and find out when studyrights have begun
 const groupStudyrightSnapshots = studyrightSnapshots => {
-  const snapshotsBystudyright = Object.entries(
-    groupBy(
-      studyrightSnapshots.filter(s => s.document_state === 'ACTIVE'),
-      'id'
-    )
-  )
+  const snapshotsBystudyright = Object.entries(groupBy(studyrightSnapshots, 'id'))
 
   return snapshotsBystudyright.reduce((res, [id, snapshots]) => {
     const byPhases = s => {
@@ -147,7 +158,7 @@ const parseTransfers = async (groupedStudyRightSnapshots, moduleGroupIdToCode, p
 const updateStudents = async personIds => {
   await loadMapsIfNeeded()
 
-  const [students, studyRightSnapshots, attainments, termRegistrations, studyRightPrimalities] = await Promise.all([
+  const [students, studyrightSnapshots, attainments, termRegistrations, studyRightPrimalities] = await Promise.all([
     selectFromByIds('persons', personIds),
     selectFromByIds('studyrights', personIds, 'person_id'),
     selectFromByIds('attainments', personIds, 'person_id'),
@@ -155,9 +166,9 @@ const updateStudents = async personIds => {
     selectFromByIds('study_right_primalities', personIds, 'student_id'),
   ])
 
-  const studyRightSnapshotsWithAddedSelectionPaths = addSelectionPaths(studyRightSnapshots)
+  const parsedStudyrightSnapshots = parseStudyrightSnapshots(studyrightSnapshots)
 
-  const groupedStudyRightSnapshots = groupStudyrightSnapshots(studyRightSnapshotsWithAddedSelectionPaths)
+  const groupedStudyRightSnapshots = groupStudyrightSnapshots(parsedStudyrightSnapshots)
 
   const personIdToStudentNumber = students.reduce((res, curr) => {
     res[curr.id] = curr.student_number
@@ -172,9 +183,7 @@ const updateStudents = async personIds => {
 
   const attainmentsToBeExluced = getAttainmentsToBeExcluded()
 
-  const mappedStudents = students.map(
-    studentMapper(attainments, studyRightSnapshotsWithAddedSelectionPaths, attainmentsToBeExluced)
-  )
+  const mappedStudents = students.map(studentMapper(attainments, parsedStudyrightSnapshots, attainmentsToBeExluced))
   await bulkCreate(Student, mappedStudents)
 
   const [moduleGroupIdToCode, formattedStudyRights] = await Promise.all([