[Credit accumulation] Graduations are shown as diamond-icon

Is not dependent on IAMs so doesn't actually need to be rerun for users table. Also takes a lot of time and leads to timeout problems

.github/workflows/release-oodikone.yaml

@@ -21,7 +21,7 @@ jobs:
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
       - name: 'Build and push'
         uses: docker/build-push-action@v3
         with:
@@ -50,13 +50,13 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: 'Get current time'
         id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
       - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
       - name: 'Build and push'
         uses: docker/build-push-action@v3
         with:

.github/workflows/test-oodikone.yaml

@@ -83,7 +83,7 @@ jobs:
       - uses: docker/setup-buildx-action@v2
       - name: 'Get current time'
         id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
       - name: 'Build image and export it'
         uses: docker/build-push-action@v3
         with:
@@ -286,7 +286,23 @@ jobs:
           username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
           spec: cypress/e2e/Student_statistics.js
-
+  test_completed_courses:
+    name: 'Test completed courses'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
+        with:
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          spec: cypress/e2e/Completed_courses.js
   test_studyprogramme_overview:
     name: 'Test studyprogramme overview'
     needs:
@@ -343,6 +359,24 @@ jobs:
           password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
           spec: cypress/e2e/Faculty_statistics.js
 
+  test_open_uni_search:
+    name: 'Test open uni search'
+    needs:
+      - build_backend
+      - build_frontend
+      - build_and_push_anon_sis_db
+      - build_and_push_anon_kone_db
+      - build_and_push_anon_user_db
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
+        with:
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          spec: cypress/e2e/Openuni.js
+
   # === Other tests ===
 
   test_service_exit_statuses:
@@ -396,9 +430,11 @@ jobs:
       - test_filters
       - test_population_statistics
       - test_student_statistics
+      - test_completed_courses
       - test_studyprogramme_overview
       - test_teachers_and_users
       - test_faculty_statistics
+      - test_open_uni_search
       - test_service_exit_statuses
     steps:
       - uses: actions/checkout@v3
@@ -463,9 +499,11 @@ jobs:
       - test_filters
       - test_population_statistics
       - test_student_statistics
+      - test_completed_courses
       - test_studyprogramme_overview
       - test_teachers_and_users
       - test_faculty_statistics
+      - test_open_uni_search
       - test_service_exit_statuses
     if: failure()
     steps:

.husky/pre-commit

@@ -2,3 +2,11 @@
 . "$(dirname "$0")/_/husky.sh"
 
 npx lint-staged
+
+if git grep -qE --cached "(it|describe)\.only" $(git diff --cached --name-only) ./cypress; then
+    echo "\e[0;31mDo not commit .only-tests."
+    exit 1
+else
+    echo "No .only-tests found"
+    exit 0
+fi

README.md

@@ -50,7 +50,11 @@ What different CLI options do:
 
 ### Architecture
 
-![Oodikone architecture](Oodikone.png)
+![Oodikone architecture](./documentation/Oodikone.png)
+
+### Documentation
+
+Folder `documentation` currently contains some notes on sis-db schema and is meant as a place for storing useful information on Oodikone to make life easier for future developers. Feel free to add anything relevant!
 
 ### Basics
 

cypress/e2e/Completed_courses.js

@@ -0,0 +1,152 @@
+/// <reference types="Cypress" />
+
+const openCompletedCoursesModal = () => {
+  cy.get('button').contains('Search completed courses of students').click()
+}
+
+const openSearch = () => {
+  cy.init('/completedcoursessearch')
+  hasLanded()
+}
+const selectSavedCourselist = name => {
+  cy.get('[data-cy="history-search"]').children().eq(0).type(name).type('{enter}')
+}
+
+const hasLanded = () => {
+  cy.contains('Search completed courses')
+  cy.contains(
+    'Here you can search by a list of student and course numbers to see whether students have completed certain courses yet'
+  )
+}
+
+const deleteAllSearches = () => {
+  cy.contains('Saved courselists')
+    .get('.dropdown')
+    .then(d => {
+      const searchItems = d.find('div[role=option] > span[class=text]')
+      for (let i = 0; i < searchItems.length; i++) {
+        if (searchItems[i].textContent.includes('TEST-')) {
+          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('button').contains('Delete').click()
+        }
+      }
+    })
+}
+
+describe('When search modal is opened', () => {
+  after(() => {
+    cy.init('/completedcoursessearch')
+    hasLanded()
+    cy.url().should('include', '/completedcoursessearch')
+    cy.contains('Search completed courses of students')
+    cy.get('[data-cy="open-completed-courses-modal-button"]').click()
+    deleteAllSearches()
+  })
+
+  it('Modal opens correctly', () => {
+    cy.init('/completedcoursessearch')
+    hasLanded()
+    openCompletedCoursesModal()
+    cy.contains('Search completed courses of students')
+    cy.contains('Insert one or more student numbers, separated by a space, a newline, or a comma.')
+    cy.contains('Insert one or more courses, separated by a space, a newline, or a comma.')
+    cy.contains('Insert name for this course list if you wish to save it')
+  })
+
+  describe('When a search is executed with invalid input', () => {
+    it('Returns empty table and notification', () => {
+      cy.init('/completedcoursessearch')
+      hasLanded()
+      openCompletedCoursesModal()
+      cy.get('[data-cy="student-no-input"]').type('1')
+      cy.get('[data-cy="course-list-input"]').type('1')
+      cy.get('[data-cy="completed-courses-search-button"]').click()
+      cy.get('[data-cy="rights-notification"]').should(
+        'contain.text',
+        'The following students information could not be displayed'
+      )
+      cy.get('[data-cy="rights-notification"]').should('contain.text', '1')
+      cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Completed courses search')
+      cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Student Number')
+      cy.get('[data-cy="completed-courses-table-div"]').should('not.contain', '1')
+    })
+  })
+
+  describe('When a search with correct data is executed', () => {
+    before(() => {
+      cy.init('/completedcoursessearch')
+      hasLanded()
+      openCompletedCoursesModal()
+      cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
+        cy.get('[data-cy="student-no-input"]').type(studentSet1.join('\n'))
+        cy.get('[data-cy="course-list-input"]').type(coursesSet1.join('\n'))
+        cy.get('[data-cy="completed-courses-search-button"]').click()
+      })
+    })
+
+    it('Pushes the query to url', () => {
+      cy.url().should(
+        'include',
+        '/completedcoursessearch?courseList=TKT10001&courseList=TKT10002&courseList=TKT10003&courseList=TKT10004&studentList=010450938&studentList=010589388&studentList=010614509'
+      )
+    })
+    it('Finds correct students and courses', () => {
+      cy.fixture('completedCoursesData').then(({ studentSet1, coursesSet1 }) => {
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[0])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[1])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', studentSet1[2])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[0])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[1])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[2])
+        cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', coursesSet1[3])
+      })
+    })
+  })
+})
+
+const courseListNames = ['TEST-courselist1', 'TEST-courselist2']
+
+describe('Courselist saving-related functions work', () => {
+  beforeEach(() => {
+    openSearch()
+    openCompletedCoursesModal()
+    cy.get('[data-cy="course-list-input"]').type('CSM14204\nTKT10004')
+    cy.get('[data-cy="search-name"]').type(courseListNames[0])
+    cy.get('button').contains('Save').click()
+  })
+
+  it('Course list can be saved', () => {
+    selectSavedCourselist(courseListNames[0])
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[0])
+    cy.get('[data-cy="course-list-input"]').contains('TKT10004').contains('CSM14204')
+  })
+
+  it('Course list can be deleted', () => {
+    openSearch()
+    openCompletedCoursesModal()
+    cy.get('[data-cy="search-name"]').type(courseListNames[1])
+    cy.get('button').contains('Save').click()
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[1])
+    cy.get('button').contains('Delete').click()
+    cy.get('[data-cy="history-search"]')
+      .type(courseListNames[1])
+      .type('{enter}')
+      .should('not.contain', courseListNames[1])
+  })
+
+  it('Course list can be updated', () => {
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[0])
+    cy.get('[data-cy="course-list-input"]').type(`,TKT10001`)
+    cy.get('button').contains('Save').click()
+    openSearch()
+    openCompletedCoursesModal()
+    selectSavedCourselist(courseListNames[0])
+    cy.get('[data-cy="course-list-input"]').contains(`TKT10001`)
+  })
+})

cypress/e2e/Faculty_statistics.js

@@ -1,11 +1,9 @@
 /// <reference types="Cypress" />
 
-// Change admin to regular user when feature is ready for general use
-
 describe('Faculty overview', () => {
   describe('Faculty can be selected', () => {
     it('Faculties are listed and one can be chosen', () => {
-      cy.init('/faculties', 'admin')
+      cy.init('/faculties')
       cy.get('[data-cy=select-faculty]').contains('td', 'H10')
       cy.contains('td', 'H99').should('not.exist')
       cy.contains('td', 'H90').click()
@@ -46,9 +44,8 @@ describe('Faculty overview', () => {
 
     it('Correct tabs are shown', () => {
       cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Basic information')
-      // Fix these when access is gained for facultyStatistics users
-      cy.get('[data-cy="FacultySegmentContainer"]').should('not.contain', 'Programmes and student populations')
-      cy.get('[data-cy="FacultySegmentContainer"]').should('not.contain', 'Graduation times')
+      cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Graduation times')
+      cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Programmes and student populations')
       cy.get('[data-cy="FacultySegmentContainer"]').should('not.contain', 'Update statistics')
     })
 
@@ -136,4 +133,168 @@ describe('Faculty overview', () => {
       cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
     })
   })
+
+  describe('Average graduation times', () => {
+    beforeEach(() => {
+      cy.init('/faculties')
+      cy.contains('td', 'H60').click()
+      cy.contains('Graduation times').click()
+    })
+
+    it('User can view graduation graphs', () => {
+      cy.get('[data-cy="Section-AverageGraduationTimes"]').should('be.visible')
+      cy.get('[data-cy="Section-bachelor"]').should('be.visible')
+      cy.get('[data-cy="Section-master"]').should('be.visible')
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.contains('Graduation year').should('be.visible')
+        cy.contains('.message', "Click a bar to view that year's programme level breakdown").should('be.visible')
+      })
+    })
+
+    it('Graphs display data', () => {
+      cy.get('[data-cy="Section-bachelor"]').within(() => {
+        cy.get('div[class="faculty-breakdown-graph"]')
+      })
+
+      cy.get('[data-cy="GraduationTimeToggle"]').click()
+
+      cy.get('[data-cy="Section-bachelor"]').within(() => {
+        cy.get('div[class="faculty-graph"]')
+        cy.contains('1 graduated').should('have.length', 1)
+        cy.contains('1 graduated').trigger('mouseover')
+        cy.contains('1 students graduated in year 2019')
+        cy.contains('median study time: 40 months')
+        cy.contains('0 graduated on time')
+        cy.contains('1 graduated max year overtime')
+
+        cy.contains('1 graduated').click()
+        cy.contains('Year 2019 by graduation year')
+        cy.get('div[class="programmes-graph"]').should('be.visible')
+        cy.get('div[class="programmes-graph"]').within(() => {
+          cy.contains('EDUK')
+          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('Kasvatustieteiden kandiohjelma')
+          cy.contains('KH60_001')
+        })
+      })
+
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.get('div[class="faculty-graph"]')
+        cy.contains('1 graduated').should('have.length', 1)
+        cy.contains('1 graduated').trigger('mouseover')
+        cy.contains('1 students graduated in year 2020')
+        cy.contains('median study time: 25 months')
+        cy.contains('0 graduated over year late')
+        cy.contains('1 graduated max year overtime')
+
+        cy.contains('1 graduated').click()
+        cy.contains('Year 2020 by graduation year')
+        cy.get('div[class="programmes-graph"]').should('be.visible')
+        cy.get('div[class="programmes-graph"]').within(() => {
+          cy.contains('EDUM')
+          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('Kasvatustieteiden maisteriohjelma')
+          cy.contains('MH60_001')
+        })
+      })
+    })
+
+    it('Graduation times grouping and time types can be toggled', () => {
+      cy.get('[data-cy="GraduationTimeToggle"]').click()
+      cy.get('[data-cy="GroupByToggle"]').click()
+      cy.get('[data-cy="Section-bachelor"]').should('not.exist')
+      cy.get('[data-cy="Section-master"]').should('be.visible')
+
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.get('div[class="faculty-graph"]')
+        cy.contains('1 graduated (100 % of class)').should('have.length', 1)
+        cy.contains('1 graduated').trigger('mouseover')
+        cy.contains('From class of 2018, 1/undefined students have graduated')
+      })
+
+      cy.get('[data-cy="GraduationTimeToggle"]').click()
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.contains('1').click()
+        cy.get('div[class="programmes-breakdown-graph"]').should('be.visible')
+        cy.get('div[class="programmes-breakdown-graph"]').within(() => {
+          cy.contains('Year 2018 by start year')
+          cy.contains('EDUM')
+        })
+      })
+    })
+  })
+
+  describe('Faculty Progress and student population -tab', () => {
+    beforeEach(() => {
+      cy.init('/faculties')
+      cy.contains('td', 'H50').click()
+      cy.contains('Programmes and student populations').click()
+    })
+
+    it('User can view studentdata table without %', () => {
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('be.visible')
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', '0 %')
+    })
+    it('User can view studentdata table with %', () => {
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('be.visible')
+      cy.get('[data-cy="HidePercentagesToggle"]').click()
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('contain', '0 %')
+    })
+
+    it('Percentage Graphs exists', () => {
+      cy.get('[data-cy="Graph-FacultyBachelorsProgress"]').should('be.visible')
+      cy.get('[data-cy="Graph-FacultyBachelorMastersProgress"]').should('be.visible')
+      cy.get('[data-cy="Graph-FacultyMastersProgress"]').should('be.visible')
+      cy.get('[data-cy="Graph-FacultyBachelorsProgress"]').should('be.visible')
+    })
+    // Following two tests just test that nothing goes broken after toggling
+    // No data for testing that data actual change as supposed
+    it('Graduations can be excluded', () => {
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('be.visible')
+      cy.get('[data-cy="StudentToggle"]').click()
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('be.visible')
+    })
+
+    it('Transfers can be excluded', () => {
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('be.visible')
+      cy.get('[data-cy="GraduatedToggle"]').click()
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('be.visible')
+    })
+
+    it('Progress tables exists', () => {
+      cy.get('[data-cy="FacultyBachelorsProgressTable"]').should('be.visible')
+      cy.get('[data-cy="FacultyBachelorMasterProgressTable"]').should('be.visible')
+      cy.get('[data-cy="FacultyMastersProgressTable"]').should('be.visible')
+      cy.get('[data-cy="FacultyBachelorsProgressTable"]').should('be.visible')
+    })
+
+    it('Progress tables can be toggled', () => {
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('not.be.visible')
+      cy.get('[data-cy="Button-Show-FacultyBachelorsProgressTable-6"]').click()
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-0"]').should('not.exist')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-1"]').should('not.exist')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-2"]').should('not.exist')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-3"]').should('not.exist')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-4"]').should('not.exist')
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('be.visible')
+      cy.get('[data-cy="Button-Hide-FacultyBachelorsProgressTable-6"]').click()
+      cy.get('[data-cy="Cell-FacultyBachelorsProgressTable-6"]').should('not.be.visible')
+    })
+
+    it('Sudent statstable can be toggled', () => {
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', 'TKT')
+      cy.get('[data-cy="Button-FacultyStudentStatsTable-3"]').click()
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('contain', 'TKT')
+      cy.get('[data-cy="Button-FacultyStudentStatsTable-3"]').click()
+      cy.get('[data-cy="FacultyStudentStatsTable"]').should('not.contain', 'TKT')
+    })
+
+    it('Progress infobox can be toggled', () => {
+      cy.get('[data-cy="InfoFacultyProgress-info-content"]').should('not.exist')
+      cy.get('[data-cy="InfoFacultyProgress-open-info"]').click()
+      cy.get('[data-cy="InfoFacultyProgress-info-content"]').should('be.visible')
+      cy.get('[data-cy="InfoFacultyProgress-close-info"]').click()
+      cy.get('[data-cy="InfoFacultyProgress-info-content"]').should('not.exist')
+    })
+  })
 })

cypress/e2e/Filters.js

@@ -1,3 +1,4 @@
+/* eslint-disable cypress/no-unnecessary-waiting */
 /// <reference types="Cypress" />
 
 // const baseUrl = Cypress.config().baseUrl
@@ -20,6 +21,7 @@ const clearSingleDropdownSelection = dataCyAttribute => {
 }
 
 const testRangeFilter = (parentEl, min, max, expected) => {
+  cy.clock(MOCKED_DATE, ['Date'])
   cy.cs(parentEl)
     .cs('range-selector-min')
     .find('input')
@@ -49,8 +51,6 @@ const createRunTestStepWithPreAndPostPartsFunction = amountWithoutFiltering => {
     const card = cy.cs(`${filterName}-filter-card`)
 
     card.invoke('attr', 'data-open').then(open => {
-      console.log('open', open)
-
       const getHeader = () => cy.cs(`${filterName}-header`)
 
       if (open === 'false') {
@@ -87,21 +87,6 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Programme defaults to "Active Study Right" mode', () => {
-    runTestStepWithPreAndPostParts('Programme', () => {
-      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
-    })
-  })
-
-  it('Programme filter works', () => {
-    runTestStepWithPreAndPostParts('Programme', () => {
-      const card = cy.cs('Programme-filter-card')
-      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(5)
-      programmeDropdown.get('i.delete').click()
-    })
-  })
-
   // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
   // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
   // So fix this
@@ -141,8 +126,8 @@ describe('Population Statistics', () => {
   it('Enrollment filter works', () => {
     runTestStepWithPreAndPostParts('EnrollmentStatus', () => {
       cy.cs('enrollmentStatusFilter-status').selectFromDropdown(0)
-      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(0)
-      checkFilteringResult(148)
+      cy.cs('enrollmentStatusFilter-semesters').selectFromDropdown(5)
+      checkFilteringResult(147)
       clearSemanticUIMultipleDropDownSelection('enrollmentStatusFilter-semesters')
     })
   })
@@ -151,6 +136,7 @@ describe('Population Statistics', () => {
     runTestStepWithPreAndPostParts('CreditsEarned', () => {
       testRangeFilter('CreditsEarned-filter-card', 50, 150, 118)
     })
+    cy.cs('reset-all-filters').click()
   })
 
   it('Age filter works', () => {
@@ -161,6 +147,7 @@ describe('Population Statistics', () => {
   })
 
   it('Gender filter works', () => {
+    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(42)
@@ -275,8 +262,8 @@ describe('Course Statistics', () => {
       programmeDropdown.get('i.delete').click()
     })
   })
-
-  it('Age filter works', () => {
+  // FIX THIS SOON!
+  it.skip('Age filter works', () => {
     cy.clock(MOCKED_DATE, ['Date'])
     runTestStepWithPreAndPostParts('Age', () => {
       testRangeFilter('Age-filter-card', 20, 40, 33)
@@ -284,6 +271,7 @@ describe('Course Statistics', () => {
   })
 
   it('Gender filter works', () => {
+    cy.cs('reset-all-filters').click()
     runTestStepWithPreAndPostParts('Gender', () => {
       cy.cs('genderFilter-dropdown').selectFromDropdown(0)
       checkFilteringResult(27)
@@ -340,6 +328,7 @@ describe('Custom Population Statistics', () => {
     runTestStepWithPreAndPostParts('Age', () => {
       testRangeFilter('Age-filter-card', 20, 30, 1)
     })
+    cy.cs('reset-all-filters').click()
   })
 
   it('Programme defaults to "Active Study Right" mode', () => {

cypress/e2e/Openuni.js

@@ -0,0 +1,116 @@
+/// <reference types="Cypress" />
+
+const deleteAllSearches = () => {
+  cy.contains('Saved populations')
+    .parent()
+    .parent()
+    .parent()
+    .parent()
+    .get('.dropdown')
+    .then(d => {
+      const searchItems = d.find('div[role=option] > span[class=text]')
+      for (let i = 0; i < searchItems.length; i++) {
+        if (searchItems[i].textContent.includes('TEST-')) {
+          cy.get('[data-cy="history-search"]').children().eq(0).type(searchItems[i].textContent).type('{enter}')
+          cy.get('button').contains('Delete').click()
+        }
+      }
+    })
+}
+
+describe('Open uni population tests', () => {
+  describe('Open uni form can be used', () => {
+    // When openUniUser is created, change it here
+    beforeEach(() => {
+      cy.init('/openunipopulation', 'admin')
+    })
+    after(() => {
+      cy.init('/openunipopulation', 'admin')
+      cy.url().should('include', '/openunipopulation')
+      cy.contains('Open uni student population')
+      cy.get('[data-cy="open-uni-search-button"]').click()
+      deleteAllSearches()
+    })
+
+    it('Finds a proper population', () => {
+      cy.fixture('openUniPopulation').then(({ courseCodesSet1 }) => {
+        cy.get('[data-cy="open-uni-search-button"]').click()
+        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
+        cy.get('[data-cy="begin-of-search"]').clear().type('01.01.2015')
+        cy.get('[data-cy="end-of-search"]').clear().type('01.01.2020')
+        cy.get('button').contains('Search population').click()
+        cy.contains('Beginning of the search for all fields:')
+        cy.contains('01.01.2015')
+        cy.contains('End of the search for enrollments:')
+        cy.contains('01.01.2020')
+        cy.url().should(
+          'include',
+          '/openunipopulation?courseCode=TKT10001&courseCode=TKT10002&enddate=01-01-2020&startdate=01-01-2015'
+        )
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
+      })
+    })
+
+    it('Textarea can handle different kinds of input', () => {
+      cy.get('[data-cy="open-uni-search-button"]').click()
+      cy.contains('Insert course code(s)')
+        .siblings()
+        .get('textarea')
+        .type('tkt10001    tkt10002,tkt10003\n TKT10004,,TKt10005\nTKT-1234')
+      cy.get('button').contains('Search population').click()
+      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10001')
+      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10002')
+      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10003')
+      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10004')
+      cy.get('[data-cy="open-uni-table-div"]').should('contain.text', 'TKT10005')
+      cy.get('[data-cy="open-uni-table-div"]').should('not.contain', 'TKT-1234')
+    })
+
+    it('Saves a custom population search', () => {
+      cy.fixture('openUniPopulation').then(({ courseCodesSet1 }) => {
+        cy.get('[data-cy="open-uni-search-button"]').click()
+        cy.get('[data-cy="search-name"]').type('TEST-Avoimen-populaatio')
+        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
+        cy.get('[data-cy="save-search"]').click()
+        cy.get('[data-cy="history-search"]').children().eq(0).type('TEST-Avoimen-populaatio').type('{enter}')
+        cy.contains('TEST-Avoimen-populaatio')
+        cy.get('button').contains('Search population').click()
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
+      })
+    })
+
+    it('Updates a custom population search', () => {
+      cy.fixture('openUniPopulation').then(({ courseCodesSet1, courseCodesSet2 }) => {
+        cy.get('[data-cy="open-uni-search-button"]').click()
+        cy.get('[data-cy="search-name"]').type('TEST-Avoimen-populaatio')
+        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet1.join('\n'))
+        cy.get('[data-cy="save-search"]').click()
+        cy.get('button').contains('Search population').click()
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
+
+        cy.get('[data-cy="open-uni-search-button"]').click()
+        cy.get('[data-cy="history-search"]').children().eq(0).type('TEST-Avoimen-populaatio').type('{enter}')
+        cy.contains('Insert course code(s)').siblings().get('textarea').should('contain', courseCodesSet1.join(', '))
+        cy.contains('Insert course code(s)').siblings().get('textarea').type('\n')
+        cy.contains('Insert course code(s)').siblings().get('textarea').type(courseCodesSet2.join('\n'))
+        cy.get('[data-cy="save-search"]').click()
+        cy.get('button').contains('Search population').click()
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[0])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet1[1])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[0])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[1])
+        cy.get('[data-cy="open-uni-table-div"]').should('contain.text', courseCodesSet2[2])
+      })
+    })
+
+    it('View does not crash with non-existing courses', () => {
+      cy.get('[data-cy="open-uni-search-button"]').click()
+      cy.contains('Insert course code(s)').siblings().get('textarea').type('cypress-fun')
+      cy.get('button').contains('Search population').click()
+      cy.get('[data-cy="open-uni-table-div"]').should('not.contain', 'cypress-fun')
+    })
+  })
+})

cypress/e2e/Population_statistics.js

@@ -1,18 +1,8 @@
 /// <reference types="Cypress" />
-
+// Now "Class statistics" in UI
 const moment = require('moment')
 const _ = require('lodash')
 
-// const setPopStatsUntil = (until, includeSettings = []) => {
-//   cy.contains('Advanced settings').siblings().get('[data-cy=advanced-toggle]').click()
-//   includeSettings.forEach(setting => {
-//     cy.contains('Advanced settings').parent().siblings().contains(setting).click()
-//   })
-//   cy.get('.adv-stats-until > .form-control').click().clear().type(until)
-//   cy.contains('Fetch population with new settings').click()
-//   cy.contains('Advanced settings')
-// }
-
 describe('Population Statistics tests', () => {
   const pathToCSBach2017 =
     '/populations?months=36&semesters=FALL&semesters=SPRING&studyRights={%22programme%22%3A%22KH50_005%22}&tag&year=2017'
@@ -23,9 +13,21 @@ describe('Population Statistics tests', () => {
       cy.init('/populations')
     })
 
+    it('Population statistics search infobox works', () => {
+      cy.get('[data-cy="PopulationSearch-info-content"]').should('not.exist')
+      cy.get('[data-cy="PopulationSearch-open-info"]').click()
+      cy.get('[data-cy="PopulationSearch-info-content"]').should('be.visible')
+      cy.get('[data-cy="PopulationSearch-info-content"]').should(
+        'contain',
+        'lukuvuosi, jolloin opiskelija on ilmoittautunut'
+      )
+      cy.get('[data-cy="PopulationSearch-close-info"]').click()
+      cy.get('[data-cy="PopulationSearch-info-content"]').should('not.exist')
+    })
+
     it('Population statistics search form is usable', () => {
-      cy.contains('See population').should('be.disabled')
-      cy.contains('Search for population')
+      cy.contains('See class').should('be.disabled')
+      cy.contains('Search for class')
       cy.contains('Class of')
         .parent()
         .within(() => {
@@ -47,7 +49,7 @@ describe('Population Statistics tests', () => {
         .children()
         .contains('Tietojenkäsittelytieteen maisteriohjelma')
         .click()
-      cy.contains('See population').should('be.enabled')
+      cy.contains('See class').should('be.enabled')
     })
 
     it('Searching for population really shows population', () => {
@@ -57,27 +59,25 @@ describe('Population Statistics tests', () => {
         .children()
         .contains('Tietojenkäsittelytieteen kandiohjelma')
         .click()
-      cy.contains('See population').click()
-      cy.contains('Population statistics')
+      cy.contains('See class').click()
+      cy.contains('class size')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma')
     })
 
     it('Population statistics is usable on general level', () => {
       cy.visit(pathToCSBach2017)
       cy.cs('filtered-students')
-      // This advanced settings does not work correctly
-      // see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3552
-      // setPopStatsUntil('toukokuu 2020')
-
-      cy.get('.card').within(() => {
+      cy.get('.header').within(() => {
         cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 170 students')
+        cy.contains('class size 170 students')
+      })
+      cy.get('.card').within(() => {
         cy.contains('Excludes exchange students')
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
       cy.cs('filtered-students')
-      cy.contains('Courses of population').click()
+      cy.contains('Courses of class').click()
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -103,12 +103,12 @@ describe('Population Statistics tests', () => {
     it.skip('Advanced settings work', () => {
       cy.visit(pathToCSBach2017)
       cy.get('[data-cy=advanced-toggle]').click()
-      cy.contains('Statistics until')
       // only spring
       cy.cs('toggle-fall').click()
-      cy.contains('Fetch population').click()
+      cy.contains('Fetch class').click()
 
       // moar waiting hack
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
       cy.wait(10000)
 
       cy.contains('No statistics found for the given query.')
@@ -118,14 +118,14 @@ describe('Population Statistics tests', () => {
       cy.cs('toggle-fall').click()
       cy.cs('toggle-spring').click()
 
-      cy.contains('Fetch population').click()
+      cy.contains('Fetch class').click()
 
       cy.contains('Credit accumulation (for 170 students)')
 
       // spring + fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-spring').click()
-      cy.contains('Fetch population').click()
+      cy.contains('Fetch class').click()
 
       cy.contains('Credit accumulation (for 170 students)')
     })
@@ -134,10 +134,9 @@ describe('Population Statistics tests', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
       cy.contains('Credits Gained').click()
-      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
-
-      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
-
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the class')
+      const today = moment().endOf('month')
+      const months = Math.round(moment.duration(moment(today).diff(moment('2017-08-01'))).asMonths())
       const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
       const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
 
@@ -185,7 +184,7 @@ describe('Population Statistics tests', () => {
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
       cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
-      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.75')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.74')
       cy.get("[data-cy='credit-stats-stdev']").should('contain', '65.53')
       cy.get("[data-cy='credit-stats-min']").should('contain', '5')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
@@ -218,17 +217,18 @@ describe('Population Statistics tests', () => {
     })
   })
 
+  /*
   describe('when using IAM user', () => {
     beforeEach(() => {
       cy.init(pathToCSBach2017, 'onlyiamrights')
     })
-
     it('Population statistics is visible', () => {
       cy.reload()
 
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('class size 170 students')
+
       cy.get('.card').within(() => {
-        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 170 students')
         cy.contains('Excludes exchange students')
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
@@ -238,9 +238,10 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit statistics')
     })
 
-    it('Students tab is not available', () => {
+    it.skip('Students tab is not available', () => {
       cy.contains('Students (170)').should('not.exist')
       // TODO: somehow check that obfuscation works
     })
   })
+  */
 })

cypress/e2e/Student_statistics.js

@@ -93,8 +93,8 @@ describe('Student Statistics tests', () => {
     cy.get('.prompt').type(student.studentnumber)
     cy.contains('td a', student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
-    cy.contains('Population statistics')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+    cy.contains('class size')
   })
 })
 

cypress/e2e/Studyprogramme_overview.js

@@ -25,12 +25,14 @@ describe('Studyprogramme overview', () => {
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
-      cy.viewport(1536, 960)
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click({ force: true })
+      // cy.viewport(1536, 960)
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
     it('Basic information -tab loads', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
       cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
       cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
@@ -39,7 +41,6 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Basic information contains correct students', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Started, Graduated, Transferred Away, Transferred to]
@@ -55,23 +56,21 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Basic information contains correct credits', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2021, 0, 0, 0, 0, 0],
         [2020, 5860, 5591, 102, 0, 167],
-        [2019, 8696, 7737, 587, 23, 349],
-        [2018, 7818, 6584, 857, 105, 272],
-        [2017, 3085, 2492, 158, 47, 388],
+        [2019, 8696, 7737, 577, 33, 349],
+        [2018, 7818, 6584, 852, 110, 272],
+        [2017, 3085, 2492, 153, 52, 388],
       ]
 
       cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
     })
 
     it('Basic information contains correct thesis writers and graduates', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Graduated, Wrote thesis]
@@ -87,7 +86,6 @@ describe('Studyprogramme overview', () => {
     })
 
     it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
       cy.get('[data-cy=StudentToggle]').click()
       const years = getEmptyYears()
       const studentTableContents = [
@@ -129,39 +127,39 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Year can be changed to academic year, and data changes accordingly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
-      cy.get('[data-cy=YearToggle]').click()
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.get('[data-cy=YearToggle]').click({ force: true })
       const isAcademicYear = true
       const years = getEmptyYears(isAcademicYear)
       const studentTableContents = [
         // [Year, Started, Graduated, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0]),
         ['2021 - 2022', 0, 0, 0, 0],
-        ['2020 - 2021', 5, 0, 0, 0],
-        ['2019 - 2020', 27, 0, 0, 0],
-        ['2018 - 2019', 4, 0, 0, 0],
-        ['2017 - 2018', 0, 0, 0, 0],
+        ['2020 - 2021', 12, 6, 3, 2],
+        ['2019 - 2020', 86, 41, 1, 0],
+        ['2018 - 2019', 162, 5, 0, 1],
+        ['2017 - 2018', 171, 1, 0, 0],
       ]
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
 
       const creditTableContents = [
         // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
-        ...years.map(year => [year, 0, 0, 0, 0]),
-        ['2021 - 2022', 0, 0, 0, 0],
-        ['2020 - 2021', 0, 0, 0, 0],
-        ['2019 - 2020', 338.5, 177, 154, 0, 7.5],
-        ['2018 - 2019', 173, 0, 173, 0, 0],
-        ['2017 - 2018', 42, 0, 42, 0, 0],
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0, 0],
+        ['2020 - 2021', 566, 486, 20, 0, 60],
+        ['2019 - 2020', 8595, 8271, 106, 0, 218],
+        ['2018 - 2019', 9992, 8855, 781, 53, 303],
+        ['2017 - 2018', 6055, 4792, 777, 142, 344],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+      //cy.wait(10000)
       cy.get('[data-cy=YearToggle]').click()
     })
 
     it('Basic information graphs render', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-
       cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
         .should('contain', 'Started')
         .should('contain', 'Graduated')
@@ -171,8 +169,8 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Major students credits')
         .should('contain', 'Non-major students credits')
         .should('contain', 'Transferred credits')
-        .should('contain', 857)
-        .should('contain', 587)
+        .should('contain', 852)
+        .should('contain', 577)
 
       cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
         .should('contain', 'Graduated students')
@@ -183,10 +181,16 @@ describe('Studyprogramme overview', () => {
         .should('contain', 25)
         .should('contain', 35)
 
-      cy.get('[data-cy=Graph-2019-AverageGraduationTimes]')
-        .should('contain', '2019')
-        .should('contain', 'n = 17')
-        .should('contain', '23 kk')
+      cy.get('[data-cy=graduation-times-graph-breakdown]')
+      cy.get('[data-cy=GraduationTimeToggle]').click()
+      cy.get('[data-cy=graduation-times-graph]').within(() => {
+        cy.contains('Graduation year')
+        cy.contains('2019')
+        cy.contains('17 graduated').trigger('mouseover')
+        cy.contains('study time: 23 months')
+        cy.contains('17 graduated on time')
+        cy.contains('0 graduated max year overtime')
+      })
 
       cy.get('[data-cy=Graph-ProgrammesBeforeOrAfter')
         .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
@@ -198,166 +202,97 @@ describe('Studyprogramme overview', () => {
     })
   })
 
+  describe('Graduation times of master programmes', () => {
+    it('are split into two graphs', () => {
+      cy.init('/study-programme', 'admin')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(5000)
+      cy.contains('a', 'Kasvatustieteiden maisteriohjelma').click({ force: true })
+
+      cy.get('[data-cy=graduation-times-graph-breakdownMaster]')
+      cy.get('[data-cy=graduation-times-graph-breakdownBachelor]')
+
+      cy.get('[data-cy=GraduationTimeToggle]').click()
+      cy.get('[data-cy=graduation-times-graphMaster]').within(() => {
+        cy.contains('Graduation year')
+        cy.contains('2020')
+        cy.contains('24 graduated').trigger('mouseover')
+        cy.contains('study time: 22 months')
+        cy.contains('20 graduated on time')
+        cy.contains('4 graduated max year overtime')
+      })
+
+      cy.get('[data-cy=graduation-times-graphBachelor]').within(() => {
+        cy.contains('Graduation year')
+        cy.contains('2020')
+        cy.contains('0 graduated').trigger('mouseover')
+        cy.contains('study time: 0 months')
+        cy.contains('0 graduated on time')
+      })
+    })
+  })
+
   /* Studytrack overview -tests*/
   describe('Studytrack overview works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and class statistics').click()
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
-    it('Studytracks and student populations -tab loads', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
+    it('Studytracks and class statistics -tab loads', () => {
       cy.get('[data-cy=Section-StudytrackOverview]')
       cy.get('[data-cy=Section-StudytrackProgress]')
-      cy.get('[data-cy=Section-AverageGraduationTimes]')
+      cy.get('[data-cy=Section-AverageGraduationTimesStudytracks]')
     })
 
     it('Students of the studyprogramme are shown correctly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
       const tableContents = [
         // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
-        [
-          '2020 - 2021',
-          12,
-          '100.0 %',
-          10,
-          '83.3 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          10,
-          '83.3 %',
-          2,
-          '16.7 %',
-          12,
-          '100.0 %',
-        ],
-        [
-          '2019 - 2020',
-          86,
-          '100.0 %',
-          86,
-          '100.0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          1,
-          '1.2 %',
-          1,
-          '1.2 %',
-          65,
-          '75.6 %',
-          21,
-          '24.4 %',
-          86,
-          '100.0 %',
-        ],
-        [
-          '2018 - 2019',
-          162,
-          '100.0 %',
-          161,
-          '99.4 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          11,
-          '6.8 %',
-          118,
-          '72.8 %',
-          44,
-          '27.2 %',
-          161,
-          '99.4 %',
-        ],
-        [
-          '2017 - 2018',
-          171,
-          '100.0 %',
-          171,
-          '100.0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          4,
-          '2.3 %',
-          42,
-          '24.6 %',
-          135,
-          '78.9 %',
-          36,
-          '21.1 %',
-          166,
-          '97.1 %',
-        ],
-        [
-          'Total',
-          431,
-          '100.0 %',
-          431,
-          '100.0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          5,
-          '1.2 %',
-          54,
-          '12.5 %',
-          328,
-          '76.1 %',
-          103,
-          '23.9 %',
-          425,
-          '98.6 %',
-        ],
+        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 12],
+        ['2019 - 2020', 86, 86, 0, 0, 1, 1, 65, 21, 86],
+        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 161],
+        ['2017 - 2018', 171, 171, 0, 0, 4, 42, 135, 36, 166],
+        ['Total', 431, 428, 0, 0, 5, 54, 328, 103, 425],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackOverview')
     })
 
     it('Student progress data is shown correctly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
       const tableContents = [
-        ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0],
-        ['2019 - 2020', 86, 26, 34, 21, 5, 0, 0, 0],
-        ['2018 - 2019', 162, 10, 27, 47, 36, 26, 12, 4],
-        ['2017 - 2018', 171, 20, 21, 26, 22, 22, 33, 27],
-        ['Total', 431, 51, 64, 97, 79, 55, 51, 34],
+        ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0, 0],
+        ['2019 - 2020', 86, 11, 15, 35, 20, 5, 0, 0, 0],
+        ['2018 - 2019', 162, 3, 9, 28, 50, 33, 25, 11, 3],
+        ['2017 - 2018', 171, 7, 13, 22, 32, 18, 23, 32, 24],
+        ['Total', 431, 33, 37, 85, 102, 56, 48, 43, 27],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackProgress')
     })
 
     it('Studytrack overview graphs render', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
-
       cy.get('[data-cy=Graph-StudytrackProgress]')
-        .should('contain', 'Less than 30 credits')
+        .should('contain', 'Less than 15 credits')
         .should('contain', '30-59 credits')
-        .should('contain', 'More than 180 credits')
-        .should('contain', '11.7%') // The percentage for less than 30 credits in 2017-2018, to check that the graph renders
-        .should('contain', '6.2%') // The percentage for less than 30 credits in 2018-2019
-        .should('contain', '30.2%') // The percentage for less than 30 credits in 2019-2020
+        .should('contain', '180 or more credits')
+        .should('contain', '4.1%') // The percentage for less than 15 credits in 2017-2018, to check that the graph renders
+        .should('contain', '5.6%') // The percentage for 15-29 credits in 2018-2019
+        .should('contain', '12.8%') // The percentage for less than 15 credits in 2019-2020
 
-      cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
-        .should('contain', '2018 - 2019')
-        .should('contain', 'n = 11 / 162')
-        .should('contain', '22 kk')
+      cy.get('[data-cy=graduation-times-graph-breakdown]')
+      cy.get('[data-cy=GraduationTimeToggle]').click()
+
+      cy.get('[data-cy=graduation-times-graph]').within(() => {
+        cy.contains('Start year')
+        cy.contains('2019 - 2020')
+        cy.contains('42 graduated').trigger('mouseover')
+        cy.contains('study time: 31.5 months')
+        cy.contains('39 graduated on time')
+      })
     })
   })
 
@@ -365,6 +300,8 @@ describe('Studyprogramme overview', () => {
   describe('Programme courses works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('.attached').contains('Programme courses').click()
     })
@@ -385,16 +322,21 @@ describe('Studyprogramme overview', () => {
     })
 
     it('calendar year -> academic year toggle works', () => {
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.get('[data-cy=fromYear]').click().contains('2018').click()
 
       cy.get('[data-cy=toYear').click().contains('2019').click()
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(20000)
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('684')
       })
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(20000)
       cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(20000)
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('772')
       })
@@ -411,7 +353,7 @@ describe('Studyprogramme overview', () => {
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         // Test sorting by different columns
         cy.get('th').eq(0).click()
-        cy.get('tr').eq(2).contains('Tietojenkäsittelytieteen kisälliopetus: Java-ohjelmointi')
+        cy.get('tr').eq(2).contains('Tietojenkäsittelytieteen kisälliopetus')
         cy.get('th').eq(1).click()
         cy.get('tr').eq(1).contains('Äidinkielinen viestintä')
         cy.get('th').eq(2).click()
@@ -427,7 +369,7 @@ describe('Studyprogramme overview', () => {
       cy.get('[data-cy=creditsStudentsToggle]').first().click()
 
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
-        cy.get('tr').eq(1).contains('267')
+        cy.get('tr').eq(2).contains('267')
       })
     })
   })
@@ -436,12 +378,14 @@ describe('Studyprogramme overview', () => {
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
     })
 
     it('can create and delete tags for population', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
       cy.contains('Create new tag').click()
@@ -452,8 +396,6 @@ describe('Studyprogramme overview', () => {
 
     it('can create personal tags', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
 
@@ -467,9 +409,6 @@ describe('Studyprogramme overview', () => {
       const name = `tag-${new Date().getTime()}`
 
       const student = '010113437'
-
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
       cy.contains('Create new tag').click()
@@ -490,7 +429,8 @@ describe('Studyprogramme overview', () => {
 
       cy.go('back')
       cy.go('back')
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       deleteTag(name)
 
       cy.contains('Student statistics').click()
@@ -500,25 +440,25 @@ describe('Studyprogramme overview', () => {
     })
   })
 
+  /*
   describe('IAM user', () => {
     beforeEach(() => {
       cy.init('/study-programme', 'onlyiamrights')
+
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').dblclick()
     })
 
     it('can access programme and correct tabs are visible', () => {
       cy.reload()
 
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-
       cy.contains('Basic information')
-      cy.contains('Studytracks and student populations')
+      cy.contains('Studytracks and class statistics')
 
       cy.contains('Update statistics').should('not.exist')
       cy.contains('Degree Courses').should('not.exist')
     })
 
     it('can access basic information', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.contains('Basic information').click()
 
       cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
@@ -529,12 +469,12 @@ describe('Studyprogramme overview', () => {
     })
 
     it('can access studytracks', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('.attached').contains('Studytracks and class statistics').click()
 
       cy.get('[data-cy=Section-StudytrackOverview]')
       cy.get('[data-cy=Section-StudytrackProgress]')
-      cy.get('[data-cy=Section-AverageGraduationTimes]')
+      cy.get('[data-cy=Section-AverageGraduationTimesStudytracks]')
     })
   })
+  */
 })

cypress/fixtures/completedCoursesData.json

@@ -0,0 +1,4 @@
+{
+  "studentSet1": ["010450938", "010589388", "010614509"],
+  "coursesSet1": ["TKT10001", "TKT10002", "TKT10003", "TKT10004"]
+}

cypress/fixtures/openUniPopulation.json

@@ -0,0 +1,4 @@
+{
+  "courseCodesSet1": ["TKT10001", "TKT10002"],
+  "courseCodesSet2": ["TKT10003", "TKT10004", "TKT10005"]
+}

cypress/support/commands.js

@@ -53,6 +53,18 @@ const onlyIamRightsUserHeaders = {
   hypersonsisuid: 'hy-hlo-1111111',
 }
 
+// TODO add user into anon-DB (check the need for changing hypersonid)
+// REMEMBER TO ADD ROLE openUniSearch
+// const openUniUser = {
+//   uid: 'openuniuser',
+//   displayname: 'Open Uni User',
+//   'shib-session-id': 'mock-cypress-session',
+//   hygroupcn: 'grp-oodikone-basics-users;hy-ypa-opa-dojo',
+//   edupersonaffiliation: 'member',
+//   mail: 'grp-toska+mockopenuniuser@helsinki.fi',
+//   hypersonsisuid: 'hy-hlo-9999999',
+// }
+
 const userHeaders = [
   adminUserHeaders,
   basicUserHeaders,
@@ -118,7 +130,7 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('navbar-studyProgramme').click()
   cy.cs('navbar-class').click()
   cy.cs('select-study-programme').click().children().contains(name).click()
-  cy.contains('See population').click()
+  cy.contains('See class').click()
 })
 
 Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {

docker-compose.ci.yml

@@ -62,6 +62,30 @@ services:
       POSTGRES_HOST_AUTH_METHOD: trust
     image: registry.toska.cs.helsinki.fi/user-db
 
+  jami:
+    <<: *defaults
+    image: toska/jami
+    environment:
+      - PORT=3003
+      - NODE_ENV=production
+      - POSTGRES_HOST=jami-db
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DATABASE=postgres
+    ports:
+      - 3003:3003
+    container_name: jami
+
+  jami-db:
+    <<: *defaults
+    image: postgres:13.2
+    environment:
+      - PGDATA=/data
+      - POSTGRES_PASSWORD=postgres
+    container_name: jami-db
+    ports:
+      - 5433:5432
+
   nginx:
     image: nginx:latest
     container_name: nginx

docker-compose.yml

@@ -168,6 +168,32 @@ services:
     volumes:
       - user-db-data:/data
 
+  jami:
+    <<: *defaults
+    image: toska/jami
+    environment:
+      - PORT=3003
+      - NODE_ENV=dev
+      - POSTGRES_HOST=jami-db
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DATABASE=postgres
+    ports:
+      - 3003:3003
+    container_name: jami
+
+  jami-db:
+    <<: *defaults
+    image: postgres:13.2
+    environment:
+      - PGDATA=/data
+      - POSTGRES_PASSWORD=postgres
+    container_name: jami-db
+    ports:
+      - 5433:5432
+    volumes:
+      - jami-data:/data
+
 volumes:
   backend-nodemod:
   frontend-nodemod:
@@ -178,3 +204,4 @@ volumes:
   sis-updater-scheduler-nodemod:
   sis-updater-worker-nodemod:
   user-db-data:
+  jami-data:

documentation/numeric_types.md

@@ -0,0 +1,48 @@
+# Mystery numbers
+
+Sis data uses some numeric classifiers that are not self-explanatory and (most) previously needed to be deciphered from existing code.
+Some mystery numbers are gathered here for convenience. Feel free to add anything that could be useful. Remember to use the spelling used in code to enable project wide search.
+
+## extentcode
+
+(from **studyright**)
+
+These are actually defined in db table studyright_extents. Tells studyright level or type, eg. bachelor's degree or non-degree studies.
+
+## prioritycode
+
+(from **studyright**)
+
+1: Primary
+
+2: Secondary
+
+5: Rescinded (Seemingly not in use anymore)
+
+6: Option
+
+30: Graduated
+
+## type
+
+(from **element_details**)
+
+20: A study programme
+
+30: Module/Studytrack/kokonaisuus? Eg. Neurology
+
+## enrollmenttype
+
+(from **semester_enrollments**)
+
+1: Present
+
+2: Absent
+
+3: Inactive (neither 1 or 2 has been declared)
+
+## credittypecode
+
+(from **credit**)
+
+Defined in credit_types db table. Completed/Improved/Transferred/Failed

package.json

@@ -29,12 +29,12 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/core": "^7.18.13",
-    "@babel/eslint-parser": "^7.17.0",
+    "@babel/core": "^7.19.3",
+    "@babel/eslint-parser": "^7.19.1",
     "@babel/plugin-syntax-jsx": "^7.16.7",
-    "concurrently": "^7.3.0",
-    "cypress": "^10.6.0",
-    "eslint": "^8.23.0",
+    "concurrently": "^7.4.0",
+    "cypress": "^10.9.0",
+    "eslint": "^8.25.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-babel": "^5.3.1",
@@ -43,13 +43,13 @@
     "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^4.2.1",
-    "eslint-plugin-react": "^7.31.1",
+    "eslint-plugin-react": "^7.31.9",
     "eslint-plugin-react-hooks": "^4.3.0",
     "husky": "^8.0.1",
     "lint-staged": "^13.0.3",
     "node-actionlint": "^1.2.2",
     "prettier": "^2.5.1",
-    "stylelint": "^14.11.0",
+    "stylelint": "^14.13.0",
     "stylelint-config-prettier": "^9.0.3",
     "stylelint-config-standard": "^26.0.0",
     "stylelint-prettier": "^2.0.0"

scripts/functions_for_setup.sh

@@ -10,6 +10,7 @@
 PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 DUMP_DIR="$PROJECT_ROOT/.databasedumps"
 USER_DATA_FILE="$DUMP_DIR/hyuserdata"
+DOCKER_COMPOSE=$PROJECT_ROOT/docker-compose.yml
 
 ## Following the naming convention in docker-compose, these are names for database
 ## services. The real data databases inside service will have names suffixed by "-real".
@@ -100,6 +101,28 @@ reset_databases() {
   successmsg "Database setup finished"
 }
 
+reset_jami_data() {
+  local database="jami-db"
+  local pannu_url="svm-85.cs.helsinki.fi:/home/toska_user/most_recent_backup_store/jami.sql.gz"
+  local dump_destination="$DUMP_DIR/$database.sqz"
+
+  infomsg "Downloading Jami data"
+  scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkki.cs.helsinki.fi" "$username@$pannu_url" "$dump_destination"
+
+  infomsg "Removing database and related volume"
+  docker volume rm oodikone_jami-data || warningmsg "This is okay, continuing"
+  docker-compose -f "$DOCKER_COMPOSE" down "$database" || warningmsg "This is okay, continuing"
+
+  infomsg "Starting postgres in the background"
+  docker-compose -f "$DOCKER_COMPOSE" up -d "$database"
+  retry docker-compose -f "$DOCKER_COMPOSE" exec "$database" pg_isready --dbname="postgres"
+
+  infomsg "Populating Jami"
+  docker exec -i "$database" /bin/bash -c "gunzip | psql -U postgres" < "$dump_destination"
+  docker-compose stop "$database"
+  msg ""
+}
+
 reset_all_real_data() {
   infomsg "Downloading real data dumps, asking for pannu password when needed"
   for i in ${!DATABASES[*]}; do
@@ -108,6 +131,7 @@ reset_all_real_data() {
     download_real_dump "$database" "$url"
   done
   reset_databases "${DATABASES[@]}"
+  reset_jami_data
 }
 
 reset_single_database() {
@@ -119,6 +143,7 @@ reset_single_database() {
     "sis-db"
     "sis-importer-db"
     "user-db"
+    "jami-db"
     "Return to main menu."
   )
 
@@ -144,6 +169,9 @@ reset_single_database() {
           local database=$USER_DB_NAME
           local url=$USER_DB_REAL_DUMP_URL
           break 2;;
+        "jami-db")
+          reset_jami_data
+          break 2;;
         "Return to main menu.")
           break 2;;
         *) msg "${RED}Invalid option:${NOFORMAT} $REPLY

services/backend/config/IAMConfig.js

@@ -1,435 +0,0 @@
-const { facultyMap } = require('./data')
-
-const joryMap = {
-  'hy-ttdk-tuk-jory': '100-K001',
-  'hy-ttdk-tum-jory': '100-M001',
-  'hy-oiktdk-on-jory': '200-K001',
-  'hy-oiktdk-otm-jory': '200-M001',
-  'hy-oiktdk-ibl-jory': '200-M002',
-  'hy-oiktdk-ggl-jory': '200-M003',
-  'hy-ltdk-psyk-jory': ['300-K001', '300-M004'],
-  'hy-ltdk-logo-jory': ['300-K002', '300-M005'],
-  'hy-ltdk-ll-jory': '300-M001',
-  'hy-ltdk-tmed-jory': '300-M002',
-  'hy-ltdk-hll-jory': '300-M003',
-  'hy-humtdk-filk-jory': '400-K001',
-  'hy-humtdk-ttk-jory': '400-K002',
-  'hy-humtdk-kik-jory': '400-K003',
-  'hy-humtdk-kok-jory': '400-K004',
-  'hy-humtdk-kuka-jory': '400-K005',
-  'hy-humtdk-hisk-jory': '400-K006',
-  'hy-humtdk-ttm-jory': '400-M001',
-  'hy-humtdk-kim-jory': '400-M002',
-  'hy-humtdk-eng-jory': '400-M003',
-  'hy-humtdk-rus-jory': '400-M004',
-  'hy-humtdk-lingdig-jory': '400-M005',
-  'hy-humtdk-tra-jory': '400-M006',
-  'hy-humtdk-suku-jory': '400-M007',
-  'hy-humtdk-nor-jory': '400-M008',
-  'hy-humtdk-kir-jory': '400-M009',
-  'hy-humtdk-kuma-jory': '400-M010',
-  'hy-humtdk-ice-jory': '400-M011',
-  'hy-humtdk-alku-jory': '400-M012',
-  'hy-humtdk-spt-jory': '400-M014',
-  'hy-humtdk-hism-jory': '400-M015',
-  'hy-mltdk-mat-jory': '500-K001',
-  'hy-mltdk-fys-jory': '500-K002',
-  'hy-mltdk-kemk-jory': '500-K003',
-  'hy-mltdk-mfkk-jory': '500-K004',
-  'hy-mltdk-tkt-jory': '500-K005',
-  'hy-mltdk-geok-jory': '500-K006',
-  'hy-mltdk-maa-jory': '500-K007',
-  'hy-mltdk-bsc-jory': '500-K008',
-  'hy-mltdk-mast-jory': '500-M001',
-  'hy-mltdk-lsi-jory': '500-M002',
-  'hy-mltdk-tcm-jory': '500-M003',
-  'hy-mltdk-paras-jory': '500-M004',
-  'hy-mltdk-matres-jory': '500-M005',
-  'hy-mltdk-atm-jory': '500-M006',
-  'hy-mltdk-kem-jory': '500-M007',
-  'hy-mltdk-mfkm-jory': '500-M008',
-  'hy-mltdk-csm-jory': '500-M009',
-  'hy-mltdk-data-jory': '500-M010',
-  'hy-mltdk-geom-jory': '500-M011',
-  'hy-mltdk-geog-jory': '500-M012',
-  'hy-mltdk-usp-jory': '500-M013',
-  'hy-ftdk-farm-jory': '550-K001',
-  'hy-ftdk-prov-jory': '550-M001',
-  'hy-ftdk-mpharm-jory': '550-M002',
-  'hy-bytdk-bio-jory': '570-K001',
-  'hy-bytdk-mole-jory': '570-K002',
-  'hy-bytdk-env-jory': '570-K003',
-  'hy-bytdk-eeb-jory': '570-M001',
-  'hy-bytdk-ips-jory': '570-M002',
-  'hy-bytdk-gmb-jory': '570-M003',
-  'hy-bytdk-neuro-jory': '570-M004',
-  'hy-bytdk-ecgs-jory': '570-M005',
-  'hy-ktdk-eduk-jory': '600-K001',
-  'hy-ktdk-edum-jory': '600-M001',
-  'hy-ktdk-ce-jory': '600-M002',
-  'hy-valttdk-pvk-jory': '700-K001',
-  'hy-valttdk-yk-jory': '700-K002',
-  'hy-valttdk-sosk-jory': '700-K003',
-  'hy-valttdk-ecok-jory': '700-K004',
-  'hy-valttdk-film-jory': '700-M001',
-  'hy-valttdk-pvm-jory': '700-M002',
-  'hy-valttdk-gpc-jory': '700-M003',
-  'hy-valttdk-ym-jory': '700-M004',
-  'hy-valttdk-cos-jory': '700-M005',
-  'hy-valttdk-ens-jory': '700-M006',
-  'hy-valttdk-msv-jory': '700-M007',
-  'hy-valttdk-sosm-jory': '700-M008',
-  'hy-valttdk-econ-jory': '700-M009',
-  'hy-valttdk-sote-jory': '700-M011',
-  'hy-sskh-ksv-jory': '740-K001',
-  'hy-mmtdk-maatk-jory': '800-K001',
-  'hy-mmtdk-metsak-jory': '800-K002',
-  'hy-mmtdk-etk-jory': '800-K003',
-  'hy-mmtdk-yet-jory': '800-K004',
-  'hy-mmtdk-agri-jory': '800-M001',
-  'hy-mmtdk-agere-jory': '800-M002',
-  'hy-mmtdk-for-jory': '800-M003',
-  'hy-mmtdk-food-jory': '800-M004',
-  'hy-mmtdk-hnfb-jory': '800-M005',
-  'hy-mmtdk-ekm-jory': '800-M006',
-  'hy-mmtdk-mmb-jory': '800-M007',
-  'hy-eltdk-elk-jory': '900-K001',
-  'hy-eltdk-ell-jory': '900-M001',
-  'hy-dp-dptheol-jory': 'T920101',
-  'hy-dp-dplaw-jory': 'T920102',
-  'hy-dp-philartsoc-jory': 'T920103',
-  'hy-dp-dphistcult-jory': 'T920104',
-  'hy-dp-dplang-jory': 'T920105',
-  'hy-dp-sky-jory': 'T920106',
-  'hy-dp-dpsocs-jory': 'T920107',
-  'hy-dp-pyam-jory': 'T920108',
-  'hy-dp-dpe-jory': 'T920109',
-  'hy-dp-seduce-jory': 'T920110',
-  'hy-dp-clic-jory': 'T920111',
-  'hy-dp-dpbm-jory': 'T921101',
-  'hy-dp-klto-jory': 'T921102',
-  'hy-dp-docpop-jory': 'T921103',
-  'hy-dp-findos-jory': 'T921104',
-  'hy-dp-dpdr-jory': 'T921105',
-  'hy-dp-ils-jory': 'T921106',
-  'hy-dp-bandm-jory': 'T921107',
-  'hy-dp-cvm-jory': 'T921108',
-  'hy-dp-dphub-jory': 'T921109',
-  'hy-dp-luova-jory': 'T922101',
-  'hy-dp-dpps-jory': 'T922102',
-  'hy-dp-denvi-jory': 'T922103',
-  'hy-dp-agforee-jory': 'T922104',
-  'hy-dp-mbdp-jory': 'T922105',
-  'hy-dp-foodhealth-jory': 'T922106',
-  'hy-dp-papu-jory': 'T923101',
-  'hy-dp-geodoc-jory': 'T923102',
-  'hy-dp-atm-dp-jory': 'T923103',
-  'hy-dp-chems-jory': 'T923104',
-  'hy-dp-domast-jory': 'T923105',
-  'hy-dp-matrena-jory': 'T923106',
-  'hy-dp-docs-jory': 'T923107',
-}
-
-const kojoMap = {
-  'hy-ttdk-tuk-jory': 'hy-ttdk-kandi-kojot',
-  'hy-ttdk-tum-jory': 'hy-ttdk-maisteri-kojot',
-  'hy-oiktdk-on-jory': 'hy-oiktdk-kandi-kojot',
-  'hy-oiktdk-otm-jory': 'hy-oiktdk-maisteri-kojot',
-  'hy-oiktdk-ibl-jory': 'hy-oiktdk-maisteri-kojot',
-  'hy-oiktdk-ggl-jory': 'hy-oiktdk-maisteri-kojot',
-  'hy-ltdk-psyk-jory': 'hy-ltdk-maisteri-kojot',
-  'hy-ltdk-logo-jory': 'hy-ltdk-maisteri-kojot',
-  'hy-ltdk-ll-jory': 'hy-ltdk-maisteri-kojot',
-  'hy-ltdk-tmed-jory': 'hy-ltdk-maisteri-kojot',
-  'hy-ltdk-hll-jory': 'hy-ltdk-maisteri-kojot',
-  'hy-humtdk-filk-jory': 'hy-humtdk-kandi-kojot',
-  'hy-humtdk-ttk-jory': 'hy-humtdk-kandi-kojot',
-  'hy-humtdk-kik-jory': 'hy-humtdk-kandi-kojot',
-  'hy-humtdk-kok-jory': 'hy-humtdk-kandi-kojot',
-  'hy-humtdk-kuka-jory': 'hy-humtdk-kandi-kojot',
-  'hy-humtdk-hisk-jory': 'hy-humtdk-kandi-kojot',
-  'hy-humtdk-ttm-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-kim-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-eng-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-rus-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-lingdig-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-tra-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-suku-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-nor-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-kir-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-kuma-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-ice-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-alku-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-spt-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-humtdk-hism-jory': 'hy-humtdk-maisteri-kojot',
-  'hy-mltdk-mat-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-fys-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-kemk-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-mfkk-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-tkt-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-geok-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-maa-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-bsc-jory': 'hy-mltdk-kandi-kojot',
-  'hy-mltdk-mast-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-lsi-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-tcm-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-paras-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-matres-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-atm-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-kem-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-mfkm-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-csm-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-data-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-geom-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-geog-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-mltdk-usp-jory': 'hy-mltdk-maisteri-kojot',
-  'hy-ftdk-farm-jory': 'hy-ftdk-kandi-kojot',
-  'hy-ftdk-prov-jory': 'hy-ftdk-maisteri-kojot',
-  'hy-ftdk-mpharm-jory': 'hy-ftdk-maisteri-kojot',
-  'hy-bytdk-bio-jory': 'hy-bytdk-kandi-kojot',
-  'hy-bytdk-mole-jory': 'hy-bytdk-kandi-kojot',
-  'hy-bytdk-env-jory': 'hy-bytdk-kandi-kojot',
-  'hy-bytdk-eeb-jory': 'hy-bytdk-maisteri-kojot',
-  'hy-bytdk-ips-jory': 'hy-bytdk-maisteri-kojot',
-  'hy-bytdk-gmb-jory': 'hy-bytdk-maisteri-kojot',
-  'hy-bytdk-neuro-jory': 'hy-bytdk-maisteri-kojot',
-  'hy-bytdk-ecgs-jory': 'hy-bytdk-maisteri-kojot',
-  'hy-ktdk-eduk-jory': 'hy-ktdk-kandi-kojot',
-  'hy-ktdk-edum-jory': 'hy-ktdk-maisteri-kojot',
-  'hy-ktdk-ce-jory': 'hy-ktdk-maisteri-kojot',
-  'hy-valttdk-pvk-jory': 'hy-valttdk-kandi-kojot',
-  'hy-valttdk-yk-jory': 'hy-valttdk-kandi-kojot',
-  'hy-valttdk-sosk-jory': 'hy-valttdk-kandi-kojot',
-  'hy-valttdk-ecok-jory': 'hy-valttdk-kandi-kojot',
-  'hy-valttdk-film-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-pvm-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-gpc-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-ym-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-cos-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-ens-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-msv-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-sosm-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-econ-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-valttdk-sote-jory': 'hy-valttdk-maisteri-kojot',
-  'hy-sskh-ksv-jory': 'hy-sskh-kandi-kojot',
-  'hy-mmtdk-maatk-jory': 'hy-mmtdk-kandi-kojot',
-  'hy-mmtdk-metsak-jory': 'hy-mmtdk-kandi-kojot',
-  'hy-mmtdk-etk-jory': 'hy-mmtdk-kandi-kojot',
-  'hy-mmtdk-yet-jory': 'hy-mmtdk-kandi-kojot',
-  'hy-mmtdk-agri-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-mmtdk-agere-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-mmtdk-for-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-mmtdk-food-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-mmtdk-hnfb-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-mmtdk-ekm-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-mmtdk-mmb-jory': 'hy-mmtdk-maisteri-kojot',
-  'hy-eltdk-elk-jory': 'hy-eltdk-kandi-kojot',
-  'hy-eltdk-ell-jory': 'hy-eltdk-maisteri-kojot',
-  'hy-dp-dptheol-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dplaw-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-philartsoc-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dphistcult-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dplang-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-sky-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dpsocs-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-pyam-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dpe-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-seduce-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-clic-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dpbm-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-klto-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-docpop-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-findos-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dpdr-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-ils-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-bandm-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-cvm-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dphub-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-luova-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-dpps-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-denvi-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-agforee-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-mbdp-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-foodhealth-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-papu-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-geodoc-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-atm-dp-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-chems-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-domast-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-matrena-jory': 'hy-tohtoriohjelma-johtajat',
-  'hy-dp-docs-jory': 'hy-tohtoriohjelma-johtajat',
-}
-
-/**
- * Maps kosu IAM to the name of TDKs, defined in data.js facultyMap
- */
-const kosuFacultyMap = {
-  'hy-ypa-opa-kosu-kumpula': 'matemaattis-luonnontieteellinen',
-  'hy-ypa-opa-kosu-meilahti': ['lääketieteellinen'],
-  'hy-ypa-opa-kosu-kruununhaka': [
-    'oikeustieteellinen',
-    'valtiotieteellinen',
-    'kasvatustieteellinen',
-    'humanistinen',
-    'teologinen',
-    'svenska',
-  ],
-  'hy-ypa-opa-kosu-metsatalo': [
-    'oikeustieteellinen',
-    'valtiotieteellinen',
-    'kasvatustieteellinen',
-    'humanistinen',
-    'teologinen',
-    'svenska',
-  ],
-  'hy-ypa-opa-kosu-porthania': [
-    'oikeustieteellinen',
-    'valtiotieteellinen',
-    'kasvatustieteellinen',
-    'humanistinen',
-    'teologinen',
-    'svenska',
-  ],
-  'hy-ypa-opa-kosu-siltavuori': [
-    'oikeustieteellinen',
-    'valtiotieteellinen',
-    'kasvatustieteellinen',
-    'humanistinen',
-    'teologinen',
-    'svenska',
-  ],
-  'hy-ypa-opa-kosu-viikki': [
-    'bio- ja ympäristötieteellinen',
-    'farmasia',
-    'maatalous -metsätieteellinen',
-    'eläinlääketieteellinen',
-  ],
-}
-
-const doctoralIams = ['hy-tohtorikoulutus-johtoryhma', 'hy-tine']
-
-const doctoralSchoolMap = {
-  'hy-tutkijakoulut-hymy-jory': [
-    'T920101',
-    'T920102',
-    'T920103',
-    'T920104',
-    'T920105',
-    'T920106',
-    'T920107',
-    'T920108',
-    'T920109',
-    'T920110',
-    'T920111',
-  ],
-  'hy-tutkijakoulut-dshealth-jory': [
-    'T921101',
-    'T921102',
-    'T921103',
-    'T921104',
-    'T921105',
-    'T921106',
-    'T921107',
-    'T921108',
-    'T921109',
-  ],
-  'hy-tutkijakoulut-yeb-jory': ['T922101', 'T922102', 'T922103', 'T922104', 'T922105', 'T922106'],
-  'hy-tutkijakoulut-donasci-jory': ['T923101', 'T923102', 'T923103', 'T923104', 'T923105', 'T923106', 'T923107'],
-}
-
-const opetusVaradekaani = 'hy-varadekaanit-opetus'
-
-const dekaaniFacultyMap = {
-  'hy-ttdk-dekanaatti': 'teologinen',
-  'hy-oiktdk-dekanaatti': 'oikeustieteellinen',
-  'hy-ltdk-dekanaatti': 'lääketieteellinen',
-  'hy-humtdk-dekanaatti': 'humanistinen',
-  'hy-mltdk-dekanaatti': 'matemaattis-luonnontieteellinen',
-  'hy-ftdk-dekanaatti': 'farmasia',
-  'hy-bytdk-dekanaatti': 'bio- ja ympäristötieteellinen',
-  'hy-ktdk-dekanaatti': 'kasvatustieteellinen',
-  'hy-valttdk-dekanaatti': 'valtiotieteellinen',
-  'hy-sskh-rehtoraatti': 'svenska',
-  'hy-mmtdk-dekanaatti': 'maatalous -metsätieteellinen',
-  'hy-eltdk-dekanaatti': 'eläinlääketieteellinen',
-}
-
-const oodikoneIams = [
-  'grp-oodikone-staging-users',
-  'grp-oodikone-basic-staging-users',
-  'grp-oodikone-users',
-  'grp-oodikone-basic-users',
-]
-
-const universityWideGroups = ['hy-ypa-opa-opintoasiainpaallikot', 'hy-rehtoraatti']
-
-const superAdminGroups = ['grp-toska']
-
-const adminGroups = ['grp-ospa']
-
-const employeeGroups = ['hy-employees']
-
-const isSuperAdminIam = iam => superAdminGroups.includes(iam)
-
-const isAdminIam = iam => adminGroups.includes(iam)
-
-const isUniversityWideIam = iam => universityWideGroups.includes(iam)
-
-const isDoctoralIam = iam => doctoralIams.includes(iam)
-
-const isEmployeeIam = iam => employeeGroups.includes(iam)
-
-const iamToDoctoralSchool = iam => doctoralSchoolMap[iam]
-
-const getStudyLeaderGroup = iam => kojoMap[iam]
-
-const iamToOrganisationCode = iam => {
-  const organisationCodes = joryMap[iam]
-  if (Array.isArray(organisationCodes)) {
-    return organisationCodes
-  }
-  return [organisationCodes]
-}
-
-const kosuIamToFaculties = iam => {
-  const faculties = kosuFacultyMap[iam]
-  if (!Array.isArray(faculties) || !faculties.length) return []
-
-  return faculties.map(f => facultyMap[f])
-}
-
-const dekaaniIamToFaculty = iam => {
-  const faculty = dekaaniFacultyMap[iam]
-  if (!faculty) return null
-  return facultyMap[faculty]
-}
-
-const relevantIAMs = []
-  .concat(Object.keys(joryMap))
-  .concat(Object.keys(kojoMap))
-  .concat(Object.values(kojoMap))
-  .concat(Object.keys(kosuFacultyMap))
-  .concat(Object.keys(dekaaniFacultyMap))
-  .concat(opetusVaradekaani)
-  .concat(doctoralIams)
-  .concat(Object.keys(doctoralSchoolMap))
-  .concat(universityWideGroups)
-  .concat(superAdminGroups)
-  .concat(adminGroups)
-  .concat(employeeGroups)
-  .concat(oodikoneIams)
-
-module.exports = {
-  isSuperAdminIam,
-  isAdminIam,
-  isUniversityWideIam,
-  isDoctoralIam,
-  isEmployeeIam,
-  iamToDoctoralSchool,
-  iamToOrganisationCode,
-  kosuIamToFaculties,
-  dekaaniIamToFaculty,
-  opetusVaradekaani,
-  getStudyLeaderGroup,
-  relevantIAMs,
-}

services/backend/config/common.js

@@ -1,19 +0,0 @@
-const mapToDegreeCode = organisationCode => {
-  if (!organisationCode) return ''
-  if (organisationCode.length < 7) return ''
-  const doctoral = organisationCode[0] === 'T'
-  if (doctoral) {
-    return organisationCode
-  }
-
-  const [start, end] = organisationCode.split('-')
-  if (end && end.length < 3) return ''
-  if (start.length < 2) return ''
-  const masters = end[0] === 'M'
-  const code = `${masters ? 'M' : 'K'}H${start.substr(0, 2)}_${end.substr(-3)}`
-  return code
-}
-
-module.exports = {
-  mapToDegreeCode,
-}

services/backend/config/programmeCodes.js

@@ -0,0 +1,139 @@
+// Map programme codes to "letter indentifier"
+
+const codes = {
+  KH10_001: 'tuk',
+  MH10_001: 'tum',
+  KH20_001: 'on',
+  MH20_001: 'otm',
+  MH20_002: 'ibl',
+  MH20_003: 'ggl',
+  KH30_001: 'psyk',
+  MH30_004: 'psym',
+  KH30_002: 'logk',
+  MH30_005: 'logm',
+  MH30_001: 'll',
+  MH30_002: 'tmed',
+  MH30_003: 'hll',
+  KH40_001: 'filk',
+  KH40_002: 'ttk',
+  KH40_003: 'kik',
+  KH40_004: 'kok',
+  KH40_005: 'kuka',
+  KH40_006: 'hisk',
+  MH40_001: 'ttm',
+  MH40_002: 'kim',
+  MH40_003: 'eng',
+  MH40_004: 'rus',
+  MH40_005: 'lingdig',
+  MH40_006: 'tra',
+  MH40_007: 'suku',
+  MH40_008: 'nor',
+  MH40_009: 'kir',
+  MH40_010: 'kuma',
+  MH40_011: 'ice',
+  MH40_012: 'alku',
+  MH40_014: 'spt',
+  MH40_015: 'hism',
+  KH50_001: 'mat',
+  KH50_002: 'fys',
+  KH50_003: 'kemk',
+  KH50_004: 'mfkk',
+  KH50_005: 'tkt',
+  KH50_006: 'geok',
+  KH50_007: 'maa',
+  KH50_008: 'bsc',
+  MH50_001: 'mast',
+  MH50_002: 'lsi',
+  MH50_003: 'tcm',
+  MH50_004: 'paras',
+  MH50_005: 'matres',
+  MH50_006: 'atm',
+  MH50_007: 'kem',
+  MH50_008: 'mfkm',
+  MH50_009: 'csm',
+  MH50_010: 'data',
+  MH50_011: 'geom',
+  MH50_012: 'geog',
+  MH50_013: 'usp',
+  KH55_001: 'farm',
+  MH55_001: 'prov',
+  MH55_002: 'mpharm',
+  KH57_001: 'bio',
+  KH57_002: 'mole',
+  KH57_003: 'env',
+  MH57_001: 'eeb',
+  MH57_002: 'ips',
+  MH57_003: 'gmb',
+  MH57_004: 'neuro',
+  MH57_005: 'ecgs',
+  KH60_001: 'eduk',
+  MH60_001: 'edum',
+  MH60_002: 'ce',
+  KH70_001: 'pvk',
+  KH70_002: 'yk',
+  KH70_003: 'sosk',
+  KH70_004: 'ecok',
+  MH70_001: 'film',
+  MH70_002: 'pvm',
+  MH70_003: 'gpc',
+  MH70_004: 'ym',
+  MH70_005: 'cos',
+  MH70_006: 'ens',
+  MH70_007: 'msv',
+  MH70_008: 'sosm',
+  MH70_009: 'econ',
+  MH70_011: 'sote',
+  KH74_001: 'ksv',
+  KH80_001: 'maatk',
+  KH80_002: 'metsak',
+  KH80_003: 'etk',
+  KH80_004: 'yet',
+  MH80_001: 'agri',
+  MH80_002: 'agere',
+  MH80_003: 'for',
+  MH80_004: 'food',
+  MH80_005: 'hnfb',
+  MH80_006: 'ekm',
+  MH80_007: 'mmb',
+  KH90_001: 'elk',
+  MH90_001: 'ell',
+  T920101: 'dptheol',
+  T920102: 'dplaw',
+  T920111: 'philartsoc',
+  T920103: 'dphistcult',
+  T920104: 'dplang',
+  T920105: 'sky',
+  T920106: 'dpsocs',
+  T920107: 'pyam',
+  T920108: 'dpe',
+  T920109: 'seduce',
+  T920110: 'clic',
+  T921101: 'dpbm',
+  T921102: 'klto',
+  T921103: 'docpop',
+  T921104: 'findos',
+  T921105: 'dpdr',
+  T921106: 'ils',
+  T921107: 'bandm',
+  T921108: 'cvm',
+  T921109: 'dphub',
+  T922101: 'luova',
+  T922102: 'dpps',
+  T922103: 'denvi',
+  T922104: 'agforee',
+  T922105: 'mbdp',
+  T922106: 'foodhealth',
+  T923101: 'papu',
+  T923102: 'geodoc',
+  T923103: 'atm-dp',
+  T923104: 'chems',
+  T923105: 'domast',
+  T923106: 'matrena',
+  T923107: 'docs',
+  // manually added programs with no IAM, either abolished, or special co-operation programmes
+  MH50_014: 'ech',
+  MH40_013: 'mkk',
+  MH70_010: 'imes',
+}
+
+module.exports = { codes }

services/backend/package-lock.json

@@ -14,7 +14,7 @@
         "bluebird": "^3.7.2",
         "cors": "^2.8.5",
         "cron": "^1.8.2",
-        "express": "^4.17.3",
+        "express": "^4.18.2",
         "express-async-errors": "^3.1.1",
         "lodash": "^4.17.21",
         "lru-cache": "^5.1.1",
@@ -23,11 +23,11 @@
         "morgan": "^1.10.0",
         "pg": "^8.8.0",
         "redis": "^3.1.2",
-        "sequelize": "^6.21.4",
+        "sequelize": "^6.24.0",
         "umzug": "^2.3.0",
         "unfuck-utf8-headers-middleware": "^1.0.1",
         "uuid": "^3.4.0",
-        "winston": "^3.6.0"
+        "winston": "^3.8.2"
       },
       "devDependencies": {
         "nodemon": "^1.19.4"
@@ -495,9 +495,9 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "node_modules/body-parser": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
-      "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
       "dependencies": {
         "bytes": "3.1.2",
         "content-type": "~1.0.4",
@@ -507,7 +507,7 @@
         "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "on-finished": "2.4.1",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "raw-body": "2.5.1",
         "type-is": "~1.6.18",
         "unpipe": "1.0.0"
@@ -1178,13 +1178,13 @@
       }
     },
     "node_modules/express": {
-      "version": "4.18.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz",
-      "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==",
+      "version": "4.18.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.0",
+        "body-parser": "1.20.1",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
         "cookie": "0.5.0",
@@ -1203,7 +1203,7 @@
         "parseurl": "~1.3.3",
         "path-to-regexp": "0.1.7",
         "proxy-addr": "~2.0.7",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "range-parser": "~1.2.1",
         "safe-buffer": "5.2.1",
         "send": "0.18.0",
@@ -1407,9 +1407,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "node_modules/get-intrinsic": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz",
-      "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
+      "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
       "dependencies": {
         "function-bind": "^1.1.1",
         "has": "^1.0.3",
@@ -2785,9 +2785,9 @@
       "dev": true
     },
     "node_modules/qs": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
-      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "dependencies": {
         "side-channel": "^1.0.4"
       },
@@ -3013,9 +3013,9 @@
       }
     },
     "node_modules/retry-as-promised": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
-      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-6.1.0.tgz",
+      "integrity": "sha512-Hj/jY+wFC+SB9SDlIIFWiGOHnNG0swYbGYsOj2BJ8u2HKUaobNKab0OIC0zOLYzDy0mb7A4xA5BMo4LMz5YtEA=="
     },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
@@ -3108,9 +3108,9 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/sequelize": {
-      "version": "6.21.4",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.21.4.tgz",
-      "integrity": "sha512-5A0+giRhGHerTDRMsZ54TYRB8oQPWxeVscbc4USG9wRtw2Eqik0Vk0p2EVDrhoq7tmNBh2nHpd9YMfvGdwPEJw==",
+      "version": "6.24.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.24.0.tgz",
+      "integrity": "sha512-mPo7Q7gWkrsstjR2aw8ahkrj8RUJCozz3kedAz2B5ZIdLoQEH1z2tvaeJONI8R6RqeuyRQosx9Yn3s9yQyo6lQ==",
       "funding": [
         {
           "type": "opencollective",
@@ -3127,7 +3127,7 @@
         "moment": "^2.29.1",
         "moment-timezone": "^0.5.34",
         "pg-connection-string": "^2.5.0",
-        "retry-as-promised": "^5.0.0",
+        "retry-as-promised": "^6.1.0",
         "semver": "^7.3.5",
         "sequelize-pool": "^7.1.0",
         "toposort-class": "^1.0.1",
@@ -3148,6 +3148,9 @@
         "mysql2": {
           "optional": true
         },
+        "oracledb": {
+          "optional": true
+        },
         "pg": {
           "optional": true
         },
@@ -4035,10 +4038,11 @@
       }
     },
     "node_modules/winston": {
-      "version": "3.8.1",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.1.tgz",
-      "integrity": "sha512-r+6YAiCR4uI3N8eQNOg8k3P3PqwAm20cLKlzVD9E66Ch39+LZC+VH1UKf9JemQj2B3QoUHfKD7Poewn0Pr3Y1w==",
+      "version": "3.8.2",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.2.tgz",
+      "integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==",
       "dependencies": {
+        "@colors/colors": "1.5.0",
         "@dabh/diagnostics": "^2.0.2",
         "async": "^3.2.3",
         "is-stream": "^2.0.0",
@@ -4526,9 +4530,9 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "body-parser": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
-      "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
       "requires": {
         "bytes": "3.1.2",
         "content-type": "~1.0.4",
@@ -4538,7 +4542,7 @@
         "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "on-finished": "2.4.1",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "raw-body": "2.5.1",
         "type-is": "~1.6.18",
         "unpipe": "1.0.0"
@@ -5085,13 +5089,13 @@
       }
     },
     "express": {
-      "version": "4.18.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz",
-      "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==",
+      "version": "4.18.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
       "requires": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.0",
+        "body-parser": "1.20.1",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
         "cookie": "0.5.0",
@@ -5110,7 +5114,7 @@
         "parseurl": "~1.3.3",
         "path-to-regexp": "0.1.7",
         "proxy-addr": "~2.0.7",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "range-parser": "~1.2.1",
         "safe-buffer": "5.2.1",
         "send": "0.18.0",
@@ -5261,9 +5265,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "get-intrinsic": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz",
-      "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
+      "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
       "requires": {
         "function-bind": "^1.1.1",
         "has": "^1.0.3",
@@ -6330,9 +6334,9 @@
       "dev": true
     },
     "qs": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
-      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "requires": {
         "side-channel": "^1.0.4"
       }
@@ -6509,9 +6513,9 @@
       "dev": true
     },
     "retry-as-promised": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
-      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-6.1.0.tgz",
+      "integrity": "sha512-Hj/jY+wFC+SB9SDlIIFWiGOHnNG0swYbGYsOj2BJ8u2HKUaobNKab0OIC0zOLYzDy0mb7A4xA5BMo4LMz5YtEA=="
     },
     "safe-buffer": {
       "version": "5.2.1",
@@ -6580,9 +6584,9 @@
       }
     },
     "sequelize": {
-      "version": "6.21.4",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.21.4.tgz",
-      "integrity": "sha512-5A0+giRhGHerTDRMsZ54TYRB8oQPWxeVscbc4USG9wRtw2Eqik0Vk0p2EVDrhoq7tmNBh2nHpd9YMfvGdwPEJw==",
+      "version": "6.24.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.24.0.tgz",
+      "integrity": "sha512-mPo7Q7gWkrsstjR2aw8ahkrj8RUJCozz3kedAz2B5ZIdLoQEH1z2tvaeJONI8R6RqeuyRQosx9Yn3s9yQyo6lQ==",
       "requires": {
         "@types/debug": "^4.1.7",
         "@types/validator": "^13.7.1",
@@ -6593,7 +6597,7 @@
         "moment": "^2.29.1",
         "moment-timezone": "^0.5.34",
         "pg-connection-string": "^2.5.0",
-        "retry-as-promised": "^5.0.0",
+        "retry-as-promised": "^6.1.0",
         "semver": "^7.3.5",
         "sequelize-pool": "^7.1.0",
         "toposort-class": "^1.0.1",
@@ -7283,10 +7287,11 @@
       }
     },
     "winston": {
-      "version": "3.8.1",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.1.tgz",
-      "integrity": "sha512-r+6YAiCR4uI3N8eQNOg8k3P3PqwAm20cLKlzVD9E66Ch39+LZC+VH1UKf9JemQj2B3QoUHfKD7Poewn0Pr3Y1w==",
+      "version": "3.8.2",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.2.tgz",
+      "integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==",
       "requires": {
+        "@colors/colors": "1.5.0",
         "@dabh/diagnostics": "^2.0.2",
         "async": "^3.2.3",
         "is-stream": "^2.0.0",

services/backend/package.json

@@ -15,7 +15,7 @@
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
-    "express": "^4.17.3",
+    "express": "^4.18.2",
     "express-async-errors": "^3.1.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
@@ -24,11 +24,11 @@
     "morgan": "^1.10.0",
     "pg": "^8.8.0",
     "redis": "^3.1.2",
-    "sequelize": "^6.21.4",
+    "sequelize": "^6.24.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.6.0"
+    "winston": "^3.8.2"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/conf-backend.js

@@ -16,6 +16,9 @@ const requiredGroup = isStaging
 // Pate
 const pateToken = process.env.PATE_API_TOKEN || ''
 
+// Jami
+const jamiUrl = isDev || runningInCI ? 'http://jami:3003/' : 'https://importer.cs.helsinki.fi/api/auth'
+
 // Importer client
 const importerToken = process.env.IMPORTER_DB_API_TOKEN || ''
 
@@ -79,6 +82,7 @@ module.exports = {
   sentryEnvironment,
   isProduction,
   pateToken,
+  jamiUrl,
   baseUrl,
   frontUrl,
   backendPort,

services/backend/src/database/migrations_kone/20221111_00_create_table_open_uni_population_search.js

@@ -0,0 +1,26 @@
+const { BIGINT, STRING, DATE, ARRAY } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable('open_uni_population_searches', {
+      id: {
+        type: BIGINT,
+        primaryKey: true,
+        autoIncrement: true,
+      },
+      user_id: {
+        type: BIGINT,
+      },
+      name: {
+        type: STRING,
+        allowNull: false,
+      },
+      course_codes: {
+        type: ARRAY(STRING),
+      },
+      created_at: DATE,
+      updated_at: DATE,
+    })
+  },
+  down: () => {},
+}

services/backend/src/database/migrations_kone/20230202_00_create_table_progress_criteria.js

@@ -0,0 +1,33 @@
+const { INTEGER, STRING, ARRAY, DATE } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable('progress_criteria', {
+      code: {
+        type: STRING,
+        primaryKey: true,
+      },
+      courses_year_one: {
+        type: ARRAY(STRING),
+      },
+      courses_year_two: {
+        type: ARRAY(STRING),
+      },
+      courses_year_three: {
+        type: ARRAY(STRING),
+      },
+      credits_year_one: {
+        type: INTEGER,
+      },
+      credits_year_two: {
+        type: INTEGER,
+      },
+      credits_year_three: {
+        type: INTEGER,
+      },
+      created_at: DATE,
+      updated_at: DATE,
+    })
+  },
+  down: () => {},
+}

services/backend/src/database/migrations_kone/20230412_00_add_columns_criteria.js

@@ -0,0 +1,23 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('progress_criteria', 'courses_year_four', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    })
+    await queryInterface.addColumn('progress_criteria', 'courses_year_five', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    })
+    await queryInterface.addColumn('progress_criteria', 'courses_year_six', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    })
+    await queryInterface.addColumn('progress_criteria', 'credits_year_four', {
+      type: Sequelize.INTEGER,
+    })
+    await queryInterface.addColumn('progress_criteria', 'credits_year_five', {
+      type: Sequelize.INTEGER,
+    })
+    await queryInterface.addColumn('progress_criteria', 'credits_year_six', {
+      type: Sequelize.INTEGER,
+    })
+  },
+  down: async () => {},
+}

services/backend/src/database/migrations_user/20221026_00_add_open_uni_dojo_access.js

@@ -0,0 +1,17 @@
+module.exports = {
+  up: queryInterface => {
+    return queryInterface.bulkInsert('access_groups', [
+      {
+        group_code: 'openUniSearch',
+        group_info: 'grants access to custom open uni course searches',
+      },
+    ])
+  },
+  down: queryInterface => {
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: 'openUniSearch',
+      },
+    ])
+  },
+}

services/backend/src/events.js

@@ -6,7 +6,7 @@ const { getAllProgrammes } = require('./services/studyrights')
 const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
 const { faculties } = require('./services/organisations')
-const { updateFacultyOverview } = require('./services/faculty/facultyUpdates')
+const { updateFacultyOverview, updateFacultyProgressOverview } = require('./services/faculty/facultyUpdates')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -26,6 +26,7 @@ const refreshFaculties = async () => {
   for (const faculty of facultyList) {
     try {
       await updateFacultyOverview(faculty.code, 'ALL')
+      await updateFacultyProgressOverview(faculty.code)
       ready += 1
     } catch (e) {
       logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: e })
@@ -37,9 +38,16 @@ const refreshFaculties = async () => {
 
 const refreshNewOverviews = async () => {
   logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
+  // Filters out old programmes and special ones like Fitech studies. Filters also out the programmes starting with
+  // 2_ or ending with _2. Those programmes are mapped to correct programme (studentProgrammeModuleFixer.js)
   const codes = (await getAllProgrammes())
     .map(p => p.code)
-    .filter(code => code.includes('KH') || code.includes('MH') || /^(T)[0-9]{6}$/.test(code))
+    .filter(
+      code =>
+        (code.includes('KH') && !code.startsWith('2_KH') && !code.endsWith('_2')) ||
+        (code.includes('MH') && !code.startsWith('2_MH') && !code.endsWith('_2')) ||
+        /^(T)[0-9]{6}$/.test(code)
+    )
   let ready = 0
   for (const code of codes) {
     try {

services/backend/src/middleware/accessLogger.js

@@ -27,12 +27,16 @@ const accessLogger = morgan((tokens, req, res) => {
     'ms',
   ].join(' ')
 
+  const usingIamRights = user.iamRights.some(programmeCode => tokens['url'](req, res).includes(programmeCode))
+  const onlyIamRights = !user.isAdmin && user.rights.length === 0
+
   logger.info(message, {
     // don't log student list which might be huge
     ..._.omit(meta, ['studentsUserCanAccess']),
     // pass this as a custom field so we can filter by it in graylog
     isUsageStats: true,
-    onlyIamRights: !user.isAdmin && user.roles.length === 0 && user.rights.length === 0,
+    // needed for Grafana IAM users panel
+    isIamRights: onlyIamRights || usingIamRights,
   })
 })
 

services/backend/src/middleware/currentUserMiddleware.js

@@ -3,8 +3,8 @@ const { ApplicationError } = require('../util/customErrors')
 const { getUser, getMockedUser } = require('../services/userService')
 const { requiredGroup } = require('../conf-backend')
 const _ = require('lodash')
-const { relevantIAMs } = require('../../config/IAMConfig')
 const { getOrganizationAccess } = require('../util/organizationAccess')
+const logger = require('../util/logger')
 
 const parseIamGroups = iamGroups => iamGroups?.split(';').filter(Boolean) ?? []
 
@@ -27,11 +27,12 @@ const currentUserMiddleware = async (req, _res, next) => {
     throw new ApplicationError('Not enough data in request headers', 403, { logoutUrl })
   }
 
-  const iamGroups = parseIamGroups(hygroupcn).filter(iam => relevantIAMs.includes(iam))
-
-  const iamRights = Object.keys(await getOrganizationAccess({ iamGroups }))
+  const iamGroups = parseIamGroups(hygroupcn)
+  const { access = {}, specialGroup = {} } = await getOrganizationAccess({ id: sisId, iamGroups })
+  const iamRights = Object.keys(access)
 
   if (!hasRequiredIamGroup(iamGroups, iamRights)) {
+    logger.error({ message: 'User does not have required iam group', meta: { iamGroups, iamRights } })
     throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
   }
 
@@ -40,6 +41,8 @@ const currentUserMiddleware = async (req, _res, next) => {
     name,
     email,
     iamGroups,
+    iamRights,
+    specialGroup,
     sisId,
   })
 

services/backend/src/models/index.js

@@ -67,6 +67,9 @@ Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code
 Studyplan.belongsTo(Student, { foreignKey: 'studentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyplan, { foreignKey: 'studentnumber', sourceKey: 'studentnumber' })
 
+Studyplan.belongsTo(Studyright, { foreignKey: 'studyrightid', targetKey: 'studyrightid' })
+Studyright.hasMany(Studyplan, { foreignKey: 'studyrightid', sourceKey: 'studyrightid' })
+
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })
 

services/backend/src/models/models_kone.js

@@ -1,3 +1,4 @@
+const { STRING } = require('sequelize')
 const Sequelize = require('sequelize')
 const { sequelizeKone } = require('../database/connection')
 
@@ -121,6 +122,76 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
   },
 })
 
+// also used in completed courses search feature
+const OpenUniPopulationSearch = sequelizeKone.define(
+  'open_uni_population_searches',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    userId: {
+      type: Sequelize.BIGINT,
+    },
+    name: {
+      type: Sequelize.STRING,
+      allowNull: false,
+    },
+    courseCodes: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
+const ProgressCriteria = sequelizeKone.define(
+  'progress_criteria',
+  {
+    code: {
+      primaryKey: true,
+      type: STRING,
+    },
+    coursesYearOne: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearTwo: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearThree: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearFour: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearFive: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    coursesYearSix: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+    creditsYearOne: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearTwo: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearThree: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearFour: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearFive: {
+      type: Sequelize.INTEGER,
+    },
+    creditsYearSix: {
+      type: Sequelize.INTEGER,
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 const StudyGuidanceGroupTag = sequelizeKone.define(
   'study_guidance_group_tags',
   {
@@ -162,7 +233,9 @@ module.exports = {
   ThesisTypeEnums,
   MandatoryCourseLabels,
   CustomPopulationSearch,
+  OpenUniPopulationSearch,
   sequelizeKone,
   ExcludedCourse,
   StudyGuidanceGroupTag,
+  ProgressCriteria,
 }

services/backend/src/models/student.js

@@ -16,14 +16,16 @@ Student.init(
     creditcount: { type: INTEGER },
     dateofuniversityenrollment: { type: DATE },
     email: { type: STRING },
+    secondary_email: { type: STRING },
     national_student_number: { type: STRING },
+    phone_number: { type: STRING },
     country_fi: { type: STRING },
     country_sv: { type: STRING },
     country_en: { type: STRING },
     home_country_fi: { type: STRING },
     home_country_sv: { type: STRING },
     home_country_en: { type: STRING },
-    gender_code: { type: INTEGER },
+    gender_code: { type: STRING },
     sis_person_id: { type: STRING },
     createdAt: {
       type: DATE,

services/backend/src/models/studyplan.js

@@ -20,6 +20,16 @@ Studyplan.init(
       onDelete: 'cascade',
       allowNull: false,
     },
+    studyrightid: {
+      type: STRING,
+      references: {
+        model: 'studyright',
+        key: 'studyrightid',
+      },
+      onUpdate: 'cascade',
+      onDelete: 'cascade',
+      allowNull: false,
+    },
     programme_code: {
       type: STRING,
       allowNull: false,
@@ -30,6 +40,9 @@ Studyplan.init(
     completed_credits: {
       type: INTEGER,
     },
+    sisu_id: {
+      type: STRING,
+    },
     createdAt: {
       type: DATE,
     },

services/backend/src/models/studyright.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class Studyright extends Model {}
@@ -27,7 +27,9 @@ Studyright.init(
     active: {
       type: INTEGER,
     },
-    // irtisanomisperuste
+    cancelled: {
+      type: BOOLEAN,
+    },
     studentStudentnumber: {
       type: STRING,
       references: {

services/backend/src/routes.js

@@ -7,7 +7,6 @@ const matomoInit = require('./routes/matomo-init')
 const currentUserMiddleware = require('./middleware/currentUserMiddleware')
 const accessLogger = require('./middleware/accessLogger')
 const auth = require('./middleware/auth')
-
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
@@ -30,8 +29,10 @@ const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 const studyProgramme = require('./routes/studyProgramme')
+const customOpenUniSearch = require('./routes/customOpenUniSearch')
+const studyProgrammeCriteria = require('./routes/studyProgrammeCriteria')
 const initializeSentry = require('./util/sentry')
-
+const completedCoursesSearch = require('./routes/completedCoursesSearch')
 const errorMiddleware = require('./middleware/errorMiddleware')
 
 module.exports = (app, url) => {
@@ -57,6 +58,9 @@ module.exports = (app, url) => {
   app.use(url, tags)
   app.use(url, programmeModules)
   app.use(url, studyProgramme)
+  app.use(`${url}/programmecriteria`, studyProgrammeCriteria)
+  app.use(`${url}/openunisearch`, auth.roles(['openUniSearch']), customOpenUniSearch)
+  app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
   app.use(`${url}/faculties`, auth.roles(['facultyStatistics']), faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)

services/backend/src/routes/completedCoursesSearch.js

@@ -0,0 +1,82 @@
+const router = require('express').Router()
+const { getCompletedCourses } = require('../services/completedCoursesSearch')
+const _ = require('lodash')
+const {
+  getOpenUniSearches,
+  createNewSearch,
+  deleteSearch,
+  updateSearch,
+} = require('../services/openUni/openUniManageSearches')
+
+router.get('/', async (req, res) => {
+  const studentNumbers = JSON.parse(req.query?.studentlist) || []
+  const courseCodes = JSON.parse(req.query?.courselist) || []
+  const {
+    user: { isAdmin, studentsUserCanAccess },
+  } = req
+  if (!Array.isArray(studentNumbers)) return res.status(400).json({ error: 'Student numbers must be of type array' })
+  if (!Array.isArray(courseCodes)) return res.status(400).json({ error: 'Courses must be of type array' })
+
+  const filteredStudentNumbers = isAdmin ? studentNumbers : _.intersection(studentNumbers, studentsUserCanAccess)
+  const completedCourses = await getCompletedCourses(filteredStudentNumbers, courseCodes)
+  const discardedStudentNumbers = _.difference(
+    studentNumbers,
+    completedCourses.students.map(s => s.studentNumber)
+  )
+  return res.json({ discardedStudentNumbers, ...completedCourses })
+})
+
+router.get('/searches', async (req, res) => {
+  const userId = req.user.id
+  const foundSearches = await getOpenUniSearches(userId)
+  return res.json(foundSearches)
+})
+
+router.post('/searches', async (req, res) => {
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  const name = req.body?.name
+
+  if (!name) return res.status(400).json({ error: 'Courselist name missing' })
+  if (courseCodes && !Array.isArray(courseCodes))
+    return res.status(400).json({ error: 'Course codes must be type of array' })
+
+  const createdSearch = await createNewSearch(userId, name, courseCodes)
+  if (!createdSearch) return res.status(400).json({ error: 'Failed to create courselist' })
+  return res.status(201).json({
+    id: createdSearch.id,
+    userId: createdSearch.userId,
+    name: createdSearch.name,
+    courseList: createdSearch.courseCodes,
+    updatedAt: createdSearch.updatedAt,
+  })
+})
+
+router.put('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+  const updatedSearch = await updateSearch(userId, id, courseCodes)
+
+  if (!updatedSearch) return res.status(404).json({ error: 'Courselist could not be found' })
+  return res.json({
+    id: updatedSearch.id,
+    userId: updatedSearch.userId,
+    name: updatedSearch.name,
+    courseList: updatedSearch.courseCodes,
+    updatedAt: updatedSearch.updatedAt,
+  })
+})
+
+router.delete('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+
+  const deletedSearch = await deleteSearch(userId, id)
+  if (!deletedSearch) return res.status(404).json({ error: 'Courselist could not be found' })
+  return res.json(id)
+})
+
+module.exports = router

services/backend/src/routes/courses.js

@@ -1,6 +1,7 @@
 const router = require('express').Router()
 const crypto = require('crypto')
 const Course = require('../services/courses')
+const { faculties } = require('../services/organisations')
 const { validateParamLength } = require('../util')
 
 router.get('/v2/coursesmulti', async (req, res) => {
@@ -43,4 +44,9 @@ router.get('/v3/courseyearlystats', async (req, res) => {
   }
 })
 
+router.get('/faculties-only', async (_req, res) => {
+  const facultyList = await faculties()
+  res.json(facultyList)
+})
+
 module.exports = router

services/backend/src/routes/customOpenUniSearch.js

@@ -0,0 +1,74 @@
+const router = require('express').Router()
+const moment = require('moment-timezone')
+const { getCustomOpenUniCourses } = require('../services/openUni/openUniStats')
+const {
+  getOpenUniSearches,
+  createNewSearch,
+  deleteSearch,
+  updateSearch,
+} = require('../services/openUni/openUniManageSearches')
+
+router.get('/', async (req, res) => {
+  const courseCodes = JSON.parse(req.query?.courselist) || []
+  const startdate = req.query?.startdate ? req.query?.startdate : moment('01-08-2017 00:00:00', 'DD-MM-YYYY')
+  const enddate = req.query?.enddate ? req.query?.enddate : moment().endOf('day')
+
+  if (!Array.isArray(courseCodes)) return res.status(400).json({ error: 'Courses must be of type array' })
+  const customOpenUniSearches = await getCustomOpenUniCourses(courseCodes, startdate, enddate)
+  return res.json(customOpenUniSearches)
+})
+
+router.get('/searches', async (req, res) => {
+  const userId = req.user.id
+  const foundSearches = await getOpenUniSearches(userId)
+  return res.json(foundSearches)
+})
+
+router.post('/searches', async (req, res) => {
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  const name = req.body?.name
+
+  if (!name) return res.status(400).json({ error: 'Name missing' })
+  if (courseCodes && !Array.isArray(courseCodes))
+    return res.status(400).json({ error: 'Course codes must be type of array' })
+
+  const createdSearch = await createNewSearch(userId, name, courseCodes)
+  if (!createdSearch) return res.status(400).json({ error: '' })
+  return res.status(201).json({
+    id: createdSearch.id,
+    userId: createdSearch.userId,
+    name: createdSearch.name,
+    courseList: createdSearch.courseCodes,
+    updatedAt: createdSearch.updatedAt,
+  })
+})
+
+router.put('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+  const updatedSearch = await updateSearch(userId, id, courseCodes)
+
+  if (!updatedSearch) return res.status(404).json({ error: `Open uni search could not be found` })
+  return res.json({
+    id: updatedSearch.id,
+    userId: updatedSearch.userId,
+    name: updatedSearch.name,
+    courseList: updatedSearch.courseCodes,
+    updatedAt: updatedSearch.updatedAt,
+  })
+})
+
+router.delete('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+
+  const deletedSearch = await deleteSearch(userId, id)
+  if (!deletedSearch) return res.status(404).json({ error: `Open uni search could not be found` })
+  return res.json(id)
+})
+
+module.exports = router

services/backend/src/routes/faculties.js

@@ -5,7 +5,10 @@ const { combineFacultyCredits } = require('../services/faculty/facultyCredits')
 const { findFacultyProgrammeCodes } = require('../services/faculty/faculty')
 const { combineFacultyThesisWriters } = require('../services/faculty/facultyThesisWriters')
 const { countGraduationTimes } = require('../services/faculty/facultyGraduationTimes')
-const { updateFacultyOverview } = require('../services/faculty/facultyUpdates')
+const { updateFacultyOverview, updateFacultyProgressOverview } = require('../services/faculty/facultyUpdates')
+const { combineFacultyStudentProgress } = require('../services/faculty/facultyStudentProgress')
+const { combineFacultyStudents } = require('../services/faculty/facultyStudents')
+
 const {
   getFacultyProgrammes,
   setFacultyProgrammes,
@@ -15,6 +18,12 @@ const {
   setCreditStats,
   getThesisWritersStats,
   setThesisWritersStats,
+  getGraduationStats,
+  setGraduationStats,
+  getFacultyStudentStats,
+  setFacultyStudentStats,
+  getFacultyProgressStats,
+  setFacultyProgressStats,
 } = require('../services/faculty/facultyService')
 const logger = require('../util/logger')
 
@@ -33,6 +42,7 @@ const getProgrammes = async (code, programmeFilter) => {
 router.get('/', async (req, res) => {
   const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
   const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
+  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
   res.json(facultyList)
 })
 
@@ -43,10 +53,8 @@ router.get('/:id/basicstats', async (req, res) => {
   const specialGroups = req.query?.special_groups
 
   if (!code) return res.status(422).end()
-
   const data = await getBasicStats(code, yearType, programmeFilter, specialGroups)
   if (data) return res.json(data)
-
   const wantedProgrammes = await getProgrammes(code, programmeFilter)
   if (!wantedProgrammes) return res.status(422).end()
 
@@ -76,20 +84,21 @@ router.get('/:id/basicstats', async (req, res) => {
 router.get('/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
-  const specialGroups = req.query?.special_groups
   const programmeFilter = req.query?.programme_filter
 
   if (!code) return res.status(422).end()
-
-  const data = await getCreditStats(code, yearType, programmeFilter, specialGroups)
+  const data = await getCreditStats(code, yearType, programmeFilter)
   if (data) return res.json(data)
 
   const programmes = await getProgrammes(code, programmeFilter)
   if (!programmes) return res.status(422).end()
 
-  let updatedStats = await combineFacultyCredits(code, programmes.data, yearType, specialGroups)
+  // list of all programmes is also needed in classification of credits
+  const allProgrammes = programmeFilter === 'ALL_PROGRAMMES' ? programmes : await getProgrammes(code, 'ALL_PROGRAMMES')
+
+  let updatedStats = await combineFacultyCredits(code, programmes.data, allProgrammes.data, yearType)
   if (updatedStats) {
-    updatedStats = await setCreditStats(updatedStats, yearType, programmeFilter, specialGroups)
+    updatedStats = await setCreditStats(updatedStats, yearType, programmeFilter)
   }
 
   return res.json(updatedStats)
@@ -104,7 +113,6 @@ router.get('/:id/thesisstats', async (req, res) => {
   if (!code) return res.status(422).end()
   const data = await getThesisWritersStats(code, yearType, programmeFilter, specialGroups)
   if (data) return res.json(data)
-
   const programmes = await getProgrammes(code, programmeFilter)
   if (!programmes) return res.status(422).end()
   let updateStats = await combineFacultyThesisWriters(code, programmes.data, yearType, specialGroups)
@@ -119,11 +127,68 @@ router.get('/:id/graduationtimes', async (req, res) => {
   const programmeFilter = req.query?.programme_filter
 
   if (!code) return res.status(422).end()
+  const data = await getGraduationStats(code, programmeFilter)
+  if (data) {
+    return res.json(data)
+  }
+  let updatedStats = await countGraduationTimes(code, programmeFilter)
+  if (updatedStats) {
+    updatedStats = await setGraduationStats(updatedStats, programmeFilter)
+  }
+  return res.json(updatedStats)
+})
 
-  // const programmes = await getProgrammes(code, programmeFilter)
-  const result = await countGraduationTimes(code, programmeFilter)
-  // currently counts all programmes
-  return res.json(result)
+router.get('/:id/progressstats', async (req, res) => {
+  const code = req.params.id
+  const programmeFilter = req.query.programme_filter
+  const specialGroups = req.query?.special_groups
+  const graduated = req.query?.graduated
+  const programmes = await getProgrammes(code, programmeFilter)
+  if (!programmes) return res.status(422).end()
+
+  //all programmes are required for correct sorting of transfers
+  let allProgrammeCodes = []
+  const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
+  allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+
+  if (!code) return res.status(422).end()
+  const data = await getFacultyProgressStats(code, specialGroups, graduated)
+  if (data) return res.json(data)
+
+  let updateStats = await combineFacultyStudentProgress(
+    code,
+    programmes.data,
+    allProgrammeCodes,
+    specialGroups,
+    graduated
+  )
+  if (updateStats) {
+    updateStats = await setFacultyProgressStats(updateStats, specialGroups, graduated)
+  }
+  return res.json(updateStats)
+})
+
+router.get('/:id/studentstats', async (req, res) => {
+  const code = req.params.id
+  const programmeFilter = req.query.programme_filter
+  const specialGroups = req.query?.special_groups
+  const graduated = req.query?.graduated
+
+  if (!code) return res.status(422).end()
+  const data = await getFacultyStudentStats(code, specialGroups, graduated)
+  if (data) return res.json(data)
+  const programmes = await getProgrammes(code, programmeFilter)
+  if (!programmes) return res.status(422).end()
+  //all programmes are required for correct sorting of transfers
+  let allProgrammeCodes = []
+  const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
+  allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+
+  let updateStats = await combineFacultyStudents(code, programmes.data, allProgrammeCodes, specialGroups, graduated)
+  if (updateStats) {
+    updateStats = await setFacultyStudentStats(updateStats, specialGroups, graduated)
+  }
+  return res.json(updateStats)
 })
 
 router.get('/:id/update_basicview', async (req, res) => {
@@ -140,4 +205,18 @@ router.get('/:id/update_basicview', async (req, res) => {
   }
   return res.status(422).end()
 })
+
+router.get('/:id/update_progressview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateFacultyProgressOverview(code)
+    } catch (e) {
+      logger.error(`Failed to update faculty ${code} progress tab stats: ${e}`)
+    }
+    return res.json(result)
+  }
+  return res.status(422).end()
+})
 module.exports = router

services/backend/src/routes/population.js

@@ -10,6 +10,7 @@ const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
 const { mapToProviders } = require('../util/utils')
 const { encrypt, decrypt } = require('../services/encrypt')
+const { mapCodesToIds } = require('../services/studyprogrammeHelpers')
 
 const { ApplicationError } = require('../util/customErrors')
 
@@ -81,20 +82,6 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
       return
     }
 
-    //These cause problems in front (where they were originally calculated)
-    // Do them here for the moment
-    for (const course of result.coursestatistics) {
-      // course.students.notParticipated = {}
-      course.students.didNotPass = {}
-
-      for (const student of Object.keys(course.students.all)) {
-        if (
-          !Object.keys(course.students.passed).includes(student) &&
-          !Object.keys(course.students.retryPassed).includes(student)
-        )
-          course.students.didNotPass[student] = true
-      }
-    }
     res.json(result)
   }
 })
@@ -252,7 +239,13 @@ router.get('/v3/populationstatistics', async (req, res) => {
 
     res.json(filterPersonalTags(result, user.id))
   } else {
-    const result = await Population.optimizedStatisticsOf({ ...req.query, studyRights })
+    // If two studyprogrammes are given, query does not work just right
+    // This solution is temporal
+    const result = await Population.optimizedStatisticsOf({
+      ...req.query,
+      studyRights: { programme: studyRights.programme },
+    })
+
     if (result.error) {
       Sentry.captureException(new Error(result.error))
       res.status(400).end()
@@ -268,6 +261,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
           ...student,
           firstnames: '',
           lastname: '',
+          phone_number: '',
           started: null,
           iv,
           studentNumber: encryptedData,
@@ -380,6 +374,7 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     studentNumber: crypto.createHash('md5').update(`${studentNumber}${randomHash}`).digest('hex'),
     firstnames: '',
     lastname: '',
+    phone_number: '',
     name: '',
     email: '',
     started: null,
@@ -400,17 +395,16 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 })
 
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
-  const { studentnumberlist } = req.body
+  const { studentnumberlist, tags } = req.body
   const {
     user: { isAdmin, id, studentsUserCanAccess },
   } = req
-
   const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
 
   const result = await Population.optimizedStatisticsOf(
     {
-      year: 1900,
-      studyRights: [],
+      year: tags?.year || 1900,
+      studyRights: tags?.studyProgramme ? [tags.studyProgramme] : [],
       semesters: ['FALL', 'SPRING'],
       months: 10000,
     },
@@ -429,9 +423,11 @@ router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
   } = req
   if (roles?.includes('admin')) {
     const studyrights = await StudyrightService.getAssociations()
+    mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   } else {
     const studyrights = await StudyrightService.getFilteredAssociations(rights.concat(iamRights))
+    mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   }
 })

services/backend/src/routes/studyProgramme.js

@@ -15,6 +15,7 @@ const {
   setStudytrackStats,
 } = require('../services/analyticsService')
 const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
+const { getProgrammeName } = require('../services/studyprogramme')
 const logger = require('../util/logger')
 
 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
@@ -143,4 +144,73 @@ router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) =>
   }
 })
 
+router.get('/v2/studyprogrammes/:id/evaluationstats', async (req, res) => {
+  const code = req.params.id
+  const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
+  const graduated = req.query?.graduated
+
+  if (!code) return res.status(422).end()
+
+  // Statistics for Tilannekuvalomake view
+
+  let gradData = await getGraduationStats(code, yearType, specialGroups)
+  if (!gradData) {
+    const updatedStats = await getGraduationStatsForStudytrack({
+      studyprogramme: req.params.id,
+      settings: {
+        isAcademicYear: yearType === 'ACADEMIC_YEAR',
+        includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
+      },
+    })
+    if (updatedStats) {
+      await setGraduationStats(updatedStats, yearType, specialGroups)
+      gradData = updatedStats
+    }
+  }
+
+  let progressData = await getStudytrackStats(code, graduated, specialGroups)
+  if (!progressData) {
+    const updated = await getStudytrackStatsForStudyprogramme({
+      studyprogramme: code,
+      settings: {
+        graduated: graduated === 'GRADUATED_INCLUDED',
+        specialGroups: specialGroups === 'SPECIAL_INCLUDED',
+      },
+    })
+    if (updated) {
+      await setStudytrackStats(updated, graduated, specialGroups)
+      progressData = updated
+    }
+  }
+
+  const programmeName = await getProgrammeName(code)
+
+  delete gradData.tableStats
+  delete gradData.graphStats
+  delete gradData.titles
+
+  const data = {
+    id: code,
+    programmeName: programmeName?.name,
+    status: gradData?.status,
+    lastUpdated: gradData.lastUpdated,
+    graduations: gradData,
+    progress: {
+      creditTableStats: {},
+      creditTableTitles: progressData?.creditTableTitles,
+      graphData: {
+        creditGraphStats: {},
+        id: progressData?.id,
+        years: progressData?.years,
+      },
+    },
+  }
+
+  data.progress.creditTableStats[code] = progressData?.creditTableStats[code]
+  data.progress.graphData.creditGraphStats[code] = progressData?.creditGraphStats[code]
+
+  return res.json(data)
+})
+
 module.exports = router

services/backend/src/routes/studyProgrammeCriteria.js

@@ -0,0 +1,30 @@
+const {
+  getCriteria,
+  saveYearlyCourseCriteria,
+  saveYearlyCreditCriteria,
+} = require('../services/studyProgrammeCriteria')
+
+const router = require('express').Router()
+
+router.get('/', async (req, res) => {
+  const studyProgramme = req.query?.programmecode
+  if (studyProgramme !== '' && !studyProgramme) return res.status(422).end()
+  const studyProgrammeCriteria = await getCriteria(studyProgramme)
+  return res.json(studyProgrammeCriteria)
+})
+
+router.post('/courses', async (req, res) => {
+  const { code, courses, year } = req.body
+  if (!code || !courses || !year) return res.status(400).end()
+  const studyProgrammeCriteria = await saveYearlyCourseCriteria(code, courses, year)
+  return res.json(studyProgrammeCriteria)
+})
+
+router.post('/credits', async (req, res) => {
+  const { code, credits } = req.body
+  if (!code || !credits) return res.status(400).end()
+  const studyProgrammeCriteria = await saveYearlyCreditCriteria(code, credits)
+  return res.json(studyProgrammeCriteria)
+})
+
+module.exports = router

services/backend/src/services/completedCoursesSearch.js

@@ -0,0 +1,162 @@
+const { Credit, Enrollment, Student, Course } = require('../models')
+const { Op } = require('sequelize')
+const _ = require('lodash')
+
+const getCompletedCourses = async (studentNumbers, courseCodes) => {
+  let courses = await Course.findAll({
+    attributes: ['code', 'name', 'substitutions'],
+    where: {
+      code: {
+        [Op.in]: courseCodes,
+      },
+    },
+  })
+
+  let fullCourseCodes = [
+    ...courseCodes,
+    ...new Set([...courses.reduce((acc, course) => [...acc, course.code, ...course.substitutions], [])]),
+  ]
+
+  let credits = await Credit.findAll({
+    attributes: ['course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+    where: {
+      course_code: {
+        [Op.in]: fullCourseCodes,
+      },
+      student_studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+    },
+  })
+
+  let enrollments = await Enrollment.findAll({
+    attributes: ['course_code', 'enrollment_date_time', 'studentnumber', 'state'],
+    where: {
+      course_code: {
+        [Op.in]: fullCourseCodes,
+      },
+      studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+      state: {
+        [Op.eq]: 'ENROLLED',
+      },
+    },
+  })
+
+  const studentInfo = await Student.findAll({
+    attributes: ['studentnumber', 'firstnames', 'lastname', 'email'],
+    where: {
+      studentnumber: {
+        [Op.in]: studentNumbers,
+      },
+    },
+  })
+
+  courses = courses.map(course => {
+    const { code, name, substitutions } = course
+    return { code, name, substitutions }
+  })
+
+  credits = credits.map(credit => {
+    const { course_code, student_studentnumber, credittypecode, attainment_date } = credit
+    const originalCode = courseCodes.includes(course_code)
+      ? null
+      : courses.find(course => course.substitutions.includes(course_code))?.code
+    return {
+      courseCode: originalCode ? originalCode : course_code,
+      substitution: originalCode ? course_code : null,
+      studentNumber: student_studentnumber,
+      creditType: credittypecode,
+      date: attainment_date,
+    }
+  })
+
+  enrollments = enrollments.map(enrollment => {
+    const originalCode = courses.find(course => course.substitutions.includes(enrollment.course_code))?.code
+    return {
+      courseCode: originalCode ? originalCode : enrollment.course_code,
+      substitution: originalCode ? enrollment.course_code : null,
+      studentNumber: enrollment.studentnumber,
+      date: enrollment.enrollment_date_time,
+    }
+  })
+
+  const studentCredits = {}
+
+  studentInfo.forEach(s => {
+    if (!studentCredits[s.studentnumber]) {
+      studentCredits[s.studentnumber] = { credits: [], enrollments: [] }
+    }
+    studentCredits[s.studentnumber].firstNames = s.firstnames
+    studentCredits[s.studentnumber].lastName = s.lastname
+    studentCredits[s.studentnumber].email = s.email
+  })
+
+  credits.forEach(credit => {
+    if (credit.creditType === 9) {
+      return
+    }
+    const previous = studentCredits[credit.studentNumber].credits?.find(c => credit.courseCode === c.courseCode)
+    if (previous && previous.date > credit.date) {
+      return
+    } else if (previous) {
+      studentCredits[credit.studentNumber].credits = studentCredits[credit.studentNumber].credits.filter(
+        c => credit.courseCode !== c.courseCode
+      )
+    }
+    studentCredits[credit.studentNumber].credits.push({
+      date: credit.date,
+      courseCode: credit.courseCode,
+      creditType: credit.creditType,
+      substitution: credit.substitution,
+    })
+  })
+
+  enrollments.forEach(enrollment => {
+    if (
+      credits.find(
+        credit => credit.courseCode === enrollment.courseCode && credit.studentNumber === enrollment.studentNumber
+      )
+    ) {
+      return
+    }
+    studentCredits[enrollment.studentNumber].enrollments.push({
+      date: enrollment.date,
+      courseCode: enrollment.courseCode,
+      state: enrollment.state,
+    })
+  })
+
+  const students = Object.keys(studentCredits).reduce(
+    (acc, student) => [
+      ...acc,
+      {
+        studentNumber: student,
+        credits: studentCredits[student].credits,
+        enrollments: studentCredits[student].enrollments,
+        firstNames: studentCredits[student].firstNames,
+        lastName: studentCredits[student].lastName,
+        email: studentCredits[student].email,
+      },
+    ],
+    []
+  )
+
+  students.forEach(student => {
+    courseCodes.forEach(courseCode => {
+      let [hits, rest] = _.partition(student.enrollments, e => e.courseCode === courseCode)
+      if (hits?.length === 0) return
+      let latest = hits[0]
+      for (let hit of hits) {
+        if (hit > latest) {
+          latest = hit
+        }
+      }
+      student.enrollments = [latest, ...rest]
+    })
+  })
+  return { students, courses }
+}
+
+module.exports = { getCompletedCourses }

services/backend/src/services/course_stats_counter.js

@@ -26,7 +26,9 @@ class CourseStatsCounter {
       failedMany: {},
       improvedPassedGrade: {},
       markedToSemester: {},
+      enrolledNoGrade: {},
     }
+
     this.stats = {
       students: 0,
       passed: 0,
@@ -38,10 +40,10 @@ class CourseStatsCounter {
       percentage: undefined,
       passedOfPopulation: undefined,
       triedOfPopulation: undefined,
+      perStudent: undefined,
       passingSemesters: this.initializePassingSemesters(),
     }
     this.enrollments = {
-      enrolledNoGrade: new Set(),
       semesters: this.initializePassingSemesters({}),
     }
     this.grades = {}
@@ -184,11 +186,20 @@ class CourseStatsCounter {
   getFinalStats() {
     const stats = this.stats
     const students = this.students
-    const allStudents = Object.keys(students.all)
-    const totalEnrolledNoGrade = [...(this.enrollments.ENROLLED || [])].filter(
-      student => !allStudents.includes(student)
-    ).length
+    const studs = this.enrollments
+      ? Object.keys(this.enrollments)
+          .filter(key => key !== 'ENROLLED' && key !== 'semesters')
+          .reduce((acc, key) => [...acc, ...[...this.enrollments[key]].map(studentnumber => studentnumber)], [])
+      : []
+    const allStudents = Object.keys(students.all).map(student => student)
+    const fileredEnrolledNoGrade = this.enrollments['ENROLLED']
+      ? [...this.enrollments['ENROLLED']]
+          .filter(student => !studs.includes(student) && !allStudents.includes(student))
+          .reduce((acc, student) => ({ ...acc, [student]: true }), {})
+      : {}
 
+    students.all = { ...students.all, ...fileredEnrolledNoGrade }
+    students.enrolledNoGrade = fileredEnrolledNoGrade
     stats.students = lengthOf(students.all)
     stats.passed = lengthOf(students.passed)
     stats.failed = lengthOf(students.failed)
@@ -200,8 +211,8 @@ class CourseStatsCounter {
     stats.triedOfPopulation = percentageOf(stats.students, this.studentsInTotal)
     stats.perStudent = stats.attempts / (stats.passed + stats.failed)
     stats.passingSemestersCumulative = this.getPassingSemestersCumulative()
-    stats.totalStudents = stats.students + totalEnrolledNoGrade
-    stats.totalEnrolledNoGrade = totalEnrolledNoGrade
+    stats.totalStudents = stats.students
+    stats.totalEnrolledNoGrade = lengthOf(fileredEnrolledNoGrade)
     stats.percentageWithEnrollments = percentageOf(stats.passed, stats.totalStudents)
 
     return {

services/backend/src/services/faculty/faculty.js

@@ -4,10 +4,9 @@ const {
   ElementDetail,
   Organization,
   ProgrammeModule,
-  Semester,
-  SemesterEnrollment,
   Studyright,
   StudyrightElement,
+  SemesterEnrollment,
   Student,
   Transfer,
   Credit,
@@ -19,23 +18,54 @@ const {
   formatFacultyTransfer,
   formatFacultyThesisWriter,
   formatOrganization,
-  formatAbsence,
   isNewProgramme,
+  mapCodesToIds,
+  facultyProgrammeStudents,
 } = require('./facultyHelpers')
 
 const getTransferredToAndAway = async (programmeCodes, allProgrammeCodes, since) => {
-  const awayTransfers = (await transferredAway(programmeCodes, allProgrammeCodes, since)).map(t => t.studyrightid)
-  const toTransfers = (await transferredTo(programmeCodes, allProgrammeCodes, since)).map(t => t.studyrightid)
-  const transferredToOrAway = [...toTransfers, ...awayTransfers]
-  return transferredToOrAway
+  const awayTransfers = await transferredAway(programmeCodes, allProgrammeCodes, since)
+  const toTransfers = await transferredTo(programmeCodes, allProgrammeCodes, since)
+  return [...toTransfers, ...awayTransfers]
 }
 
 const getTransferredInside = async (programmeCodes, allProgrammeCodes, since) => {
-  const transfersInside = (await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)).map(
-    t => t.studyrightid
-  )
-  return transfersInside
+  return await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
 }
+
+const getTransfers = async (programmeCode, allProgrammeCodes, since, end) => {
+  return await transferredFaculty(programmeCode, allProgrammeCodes, since, end)
+}
+
+const transferredFaculty = async (programme, allProgrammeCodes, since, end) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.between]: [since, end],
+        },
+        [Op.or]: [
+          {
+            sourcecode: allProgrammeCodes,
+            targetcode: programme,
+          },
+          {
+            sourcecode: programme,
+            targetcode: {
+              [Op.notIn]: allProgrammeCodes,
+            },
+          },
+          {
+            sourcecode: {
+              [Op.notIn]: allProgrammeCodes,
+            },
+            targetcode: programme,
+          },
+        ],
+      },
+    })
+  ).map(formatFacultyTransfer)
+
 const startedStudyrights = async (faculty, since, studyRightWhere) =>
   (
     await Studyright.findAll({
@@ -95,12 +125,93 @@ const graduatedStudyrights = async (faculty, since, studyrightWhere) =>
     })
   ).map(facultyFormatStudyright)
 
-const bachelorStudyright = async id => {
+const studyrightsByRightStartYear = async (faculty, since, graduated = 1) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+          },
+        },
+      ],
+      where: {
+        faculty_code: faculty,
+        startdate: {
+          [Op.gte]: since,
+        },
+        graduated: graduated,
+        student_studentnumber: { [Op.not]: null },
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const getStudyRightsByExtent = async (faculty, elementStart, studyrightStart, code, extents, graduated) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+        {
+          model: StudyrightElement,
+          required: true,
+          where: {
+            code: code,
+            ...elementStart,
+          },
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              type: 20,
+            },
+          },
+        },
+      ],
+      where: {
+        faculty_code: faculty,
+        extentcode: {
+          [Op.in]: extents,
+        },
+        graduated: {
+          [Op.in]: graduated,
+        },
+        studystartdate: { [Op.not]: null },
+        ...studyrightStart,
+        student_studentnumber: { [Op.not]: null },
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const getStudentsByStudentnumbers = async studentnumbers =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: studentnumbers,
+      },
+      include: {
+        model: SemesterEnrollment,
+        attributes: ['semestercode', 'enrollmenttype'],
+      },
+    })
+  ).map(facultyProgrammeStudents)
+
+const hasMasterRight = async id => {
   return await Studyright.findOne({
-    attributes: ['studystartdate'],
     where: {
       studyrightid: id,
-      extentcode: 1,
+      extentcode: 2,
     },
   })
 }
@@ -217,29 +328,6 @@ const thesisWriters = async (provider, since, thesisTypes, students) =>
     })
   ).map(formatFacultyThesisWriter)
 
-const statutoryAbsences = async (studentnumber, startdate, enddate) =>
-  (
-    await SemesterEnrollment.findAll({
-      attributes: ['semestercode'],
-      include: {
-        model: Semester,
-        attributes: ['startdate', 'enddate'],
-        where: {
-          startdate: {
-            [Op.gte]: startdate,
-          },
-          enddate: {
-            [Op.lte]: enddate,
-          },
-        },
-      },
-      where: {
-        studentnumber: studentnumber,
-        statutory_absence: true,
-      },
-    })
-  ).map(formatAbsence)
-
 // Some programme modules are not directly associated to a faculty (organization).
 // Some have intermediate organizations, such as department, so the connection must be digged up
 const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
@@ -288,13 +376,16 @@ const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
     allProgrammes = allProgrammes.filter(prog => isNewProgramme(prog.code))
   }
 
+  mapCodesToIds(allProgrammes)
+
   return allProgrammes
 }
 
 module.exports = {
   startedStudyrights,
   graduatedStudyrights,
-  bachelorStudyright,
+  studyrightsByRightStartYear,
+  hasMasterRight,
   transferredInsideFaculty,
   transferredAway,
   transferredTo,
@@ -302,7 +393,9 @@ module.exports = {
   thesisWriters,
   findFacultyProgrammeCodes,
   facultyOgranizationId,
-  statutoryAbsences,
   getTransferredToAndAway,
   getTransferredInside,
+  getStudyRightsByExtent,
+  getStudentsByStudentnumbers,
+  getTransfers,
 }

services/backend/src/services/faculty/facultyBasics.js

@@ -1,4 +1,5 @@
 const { indexOf } = require('lodash')
+const moment = require('moment')
 const {
   startedStudyrights,
   graduatedStudyrights,
@@ -7,9 +8,11 @@ const {
   transferredTo,
   getTransferredToAndAway,
   getTransferredInside,
+  studyrightsByRightStartYear,
 } = require('./faculty')
 const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogrammeHelpers')
 const { findRightProgramme, isNewProgramme, checkTransfers, getExtentFilter } = require('./facultyHelpers')
+const { codes } = require('../../../config/programmeCodes')
 
 const filterDuplicateStudyrights = studyrights => {
   // bachelor+master students have two studyrights (separated by two last digits in studyrightid)
@@ -40,19 +43,18 @@ const addProgramme = async (
   facultyProgrammes,
   includeAllSpecials
 ) => {
-  if (!(programme in stats)) {
-    stats[programme] = {}
+  let { progId, name, code } = facultyProgrammes.filter(prog => prog.code === programme)[0]
+  if (!(progId in stats)) {
+    stats[progId] = {}
     if (includeAllSpecials) {
-      Object.keys(tableStats).forEach(year => (stats[programme][year] = [0, 0, 0]))
+      Object.keys(tableStats).forEach(year => (stats[progId][year] = [0, 0, 0]))
     } else {
-      Object.keys(tableStats).forEach(year => (stats[programme][year] = [0]))
+      Object.keys(tableStats).forEach(year => (stats[progId][year] = [0]))
     }
   }
-  stats[programme][transferYear][index] += 1
-
-  if (!(programme in allBasics.programmeNames)) {
-    const name = facultyProgrammes.filter(prog => prog.code === programme)[0]?.name
-    allBasics.programmeNames[programme] = name
+  stats[progId][transferYear][index] += 1
+  if (!(progId in allBasics.programmeNames)) {
+    allBasics.programmeNames[progId] = { ...name, code: code }
   }
 }
 
@@ -76,6 +78,7 @@ const getFacultyStarters = async (
   const studyrightWhere = getExtentFilter(includeAllSpecials)
   const studyrights = await startedStudyrights(faculty, since, studyrightWhere)
   let filteredStudyrights = filterDuplicateStudyrights(studyrights)
+  const keys = Object.keys(codes)
   const programmeTableStats = {}
   if (!includeAllSpecials) {
     filteredStudyrights = filteredStudyrights.filter(
@@ -85,6 +88,11 @@ const getFacultyStarters = async (
 
   filteredStudyrights.forEach(({ studystartdate, studyrightElements }) => {
     const { programme, programmeName } = findRightProgramme(studyrightElements, 'started')
+    let programmeId = programme
+
+    if (keys.includes(programme)) {
+      programmeId = codes[programme].toUpperCase()
+    }
 
     if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
       const startYear = defineYear(studystartdate, isAcademicYear)
@@ -92,18 +100,18 @@ const getFacultyStarters = async (
       startedGraphStats[indexOf(yearsArray, startYear)] += 1
       startedTableStats[startYear] += 1
 
-      if (!(programme in programmeTableStats)) {
-        programmeTableStats[programme] = { ...tableStats }
+      if (!(programmeId in programmeTableStats)) {
+        programmeTableStats[programmeId] = { ...tableStats }
       }
-      programmeTableStats[programme][startYear] += 1
+      programmeTableStats[programmeId][startYear] += 1
 
-      if (!(programme in allBasics.programmeNames)) {
-        allBasics.programmeNames[programme] = programmeName
+      if (!(programmeId in allBasics.programmeNames)) {
+        allBasics.programmeNames[programmeId] = { ...programmeName, code: programme }
       }
     }
   })
 
-  allBasics.studentInfo.graphStats.push({ name: 'Started studying', data: startedGraphStats })
+  allBasics.studentInfo.graphStats.push({ name: 'Started studying (new in faculty)', data: startedGraphStats })
   programmeData['started'] = programmeTableStats
 
   Object.keys(startedTableStats).forEach(year => {
@@ -129,7 +137,7 @@ const getFacultyGraduates = async (
 ) => {
   const graduatedGraphStats = [[...graphStats], [...graphStats], [...graphStats], [...graphStats], [...graphStats]]
   const graduatedTableStats = {}
-
+  const keys = Object.keys(codes)
   let studyrightWhere = getExtentFilter(includeAllSpecials)
   let graduatedRights = await graduatedStudyrights(faculty, since, studyrightWhere)
   const programmeTableStats = {}
@@ -140,40 +148,45 @@ const getFacultyGraduates = async (
       s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
     )
   }
-
   graduatedRights.forEach(({ enddate, extentcode, studyrightElements }) => {
     const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
+    let programmeId = programme
+
+    if (keys.includes(programme)) {
+      programmeId = codes[programme].toUpperCase()
+    }
+
     if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
       const endYear = defineYear(enddate, isAcademicYear)
       graduatedGraphStats[0][indexOf(yearsArray, endYear)] += 1
       graduatedTableStats[endYear][0] += 1
 
-      if (!(programme in programmeTableStats)) {
-        programmeTableStats[programme] = {}
-        Object.keys(tableStats).forEach(year => (programmeTableStats[programme][year] = [0, 0, 0, 0, 0]))
+      if (!(programmeId in programmeTableStats)) {
+        programmeTableStats[programmeId] = {}
+        Object.keys(tableStats).forEach(year => (programmeTableStats[programmeId][year] = [0, 0, 0, 0, 0]))
       }
-      programmeTableStats[programme][endYear][0] += 1
+      programmeTableStats[programmeId][endYear][0] += 1
 
       if (extentcode === 1) {
         graduatedGraphStats[1][indexOf(yearsArray, endYear)] += 1
         graduatedTableStats[endYear][1] += 1
-        programmeTableStats[programme][endYear][1] += 1
+        programmeTableStats[programmeId][endYear][1] += 1
       } else if (extentcode === 2) {
         graduatedGraphStats[2][indexOf(yearsArray, endYear)] += 1
         graduatedTableStats[endYear][2] += 1
-        programmeTableStats[programme][endYear][2] += 1
+        programmeTableStats[programmeId][endYear][2] += 1
       } else if (extentcode === 4) {
         graduatedGraphStats[3][indexOf(yearsArray, endYear)] += 1
         graduatedTableStats[endYear][3] += 1
-        programmeTableStats[programme][endYear][3] += 1
+        programmeTableStats[programmeId][endYear][3] += 1
       } else {
         graduatedGraphStats[4][indexOf(yearsArray, endYear)] += 1
         graduatedTableStats[endYear][4] += 1
-        programmeTableStats[programme][endYear][4] += 1
+        programmeTableStats[programmeId][endYear][4] += 1
       }
 
-      if (!(programme in allBasics.programmeNames)) {
-        allBasics.programmeNames[programme] = programmeName
+      if (!(programmeId in allBasics.programmeNames)) {
+        allBasics.programmeNames[programmeId] = { code: programme, ...programmeName }
       }
     }
   })
@@ -192,7 +205,18 @@ const getFacultyGraduates = async (
     countsGraduations[year] = graduatedTableStats[year]
   })
 }
-
+// If includeAllSpecial false, insidetransfers should contain only students started after 1.8.2017.
+const getInsideTransfers = async (programmeCodes, allProgrammeCodes, since, includeAllSpecials, faculty) => {
+  if (includeAllSpecials) {
+    return await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
+  }
+  const insiders = await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
+  const studyrights = (await studyrightsByRightStartYear(faculty, moment('2017-08-01', 'YYYY-MM-DD'))).map(
+    sr => sr.studyrightid
+  )
+  const filteredTransfers = insiders.filter(tr => studyrights.includes(tr.studyrightid))
+  return filteredTransfers
+}
 const getFacultyTransfers = async (
   programmes,
   programmeCodes,
@@ -205,10 +229,16 @@ const getFacultyTransfers = async (
   allBasics,
   counts,
   programmeData,
-  includeAllSpecials
+  includeAllSpecials,
+  faculty
 ) => {
-  const insideTransfers = await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
-  // Filter still away those  that have started studying before 1.8.2017, when include all specials false.
+  const insideTransfers = await getInsideTransfers(
+    programmeCodes,
+    allProgrammeCodes,
+    since,
+    includeAllSpecials,
+    faculty
+  )
   let transferGraphStats = []
   let transferTableStats = {}
   if (includeAllSpecials) {
@@ -224,17 +254,19 @@ const getFacultyTransfers = async (
   for (const { transferdate, targetcode } of insideTransfers) {
     const transferYear = defineYear(transferdate, isAcademicYear)
     transferGraphStats[0][indexOf(yearsArray, transferYear)] += 1
-    transferTableStats[transferYear][0] += 1
-    await addProgramme(
-      programmeTableStats,
-      targetcode,
-      tableStats,
-      transferYear,
-      0,
-      allBasics,
-      programmes,
-      includeAllSpecials
-    )
+    if (isAcademicYear || !(transferYear > moment().year())) {
+      transferTableStats[transferYear][0] += 1
+      await addProgramme(
+        programmeTableStats,
+        targetcode,
+        tableStats,
+        transferYear,
+        0,
+        allBasics,
+        programmes,
+        includeAllSpecials
+      )
+    }
   }
   if (includeAllSpecials) {
     const awayTransfers = await transferredAway(programmeCodes, allProgrammeCodes, since)
@@ -242,34 +274,38 @@ const getFacultyTransfers = async (
 
     for (const { transferdate, sourcecode } of awayTransfers) {
       const transferYear = defineYear(transferdate, isAcademicYear)
-      transferGraphStats[1][indexOf(yearsArray, transferYear)] += 1
-      transferTableStats[transferYear][1] += 1
-      await addProgramme(
-        programmeTableStats,
-        sourcecode,
-        tableStats,
-        transferYear,
-        1,
-        allBasics,
-        programmes,
-        includeAllSpecials
-      )
+      if (isAcademicYear || !(transferYear > moment().year())) {
+        transferGraphStats[1][indexOf(yearsArray, transferYear)] += 1
+        transferTableStats[transferYear][1] += 1
+        await addProgramme(
+          programmeTableStats,
+          sourcecode,
+          tableStats,
+          transferYear,
+          1,
+          allBasics,
+          programmes,
+          includeAllSpecials
+        )
+      }
     }
 
     for (const { transferdate, targetcode } of toTransfers) {
       const transferYear = defineYear(transferdate, isAcademicYear)
-      transferGraphStats[2][indexOf(yearsArray, transferYear)] += 1
-      transferTableStats[transferYear][2] += 1
-      await addProgramme(
-        programmeTableStats,
-        targetcode,
-        tableStats,
-        transferYear,
-        2,
-        allBasics,
-        programmes,
-        includeAllSpecials
-      )
+      if (isAcademicYear || !(transferYear > moment().year())) {
+        transferGraphStats[2][indexOf(yearsArray, transferYear)] += 1
+        transferTableStats[transferYear][2] += 1
+        await addProgramme(
+          programmeTableStats,
+          targetcode,
+          tableStats,
+          transferYear,
+          2,
+          allBasics,
+          programmes,
+          includeAllSpecials
+        )
+      }
     }
   }
 
@@ -367,7 +403,8 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
     allBasics,
     counts,
     programmeData,
-    includeAllSpecials
+    includeAllSpecials,
+    faculty
   )
 
   // combine tableStats from all categories
@@ -413,7 +450,11 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
       if (code in programmeData.transferred) {
         studentInfo[code][year] = studentInfo[code][year].concat(programmeData.transferred[code][year])
       } else {
-        studentInfo[code][year] = studentInfo[code][year].concat([0, 0, 0])
+        if (includeAllSpecials) {
+          studentInfo[code][year] = studentInfo[code][year].concat([0, 0, 0])
+        } else {
+          studentInfo[code][year] = studentInfo[code][year].concat([0])
+        }
       }
     }
   })

services/backend/src/services/faculty/facultyCredits.js

@@ -7,14 +7,8 @@ const {
   isNonMajorCredit,
   getYearsArray,
   getStartDate,
-  isSpecialGroupCredit,
 } = require('../studyprogrammeHelpers')
-const {
-  getStudyRights,
-  getCreditsForStudyProgramme,
-  allTransfers,
-  getCourseCodesForStudyProgramme,
-} = require('../studyprogramme')
+const { getStudyRights, getCreditsForStudyProgramme, getCourseCodesForStudyProgramme } = require('../studyprogramme')
 
 const isFacultyNonMajorCredit = (studyrights, attainment_date, facultyProgrammes) => {
   let right = ''
@@ -42,25 +36,14 @@ const isFacultyNonMajorCredit = (studyrights, attainment_date, facultyProgrammes
 // Fetches all the credits for the studyprogramme and divides them into major-students, non-major faculty students,
 // non-major other faculty students adn non-degree student credits
 // Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
-const getFacultyRegularCreditStats = async ({
-  studyprogramme,
-  since,
-  years,
-  isAcademicYear,
-  includeAllSpecials,
-  facultyProgrammes,
-}) => {
+const getFacultyRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, facultyProgrammes }) => {
   const providercode = mapToProviders([studyprogramme])[0]
   const courses = await getCourseCodesForStudyProgramme(providercode)
+  // console.log(JSON.stringify(courses))
   const credits = await getCreditsForStudyProgramme(courses, since)
   const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
 
   let studyrights = await getStudyRights(students)
-  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid).filter(s => s !== 'undefined')
-  if (!includeAllSpecials) {
-    studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
-  }
-
   let majors = getStatsBasis(years)
   let allNonMajors = getStatsBasis(years)
   let facultyNonMajors = getStatsBasis(years)
@@ -71,10 +54,6 @@ const getFacultyRegularCreditStats = async ({
     const studentStudyrights = studyrights.filter(studyright => studyright.studentnumber === student_studentnumber)
     const attainmentYear = defineYear(attainment_date, isAcademicYear)
 
-    if (!includeAllSpecials && isSpecialGroupCredit(studentStudyrights, attainment_date, transfers)) {
-      return
-    }
-
     if (isMajorStudentCredit(studentStudyrights, attainment_date, studyprogramme)) {
       majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
       majors.tableStats[attainmentYear] += credits || 0
@@ -95,6 +74,7 @@ const getFacultyRegularCreditStats = async ({
       nonDegree.tableStats[attainmentYear] += credits || 0
     }
   })
+
   if (
     majors.graphStats.every(year => year === 0) &&
     facultyNonMajors.graphStats.every(year => year === 0) &&
@@ -115,12 +95,11 @@ const getFacultyRegularCreditStats = async ({
 // Fetches all credits for the studytrack and combines them into statistics
 // These are for the faculty view which differs from study programme overview
 // (no transfers, non majors split to in- and outside-faculty)
-const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgrammes, settings }) => {
-  const { isAcademicYear, includeAllSpecials } = settings
+const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgrammes, isAcademicYear }) => {
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
 
-  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials, facultyProgrammes }
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, facultyProgrammes }
   const { majors, facultyNonMajors, otherNonMajors, nonDegree } = await getFacultyRegularCreditStats(queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
@@ -128,47 +107,36 @@ const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgr
 
   if (!dataFound) return null
 
-  const tableStats = reversedYears.map(year =>
-    includeAllSpecials
-      ? [
-          year,
-          majors.tableStats[year] +
-            facultyNonMajors.tableStats[year] +
-            otherNonMajors.tableStats[year] +
-            nonDegree.tableStats[year],
-          majors.tableStats[year],
-          facultyNonMajors.tableStats[year],
-          otherNonMajors.tableStats[year],
-          nonDegree.tableStats[year],
-        ]
-      : [year, majors.tableStats[year], majors.tableStats[year]]
-  )
+  const tableStats = reversedYears.map(year => [
+    year,
+    majors.tableStats[year] +
+      facultyNonMajors.tableStats[year] +
+      otherNonMajors.tableStats[year] +
+      nonDegree.tableStats[year],
+    majors.tableStats[year],
+    facultyNonMajors.tableStats[year],
+    otherNonMajors.tableStats[year],
+    nonDegree.tableStats[year],
+  ])
 
-  const graphStats = includeAllSpecials
-    ? [
-        {
-          name: 'Major students credits',
-          data: majors.graphStats,
-        },
-        {
-          name: 'Non-major faculty students credits',
-          data: facultyNonMajors.graphStats,
-        },
-        {
-          name: 'Non-major other faculty students credits 2',
-          data: otherNonMajors.graphStats,
-        },
-        {
-          name: 'Non-degree credits',
-          data: nonDegree.graphStats,
-        },
-      ]
-    : [
-        {
-          name: 'Major students credits',
-          data: majors.graphStats,
-        },
-      ]
+  const graphStats = [
+    {
+      name: 'Major students credits',
+      data: majors.graphStats,
+    },
+    {
+      name: 'Non-major faculty students credits',
+      data: facultyNonMajors.graphStats,
+    },
+    {
+      name: 'Non-major other faculty students credits 2',
+      data: otherNonMajors.graphStats,
+    },
+    {
+      name: 'Non-degree credits',
+      data: nonDegree.graphStats,
+    },
+  ]
 
   return {
     id: studyprogramme,
@@ -178,33 +146,27 @@ const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgr
   }
 }
 
-const getProgrammeCredits = async (code, yearType, specialGroups, facultyProgrammes) => {
+const getProgrammeCredits = async (code, yearType, facultyProgrammes) => {
   const updatedStats = await getFacultyCreditStatsForStudytrack({
     studyprogramme: code,
     facultyProgrammes,
-    settings: {
-      isAcademicYear: yearType === 'ACADEMIC_YEAR',
-      includeAllSpecials: specialGroups === 'SPECIAL_INCLUDED',
-    },
+    isAcademicYear: yearType === 'ACADEMIC_YEAR',
   })
 
   return updatedStats
 }
 
-const combineFacultyCredits = async (faculty, programmes, yearType, specialGroups) => {
+const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearType) => {
   let counts = {}
   let years = []
-  const titles =
-    specialGroups === 'SPECIAL_INCLUDED'
-      ? [
-          '',
-          'Total',
-          'Major students credits',
-          'Non-major faculty students credits',
-          'Non-major other faculty students credits',
-          'Non-degree student credits',
-        ]
-      : ['', 'Total', 'Major students credits']
+  const titles = [
+    '',
+    'Total',
+    'Major students credits',
+    'Non-major faculty students credits',
+    'Non-major other faculty students credits',
+    'Non-degree student credits',
+  ]
 
   let allCredits = {
     id: faculty,
@@ -216,12 +178,13 @@ const combineFacultyCredits = async (faculty, programmes, yearType, specialGroup
     titles,
   }
 
-  const progCodes = programmes.map(p => p.code)
+  const progCodes = allProgrammes.map(p => p.code)
   for (const prog of programmes) {
-    const data = await getProgrammeCredits(prog.code, yearType, specialGroups, progCodes)
+    if (prog.code === 'MH70_008_2') continue
+    const data = await getProgrammeCredits(prog.code, yearType, progCodes)
     if (data) {
-      allCredits.programmeNames[prog.code] = prog.name
-      allCredits.programmeTableStats[prog.code] = data.tableStats
+      allCredits.programmeNames[prog.progId] = { code: prog.code, ...prog.name }
+      allCredits.programmeTableStats[prog.progId] = data.tableStats
       if (!allCredits.lastUpdated || new Date(data.lastUpdated) > new Date(allCredits.lastUpdated))
         allCredits.lastUpdated = data.lastUpdated
 
@@ -237,6 +200,7 @@ const combineFacultyCredits = async (faculty, programmes, yearType, specialGroup
       })
     }
   }
+
   // save table stats and graph stats
   years.forEach(year => {
     allCredits.tableStats.push([year, ...counts[year]])
@@ -257,16 +221,12 @@ const combineFacultyCredits = async (faculty, programmes, yearType, specialGroup
     nonDegree.push(counts[year][4])
   })
 
-  allCredits.graphStats =
-    specialGroups === 'SPECIAL_INCLUDED'
-      ? [
-          { name: 'Major students credits', data: majors },
-          { name: 'Non-major faculty students credits', data: facultyNonMajor },
-          { name: 'Non-major other faculty students credits', data: otherNonMajor },
-          { name: 'Non-degree credits', data: nonDegree },
-        ]
-      : [{ name: 'Major students credits', data: majors }]
-
+  allCredits.graphStats = [
+    { name: 'Major students credits', data: majors },
+    { name: 'Non-major faculty students credits', data: facultyNonMajor },
+    { name: 'Non-major other faculty students credits', data: otherNonMajor },
+    { name: 'Non-degree credits', data: nonDegree },
+  ]
   return allCredits
 }
 

services/backend/src/services/faculty/facultyGraduationTimes.js

@@ -1,83 +1,136 @@
 const moment = require('moment')
-const { graduatedStudyrights, bachelorStudyright, statutoryAbsences } = require('./faculty')
+const { graduatedStudyrights, studyrightsByRightStartYear, hasMasterRight } = require('./faculty')
 const { findRightProgramme, isNewProgramme } = require('./facultyHelpers')
-const { getYearsArray, getYearsObject, getMean, getMedian, defineYear } = require('../studyprogrammeHelpers')
+const { getYearsArray, getYearsObject, getMedian, defineYear } = require('../studyprogrammeHelpers')
+const { codes } = require('../../../config/programmeCodes')
+const { bachelorStudyright, countTimeCategories, getStatutoryAbsences } = require('../graduationHelpers')
+
+const sortProgrammes = data => {
+  const check = name => {
+    if (isNaN(name[0])) return -1
+    return 1
+  }
+  data.sort((a, b) => {
+    if (check(a.name) === check(b.name)) return a.name.localeCompare(b.name)
+    return check(a.name) - check(b.name)
+  })
+}
+
+const getSortedProgIds = progs => {
+  let programmes = []
+
+  for (const prog of progs) {
+    if (Object.keys(codes).includes(prog)) {
+      const name = prog === 'T923103' ? codes[prog].toUpperCase() + '-DP' : codes[prog].toUpperCase()
+      programmes = [...programmes, { name: name, code: prog }]
+    } else {
+      programmes = [...programmes, { name: prog, code: prog }]
+    }
+  }
+  sortProgrammes(programmes)
+  return programmes
+}
 
 const findBachelorStartdate = async id => {
   const studyright = await bachelorStudyright(id)
-  if (studyright) return studyright.studystartdate
+  if (studyright) return studyright.startdate
   return null
 }
-const getStatutoryAbsences = async (studentnumber, startdate, enddate) => {
-  const absences = await statutoryAbsences(studentnumber, startdate, enddate)
-  if (absences.length) {
-    const absentMonths = absences.reduce((sum, ab) => sum + moment(ab.end).diff(moment(ab.start), 'months'), 0)
-    return absentMonths
-  }
-  return 0
+
+const getProgrammeObjectBasis = (years, levels, emptyObject = true) => {
+  return levels.reduce(
+    (acc, level) => ({ ...acc, [level]: years.reduce((acc, year) => ({ ...acc, [year]: emptyObject ? {} : 0 }), {}) }),
+    {}
+  )
 }
 
-const countGraduationTimes = async (faculty, programmeFilter) => {
-  const isAcademicYear = false // = yearType === 'ACADEMIC_YEAR'
-  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const levels = ['bachelor', 'bcMsCombo', 'master', 'doctor', 'licentiate']
+const hasMS = async (programme, elements, studyrightid) => {
+  // Only bachelor studyright: socCom, kasvatustiede - varhaiskasvatus track, farmasia - applied only to Bc = farmaseutti,
+  if (['KH74_001', '620050-ba', '620030-ba', '520091-ba'].includes(programme)) {
+    return false
+  }
+  if (programme === 'KH60_001') {
+    return !elements.some(el => ['EDUK-VO', '0371', '620050-ba', '620030-ba'].includes(el.dataValues.code))
+  }
+  if (programme === 'KH55_001') {
+    const result = await hasMasterRight(studyrightid.replace(/-1$/, '-2'))
+    if (!result) {
+      return false
+    }
+  }
+  return true
+}
 
-  let graduationAmounts = {}
-  let graduationTimes = {}
-  let medians = {}
-  let means = {}
+const addGraduation = async (
+  extentcode,
+  startdate,
+  studyrightid,
+  enddate,
+  studentnumber,
+  graduationAmounts,
+  graduationTimes,
+  year,
+  programmes,
+  programme
+) => {
+  let actualStartdate = startdate
+  let level = null
 
-  let result = {
-    years: [...years].reverse(),
-    means: {},
-    medians: {},
-    goals: {
-      bachelor: 36,
-      bcMsCombo: 60,
-      master: 24,
-      doctor: 48,
-      // Fix differrent cases?
-      licentiate: 78,
-    },
+  if (programme === 'MH30_001' || programme === 'MH30_003') {
+    level = 'bcMsCombo'
+  } else if (extentcode === 1) {
+    level = 'bachelor'
+  } else if (extentcode === 2) {
+    if (studyrightid.slice(-2) === '-2') {
+      level = 'bcMsCombo'
+      actualStartdate = await findBachelorStartdate(studyrightid.replace(/-2$/, '-1'))
+    } else {
+      level = 'master'
+    }
+  } else if (extentcode === 4) {
+    level = 'doctor'
+  } else if (extentcode === 3) {
+    level = 'licentiate'
+  } else {
+    return
   }
 
-  levels.forEach(level => {
-    graduationAmounts[level] = getYearsObject({ years })
-    graduationTimes[level] = getYearsObject({ years, emptyArrays: true })
-    medians[level] = getYearsObject({ years, emptyArrays: true })
-    means[level] = getYearsObject({ years, emptyArrays: true })
-    result.medians[level] = []
-    result.means[level] = []
-  })
+  const absoluteTimeToGraduation = moment(enddate).diff(moment(actualStartdate), 'months')
+  const statutoryAbsences = await getStatutoryAbsences(studentnumber, actualStartdate, enddate)
+  const timeToGraduation = absoluteTimeToGraduation - statutoryAbsences
 
-  // the max amount of months in the graph, depends on programme type
-  // (times 2 to set goal time at the halfway point on graphs)
-  const comparisonValues = {
-    bachelor: 36 * 2,
-    bcMsCombo: 60 * 2,
-    master: 24 * 2,
-    doctor: 48 * 2,
-    // Fix differrent cases?
-    licentiate: 78 * 2,
-  }
+  graduationAmounts[level][year] += 1
+  graduationTimes[level][year] = [...graduationTimes[level][year], timeToGraduation]
 
-  // We count studyrights (vs. studyright_elements)
-  // This way we get the whole time for a degree, even if the student was transferred to a new programme
-  // E.g. started 8/2016 in old Bc, transferred to new 10/2020, graduated from new 1/2021 --> total 53 months (not 3)
+  if (!(programme in programmes[level][year]))
+    programmes[level][year][programme] = { graduationTimes: [], graduationAmounts: 0 }
 
-  const graduatedRights = await graduatedStudyrights(faculty, since)
+  programmes[level][year][programme].graduationAmounts += 1
+  programmes[level][year][programme].graduationTimes = [
+    ...programmes[level][year][programme].graduationTimes,
+    timeToGraduation,
+  ]
+}
 
-  for (const right of graduatedRights) {
-    const { enddate, startdate, studyrightid, extentcode, studyrightElements, studentnumber } = right
+const getClassSizes = async (faculty, since, classSizes, programmeFilter, years) => {
+  const studyrights = await studyrightsByRightStartYear(faculty, since, [0, 1])
+
+  // get all received studyrights for each year
+  // a transferred student is counted into new programmes class size i.e.
+  // students who received studyright on year X and graduated/will be graduating from programme Y
+  // if we only counted those who started fresh in the new programme we could get a bigger number of graduates
+  // than the class size, as the graduatation time count only looks at a degree as whole studyright
+  for (const right of studyrights) {
+    const { startdate, studyrightid, extentcode, studyrightElements } = right
     const { programme } = findRightProgramme(studyrightElements, 'graduated')
     if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
 
-    const graduationYear = defineYear(enddate, isAcademicYear)
     let actualStartdate = startdate
     let level = null
 
-    if (extentcode == 1) {
+    if (programme === 'MH30_001' || programme === 'MH30_003') {
+      level = 'bcMsCombo'
+    } else if (extentcode === 1) {
       level = 'bachelor'
     } else if (extentcode === 2) {
       if (studyrightid.slice(-2) === '-2') {
@@ -94,48 +147,198 @@ const countGraduationTimes = async (faculty, programmeFilter) => {
       continue
     }
 
-    const absoluteTimeToGraduation = moment(enddate).diff(moment(actualStartdate), 'months')
-    const statutoryAbsences = await getStatutoryAbsences(studentnumber, actualStartdate, enddate)
-    const timeToGraduation = absoluteTimeToGraduation - statutoryAbsences
+    const startYear = defineYear(actualStartdate, false)
 
-    graduationAmounts[level][graduationYear] += 1
-    graduationTimes[level][graduationYear] = [...graduationTimes[level][graduationYear], timeToGraduation]
+    // On faculty level, count started bachelor rights to BcMsCombo as well  --> useful class size even if master studies have yet not started
+    // On programme level BcMsCombo sizes are iffy as many students haven't started masters yet --> no Ms programme element to count
+    if (level !== 'bcMsCombo') {
+      classSizes[level][startYear] += 1
+      if (level === 'bachelor' && (await hasMS(programme, studyrightElements, studyrightid))) {
+        classSizes['bcMsCombo'][startYear] += 1
+      }
+    }
+
+    if (!(programme in classSizes.programmes)) {
+      if (extentcode === 2) {
+        classSizes.programmes[programme] = {}
+        classSizes.programmes[programme]['bcMsCombo'] = getYearsObject({ years, emptyArrays: false })
+        classSizes.programmes[programme]['master'] = getYearsObject({ years, emptyArrays: false })
+      } else {
+        classSizes.programmes[programme] = getYearsObject({ years, emptyArrays: false })
+      }
+    }
+
+    if (extentcode === 2) {
+      classSizes.programmes[programme][level][startYear] += 1
+    } else {
+      classSizes.programmes[programme][startYear] += 1
+    }
+  }
+}
+
+const count = async (faculty, since, years, yearsList, levels, programmeFilter, programmeNames, goals, mode) => {
+  let graduationAmounts = {}
+  let graduationTimes = {}
+  let programmes = getProgrammeObjectBasis(yearsList, levels)
+  let data = {
+    medians: {},
+    programmes: {
+      medians: getProgrammeObjectBasis(yearsList, levels),
+    },
   }
 
-  result.years.forEach(year => {
+  levels.forEach(level => {
+    graduationAmounts[level] = getYearsObject({ years: yearsList })
+    graduationTimes[level] = getYearsObject({ years: yearsList, emptyArrays: true })
+    data.medians[level] = []
+  })
+
+  let studyrights = null
+  if (mode === 'gradYear') {
+    studyrights = await graduatedStudyrights(faculty, since)
+  } else {
+    studyrights = await studyrightsByRightStartYear(faculty, since)
+  }
+
+  for (const right of studyrights) {
+    const { enddate, startdate, studyrightid, extentcode, studyrightElements, studentnumber } = right
+    const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
+    if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
+
+    if (!(programme in programmeNames)) {
+      programmeNames[programme] = programmeName
+    }
+
+    let year = null
+    if (mode === 'gradYear') {
+      year = defineYear(enddate, false)
+    } else {
+      year = defineYear(startdate, false)
+    }
+
+    await addGraduation(
+      extentcode,
+      startdate,
+      studyrightid,
+      enddate,
+      studentnumber,
+      graduationAmounts,
+      graduationTimes,
+      year,
+      programmes,
+      programme
+    )
+  }
+
+  years.forEach(year => {
     levels.forEach(level => {
       const median = getMedian(graduationTimes[level][year])
-      const mean = getMean(graduationTimes[level][year])
-      result.medians[level] = [...result.medians[level], { y: median, amount: graduationAmounts[level][year] }]
-      result.means[level] = [...result.means[level], { y: mean, amount: graduationAmounts[level][year] }]
-      medians[level][year] = [
-        ['', median],
-        ['', comparisonValues[level] - median],
-      ]
-      means[level][year] = [
-        ['', mean],
-        ['', comparisonValues[level] - mean],
+      const statistics = { onTime: 0, yearOver: 0, wayOver: 0 }
+
+      // Programme level breakdown
+      data.programmes.medians[level][year] = { programmes: [], data: [] }
+      const programmeCodes = Object.keys(programmes[level][year])
+      const programmeIds = getSortedProgIds(programmeCodes)
+
+      if (programmeIds.length > 0) {
+        for (const prog of programmeIds) {
+          const progMedian = getMedian(programmes[level][year][prog.code].graduationTimes)
+
+          let goal = goals[level]
+          if (faculty === 'H30' && Object.keys(goals.exceptions).includes(prog.code)) {
+            goal += goals.exceptions[prog.code]
+          }
+
+          const progStatistics = countTimeCategories(programmes[level][year][prog.code].graduationTimes, goal)
+          Object.keys(progStatistics).forEach(key => (statistics[key] += progStatistics[key]))
+
+          data.programmes.medians[level][year].programmes = [
+            ...data.programmes.medians[level][year].programmes,
+            prog.code,
+          ]
+          data.programmes.medians[level][year].data = [
+            ...data.programmes.medians[level][year].data,
+            {
+              y: progMedian,
+              amount: programmes[level][year][prog.code].graduationAmounts,
+              statistics: progStatistics,
+              name: prog.name,
+              code: prog.code,
+            },
+          ]
+        }
+      }
+      data.medians[level] = [
+        ...data.medians[level],
+        { y: median, amount: graduationAmounts[level][year], statistics, name: year },
       ]
     })
   })
 
-  // // HighCharts graph require the data to have this format (ie. actual value, "empty value")
-  // years.forEach(year => {
-  //   levels.forEach(level => {
-  //     const median = getMedian(graduationTimes[level][year])
-  //     const mean = getMean(graduationTimes[level][year])
-  //     medians[level][year] = [
-  //       ['', median],
-  //       ['', comparisonValues[level] - median],
-  //     ]
-  //     means[level][year] = [
-  //       ['', mean],
-  //       ['', comparisonValues[level] - mean],
-  //     ]
-  //   })
-  // })
+  return data
+}
 
-  return { medians, means, graduationAmounts, years, levels, result }
+const countGraduationTimes = async (faculty, programmeFilter) => {
+  const isAcademicYear = false // = yearType === 'ACADEMIC_YEAR'
+  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
+  const yearsList = getYearsArray(since.getFullYear(), isAcademicYear)
+  const levels = ['bachelor', 'bcMsCombo', 'master', 'doctor', 'licentiate']
+  const programmeNames = {}
+
+  const years = [...yearsList].reverse()
+  const goals = {
+    bachelor: 36,
+    bcMsCombo: 60,
+    master: 24,
+    doctor: 48,
+    licentiate: 78,
+    exceptions: {
+      MH30_004: 6, //months more
+      '420420-ma': 6,
+      MH30_001: 12,
+      '320011-ma': 12,
+      '320001-ma': 12,
+      MH30_003: 12,
+      '320002-ma': 12,
+    },
+  }
+
+  if (faculty === 'H90') {
+    goals.bcMsCombo += 12
+  }
+
+  // We count studyrights (vs. studyright_elements)
+  // This way we get the whole time for a degree, even if the student was transferred to a new programme
+  // E.g. started 8/2016 in old Bc, transferred to new 10/2020, graduated from new 1/2021 --> total 53 months (not 3)
+
+  const byGradYear = await count(
+    faculty,
+    since,
+    years,
+    yearsList,
+    levels,
+    programmeFilter,
+    programmeNames,
+    goals,
+    'gradYear'
+  )
+  const byStartYear = await count(
+    faculty,
+    since,
+    years,
+    yearsList,
+    levels,
+    programmeFilter,
+    programmeNames,
+    goals,
+    'startYear'
+  )
+
+  const classSizes = getProgrammeObjectBasis(yearsList, levels, false)
+  classSizes['programmes'] = {}
+  await getClassSizes(faculty, since, classSizes, programmeFilter, years)
+
+  return { id: faculty, goals, byGradYear, byStartYear, programmeNames, classSizes }
 }
 
 module.exports = { countGraduationTimes }

services/backend/src/services/faculty/facultyHelpers.js

@@ -1,5 +1,6 @@
 const Sequelize = require('sequelize')
 const { Op } = Sequelize
+const { codes } = require('../../../config/programmeCodes')
 
 const findRightProgramme = (studyrightElements, mode) => {
   let programme = ''
@@ -55,14 +56,23 @@ const facultyFormatStudyright = studyright => {
     studyrightElements: studyright_elements,
   }
 }
-
+const facultyProgrammeStudents = student => {
+  const { studentnumber, home_country_en, gender_code, semester_enrollments } = student
+  return {
+    studentnumber,
+    home_country_en,
+    gender_code,
+    semesters: semester_enrollments.map(s => s.dataValues),
+  }
+}
 const formatFacultyTransfer = transfer => {
-  const { sourcecode, targetcode, transferdate, studyrightid } = transfer
+  const { sourcecode, targetcode, transferdate, studyrightid, studentnumber } = transfer
   return {
     sourcecode,
     targetcode,
     transferdate,
     studyrightid,
+    studentnumber,
   }
 }
 
@@ -87,11 +97,6 @@ const formatOrganization = org => {
   return { id, name, code, parentId: parent_id }
 }
 
-const formatAbsence = absence => {
-  const { semestercode, semester } = absence
-  return { semestercode, start: semester.startdate, end: semester.enddate }
-}
-
 const newProgrammes = [/^KH/, /^MH/, /^T/, /^LI/, /^K-/, /^FI/, /^00901$/, /^00910$/]
 
 const isNewProgramme = code => {
@@ -104,10 +109,11 @@ const isNewProgramme = code => {
 }
 
 const checkTransfers = (s, insideTransfersStudyrights, transfersToOrAwayStudyrights) => {
-  if (transfersToOrAwayStudyrights.includes(s.studyrightid)) return true
-  if (insideTransfersStudyrights.includes(s.studyrightid) && new Date(s.studystartdate) < new Date('2017-08-01'))
-    return true
-  return false
+  const allTransfers = [
+    ...insideTransfersStudyrights.map(sr => sr.studentnumber),
+    ...transfersToOrAwayStudyrights.map(sr => sr.studentnumber),
+  ]
+  return allTransfers.includes(s.studentnumber)
 }
 
 const getExtentFilter = includeAllSpecials => {
@@ -123,6 +129,21 @@ const getExtentFilter = includeAllSpecials => {
   return studyrightWhere
 }
 
+const mapCodesToIds = data => {
+  // Add programme id e.g. TKT
+  const keys = Object.keys(codes)
+  for (const prog of data) {
+    if (keys.includes(prog.code)) {
+      prog.progId = codes[prog.code].toUpperCase()
+      if (prog.code === 'T923103') {
+        prog.progId = prog.progId + '-DP'
+      }
+    } else {
+      prog.progId = prog.code
+    }
+  }
+}
+
 module.exports = {
   findRightProgramme,
   facultyFormatStudyright,
@@ -130,8 +151,9 @@ module.exports = {
   formatFacultyTransfer,
   formatFacultyThesisWriter,
   formatOrganization,
-  formatAbsence,
   isNewProgramme,
   checkTransfers,
   getExtentFilter,
+  mapCodesToIds,
+  facultyProgrammeStudents,
 }

services/backend/src/services/faculty/facultyService.js

@@ -6,10 +6,14 @@ const createRedisKeyForBasicStats = (id, yearType, programmeFilter, specialGroup
   `FACULTY_BASIC_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
 const createRedisKeyForCreditStats = (id, yearType, programmeFilter, specialGroups) =>
   `FACULTY_CREDIT_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
-// const createRedisKeyForGraduationTimeStats = (id, mode, programmeFilter) =>
-//  `FACULTY_GRADUATION_TIME_STATS_${id}_${mode}__${programmeFilter}`
 const createRedisKeyForThesiswriters = (id, yearType, programmeFilter, specialGroups) =>
   `FACULTY_THESIS_WRITERS_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
+const createRedisKeyForGraduationTimeStats = (id, programmeFilter) =>
+  `FACULTY_GRADUATION_TIME_STATS_${id}_${programmeFilter}`
+const createRediskeyForFacultyProgress = (id, special_groups, graduated) =>
+  `FACULTY_PROGRESS_STATS_${id}_${special_groups}_${graduated}`
+const createRediskeyForFacultyStudents = (id, special_groups, graduated) =>
+  `FACULTY_STUDENTS_STATS_${id}_${special_groups}_${graduated}`
 
 const setFacultyProgrammes = async (id, data, programmeFilter) => {
   const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
@@ -50,9 +54,9 @@ const getBasicStats = async (id, yearType, programmeFilter, specialGroups) => {
   return JSON.parse(dataFromRedis)
 }
 
-const setCreditStats = async (data, yearType, programmeFilter, specialGroups) => {
+const setCreditStats = async (data, yearType, programmeFilter) => {
   const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter, specialGroups)
+  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -89,6 +93,66 @@ const getThesisWritersStats = async (id, yearType, programmeFilter, specialGroup
   return JSON.parse(dataFromRedis)
 }
 
+const setGraduationStats = async (data, programmeFilter) => {
+  const { id } = data
+  const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getGraduationStats = async (id, programmeFilter) => {
+  const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const getFacultyProgressStats = async (id, specialGroups, graduated) => {
+  const redisKey = createRediskeyForFacultyProgress(id, specialGroups, graduated)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setFacultyProgressStats = async (data, specialGroups, graduated) => {
+  const { id } = data
+  const redisKey = createRediskeyForFacultyProgress(id, specialGroups, graduated)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getFacultyStudentStats = async (id, specialGroups, graduated) => {
+  const redisKey = createRediskeyForFacultyStudents(id, specialGroups, graduated)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setFacultyStudentStats = async (data, specialGroups, graduated) => {
+  const { id } = data
+  const redisKey = createRediskeyForFacultyStudents(id, specialGroups, graduated)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
 module.exports = {
   setFacultyProgrammes,
   getFacultyProgrammes,
@@ -98,4 +162,10 @@ module.exports = {
   getCreditStats,
   setThesisWritersStats,
   getThesisWritersStats,
+  setGraduationStats,
+  getGraduationStats,
+  getFacultyProgressStats,
+  setFacultyProgressStats,
+  getFacultyStudentStats,
+  setFacultyStudentStats,
 }

services/backend/src/services/faculty/facultyStudentProgress.js

@@ -0,0 +1,322 @@
+const { indexOf } = require('lodash')
+const { Op } = require('sequelize')
+const moment = require('moment')
+const { getAcademicYearDates } = require('../../util/semester')
+const {
+  getCreditThresholds,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
+  getDoctoralCreditGraphStats,
+  getYearsArray,
+  tableTitles,
+  getOnlyMasterCreditGraphStats,
+  getBcMsThresholds,
+  getVetenaryCreditGraphStats,
+} = require('../studyprogrammeHelpers')
+const { studytrackStudents } = require('../studyprogramme')
+const { getStudyRightsByExtent, getTransferredInside, getTransferredToAndAway } = require('./faculty')
+const { checkTransfers } = require('./facultyHelpers')
+
+const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts, limits = [], limitKeys = [], prog) => {
+  let data = {}
+  let programmeData = {}
+  thresholdKeys.forEach(t => (data[t] = 0))
+  limitKeys.forEach(t => (programmeData[t] = 0))
+  students.forEach(({ credits }) => {
+    const creditcount = credits
+      .filter(credit => moment(credit.attainment_date).isSameOrAfter(startDate))
+      .reduce((prev, curr) => prev + curr.credits, 0)
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    if (thresholdKeys.length === 8) {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] && creditcount < thresholdAmounts[6] ? 1 : 0
+      data[thresholdKeys[7]] += creditcount >= thresholdAmounts[6] ? 1 : 0
+    } else if (thresholdKeys.length === 7) {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] ? 1 : 0
+    } else if (thresholdKeys.length === 6) {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] ? 1 : 0
+    } else {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] ? 1 : 0
+    }
+    if (['KH', 'MH'].includes(prog)) {
+      programmeData[limitKeys[0]] += creditcount <= limits[5][0] ? 1 : 0
+      programmeData[limitKeys[1]] += limits[4][0] <= creditcount && limits[4][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[2]] += limits[3][0] <= creditcount && limits[3][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[3]] += limits[2][0] <= creditcount && limits[2][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[4]] += limits[1][0] <= creditcount && limits[1][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[5]] += creditcount >= limits[0][0] ? 1 : 0
+    }
+  })
+
+  return { data, progData: programmeData }
+}
+
+const createLimits = (months, creditsToAdd) => {
+  return [
+    [Math.ceil(months * (60 / 12)) + creditsToAdd, null],
+    [Math.ceil(months * (45 / 12)) + creditsToAdd, Math.ceil(months * (60 / 12)) + creditsToAdd],
+    [Math.ceil(months * (30 / 12)) + creditsToAdd, Math.ceil(months * (45 / 12)) + creditsToAdd],
+    [Math.ceil(months * (15 / 12)) + creditsToAdd, Math.ceil(months * (30 / 12)) + creditsToAdd],
+    [creditsToAdd + 1, Math.ceil(months * (15 / 12)) + creditsToAdd],
+    [creditsToAdd, null],
+  ]
+}
+
+const createYearlyTitles = (start, limitList) => {
+  return [
+    `${start} Credits`,
+    `${limitList[4][0]} ≤ Credits < ${limitList[4][1]}`,
+    `${limitList[3][0]} ≤ Credits < ${limitList[3][1]}`,
+    `${limitList[2][0]} ≤ Credits < ${limitList[2][1]}`,
+    `${limitList[1][0]} ≤ Credits < ${limitList[1][1]}`,
+    `${limitList[0][0]} ≤ Credits`,
+  ]
+}
+
+const filterOutTransfers = async (studyrights, programmes, allProgrammeCodes, since) => {
+  const transferredInAndOut = await getTransferredToAndAway(programmes, allProgrammeCodes, since)
+  const insideTransfers = await getTransferredInside(programmes, allProgrammeCodes, since)
+  const filteredStudyrights = studyrights.filter(sr => !checkTransfers(sr, insideTransfers, transferredInAndOut))
+  return filteredStudyrights
+}
+
+const combineFacultyStudentProgress = async (faculty, programmes, allProgrammeCodes, specialGroups, graduated) => {
+  const since = new Date('2017-08-01')
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
+  const graduationStatus = [0]
+  if (includeGraduated) {
+    graduationStatus.push(1)
+  }
+  const yearsArray = getYearsArray(since.getFullYear(), true, true)
+  const titlesVet = [
+    '',
+    'All',
+    '< 210 credits',
+    '210-239 credits',
+    '240-269 credits',
+    '270-299 credits',
+    '300-329 credits',
+    '330-359 credits',
+    ' 360 ≤ credits ',
+  ]
+  const progressStats = {
+    id: faculty,
+    years: yearsArray,
+    bachelorsTableStats: new Array(yearsArray.length),
+    bcMsTableStats: new Array(yearsArray.length),
+    mastersTableStats: new Array(yearsArray.length),
+    doctoralTableStats: new Array(yearsArray.length),
+    bachelorsGraphStats: getBachelorCreditGraphStats(yearsArray),
+    bcMsGraphStats: faculty === 'H90' ? getVetenaryCreditGraphStats(yearsArray) : getMasterCreditGraphStats(yearsArray),
+    mastersGraphStats:
+      faculty === 'H90' ? getBachelorCreditGraphStats(yearsArray) : getOnlyMasterCreditGraphStats(yearsArray),
+    doctoralGraphStats: getDoctoralCreditGraphStats(yearsArray),
+    bachelorTitles: tableTitles.creditProgress.bachelor,
+    bcMsTitles: faculty === 'H90' ? titlesVet : tableTitles.creditProgress.master,
+    mastersTitles: faculty === 'H90' ? tableTitles.creditProgress.bachelor : tableTitles.creditProgress.masterOnly,
+    doctoralTitles: tableTitles.creditProgress.doctoral,
+    yearlyBachelorTitles: [],
+    yearlyBcMsTitles: [],
+    yearlyMasterTitles: [],
+    programmeNames: {},
+    bachelorsProgStats: {},
+    bcMsProgStats: {},
+    mastersProgStats: {},
+    doctoralProgStats: {},
+  }
+
+  const reversedYears = [...yearsArray].reverse()
+  reversedYears.forEach(year => {
+    progressStats.bachelorsTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    progressStats.bcMsTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0, 0]
+    progressStats.mastersTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0]
+    progressStats.doctoralTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0]
+  })
+
+  const limitKeys = ['zero', 't5', 't4', 't3', 't2', 't1']
+  let bachelorlimits = []
+  let masterlimits = []
+  let bcmslimits = []
+  const bachelorProgress = {}
+  const masterProgress = {}
+  const bcmsProgress = {}
+  const doctoralProgress = {}
+  for (const year of reversedYears) {
+    if (year === 'Total') continue
+    const { startDate, endDate } = getAcademicYearDates(year, since)
+    const lastDayOfMonth = moment().endOf('month')
+    let months = Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(startDate))).asMonths())
+    if (months >= 36) {
+      bachelorlimits = createLimits(36, 0)
+    } else {
+      bachelorlimits = createLimits(months, 0)
+    }
+    if (months >= 24) {
+      masterlimits = createLimits(24, 0)
+    } else {
+      masterlimits = createLimits(months, 0)
+    }
+    if (months >= 72 && faculty === 'H90') {
+      bcmslimits = createLimits(72, 0)
+    } else if (months >= 60) {
+      bcmslimits = createLimits(60, 0)
+    } else {
+      bcmslimits = createLimits(months, 0)
+    }
+    progressStats.yearlyBachelorTitles = [...progressStats.yearlyBachelorTitles, createYearlyTitles(0, bachelorlimits)]
+    progressStats.yearlyMasterTitles = [...progressStats.yearlyMasterTitles, createYearlyTitles(0, masterlimits)]
+    progressStats.yearlyBcMsTitles = [...progressStats.yearlyBcMsTitles, createYearlyTitles(0, bcmslimits)]
+    const programmeCodes = programmes.map(prog => prog.code)
+
+    for (const { progId, code, name } of programmes) {
+      let extentcodes = [1, 2, 3, 4]
+      if (includeAllSpecials) {
+        extentcodes = [...extentcodes, ...[7, 9, 34, 22, 99, 14, 13]]
+      }
+      const startDateWhere = {
+        startdate: {
+          [Op.and]: {
+            [Op.gte]: startDate,
+            [Op.lte]: endDate,
+          },
+        },
+      }
+      let studyrights = await getStudyRightsByExtent(faculty, {}, startDateWhere, code, extentcodes, graduationStatus)
+      let { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(code, true)
+      if (!creditThresholdKeys) continue
+      if (!(progId in progressStats.programmeNames)) {
+        progressStats.programmeNames[progId] = { code: code, ...name }
+      }
+      if (!includeAllSpecials) {
+        studyrights = await filterOutTransfers(studyrights, programmeCodes, allProgrammeCodes, since)
+      }
+      if (code.includes('KH')) {
+        const allBachelors = studyrights.filter(sr => sr.extentcode === 1).map(sr => sr.studentnumber)
+        const students = await studytrackStudents(allBachelors)
+        const { data: studentData, progData } = getStudentData(
+          startDate,
+          students,
+          creditThresholdKeys,
+          creditThresholdAmounts,
+          bachelorlimits,
+          limitKeys,
+          'KH'
+        )
+
+        if (!(progId in bachelorProgress)) {
+          bachelorProgress[progId] = new Array(reversedYears.length - 1)
+        }
+        progressStats.bachelorsTableStats[indexOf(reversedYears, year)][1] += students.length || 0
+        progressStats.bachelorsTableStats[indexOf(reversedYears, 'Total')][1] += students.length || 0
+        bachelorProgress[progId][indexOf(reversedYears, year)] = limitKeys.map(key => progData[key])
+        for (const key of Object.keys(studentData)) {
+          progressStats.bachelorsTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeys, key) + 2] +=
+            studentData[key] || 0
+          progressStats.bachelorsGraphStats[key].data[indexOf(yearsArray, year)] += studentData[key] || 0
+          progressStats.bachelorsTableStats[indexOf(reversedYears, 'Total')][indexOf(creditThresholdKeys, key) + 2] +=
+            studentData[key] || 0
+          progressStats.bachelorsGraphStats[key].data[indexOf(yearsArray, 'Total')] += studentData[key] || 0
+        }
+      } else if (code.includes('MH')) {
+        const allMsStudents = studyrights
+          .filter(sr => sr.extentcode === 2 && sr.studyrightid.slice(-2) !== '-2')
+          .map(sr => sr.studentnumber)
+        const allBcMsStudents = studyrights
+          .filter(sr => sr.extentcode === 2 && sr.studyrightid.slice(-2) === '-2')
+          .map(sr => sr.studentnumber)
+        const bcMsStudents = await studytrackStudents(allBcMsStudents)
+        const msStudents = await studytrackStudents(allMsStudents)
+
+        progressStats.mastersTableStats[indexOf(reversedYears, year)][1] += msStudents.length || 0
+        progressStats.mastersTableStats[indexOf(reversedYears, 'Total')][1] += msStudents.length || 0
+        progressStats.bcMsTableStats[indexOf(reversedYears, year)][1] += bcMsStudents.length || 0
+        progressStats.bcMsTableStats[indexOf(reversedYears, 'Total')][1] += bcMsStudents.length || 0
+        const { creditThresholdKeysBcMs, creditThresholdAmountsBcMs } = getBcMsThresholds(code)
+        const { data: bcMsStudentdata, progData: bcMsProgdata } = getStudentData(
+          startDate,
+          bcMsStudents,
+          creditThresholdKeysBcMs,
+          creditThresholdAmountsBcMs,
+          bcmslimits,
+          limitKeys,
+          'MH'
+        )
+
+        const { data: msStudentdata, progData: msProgdata } = getStudentData(
+          startDate,
+          msStudents,
+          creditThresholdKeys,
+          creditThresholdAmounts,
+          masterlimits,
+          limitKeys,
+          'MH'
+        )
+        if (!(progId in masterProgress)) {
+          masterProgress[progId] = new Array(reversedYears.length - 1)
+          bcmsProgress[progId] = new Array(reversedYears.length - 1)
+        }
+        masterProgress[progId][indexOf(reversedYears, year)] = limitKeys.map(key => msProgdata[key])
+        bcmsProgress[progId][indexOf(reversedYears, year)] = limitKeys.map(key => bcMsProgdata[key])
+        for (const key of Object.keys(msStudentdata)) {
+          progressStats.mastersTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeys, key) + 2] +=
+            msStudentdata[key] || 0
+          progressStats.mastersTableStats[indexOf(reversedYears, 'Total')][indexOf(creditThresholdKeys, key) + 2] +=
+            msStudentdata[key] || 0
+          progressStats.mastersGraphStats[key].data[indexOf(yearsArray, year)] += msStudentdata[key] || 0
+          progressStats.mastersGraphStats[key].data[indexOf(yearsArray, 'Total')] += msStudentdata[key] || 0
+        }
+        for (const key of Object.keys(bcMsStudentdata)) {
+          progressStats.bcMsTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeysBcMs, key) + 2] +=
+            bcMsStudentdata[key] || 0
+          progressStats.bcMsTableStats[indexOf(reversedYears, 'Total')][indexOf(creditThresholdKeysBcMs, key) + 2] +=
+            bcMsStudentdata[key] || 0
+          progressStats.bcMsGraphStats[key].data[indexOf(yearsArray, year)] += bcMsStudentdata[key] || 0
+          progressStats.bcMsGraphStats[key].data[indexOf(yearsArray, 'Total')] += bcMsStudentdata[key] || 0
+        }
+      } else {
+        const all = studyrights.filter(sr => sr.extentcode === 4 || sr.extentcode === 3).map(sr => sr.studentnumber)
+        const doctoralStudents = await studytrackStudents(all)
+        const { data } = getStudentData(startDate, doctoralStudents, creditThresholdKeys, creditThresholdAmounts, 'T')
+
+        progressStats.doctoralTableStats[indexOf(reversedYears, year)][1] += doctoralStudents.length || 0
+        progressStats.doctoralTableStats[indexOf(reversedYears, 'Total')][1] += doctoralStudents.length || 0
+        if (!(progId in doctoralProgress)) {
+          doctoralProgress[progId] = new Array(reversedYears.length - 1)
+        }
+        doctoralProgress[progId][indexOf(reversedYears, year)] = creditThresholdKeys.map(key => data[key])
+
+        for (const key of Object.keys(data)) {
+          progressStats.doctoralTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeys, key) + 2] +=
+            data[key] || 0
+          progressStats.doctoralTableStats[indexOf(reversedYears, 'Total')][indexOf(creditThresholdKeys, key) + 2] +=
+            data[key] || 0
+          progressStats.doctoralGraphStats[key].data[indexOf(yearsArray, year)] += data[key] || 0
+          progressStats.doctoralGraphStats[key].data[indexOf(yearsArray, 'Total')] += data[key] || 0
+        }
+      }
+    }
+  }
+
+  progressStats.bachelorsProgStats = Object.keys(bachelorProgress)
+    .filter(prog => !bachelorProgress[prog].every(year => year.every(value => value === 0)))
+    .reduce((result, prog) => ({ ...result, [prog]: bachelorProgress[prog] }), {})
+  progressStats.mastersProgStats = Object.keys(masterProgress)
+    .filter(prog => !masterProgress[prog].every(year => year.every(value => value === 0)))
+    .reduce((result, prog) => ({ ...result, [prog]: masterProgress[prog] }), {})
+  progressStats.bcMsProgStats = Object.keys(bcmsProgress)
+    .filter(prog => !bcmsProgress[prog].every(year => year.every(value => value === 0)))
+    .reduce((result, prog) => ({ ...result, [prog]: bcmsProgress[prog] }), {})
+  progressStats.doctoralProgStats = Object.keys(doctoralProgress)
+    .filter(prog => !doctoralProgress[prog].every(year => year.every(value => value === 0)))
+    .reduce((result, prog) => ({ ...result, [prog]: doctoralProgress[prog] }), {})
+  return progressStats
+}
+module.exports = { combineFacultyStudentProgress }

services/backend/src/services/faculty/facultyStudents.js

@@ -0,0 +1,279 @@
+const moment = require('moment')
+const { Op } = require('sequelize')
+const { getStartDate, getYearsArray, getPercentage, getYearsObject, tableTitles } = require('../studyprogrammeHelpers')
+const { getAcademicYearDates } = require('../../util/semester')
+const { getStudyRightsByExtent, getStudentsByStudentnumbers, getTransfers } = require('./faculty')
+const { getCurrentSemester } = require('../semesters')
+const { checkTransfers } = require('./facultyHelpers')
+const { inactiveStudyrights, absentStudents } = require('../studyprogramme')
+const emptyTotals = () => {
+  return {
+    total: 0,
+    allStarted: 0,
+    allEnrolled: 0,
+    allAbsent: 0,
+    allInactive: 0,
+    allGraduated: 0,
+    allFinnish: 0,
+    allFemale: 0,
+    allMale: 0,
+    allOtherUnknown: 0,
+    allOtherCountries: 0,
+  }
+}
+const toltalsCalculation = totals => {
+  return [
+    totals.total,
+    totals.allStarted,
+    getPercentage(totals.allStarted, totals.total),
+    totals.allEnrolled,
+    getPercentage(totals.allEnrolled, totals.total),
+    totals.allAbsent,
+    getPercentage(totals.allAbsent, totals.total),
+    totals.allInactive,
+    getPercentage(totals.allInactive, totals.total),
+    totals.allGraduated,
+    getPercentage(totals.allGraduated, totals.total),
+    totals.allMale,
+    getPercentage(totals.allMale, totals.total),
+    totals.allFemale,
+    getPercentage(totals.allFemale, totals.total),
+    totals.allOtherUnknown,
+    getPercentage(totals.allOtherUnknown, totals.total),
+    totals.allFinnish,
+    getPercentage(totals.allFinnish, totals.total),
+    totals.allOtherCountries,
+    getPercentage(totals.allOtherCountries, totals.total),
+  ]
+}
+const addTotals = (facultyTableStats, year, totals) => {
+  facultyTableStats[year] = [year, ...toltalsCalculation(totals)]
+}
+const addTotalsProgramme = (programmeTableStats, progId, year, totals) => {
+  programmeTableStats[progId][year] = toltalsCalculation(totals)
+}
+
+const getStudentData = (students, facultyExtra, year, code) => {
+  let data = { female: 0, male: 0, finnish: 0, otherCountries: 0, otherUnknown: 0 }
+  students.forEach(({ gender_code, home_country_en }) => {
+    data.male += gender_code === '1' ? 1 : 0
+    data.female += gender_code === '2' ? 1 : 0
+    data.otherUnknown += ['0', '3'].includes(gender_code) ? 1 : 0
+    facultyExtra[year][code].otherG += gender_code === '3' ? 1 : 0
+    facultyExtra[year][code].unknownG += gender_code === '0' ? 1 : 0
+    facultyExtra['Total'][code].otherG += gender_code === '3' ? 1 : 0
+    facultyExtra['Total'][code].unknownG += gender_code === '0' ? 1 : 0
+    data.finnish += home_country_en === 'Finland' ? 1 : 0
+    data.otherCountries += home_country_en !== 'Finland' ? 1 : 0
+    if (!Object.keys(facultyExtra[year][code].countries).includes(home_country_en))
+      facultyExtra[year][code].countries[home_country_en] = 0
+    if (!Object.keys(facultyExtra['Total'][code].countries).includes(home_country_en))
+      facultyExtra['Total'][code].countries[home_country_en] = 0
+    facultyExtra[year][code].countries[home_country_en] += 1
+    facultyExtra['Total'][code].countries[home_country_en] += 1
+  })
+  return data
+}
+
+// Goes through the faculty and its study programmes for the year
+const getFacultyDataForYear = async ({
+  faculty,
+  programmes,
+  allProgrammeCodes,
+  since,
+  settings,
+  year,
+  years,
+  programmeTableStats,
+  facultyExtra,
+  allTotals,
+  progTotals,
+  programmeNames,
+}) => {
+  const { includeAllSpecials, includeGraduated } = settings
+  const { startDate, endDate } = getAcademicYearDates(year, since)
+  // Totals for the year into table stats
+  let extents = [1, 2, 3, 4]
+  let graduated = [0]
+  if (includeAllSpecials) {
+    extents = [...extents, ...[7, 9, 34, 22, 99, 14, 13]]
+  }
+  if (includeGraduated) {
+    graduated = [...graduated, 1]
+  }
+  const totals = emptyTotals()
+
+  for (const { progId, code, name } of programmes) {
+    let elementStart = {}
+    let studyrightStart = {}
+    const start = moment(`${year.slice(0, 4)}-08-01`, 'YYYY-MM-DD')
+    const end = moment(`${year.slice(-4)}-07-31`, 'YYYY-MM-DD').endOf('day')
+    const allTransfers = await getTransfers(code, allProgrammeCodes, start, end)
+    if (!Object.keys(facultyExtra[year]).includes(code))
+      facultyExtra[year][code] = { unknownG: 0, otherG: 0, countries: {} }
+    if (!Object.keys(facultyExtra['Total']).includes(code))
+      facultyExtra['Total'][code] = { unknownG: 0, otherG: 0, countries: {} }
+    const startDateWhere = {
+      startdate: {
+        [Op.and]: {
+          [Op.gte]: startDate,
+          [Op.lte]: endDate,
+        },
+      },
+    }
+    if (code.includes('MH')) {
+      elementStart = startDateWhere
+    } else {
+      studyrightStart = startDateWhere
+    }
+    const allStudyrights = await getStudyRightsByExtent(
+      faculty,
+      elementStart,
+      studyrightStart,
+      code,
+      extents,
+      graduated
+    )
+    let studyrights = allStudyrights
+    let allStudents = []
+
+    studyrights = allStudyrights.filter(s => !checkTransfers(s, allTransfers, allTransfers))
+    if (includeAllSpecials) {
+      allStudents = [...new Set([...allTransfers.map(sr => sr.studentnumber).filter(smbr => smbr !== null)])]
+    }
+    // Get all the studyrights and students for the calculations
+    const currentSemester = await getCurrentSemester()
+    const studentNbrs = studyrights.map(sr => sr.studentnumber)
+    allStudents = [...studentNbrs, ...allStudents]
+
+    const students = await getStudentsByStudentnumbers(allStudents)
+    const studentData = getStudentData(students, facultyExtra, year, code)
+    const started = await getStudentsByStudentnumbers(studentNbrs)
+    const inactive = await inactiveStudyrights(code, studentNbrs)
+    const graduatedStudents = studyrights.filter(sr => sr.graduated === 1)
+    const absent = await absentStudents(code, studentNbrs)
+    const enrolled = students.filter(
+      s =>
+        s.semesters.filter(semester => semester.semestercode === currentSemester.semestercode)[0]?.enrollmenttype === 1
+    )
+    // Count stats for the programme
+    if (students.length === 0) continue
+    if (!(progId in programmeTableStats)) {
+      programmeTableStats[progId] = years.reduce((resultObj, yearKey) => {
+        return { ...resultObj, [yearKey]: [] }
+      }, {})
+      progTotals[progId] = emptyTotals()
+    }
+    if (!(progId in programmeNames)) {
+      programmeNames[progId] = { code: code, ...name }
+      progTotals[progId] = emptyTotals()
+    }
+
+    const yearTotals = {
+      total: students.length,
+      allStarted: started.length,
+      allEnrolled: enrolled.length,
+      allAbsent: absent.length,
+      allInactive: inactive.length,
+      allGraduated: graduatedStudents.length,
+      allMale: studentData.male,
+      allFemale: studentData.female,
+      allOtherUnknown: studentData.otherUnknown,
+      allFinnish: studentData.finnish,
+      allOtherCountries: studentData.otherCountries,
+    }
+    addTotalsProgramme(programmeTableStats, progId, year, yearTotals)
+
+    totals.total += students.length
+    totals.allStarted += started.length
+    totals.allEnrolled += enrolled.length
+    totals.allAbsent += absent.length
+    totals.allInactive += inactive.length
+    totals.allGraduated += graduatedStudents.length
+    totals.allFemale += studentData.female
+    totals.allMale += studentData.male
+    totals.allOtherUnknown += studentData.otherUnknown
+    totals.allFinnish += studentData.finnish
+    totals.allOtherCountries += studentData.otherCountries
+
+    allTotals.total += students.length
+    allTotals.allStarted += started.length
+    allTotals.allEnrolled += enrolled.length
+    allTotals.allAbsent += absent.length
+    allTotals.allInactive += inactive.length
+    allTotals.allGraduated += graduatedStudents.length
+    allTotals.allFemale += studentData.female
+    allTotals.allMale += studentData.male
+    allTotals.allOtherUnknown += studentData.otherUnknown
+    allTotals.allFinnish += studentData.finnish
+    allTotals.allOtherCountries += studentData.otherCountries
+
+    progTotals[progId].total += students.length
+    progTotals[progId].allStarted += started.length
+    progTotals[progId].allEnrolled += enrolled.length
+    progTotals[progId].allAbsent += absent.length
+    progTotals[progId].allInactive += inactive.length
+    progTotals[progId].allGraduated += graduatedStudents.length
+    progTotals[progId].allFemale += studentData.female
+    progTotals[progId].allMale += studentData.male
+    progTotals[progId].allOtherUnknown += studentData.otherUnknown
+    progTotals[progId].allFinnish += studentData.finnish
+    progTotals[progId].allOtherCountries += studentData.otherCountries
+  }
+  return totals
+}
+
+// Combines all the data for faculty students table
+const combineFacultyStudents = async (code, programmes, allProgrammeCodes, specialGroups, graduated) => {
+  // Only academic years are considered
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
+  const since = getStartDate(code, true)
+  const years = getYearsArray(since.getFullYear(), true, true)
+  const settings = { includeAllSpecials, includeGraduated }
+  const facultyTableStats = getYearsObject({ years, emptyArrays: true })
+  const programmeTableStats = {}
+  const facultyExtra = years.reduce((acc, year) => ({ ...acc, [year]: {} }), {})
+  const progTotals = {}
+  const allTotals = emptyTotals()
+  const programmeNames = {}
+  for (const year of years.reverse()) {
+    if (year === 'Total') continue
+    const queryParams = {
+      faculty: code,
+      programmes,
+      allProgrammeCodes,
+      since,
+      settings,
+      year,
+      years,
+      programmeTableStats,
+      facultyExtra,
+      allTotals,
+      progTotals,
+      programmeNames,
+    }
+    const totals = await getFacultyDataForYear(queryParams)
+    if (totals.total !== 0) {
+      addTotals(facultyTableStats, year, totals)
+    }
+  }
+
+  addTotals(facultyTableStats, 'Total', allTotals)
+  Object.keys(progTotals).forEach(programme =>
+    addTotalsProgramme(programmeTableStats, programme, 'Total', progTotals[programme])
+  )
+
+  const studentsData = {
+    id: code,
+    years: years,
+    facultyTableStats: facultyTableStats,
+    facultyTableStatsExtra: facultyExtra,
+    programmeStats: programmeTableStats,
+    titles: tableTitles['programmes'],
+    programmeNames: programmeNames,
+  }
+  return studentsData
+}
+
+module.exports = { combineFacultyStudents }

services/backend/src/services/faculty/facultyThesisWriters.js

@@ -11,21 +11,15 @@ const {
 } = require('../studyprogrammeHelpers')
 
 const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }) => {
-  const thesisTypes = [
-    'urn:code:course-unit-type:bachelors-thesis',
-    'urn:code:course-unit-type:masters-thesis',
-    'urn:code:course-unit-type:doctors-thesis',
-    'urn:code:course-unit-type:licentiate-thesis',
-  ]
+  const thesisTypes = ['urn:code:course-unit-type:bachelors-thesis', 'urn:code:course-unit-type:masters-thesis']
 
   let bachelors = getStatsBasis(years)
   let masters = getStatsBasis(years)
-  let doctors = getStatsBasis(years)
-  let licentiates = getStatsBasis(years)
   let programmeCounts = {}
   let programmeNames = {}
 
-  for (const { code, name } of facultyProgrammes) {
+  for (const { progId, code, name } of facultyProgrammes) {
+    if (code === 'MH70_008_2' || code.startsWith('LIS') || code.startsWith('T')) continue
     const provider = mapToProviders([code])[0]
     const students = await getCorrectStudentnumbers({
       codes: [code],
@@ -33,42 +27,35 @@ const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyPr
       endDate: alltimeEndDate,
       includeAllSpecials,
     })
+
     const thesisCourseCodes = await thesisWriters(provider, since, thesisTypes, students)
     thesisCourseCodes?.forEach(({ attainment_date, courseUnitType }) => {
       const thesisYear = defineYear(attainment_date, isAcademicYear)
 
-      if (!(code in programmeCounts)) {
-        programmeCounts[code] = {}
+      if (!(progId in programmeCounts)) {
+        programmeCounts[progId] = {}
 
-        Object.keys(bachelors.tableStats).forEach(year => (programmeCounts[code][year] = [0, 0, 0, 0, 0]))
-        programmeNames[code] = name
+        Object.keys(bachelors.tableStats).forEach(year => (programmeCounts[progId][year] = [0, 0, 0]))
+        programmeNames[progId] = { ...name, code: code }
       }
-      programmeCounts[code][thesisYear][0] += 1
+      programmeCounts[progId][thesisYear][0] += 1
 
       if (courseUnitType === thesisTypes[0]) {
         bachelors.graphStats[indexOf(years, thesisYear)] += 1
         bachelors.tableStats[thesisYear] += 1
-        programmeCounts[code][thesisYear][1] += 1
+        programmeCounts[progId][thesisYear][1] += 1
       } else if (courseUnitType === thesisTypes[1]) {
         masters.graphStats[indexOf(years, thesisYear)] += 1
         masters.tableStats[thesisYear] += 1
-        programmeCounts[code][thesisYear][2] += 1
-      } else if (courseUnitType === thesisTypes[2]) {
-        doctors.graphStats[indexOf(years, thesisYear)] += 1
-        doctors.tableStats[thesisYear] += 1
-        programmeCounts[code][thesisYear][3] += 1
-      } else if (courseUnitType === thesisTypes[3]) {
-        licentiates.graphStats[indexOf(years, thesisYear)] += 1
-        licentiates.tableStats[thesisYear] += 1
-        programmeCounts[code][thesisYear][4] += 1
+        programmeCounts[progId][thesisYear][2] += 1
       }
     })
   }
 
-  return { bachelors, masters, doctors, licentiates, programmeCounts, programmeNames }
+  return { bachelors, masters, programmeCounts, programmeNames }
 }
 
-const getFacultyThesisWritersForStudyTrack = async (
+const getFacultyThesisWritersForProgrammes = async (
   allThesisWriters,
   facultyProgrammes,
   isAcademicYear,
@@ -77,33 +64,27 @@ const getFacultyThesisWritersForStudyTrack = async (
   const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
   const years = getYearsArray(since.getFullYear(), isAcademicYear)
   const queryParameters = { since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }
-  const { bachelors, masters, doctors, licentiates, programmeCounts, programmeNames } = await getFacultyThesisWriters(
-    queryParameters
-  )
+  const { bachelors, masters, programmeCounts, programmeNames } = await getFacultyThesisWriters(queryParameters)
 
   allThesisWriters.years = years.map(year => year.toString())
   const reversedYears = years.reverse()
   allThesisWriters.tableStats = reversedYears.map(year => [
     year,
-    bachelors.tableStats[year] + masters.tableStats[year] + doctors.tableStats[year] + licentiates.tableStats[year],
+    bachelors.tableStats[year] + masters.tableStats[year],
     bachelors.tableStats[year],
     masters.tableStats[year],
-    doctors.tableStats[year],
-    licentiates.tableStats[year],
   ])
   allThesisWriters.graphStats.push({ name: 'Bachelors', data: bachelors.graphStats })
   allThesisWriters.graphStats.push({ name: 'Masters', data: masters.graphStats })
-  allThesisWriters.graphStats.push({ name: 'Doctors', data: doctors.graphStats })
-  allThesisWriters.graphStats.push({ name: 'Licentiates', data: licentiates.graphStats })
 
   const programmes = programmeNames ? Object.keys(programmeNames) : facultyProgrammes.map(programme => programme.code)
-  programmes.forEach(programmeCode => {
+  programmes.forEach(programmeId => {
     reversedYears.forEach(year => {
-      if (programmeCounts && programmeCode in programmeCounts) {
-        if (!(programmeCode in allThesisWriters.programmeTableStats)) {
-          allThesisWriters.programmeTableStats[programmeCode] = []
+      if (programmeCounts && programmeId in programmeCounts) {
+        if (!(programmeId in allThesisWriters.programmeTableStats)) {
+          allThesisWriters.programmeTableStats[programmeId] = []
         }
-        allThesisWriters.programmeTableStats[programmeCode].push([year, ...programmeCounts[programmeCode][year]])
+        allThesisWriters.programmeTableStats[programmeId].push([year, ...programmeCounts[programmeId][year]])
       }
     })
   })
@@ -118,7 +99,7 @@ const combineFacultyThesisWriters = async (faculty, facultyProgrammes, yearType,
     tableStats: [],
     graphStats: [],
     programmeTableStats: {},
-    titles: ['', 'All', 'Bachelors', 'Masters', 'Doctors', 'Licentiates'],
+    titles: ['', 'All', 'Bachelors', 'Masters'],
     programmeNames: {},
     status: 'Done',
     lastUpdated: '',
@@ -126,7 +107,7 @@ const combineFacultyThesisWriters = async (faculty, facultyProgrammes, yearType,
   const isAcademicYear = yearType === 'ACADEMIC_YEAR'
   const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
 
-  await getFacultyThesisWritersForStudyTrack(allThesisWriters, facultyProgrammes, isAcademicYear, includeAllSpecials)
+  await getFacultyThesisWritersForProgrammes(allThesisWriters, facultyProgrammes, isAcademicYear, includeAllSpecials)
 
   return allThesisWriters
 }

services/backend/src/services/faculty/facultyUpdates.js

@@ -3,7 +3,18 @@ const { findFacultyProgrammeCodes } = require('./faculty')
 const { combineFacultyBasics } = require('./facultyBasics')
 const { combineFacultyCredits } = require('./facultyCredits')
 const { combineFacultyThesisWriters } = require('./facultyThesisWriters')
-const { setFacultyProgrammes, setBasicStats, setCreditStats, setThesisWritersStats } = require('./facultyService')
+const { countGraduationTimes } = require('./facultyGraduationTimes')
+const {
+  setFacultyProgrammes,
+  setBasicStats,
+  setCreditStats,
+  setThesisWritersStats,
+  setGraduationStats,
+  setFacultyProgressStats,
+  setFacultyStudentStats,
+} = require('./facultyService')
+const { combineFacultyStudentProgress } = require('./facultyStudentProgress')
+const { combineFacultyStudents } = require('./facultyStudents')
 
 const updateFacultyOverview = async (faculty, statsType) => {
   const calendarNewSpecial = {
@@ -89,12 +100,12 @@ const updateFacultyOverview = async (faculty, statsType) => {
         )
         await setBasicStats(updatedStudentInfo, option.yearType, option.programmeFilter, option.specialGroups)
       }
-      if (statsType === 'ALL' || statsType === 'CREDITS') {
+      if ((statsType === 'ALL' || statsType === 'CREDITS') && specialGroups !== 'SPECIAL_EXCLUDED') {
         const updatedCredits = await combineFacultyCredits(
           faculty,
           programmeFilter === 'ALL_PROGRAMMES' ? allProgrammes.data : newProgrammes.data,
-          yearType,
-          specialGroups
+          allProgrammes.data,
+          yearType
         )
         await setCreditStats(updatedCredits, yearType, programmeFilter, specialGroups)
       }
@@ -111,7 +122,74 @@ const updateFacultyOverview = async (faculty, statsType) => {
       logger.error(e)
     }
   }
+
+  try {
+    const updatedTimesAll = await countGraduationTimes(faculty, 'ALL_PROGRAMMES')
+    await setGraduationStats(updatedTimesAll, 'ALL_PROGRAMMES')
+    const updatedTimesNew = await countGraduationTimes(faculty, 'NEW_STUDY_PROGRAMMES')
+    await setGraduationStats(updatedTimesNew, 'NEW_STUDY_PROGRAMMES')
+  } catch (e) {
+    logger.error(e)
+  }
   return 'OK'
 }
 
-module.exports = { updateFacultyOverview }
+const updateFacultyProgressOverview = async faculty => {
+  const specialGraduated = {
+    specialGroups: 'SPECIAL_INCLUDED',
+    graduated: 'GRADUATED_INCLUDED',
+  }
+  const specialNotGraduated = {
+    specialGroups: 'SPECIAL_INCLUDED',
+    graduated: 'GRADUATED_EXCLUDED',
+  }
+  const notSpecialGraduated = {
+    specialGroups: 'SPECIAL_EXCLUDED',
+    graduated: 'GRADUATED_INCLUDED',
+  }
+  const notSpecialNotGraduated = {
+    specialGroups: 'SPECIAL_EXCLUDED',
+    graduated: 'GRADUATED_EXCLUDED',
+  }
+  const options = [specialGraduated, specialNotGraduated, notSpecialGraduated, notSpecialNotGraduated]
+
+  let newProgrammes = []
+  let allProgrammes = []
+  let allProgrammeCodes = []
+  try {
+    const all = await findFacultyProgrammeCodes(faculty, 'ALL_PROGRAMMES')
+    const onlyNew = await findFacultyProgrammeCodes(faculty, 'NEW_STUDY_PROGRAMMES')
+    allProgrammes = await setFacultyProgrammes(faculty, all, 'ALL_PROGRAMMES')
+    newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
+    allProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+  } catch (e) {
+    logger.error(e)
+  }
+
+  for (const option of options) {
+    const { specialGroups, graduated } = option
+    try {
+      const updateFacultyStudentStats = await combineFacultyStudents(
+        faculty,
+        newProgrammes.data,
+        allProgrammeCodes,
+        specialGroups,
+        graduated
+      )
+      await setFacultyStudentStats(updateFacultyStudentStats, specialGroups, graduated)
+      const updateFacultyProgressStats = await combineFacultyStudentProgress(
+        faculty,
+        newProgrammes.data,
+        allProgrammeCodes,
+        specialGroups,
+        graduated
+      )
+      await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
+    } catch (e) {
+      logger.error(e)
+    }
+  }
+  return 'OK'
+}
+
+module.exports = { updateFacultyOverview, updateFacultyProgressOverview }

services/backend/src/services/graduationHelpers.js

@@ -0,0 +1,73 @@
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
+const moment = require('moment')
+const { Semester, SemesterEnrollment, Studyright } = require('../models')
+
+// categorize graduation times
+
+const countTimeCategories = (times, goal) => {
+  const statistics = { onTime: 0, yearOver: 0, wayOver: 0 }
+  times.forEach(time => {
+    if (time <= goal) statistics.onTime += 1
+    else if (time <= goal + 12) statistics.yearOver += 1
+    else statistics.wayOver += 1
+  })
+  return statistics
+}
+
+// discount staturory absences from graduation times
+
+const formatAbsence = absence => {
+  const { semestercode, semester } = absence
+  return { semestercode, start: semester.startdate, end: semester.enddate }
+}
+
+const statutoryAbsences = async (studentnumber, startdate, enddate) =>
+  (
+    await SemesterEnrollment.findAll({
+      attributes: ['semestercode'],
+      include: {
+        model: Semester,
+        attributes: ['startdate', 'enddate'],
+        where: {
+          startdate: {
+            [Op.gte]: startdate,
+          },
+          enddate: {
+            [Op.lte]: enddate,
+          },
+        },
+      },
+      where: {
+        studentnumber: studentnumber,
+        statutory_absence: true,
+      },
+    })
+  ).map(formatAbsence)
+
+const getStatutoryAbsences = async (studentnumber, startdate, enddate) => {
+  const absences = await statutoryAbsences(studentnumber, startdate, enddate)
+  if (absences.length) {
+    const absentMonths = absences.reduce((sum, ab) => sum + moment(ab.end).diff(moment(ab.start), 'months'), 0)
+    return absentMonths
+  }
+  return 0
+}
+
+// count bachelor time in Bc+Ms studyrights to master graduation times
+
+const bachelorStudyright = async id => {
+  return await Studyright.findOne({
+    attributes: ['startdate'],
+    where: {
+      studyrightid: id,
+      extentcode: 1,
+    },
+  })
+}
+
+module.exports = {
+  countTimeCategories,
+  getStatutoryAbsences,
+  bachelorStudyright,
+}

services/backend/src/services/openUni/openUniHelpers.js

@@ -0,0 +1,46 @@
+const mapOpenCredits = credit => {
+  const { course_code, attainment_date, student_studentnumber, grade } = credit
+  return {
+    course_code,
+    attainment_date,
+    student_studentnumber,
+    grade,
+  }
+}
+
+const mapStundentInfo = student => {
+  const { email, secondary_email, studentnumber } = student
+  return {
+    studentnumber,
+    email,
+    secondary_email,
+  }
+}
+const mapOpenEnrollments = enrollment => {
+  const { studentnumber, course_code, state, enrollment_date_time } = enrollment
+  return {
+    enrollmentStudentnumber: studentnumber,
+    course_code,
+    state,
+    enrollment_date_time,
+  }
+}
+
+const mapStudyRights = studyright => {
+  const { enddate, startdate, student } = studyright
+  return {
+    startdate,
+    enddate,
+    studyrightStudentnumber: student.studentnumber,
+  }
+}
+
+const mapCourseInfo = course => {
+  const { code, name } = course
+  return {
+    label: code,
+    name,
+  }
+}
+
+module.exports = { mapOpenCredits, mapOpenEnrollments, mapStundentInfo, mapStudyRights, mapCourseInfo }

services/backend/src/services/openUni/openUniManageSearches.js

@@ -0,0 +1,65 @@
+const logger = require('../../util/logger')
+const {
+  getOpenUniSearchesByUser,
+  createOpenUniPopulationSearch,
+  updateOpenUniPopulationSearch,
+  deleteOpenUniSearch,
+} = require('./openUniSearches')
+
+const getOpenUniSearches = async userId => {
+  const savedSearches = await getOpenUniSearchesByUser(userId)
+  if (savedSearches === undefined) return []
+  const foundSearches = savedSearches.reduce(
+    (res, search) => [
+      ...res,
+      {
+        id: search.id,
+        userId: search.userId,
+        name: search.name,
+        courseList: search.courseCodes,
+        updatedAt: search.updatedAt,
+      },
+    ],
+    []
+  )
+  return foundSearches
+}
+
+const createNewSearch = async (userId, name, courseCodes) => {
+  const savedSearches = await getOpenUniSearchesByUser(userId)
+  if (savedSearches !== undefined) {
+    const searchNames = savedSearches.map(search => search.name)
+    if (searchNames.includes(name)) return null
+  }
+  try {
+    const created = await createOpenUniPopulationSearch(userId, name, courseCodes)
+    if (!created) return null
+    return created
+  } catch (e) {
+    logger.error(`Couldn't create new open uni search instance: ${e}`)
+    return null
+  }
+}
+
+const deleteSearch = async (userId, id) => {
+  try {
+    const deleted = await deleteOpenUniSearch(userId, id)
+    if (!deleted) return null
+    return deleted
+  } catch (e) {
+    logger.error(`Couldn't delete open uni search instance: ${e}`)
+    return null
+  }
+}
+
+const updateSearch = async (userId, id, courseCodes) => {
+  try {
+    const updated = await updateOpenUniPopulationSearch(userId, id, courseCodes)
+    if (!updated) return null
+    return updated
+  } catch (e) {
+    logger.error(`Couldn't update open uni search instance: ${e}`)
+  }
+}
+
+module.exports = { getOpenUniSearches, createNewSearch, deleteSearch, updateSearch }

services/backend/src/services/openUni/openUniSearches.js

@@ -0,0 +1,134 @@
+const { Credit, Enrollment, Studyright, Student, Course } = require('../../models')
+const { OpenUniPopulationSearch } = require('../../models/models_kone')
+const {
+  mapOpenCredits,
+  mapOpenEnrollments,
+  mapStundentInfo,
+  mapStudyRights,
+  mapCourseInfo,
+} = require('./openUniHelpers')
+const { Op } = require('sequelize')
+
+const getCredits = async (courseCodes, startdate) =>
+  (
+    await Credit.findAll({
+      attributes: ['course_code', 'student_studentnumber', 'grade', 'attainment_date'],
+      where: {
+        course_code: {
+          [Op.in]: courseCodes,
+        },
+        is_open: true,
+        attainment_date: {
+          [Op.gte]: startdate,
+        },
+      },
+    })
+  ).map(mapOpenCredits)
+
+const getStudentInfo = async students =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: students,
+        },
+      },
+    })
+  ).map(mapStundentInfo)
+
+const getEnrollments = async (courseCodes, startdate, enddate) =>
+  (
+    await Enrollment.findAll({
+      where: {
+        course_code: {
+          [Op.in]: courseCodes,
+        },
+        is_open: true,
+        enrollment_date_time: {
+          [Op.and]: {
+            [Op.lte]: enddate,
+            [Op.gte]: startdate,
+          },
+        },
+      },
+    })
+  ).map(mapOpenEnrollments)
+
+const getCourseNames = async courseCodes =>
+  (
+    await Course.findAll({
+      attributes: ['code', 'name'],
+      where: {
+        code: {
+          [Op.in]: courseCodes,
+        },
+      },
+    })
+  ).map(mapCourseInfo)
+
+const getStudyRights = async students =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        student_studentnumber: {
+          [Op.in]: students.length > 0 ? students : { [Op.not]: null },
+        },
+      },
+    })
+  ).map(mapStudyRights)
+
+const getOpenUniSearchesByUser = async userId => {
+  return await OpenUniPopulationSearch.findAll({
+    where: {
+      userId,
+    },
+  })
+}
+
+const createOpenUniPopulationSearch = async (userId, name, courseCodes) => {
+  return await OpenUniPopulationSearch.create({
+    userId,
+    name,
+    courseCodes,
+  })
+}
+
+const updateOpenUniPopulationSearch = async (userId, id, courseCodes) => {
+  const searchToUpdate = await OpenUniPopulationSearch.findOne({
+    where: {
+      userId: userId,
+      id: id,
+    },
+  })
+
+  if (!searchToUpdate) return null
+  return await searchToUpdate.update({ courseCodes })
+}
+
+const deleteOpenUniSearch = async (userId, id) => {
+  return await OpenUniPopulationSearch.destroy({
+    where: {
+      userId: userId,
+      id: id,
+    },
+  })
+}
+
+module.exports = {
+  getCredits,
+  getEnrollments,
+  getStudyRights,
+  getStudentInfo,
+  getCourseNames,
+  getOpenUniSearchesByUser,
+  createOpenUniPopulationSearch,
+  updateOpenUniPopulationSearch,
+  deleteOpenUniSearch,
+}

services/backend/src/services/openUni/openUniStats.js

@@ -0,0 +1,110 @@
+const moment = require('moment')
+const { getCredits, getStudyRights, getEnrollments, getStudentInfo, getCourseNames } = require('./openUniSearches')
+
+const uniq = objects => [...new Set(objects)]
+
+const calulateTotalsForStudent = async (studentStats, studentnumber) => {
+  Object.keys(studentStats[studentnumber].courseInfo).forEach(course => {
+    if (studentStats[studentnumber].courseInfo[course].status.passed) {
+      studentStats[studentnumber].totals.passed += 1
+    } else if (studentStats[studentnumber].courseInfo[course].status.failed) {
+      studentStats[studentnumber].totals.failed += 1
+    } else if (studentStats[studentnumber].courseInfo[course].status.unfinished) {
+      studentStats[studentnumber].totals.unfinished += 1
+    }
+  })
+}
+const getCustomOpenUniCourses = async (courseCodes, startdate, enddate) => {
+  const ayCourseCodes = courseCodes.map(courseCode => 'AY' + courseCode)
+  const allCourseCodes = courseCodes.concat(ayCourseCodes)
+  const allCredits = await getCredits(allCourseCodes, startdate)
+  const allEnrollments = await getEnrollments(allCourseCodes, startdate, enddate)
+  const students = uniq(allEnrollments.map(enrollment => enrollment.enrollmentStudentnumber))
+  const courses = await getCourseNames(courseCodes)
+  const passedGrades = ['1', '2', '3', '4', '5', 'Hyv.', 'hyv.', 'HT', 'TT']
+  const failedGrades = ['Hyl.', 'HYL', '0']
+
+  const allStudyrights = await getStudyRights(students)
+  const studentInfo = await getStudentInfo(students)
+  // Filter out current studyrights:
+  // Case 1: Both startdate and enddate are outside of the given interval
+  // Case 2: Startdate is inside of the given interval and enddate is outside
+  // Case 3: Startdate is before the interval start and the enddate is within the interval
+  const studentsWithCurrentStudyRight = allStudyrights
+    .filter(
+      right =>
+        moment(right.startdate).isBetween(startdate, moment()) ||
+        (moment(right.startdate).isSameOrBefore(startdate) && moment(right.enddate).isSameOrAfter(moment())) ||
+        moment(right.enddate).isSameOrBefore(moment())
+    )
+    .map(right => right.studyrightStudentnumber)
+  const uniqueStudentsWithCurrentStudyRight = uniq(studentsWithCurrentStudyRight)
+
+  const studentStats = {}
+  const getEmptyCourseInfo = () => {
+    return {
+      status: {
+        passed: null,
+        failed: null,
+        unfinished: null,
+      },
+    }
+  }
+
+  for (const { studentnumber, email, secondary_email } of studentInfo) {
+    // Check if the student has existing studyright: if yes, then stop here
+    if (!uniqueStudentsWithCurrentStudyRight.includes(studentnumber)) {
+      if (!(studentnumber in studentStats)) {
+        studentStats[studentnumber] = {
+          courseInfo: courseCodes.reduce(
+            (acc, code) => ({ ...acc, [code.replace('AY', '')]: getEmptyCourseInfo() }),
+            {}
+          ),
+          email: email,
+          secondaryEmail: secondary_email,
+          totals: { passed: 0, failed: 0, unfinished: 0 },
+        }
+      }
+      for (const { course_code, attainment_date, student_studentnumber, grade } of allCredits) {
+        if (student_studentnumber === studentnumber) {
+          const courseCode = course_code.replace('AY', '')
+          if (
+            passedGrades.includes(grade) &&
+            (!studentStats[studentnumber].courseInfo[courseCode].status.passed ||
+              moment(studentStats[studentnumber].courseInfo[courseCode].status.passed).isBefore(attainment_date, 'day'))
+          ) {
+            studentStats[studentnumber].courseInfo[courseCode].status.passed = attainment_date
+          } else if (
+            failedGrades.includes(grade) &&
+            !studentStats[studentnumber].courseInfo[courseCode].status.passed &&
+            (!studentStats[studentnumber].courseInfo[courseCode].status.failed ||
+              moment(studentStats[studentnumber].courseInfo[courseCode].status.failed).isBefore(attainment_date, 'day'))
+          ) {
+            studentStats[studentnumber].courseInfo[courseCode].status.failed = attainment_date
+          }
+        }
+      }
+      for (const { enrollmentStudentnumber, course_code, enrollment_date_time } of allEnrollments) {
+        if (enrollmentStudentnumber === studentnumber) {
+          const courseCode = course_code.replace('AY', '')
+          if (
+            !studentStats[studentnumber].courseInfo[courseCode].status.passed &&
+            !studentStats[studentnumber].courseInfo[courseCode].status.failed &&
+            (!studentStats[studentnumber].courseInfo[courseCode].status.unfinished ||
+              moment(studentStats[studentnumber].courseInfo[courseCode].status.unfinished).isBefore(
+                enrollment_date_time,
+                'day'
+              ))
+          ) {
+            studentStats[studentnumber].courseInfo[courseCode].status.unfinished = enrollment_date_time
+          }
+        }
+      }
+      await calulateTotalsForStudent(studentStats, studentnumber)
+    }
+  }
+  const openUniStats = { students: studentStats, courses: courses }
+  return openUniStats
+}
+
+module.exports = { getCustomOpenUniCourses }

services/backend/src/services/populations.js

@@ -25,6 +25,7 @@ const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
 const { getAllProgrammes } = require('./studyrights')
 const { encrypt } = require('../services/encrypt')
+const { getCriteria } = require('./studyProgrammeCriteria')
 
 const enrolmentDates = () => {
   const query = 'SELECT DISTINCT s.dateOfUniversityEnrollment as date FROM Student s'
@@ -38,6 +39,61 @@ const universityEnrolmentDates = async () => {
     .filter(d => d)
     .sort()
 }
+// Progress tab related helper functions.
+const createEmptyCriteriaYear = (criteria, year) => {
+  return {
+    credits: false,
+    totalSatisfied: 0,
+    coursesSatisfied:
+      criteria?.courses && criteria?.courses[year]
+        ? criteria.courses[year].reduce((acc, course) => {
+            acc[course] = null
+            return acc
+          }, {})
+        : {},
+  }
+}
+
+const getCreditAmount = (course, hops, courseCode, startDate, addition) => {
+  return moment(course.date).isBetween(moment(startDate), moment(startDate).add(addition, 'year')) &&
+    hops.length > 0 &&
+    (hops[0].included_courses.includes(courseCode) || hops[0].included_courses.includes(course.course_code))
+    ? course.credits
+    : 0
+}
+
+const updateCourseByYear = (criteria, criteriaYear, course, criteriaChecked, yearToAdd, correctCode) => {
+  if (
+    criteria?.courses &&
+    criteria?.courses[criteriaYear] &&
+    (criteria.courses[criteriaYear].includes(course.course_code) ||
+      criteria.courses[criteriaYear].some(
+        c => criteria.allCourses[c] && criteria.allCourses[c].includes(course.course_code)
+      ))
+  ) {
+    if (
+      !criteriaChecked[yearToAdd].coursesSatisfied[correctCode] ||
+      new Date(criteriaChecked[yearToAdd].coursesSatisfied[correctCode]) > new Date(course.date)
+    ) {
+      criteriaChecked[yearToAdd].coursesSatisfied[correctCode] = course.date
+    }
+  }
+}
+
+const updateCreditcriteriaInfo = (criteria, criteriaYear, criteriaChecked, yearToAdd, academicYears, academicYear) => {
+  if (criteria.courses) {
+    if (criteriaChecked?.[yearToAdd]) {
+      criteriaChecked[yearToAdd].totalSatisfied +=
+        Object.keys(criteriaChecked?.[yearToAdd].coursesSatisfied).filter(
+          course => criteriaChecked?.[yearToAdd].coursesSatisfied[course] !== null
+        ).length || 0
+    }
+    if (academicYears[academicYear] >= criteria?.credits[criteriaYear] && criteria?.credits[criteriaYear] > 0) {
+      criteriaChecked[yearToAdd].credits = true
+      criteriaChecked[yearToAdd].totalSatisfied += 1
+    }
+  }
+}
 
 const formatStudentForPopulationStatistics = (
   {
@@ -48,6 +104,8 @@ const formatStudentForPopulationStatistics = (
     creditcount,
     abbreviatedname,
     email,
+    secondary_email,
+    phone_number,
     studyrights,
     studyplans,
     semester_enrollments,
@@ -66,9 +124,10 @@ const formatStudentForPopulationStatistics = (
   enrollments,
   credits,
   startDate,
-  endDate,
   startDateMoment,
-  endDateMoment
+  endDateMoment,
+  criteria,
+  code
 ) => {
   const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
     const attainment_date_normailized =
@@ -86,10 +145,58 @@ const formatStudentForPopulationStatistics = (
       language,
     }
   }
+  const criteriaCoursesBySubstitutions = criteria?.allCourses
+    ? Object.keys(criteria.allCourses).reduce((acc, code) => {
+        acc[code] = code
+        criteria.allCourses[code].map(subst => (acc[subst] = code))
+        return acc
+      }, {})
+    : {}
+
+  const toProgressCriteria = () => {
+    const criteriaChecked = {
+      year1: createEmptyCriteriaYear(criteria, 'yearOne'),
+      year2: createEmptyCriteriaYear(criteria, 'yearTwo'),
+      year3: createEmptyCriteriaYear(criteria, 'yearThree'),
+      year4: createEmptyCriteriaYear(criteria, 'yearFour'),
+      year5: createEmptyCriteriaYear(criteria, 'yearFive'),
+      year6: createEmptyCriteriaYear(criteria, 'yearSix'),
+    }
+    const correctStudyplan = studyplans ? studyplans.filter(plan => plan.programme_code === code) : []
+    const academicYears = { first: 0, second: 0, third: 0, fourth: 0, fifth: 0, sixth: 0 }
+    if (criteria.courses || criteria.credits) {
+      const courses = credits[studentnumber] ? credits[studentnumber].map(toCourse) : []
+      courses.forEach(course => {
+        if (course.passed) {
+          const correctCode = criteriaCoursesBySubstitutions[course.course_code]
+          updateCourseByYear(criteria, 'yearOne', course, criteriaChecked, 'year1', correctCode)
+          updateCourseByYear(criteria, 'yearTwo', course, criteriaChecked, 'year2', correctCode)
+          updateCourseByYear(criteria, 'yearThree', course, criteriaChecked, 'year3', correctCode)
+          updateCourseByYear(criteria, 'yearFour', course, criteriaChecked, 'year4', correctCode)
+          updateCourseByYear(criteria, 'yearFive', course, criteriaChecked, 'year5', correctCode)
+          updateCourseByYear(criteria, 'yearSix', course, criteriaChecked, 'year6', correctCode)
+          academicYears.first += getCreditAmount(course, correctStudyplan, correctCode, startDate, 1)
+          academicYears.second += getCreditAmount(course, correctStudyplan, correctCode, startDate, 2)
+          academicYears.third += getCreditAmount(course, correctStudyplan, correctCode, startDate, 3)
+          academicYears.fourth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 4)
+          academicYears.fifth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 5)
+          academicYears.sixth += getCreditAmount(course, correctStudyplan, correctCode, startDate, 6)
+        }
+      })
+    }
+
+    updateCreditcriteriaInfo(criteria, 'yearOne', criteriaChecked, 'year1', academicYears, 'first')
+    updateCreditcriteriaInfo(criteria, 'yearTwo', criteriaChecked, 'year2', academicYears, 'second')
+    updateCreditcriteriaInfo(criteria, 'yearThree', criteriaChecked, 'year3', academicYears, 'third')
+    updateCreditcriteriaInfo(criteria, 'yearFour', criteriaChecked, 'year4', academicYears, 'fourth')
+    updateCreditcriteriaInfo(criteria, 'yearFive', criteriaChecked, 'year5', academicYears, 'fifth')
+    updateCreditcriteriaInfo(criteria, 'yearSix', criteriaChecked, 'year6', academicYears, 'sixth')
+    return criteriaChecked
+  }
 
   studyrights = studyrights || []
-
   const started = dateofuniversityenrollment
+
   return {
     firstnames,
     lastname,
@@ -104,7 +211,11 @@ const formatStudentForPopulationStatistics = (
     gender_code,
     gender: { gender_en, gender_fi, gender_sv },
     email,
-    semesterenrollments: semester_enrollments || [],
+    secondaryEmail: secondary_email,
+    phoneNumber: phone_number,
+    semesterenrollments: semester_enrollments
+      ? semester_enrollments.sort((a, b) => a.semestercode - b.semestercode)
+      : [],
     updatedAt: updatedAt || createdAt,
     tags: tags || [],
     studyrightStart: startDate,
@@ -113,6 +224,7 @@ const formatStudentForPopulationStatistics = (
     birthdate,
     studyplans,
     sis_person_id,
+    criteriaProgress: toProgressCriteria(),
   }
 }
 
@@ -232,6 +344,8 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'creditcount',
         'abbreviatedname',
         'email',
+        'secondary_email',
+        'phone_number',
         'updatedAt',
         'gender_code',
         'birthdate',
@@ -259,6 +373,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             'faculty_code',
             'studystartdate',
             'admission_type',
+            'cancelled',
           ],
           separate: true,
           include: [
@@ -289,7 +404,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         },
         {
           model: Studyplan,
-          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+          attributes: ['included_courses', 'programme_code', 'completed_credits', 'studyrightid'],
           separate: true,
         },
       ],
@@ -410,7 +525,6 @@ const studentnumbersWithAllStudyrightElements = async (
   graduatedStudents
 ) => {
   // eslint-disable-line
-
   // db startdate is formatted to utc so need to change it when querying
   const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
 
@@ -747,7 +861,9 @@ const formatStudentsForApi = async (
   startDate,
   endDate,
   { studyRights },
-  optionData
+  optionData,
+  criteria,
+  code
 ) => {
   const startDateMoment = moment(startDate)
   const endDateMoment = moment(endDate)
@@ -781,15 +897,17 @@ const formatStudentsForApi = async (
       })
       if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
       else student.option = null
+
       stats.students.push(
         formatStudentForPopulationStatistics(
           student,
           enrollments,
           credits,
           startDate,
-          endDate,
           startDateMoment,
-          endDateMoment
+          endDateMoment,
+          criteria,
+          code
         )
       )
       return stats
@@ -874,16 +992,17 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
         nondegreeStudents,
         transferredStudents
       )
-  // wtf
-  // plz
   const code = studyRights[0] || ''
   let optionData = {}
+  let criteria = {}
   if (code.includes('MH')) {
     optionData = await getOptionsForStudents(studentnumbers, code, 'MSC')
   } else if (code.includes('KH')) {
     optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
   }
-
+  if (code.includes('KH') || ['MH30_001', 'MH30_003'].includes(code)) {
+    criteria = await getCriteria(code)
+  }
   const { students, enrollments, credits, extents, semesters, elementdetails, courses } =
     await getStudentsIncludeCoursesBetween(
       studentnumbers,
@@ -898,7 +1017,9 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     startDate,
     endDate,
     formattedQueryParams,
-    optionData
+    optionData,
+    criteria,
+    code
   )
 
   return formattedStudents
@@ -908,10 +1029,52 @@ const getSubstitutions = async codes => {
   const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
   return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
 }
-
+// This duplicate code is added here to ensure that we get the enrollments in cases no credits found for the selected students.
+const findCourseEnrollments = async (studentnumbers, beforeDate, courses = [], studentCountLimit = 0) => {
+  const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
+  const res = await sequelize.query(
+    `
+      SELECT DISTINCT ON (course.id)
+        course.code,
+        course.name,
+        course.coursetypecode,
+        course.substitutions,
+        course.main_course_code,
+        course_types.name AS course_type,
+        enrollment.data AS enrollments
+      FROM course
+      INNER JOIN course_types ON course_types.coursetypecode = course.coursetypecode
+      LEFT JOIN (
+        SELECT
+          course_id,
+          ARRAY_AGG(JSON_BUILD_OBJECT(
+            'studentnumber', studentnumber,
+            'state', state,
+            'enrollment_date_time', enrollment_date_time
+          )) AS data
+        FROM enrollment
+        WHERE enrollment.studentnumber IN (:studentnumbers) AND enrollment.enrollment_date_time < :beforeDate
+        GROUP BY enrollment.course_id
+        HAVING COUNT(DISTINCT enrollment.studentnumber) >= :studentCountLimit
+      ) enrollment ON enrollment.course_id = course.id 
+      WHERE :skipCourseCodeFilter OR course.code IN (:courseCodes)
+      -- GROUP BY 1, 2, 3, 4, 5, 6
+    `,
+    {
+      replacements: {
+        studentnumbers: studentnumbers.length > 0 ? studentnumbers : ['DUMMY'],
+        beforeDate,
+        studentCountLimit,
+        courseCodes,
+        skipCourseCodeFilter: courses.length === 0,
+      },
+      type: sequelize.QueryTypes.SELECT,
+    }
+  )
+  return res
+}
 const findCourses = async (studentnumbers, beforeDate, courses = [], studentCountLimit = 0) => {
   const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
-
   const res = await sequelize.query(
     `
       SELECT DISTINCT ON (course.id)
@@ -1036,14 +1199,22 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
         numbers[num] = true
         return numbers
       }, {})
-
       const courses = await findCourses(
         studentnumbers,
         dateMonthsFromNow(startDate, months),
         courseCodes,
         query.studentCountLimit
       )
-      return [allstudents, courses]
+      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
+
+      const courseEnrollements = await findCourseEnrollments(
+        studentnumbers,
+        dateMonthsFromNow(startDate, months),
+        filteredCourseCodes,
+        query.studentCountLimit
+      )
+      return [allstudents, courses, courseEnrollements]
     } else {
       const { months, startDate } = params
       const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
@@ -1053,7 +1224,16 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
         return numbers
       }, {})
       const courses = await findCourses(studentnumberlist, beforeDate, courseCodes, query.studentCountLimit)
-      return [allstudents, courses]
+      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
+
+      const courseEnrollements = await findCourseEnrollments(
+        studentnumberlist,
+        dateMonthsFromNow(startDate, months),
+        filteredCourseCodes,
+        query.studentCountLimit
+      )
+      return [allstudents, courses, courseEnrollements]
     }
   }
 
@@ -1073,24 +1253,52 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
   }
 
   const params = parseQueryParams(query)
-  const [[allstudents, courses], error] = await Promise.all([
+  const allStudentsByYears = query?.selectedStudentsByYear
+    ? Object.keys(query.selectedStudentsByYear).reduce(
+        (res, year) => [...res, ...query.selectedStudentsByYear[year]],
+        []
+      )
+    : []
+  // To fix failed and enrolled, no grade filter options some not so clean and nice solutions were added
+  // Get the data with actual 1. courses and filtered students. 2. all students by year, if provided.
+  const [[allstudents, courses, courseEnrollements], [, allCourses], error] = await Promise.all([
     getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
+    getStudentsAndCourses(allStudentsByYears, null, query.courses),
     isValidRequest(query, params),
   ])
-  if (error) return error
 
+  if (error) return error
+  // Get the substitution codes for the fetch data by selscted students
+  const substitutionCodes = Object.entries(courses).reduce(
+    (res, [, obj]) => [...res, ...(obj?.substitutions || [])],
+    []
+  )
+  const codes = Object.keys(keyBy(courses, 'code')).map(code => code)
+  // Filter substitution courses for fetched courses -> this avoid the situation in which there is only
+  // courses with old course codes. Frontend NEEDS in most cases the current course.
+  const substitutionCourses = allCourses.filter(
+    obj => substitutionCodes.includes(obj.code) && !codes.includes(obj.code)
+  )
   const bottlenecks = {
     disciplines: {},
     coursetypes: {},
   }
 
   const stats = {}
-
   const startYear = parseInt(query.year, 10)
   const allstudentslength = Object.keys(allstudents).length
-  const coursesByCode = keyBy(courses, 'code')
+  let coursesToLoop = courses.concat(substitutionCourses)
+  const codesList = coursesToLoop.map(course => course.code)
+  // This fix problem when Enrolled no grade is chosen. The sql query for fetching
+  // credits do not fetch enrollments if no credits found for selected students.
+  // This and other sql query ensures that enrollments are added.
+  const coursesOnlyWithEnrollments = courseEnrollements.filter(
+    course => !codesList.includes(course.code) && course.enrollments
+  )
+  coursesToLoop = coursesToLoop.concat(coursesOnlyWithEnrollments)
 
-  for (const course of courses) {
+  const coursesByCode = keyBy(coursesToLoop, 'code')
+  for (const course of coursesToLoop) {
     let { course_type } = course
     let maincourse = course
 
@@ -1107,28 +1315,32 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
     }
 
     const coursestats = stats[maincourse.code]
-
     coursestats.addCourseType(course_type.coursetypecode, course_type.name)
     coursestats.addCourseSubstitutions(course.substitutions)
     bottlenecks.coursetypes[course_type.coursetypecode] = course_type.name
-
-    course.credits.forEach(credit => {
-      const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
-      const semester = getPassingSemester(startYear, date)
-      coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
-    })
-
     if (course.enrollments) {
       course.enrollments.forEach(({ studentnumber, state, enrollment_date_time }) => {
-        const semester = getPassingSemester(startYear, enrollment_date_time)
-        coursestats.markEnrollment(studentnumber, state, semester)
+        if ((query?.selectedStudents && query?.selectedStudents.includes(studentnumber)) || !query?.selectedStudents) {
+          const semester = getPassingSemester(startYear, enrollment_date_time)
+          coursestats.markEnrollment(studentnumber, state, semester, enrollment_date_time)
+        }
+      })
+    }
+    if (course.credits) {
+      course.credits.forEach(credit => {
+        const { studentnumber, passingGrade, improvedGrade, failingGrade, grade, date } = parseCreditInfo(credit)
+        if ((query?.selectedStudents && query?.selectedStudents.includes(studentnumber)) || !query?.selectedStudents) {
+          const semester = getPassingSemester(startYear, date)
+          coursestats.markCredit(studentnumber, grade, passingGrade, failingGrade, improvedGrade, semester)
+        }
       })
     }
 
     stats[maincourse.code] = coursestats
   }
 
-  bottlenecks.coursestatistics = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
+  const allStats = Object.values(stats).map(coursestatistics => coursestatistics.getFinalStats())
+  bottlenecks.coursestatistics = allStats.filter(course => course.stats.students > 0)
   bottlenecks.allStudents = allstudentslength
 
   if (encryptdata) encryptStudentnumbers(bottlenecks)

services/backend/src/services/programmeModules.js

@@ -38,8 +38,14 @@ const byProgrammeCode = async code => {
     },
   })
 
+  // filter out possible duplicates
+  const filtered = result.filter(
+    (course, index, array) =>
+      array.findIndex(c => c.id == course.id && c.code == course.code && c.label_code == course.label_code) == index
+  )
+
   // this labels the modules to match the old system in frontend
-  const labeled = result.map(module => {
+  const labeled = filtered.map(module => {
     const label = {
       id: module.label_name.fi,
       label: `${module.label_code}\n${module.label_name.fi}`,

services/backend/src/services/students.js

@@ -13,6 +13,7 @@ const {
   SemesterEnrollment,
   Semester,
   Studyplan,
+  Transfer,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
 const Op = Sequelize.Op
@@ -60,7 +61,10 @@ const byId = async id => {
         },
         {
           model: Studyplan,
-          attributes: ['included_courses', 'programme_code', 'completed_credits'],
+          attributes: ['included_courses', 'programme_code', 'completed_credits', 'studyrightid'],
+        },
+        {
+          model: Transfer,
         },
       ],
     }),
@@ -347,6 +351,20 @@ const studentnumberLike = terms => {
 const bySearchTerm = async searchterm => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
+    include: {
+      model: Studyright,
+      include: {
+        model: StudyrightElement,
+        include: {
+          model: ElementDetail,
+          where: {
+            type: {
+              [Op.in]: [10, 20, 30],
+            },
+          },
+        },
+      },
+    },
     where: {
       [Op.or]: [nameLike(terms), studentnumberLike(terms)],
     },
@@ -357,6 +375,20 @@ const bySearchTerm = async searchterm => {
 const bySearchTermAndStudentNumbers = async (searchterm, studentnumbers) => {
   const terms = splitByEmptySpace(searchterm)
   const matches = await Student.findAll({
+    include: {
+      model: Studyright,
+      include: {
+        model: StudyrightElement,
+        include: {
+          model: ElementDetail,
+          where: {
+            type: {
+              [Op.in]: [10, 20, 30],
+            },
+          },
+        },
+      },
+    },
     where: {
       [Op.and]: {
         [Op.or]: [nameLike(terms), studentnumberLike(terms)],

services/backend/src/services/studyProgrammeCriteria.js

@@ -0,0 +1,148 @@
+const { ProgressCriteria } = require('../models/models_kone')
+const { Course } = require('../models')
+const logger = require('../util/logger')
+
+const getCriteriaByStudyProgramme = async code => {
+  if (code === '') return null
+  return await ProgressCriteria.findOne({
+    where: {
+      code: code,
+    },
+  })
+}
+
+const getSubstitutions = async codes => {
+  if (codes === []) return []
+  const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
+  return courses.reduce((acc, { code, substitutions }) => ({ ...acc, [code]: substitutions }), {})
+}
+
+const formattedData = async data => {
+  const yearOne = data.coursesYearOne ? data.coursesYearOne : []
+  const yearTwo = data.coursesYearTwo ? data.coursesYearTwo : []
+  const yearThree = data.coursesYearThree ? data.coursesYearThree : []
+  const yearFour = data.coursesYearFour ? data.coursesYearFour : []
+  const yearFive = data.coursesYearFive ? data.coursesYearFive : []
+  const yearSix = data.coursesYearSix ? data.coursesYearSix : []
+  const courseCodes = [...yearOne, ...yearTwo, ...yearThree, ...yearFour, ...yearFive, ...yearSix]
+  const allCourses = await getSubstitutions(courseCodes)
+
+  const criteria = {
+    courses: { yearOne, yearTwo, yearThree, yearFour, yearFive, yearSix },
+    allCourses,
+    credits: {
+      yearOne: data.creditsYearOne ? data.creditsYearOne : 0,
+      yearTwo: data.creditsYearTwo ? data.creditsYearTwo : 0,
+      yearThree: data.creditsYearThree ? data.creditsYearThree : 0,
+      yearFour: data.creditsYearFour ? data.creditsYearFour : 0,
+      yearFive: data.creditsYearFive ? data.creditsYearFive : 0,
+      yearSix: data.creditsYearSix ? data.creditsYearSix : 0,
+    },
+  }
+  return criteria
+}
+
+const createCriteria = async (studyProgramme, courseObj, creditsObj) => {
+  const newProgrammeCriteria = {
+    code: studyProgramme,
+    coursesYearOne: courseObj.year1,
+    coursesYearTwo: courseObj.year2,
+    coursesYearThree: courseObj.year3,
+    coursesYearFour: courseObj.year4,
+    coursesYearFive: courseObj.year5,
+    coursesYearSix: courseObj.year6,
+    creditsYearOne: creditsObj.year1,
+    creditsYearTwo: creditsObj.year2,
+    creditsYearThree: creditsObj.year3,
+    creditsYearFour: creditsObj.year4,
+    creditsYearFive: creditsObj.year5,
+    creditsYearSix: creditsObj.year6,
+  }
+  try {
+    const createdData = await ProgressCriteria.create(newProgrammeCriteria)
+    return await formattedData(createdData)
+  } catch (error) {
+    logger.error(`Create criteria failed: ${error}`)
+  }
+}
+
+const saveYearlyCreditCriteria = async (studyProgramme, credits) => {
+  const studyProgrammeToUpdate = await getCriteriaByStudyProgramme(studyProgramme)
+  if (!studyProgrammeToUpdate) {
+    const courseObj = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
+    const creditsObj = {
+      year1: parseInt(credits.year1),
+      year2: parseInt(credits.year2),
+      year3: parseInt(credits.year3),
+      year4: parseInt(credits.year4),
+      year5: parseInt(credits.year5),
+      year6: parseInt(credits.year6),
+    }
+    return await createCriteria(studyProgramme, courseObj, creditsObj)
+  }
+  const yearlyCredits = {
+    creditsYearOne: parseInt(credits.year1),
+    creditsYearTwo: parseInt(credits.year2),
+    creditsYearThree: parseInt(credits.year3),
+    creditsYearFour: parseInt(credits.year4),
+    creditsYearFive: parseInt(credits.year5),
+    creditsYearSix: parseInt(credits.year6),
+  }
+  try {
+    const updatedData = await studyProgrammeToUpdate.update({ ...yearlyCredits })
+    return await formattedData(updatedData)
+  } catch (error) {
+    logger.error(`Update yearly credit criteria failed: ${error}`)
+  }
+}
+
+const saveYearlyCourseCriteria = async (studyProgramme, courses, year) => {
+  const studyProgrammeToUpdate = await getCriteriaByStudyProgramme(studyProgramme)
+  if (!studyProgrammeToUpdate) {
+    const courseObj = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
+    const creditObj = { year1: 0, year2: 0, year3: 0, year4: 0, year5: 0, year6: 0 }
+    if (year === 1) {
+      courseObj.year1 = courses
+    } else if (year === 2) {
+      courseObj.year2 = courses
+    } else if (year === 3) {
+      courseObj.year3 = courses
+    } else if (year === 4) {
+      courseObj.year4 = courses
+    } else if (year === 5) {
+      courseObj.year5 = courses
+    } else {
+      courseObj.year6 = courses
+    }
+    return await createCriteria(studyProgramme, courseObj, creditObj)
+  }
+
+  try {
+    let updatedData = {}
+    if (year === 1) updatedData = await studyProgrammeToUpdate.update({ coursesYearOne: courses })
+    if (year === 2) updatedData = await studyProgrammeToUpdate.update({ coursesYearTwo: courses })
+    if (year === 3) updatedData = await studyProgrammeToUpdate.update({ coursesYearThree: courses })
+    if (year === 4) updatedData = await studyProgrammeToUpdate.update({ coursesYearFour: courses })
+    if (year === 5) updatedData = await studyProgrammeToUpdate.update({ coursesYearFive: courses })
+    if (year === 6) updatedData = await studyProgrammeToUpdate.update({ coursesYearSix: courses })
+    return await formattedData(updatedData)
+  } catch (error) {
+    logger.error(`Update yearly credit criteria failed: ${error}`)
+  }
+}
+
+const getCriteria = async studyProgramme => {
+  const studyProgrammeCriteria = await getCriteriaByStudyProgramme(studyProgramme)
+  const criteria = {
+    courses: { yearOne: [], yearTwo: [], yearThree: [], yearFour: [], yearFive: [], yearSix: [] },
+    allCourses: [],
+    credits: { yearOne: 0, yearTwo: 0, yearThree: 0, yearFour: 0, yearFive: 0, yearSix: 0 },
+  }
+  return studyProgrammeCriteria ? await formattedData(studyProgrammeCriteria) : criteria
+}
+
+module.exports = {
+  getCriteria,
+  saveYearlyCourseCriteria,
+  saveYearlyCreditCriteria,
+}

services/backend/src/services/studyprogramme.js

@@ -12,13 +12,16 @@ const {
   Studyright,
   StudyrightElement,
   ElementDetail,
+  ProgrammeModule,
   Transfer,
   Student,
   SemesterEnrollment,
   Semester,
+  Enrollment,
 } = require('../models')
 const { formatStudyright, formatStudent, formatTransfer } = require('./studyprogrammeHelpers')
 const { getCurrentSemester } = require('./semesters')
+const logger = require('../util/logger')
 
 const whereStudents = studentnumbers => {
   return studentnumbers ? studentnumbers : { [Op.not]: null }
@@ -36,9 +39,7 @@ const studytrackStudents = async studentnumbers =>
         separate: true,
         attributes: ['credits', 'attainment_date'],
         where: {
-          isStudyModule: {
-            [Op.eq]: null,
-          },
+          [Op.or]: [{ isStudyModule: null }, { isStudyModule: false }],
           credittypecode: {
             [Op.in]: [4, 9],
           },
@@ -358,7 +359,7 @@ const allTransfers = async (studytrack, since) => {
 const getProgrammesStudyrights = async studyprogramme =>
   (
     await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode', 'cancelled'],
       include: [
         {
           model: StudyrightElement,
@@ -381,7 +382,16 @@ const getProgrammesStudyrights = async studyprogramme =>
 const getStudyRights = async students =>
   (
     await Studyright.findAll({
-      attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
+      attributes: [
+        'studyrightid',
+        'startdate',
+        'studystartdate',
+        'enddate',
+        'graduated',
+        'prioritycode',
+        'extentcode',
+        'cancelled',
+      ],
       where: {
         studentStudentnumber: students,
       },
@@ -417,9 +427,7 @@ const getCreditsForStudyProgramme = async (codes, since) => {
       },
     },
     where: {
-      credittypecode: {
-        [Op.notIn]: [10, 9, 7],
-      },
+      credittypecode: 4,
       isStudyModule: {
         [Op.not]: true,
       },
@@ -500,16 +508,110 @@ const getStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
       code: course.code,
       name: course.course_name,
       year: course.year,
-      totalAllStudents: parseInt(course.total_students),
+      totalPassed: parseInt(course.total_students),
       totalAllcredits: parseInt(course.total_credits),
-      type: 'total',
+      type: 'passed',
     }))
   } catch (e) {
     // eslint-disable-next-line no-console
-    console.log(e)
+    logger.log(e)
   }
 }
+const getNotCompletedForProgrammeCourses = async (from, to, programmeCourses) => {
+  try {
+    const enrollmentsCourses = await Enrollment.findAll({
+      attributes: ['studentnumber', 'course_code'],
+      where: {
+        course_code: {
+          [Op.in]: programmeCourses,
+        },
+        studentnumber: {
+          [Op.ne]: null,
+        },
+        enrollment_date_time: {
+          [Op.between]: [from, to],
+        },
+        state: 'ENROLLED',
+      },
+    })
+    const allEnrollments = {}
+    for (const { studentnumber, course_code } of enrollmentsCourses) {
+      if (!(course_code in allEnrollments)) {
+        allEnrollments[course_code] = []
+      }
+      allEnrollments[course_code].push(studentnumber)
+    }
 
+    const credits = await Credit.findAll({
+      attributes: ['id', 'course_code', 'student_studentnumber', 'credittypecode', 'attainment_date'],
+      include: {
+        model: Course,
+        attributes: ['code', 'name'],
+        required: true,
+        where: {
+          is_study_module: false,
+          code: programmeCourses,
+        },
+      },
+      where: {
+        credittypecode: [4, 7, 10],
+        isStudyModule: {
+          [Op.not]: true,
+        },
+        attainment_date: {
+          [Op.between]: [from, to],
+        },
+      },
+    })
+    const creditsObj = credits.map(credit => {
+      return {
+        code: credit.course_code,
+        studentnumber: credit.student_studentnumber,
+        credittype: credit.credittypecode,
+        courseName: credit.course.name,
+        year: moment(credit.attainment_date).year(),
+      }
+    })
+
+    const passedByCourseCodes = {}
+    const notCompletedByCourseCodes = {}
+    const courses = {}
+    for (const course of creditsObj) {
+      if (!(course.code in courses)) {
+        courses[course.code] = {
+          code: course.code,
+          name: course.courseName,
+          year: course.year,
+        }
+
+        passedByCourseCodes[course.code] = []
+        notCompletedByCourseCodes[course.code] = []
+      }
+      if ([4, 7].includes(course.credittype)) {
+        passedByCourseCodes[course.code].push(course.studentnumber)
+      }
+      if (
+        (allEnrollments[course.code]?.includes(course.studentnumber) || course.credittype === 10) &&
+        !passedByCourseCodes[course.code].includes(course.studentnumber)
+      ) {
+        notCompletedByCourseCodes[course.code].push(course.studentnumber)
+      }
+    }
+
+    return Object.keys(courses)
+      .reduce((acc, val) => [...acc, { ...courses[val] }], [])
+      .map(course => ({
+        code: course.code,
+        name: course.name,
+        year: course.year,
+        totalNotCompleted: [...new Set(notCompletedByCourseCodes[course.code])].length,
+        type: 'notCompleted',
+      }))
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    logger.log(e)
+  }
+}
 const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, studyprogramme) => {
   const res = await sequelize.query(
     `
@@ -536,7 +638,6 @@ const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, stu
     }
   )
   return res.map(course => {
-    // console.log('course: ', course)
     const res = {
       code: course.code,
       name: course.course_name,
@@ -734,6 +835,15 @@ const getCurrentStudyYearStartDate = _.memoize(
     )
 )
 
+const getProgrammeName = async code => {
+  return ProgrammeModule.findOne({
+    attributes: ['name'],
+    where: {
+      code,
+    },
+  })
+}
+
 module.exports = {
   studytrackStudents,
   enrolledStudents,
@@ -760,4 +870,6 @@ module.exports = {
   getStudyRights,
   getCourseCodesForStudyProgramme,
   getTransferStudentsForProgrammeCourses,
+  getProgrammeName,
+  getNotCompletedForProgrammeCourses,
 }

services/backend/src/services/studyprogrammeCourses.js

@@ -4,6 +4,7 @@ const {
   getStudentsForProgrammeCourses,
   getCurrentStudyYearStartDate,
   getOwnStudentsForProgrammeCourses,
+  getNotCompletedForProgrammeCourses,
   getStudentsWithoutStudyrightForProgrammeCourses,
   getTransferStudentsForProgrammeCourses,
   getOtherStudentsForProgrammeCourses,
@@ -39,9 +40,12 @@ const makeYearlyPromises = (years, academicYear, type, programmeCourses, studypr
         let result = null
 
         switch (type) {
-          case 'allStudents':
+          case 'passed':
             result = await getStudentsForProgrammeCourses(from, to, programmeCourses)
             break
+          case 'notCompleted':
+            result = await getNotCompletedForProgrammeCourses(from, to, programmeCourses)
+            break
           case 'ownStudents':
             result = await getOwnStudentsForProgrammeCourses(from, to, programmeCourses, studyprogramme)
             break
@@ -74,7 +78,13 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
   const yearRange = _.range(2017, startYear + 1)
   const programmeCourses = await getAllStudyprogrammeCourses(studyprogramme)
 
-  const yearlyStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'allStudents', programmeCourses)
+  const yearlyPassedStudentByCoursePromises = makeYearlyPromises(yearRange, academicYear, 'passed', programmeCourses)
+  const yearlyNotCompletedStudentByCoursePromises = makeYearlyPromises(
+    yearRange,
+    academicYear,
+    'notCompleted',
+    programmeCourses
+  )
   const yearlyProgrammeStudentsPromises = makeYearlyPromises(
     yearRange,
     academicYear,
@@ -105,22 +115,24 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
   )
 
   const [
-    yearlyStudentByCourse,
+    yearlyPassedStudentByCourse,
+    yearlyNotCompletedStudentByCourse,
     yearlyProgrammeStudents,
     yearlyStudentsWithoutStudyright,
     yearlyOtherProgrammeStudents,
     yearlyTransferStudents,
   ] = await Promise.all([
-    Promise.all(yearlyStudentByCoursePromises),
+    Promise.all(yearlyPassedStudentByCoursePromises),
+    Promise.all(yearlyNotCompletedStudentByCoursePromises),
     Promise.all(yearlyProgrammeStudentsPromises),
     Promise.all(yearlyStudentsWithoutStudyrightPromises),
     Promise.all(yearlyOtherProgrammeStudentsPromises),
     Promise.all(yearlyTransferStudentsPromises),
   ])
-
   let maxYear = 0
   const allCourses = [
-    ...yearlyStudentByCourse.flat(),
+    ...yearlyPassedStudentByCourse.flat(),
+    ...yearlyNotCompletedStudentByCourse.flat(),
     ...yearlyProgrammeStudents.flat(),
     ...yearlyStudentsWithoutStudyright.flat(),
     ...yearlyOtherProgrammeStudents.flat(),
@@ -134,9 +146,12 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
         years: {},
       }
     }
+
     if (!acc[curr.code]['years'][curr.year]) {
       acc[curr.code]['years'][curr.year] = {
         totalAllStudents: 0,
+        totalPassed: 0,
+        totalNotCompleted: 0,
         totalAllCredits: 0,
         totalProgrammeStudents: 0,
         totalProgrammeCredits: 0,
@@ -149,10 +164,16 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
       }
     }
     switch (curr.type) {
-      case 'total':
-        acc[curr.code]['years'][curr.year]['totalAllStudents'] += curr.totalAllStudents
+      case 'passed':
+        acc[curr.code]['years'][curr.year]['totalPassed'] += curr.totalPassed
+        acc[curr.code]['years'][curr.year]['totalAllStudents'] += acc[curr.code]['years'][curr.year]['totalPassed']
         acc[curr.code]['years'][curr.year]['totalAllCredits'] += curr.totalAllcredits
         break
+      case 'notCompleted':
+        acc[curr.code]['years'][curr.year]['totalNotCompleted'] += curr.totalNotCompleted
+        acc[curr.code]['years'][curr.year]['totalAllStudents'] +=
+          acc[curr.code]['years'][curr.year]['totalNotCompleted']
+        break
       case 'ownProgramme':
         acc[curr.code]['years'][curr.year]['totalProgrammeStudents'] += curr.totalProgrammeStudents
         acc[curr.code]['years'][curr.year]['totalProgrammeCredits'] += curr.totalProgrammeCredits
@@ -175,6 +196,8 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
   const ayCourses = Object.keys(allCourses).filter(c => c.startsWith('AY'))
   const properties = [
     'totalAllStudents',
+    'totalPassed',
+    'totalNotCompleted',
     'totalAllCredits',
     'totalProgrammeStudents',
     'totalProgrammeCredits',
@@ -200,6 +223,8 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
           if (!allCourses[normCode]['years'][year]) {
             mergedCourse['years'][year] = {
               totalAllStudents: 0,
+              totalPassed: 0,
+              totalNotCompleted: 0,
               totalAllCredits: 0,
               totalProgrammeStudents: 0,
               totalProgrammeCredits: 0,

services/backend/src/services/studyprogrammeGraduations.js

@@ -9,11 +9,12 @@ const {
   getYearsObject,
   getYearsArray,
   getMedian,
-  getMean,
   getStartDate,
   getThesisType,
   alltimeStartDate,
   alltimeEndDate,
+  getId,
+  getGoal,
 } = require('./studyprogrammeHelpers')
 const {
   graduatedStudyRights,
@@ -23,8 +24,30 @@ const {
   allStudyrights,
   allTransfers,
 } = require('./studyprogramme')
+const { countTimeCategories, bachelorStudyright, getStatutoryAbsences } = require('./graduationHelpers')
 const { getAllProgrammes } = require('./studyrights')
 
+const checkStartdate = async (id, startdate) => {
+  if (id.slice(-2) === '-1') {
+    return startdate
+  }
+  const studyright = await bachelorStudyright(id.replace(/-2$/, '-1'))
+  if (studyright) return studyright.startdate
+  return null
+}
+
+const addGraduation = async (studyright, isAcademicYear, amounts, times) => {
+  const startdate = await checkStartdate(studyright.studyrightid, studyright.startdate)
+  const graduationYear = defineYear(studyright.enddate, isAcademicYear)
+
+  const totalTimeToGraduation = moment(studyright.enddate).diff(moment(startdate), 'months')
+  const statutoryAbsences = await getStatutoryAbsences(studyright.studentnumber, startdate, studyright.enddate)
+  const timeToGraduation = totalTimeToGraduation - statutoryAbsences
+
+  amounts[graduationYear] += 1
+  times[graduationYear] = [...times[graduationYear], timeToGraduation]
+}
+
 const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   const { graphStats, tableStats } = getStatsBasis(years)
 
@@ -77,6 +100,10 @@ const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, in
 const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
   let graduationAmounts = getYearsObject({ years })
   let graduationTimes = getYearsObject({ years, emptyArrays: true })
+  // for bc+ms combo
+  const doCombo = studyprogramme.startsWith('MH') && !['MH30_001', 'MH30_003'].includes(studyprogramme)
+  let graduationAmountsCombo = getYearsObject({ years })
+  let graduationTimesCombo = getYearsObject({ years, emptyArrays: true })
 
   const studentnumbers = await getCorrectStudentnumbers({
     codes: [studyprogramme],
@@ -87,32 +114,35 @@ const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademic
 
   const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbers)
 
-  studyrights.forEach(({ enddate, studystartdate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-    graduationAmounts[graduationYear] += 1
-    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
-  })
-  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
-  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+  // for masters, separate bc+ms combo from just ms students (except some medical progs)
+  for (const right of studyrights) {
+    if (doCombo && right.studyrightid.slice(-2) === '-2') {
+      await addGraduation(right, isAcademicYear, graduationAmountsCombo, graduationTimesCombo)
+    } else {
+      await addGraduation(right, isAcademicYear, graduationAmounts, graduationTimes)
+    }
+  }
 
-  const medians = getYearsObject({ years, emptyArrays: true })
-  const means = getYearsObject({ years, emptyArrays: true })
+  const goal = getGoal(studyprogramme)
+  const times = { medians: [], goal }
+  const comboTimes = { medians: [], goal: goal + 36 }
 
-  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
-  years.forEach(year => {
+  const rev = [...years].reverse()
+  for (const year of rev) {
     const median = getMedian(graduationTimes[year])
-    const mean = getMean(graduationTimes[year])
-    medians[year] = [
-      ['', median],
-      ['', comparisonValue - median],
-    ]
-    means[year] = [
-      ['', mean],
-      ['', comparisonValue - mean],
-    ]
-  })
-  return { medians, means, graduationAmounts }
+    const statistics = countTimeCategories(graduationTimes[year], goal)
+    times.medians = [...times.medians, { y: median, amount: graduationAmounts[year], name: year, statistics }]
+
+    if (doCombo) {
+      const median = getMedian(graduationTimesCombo[year])
+      const statistics = countTimeCategories(graduationTimesCombo[year], goal + 36)
+      comboTimes.medians = [
+        ...comboTimes.medians,
+        { y: median, amount: graduationAmountsCombo[year], name: year, statistics },
+      ]
+    }
+  }
+  return { times, doCombo, comboTimes }
 }
 
 // Creates (studyrightid, transfer)-map out of programmes transfers
@@ -126,7 +156,7 @@ const getTransferStudyrightMap = async (studyprogramme, since) => {
 const formatStats = (stats, years) => {
   const tableStats = Object.values(stats)
     .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
-    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+    .map(p => [p.code, getId(p.code), p.name['fi'], ...years.map(year => p[year])])
 
   const graphStats = Object.values(stats)
     .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
@@ -223,33 +253,49 @@ const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) =>
   const thesis = await getThesisStats(queryParameters)
 
   const graduated = await getGraduatedStats(queryParameters)
+
   const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+
   const programmesBeforeOrAfter = await getProgrammesBeforeOrAfter(studyprogramme, queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
 
-  const titles = ['', 'Graduated', 'Wrote thesis']
-  const programmesBeforeOrAfterTitles = ['Code', 'Programme', ...years]
-  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+  const titles =
+    studyprogramme.includes('LIS') || studyprogramme.includes('T')
+      ? ['', 'Graduated']
+      : ['', 'Graduated', 'Wrote thesis']
+  const programmesBeforeOrAfterTitles = ['Code', 'Id', 'Programme', ...years]
+  const tableStats =
+    studyprogramme.includes('LIS') || studyprogramme.includes('T')
+      ? reversedYears.map(year => [year, graduated.tableStats[year]])
+      : reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
 
   return {
     id: studyprogramme,
     years,
     tableStats,
     titles,
-    graphStats: [
-      {
-        name: 'Graduated students',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Wrote thesis',
-        data: thesis.graphStats,
-      },
-    ],
-    graduationMedianTime: graduationTimeStats.medians,
-    graduationMeanTime: graduationTimeStats.means,
-    graduationAmounts: graduationTimeStats.graduationAmounts,
+    graphStats:
+      studyprogramme.includes('LIS') || studyprogramme.includes('T')
+        ? [
+            {
+              name: 'Graduated students',
+              data: graduated.graphStats,
+            },
+          ]
+        : [
+            {
+              name: 'Graduated students',
+              data: graduated.graphStats,
+            },
+            {
+              name: 'Wrote thesis',
+              data: thesis.graphStats,
+            },
+          ],
+    graduationTimes: graduationTimeStats.times,
+    doCombo: graduationTimeStats.doCombo,
+    comboTimes: graduationTimeStats.comboTimes,
     programmesBeforeOrAfterTableStats: programmesBeforeOrAfter?.tableStats,
     programmesBeforeOrAfterGraphStats: programmesBeforeOrAfter?.graphStats,
     programmesBeforeOrAfterTitles,

services/backend/src/services/studyprogrammeHelpers.js

@@ -1,8 +1,7 @@
-const { mean } = require('lodash')
 const { studentnumbersWithAllStudyrightElements } = require('./populations')
+const { codes } = require('../../config/programmeCodes')
 
 // Helper functions
-
 const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
   let studentnumbers = []
   const exchangeStudents = includeAllSpecials
@@ -45,6 +44,7 @@ const resolveStudyRightCode = studyright_elements => {
 const formatStudyright = studyright => {
   const {
     studyrightid,
+    startdate,
     studystartdate,
     enddate,
     givendate,
@@ -54,10 +54,12 @@ const formatStudyright = studyright => {
     extentcode,
     student,
     studyright_elements,
+    cancelled,
   } = studyright
 
   return {
     studyrightid,
+    startdate,
     studystartdate: defineStartDate(studyright_elements, studystartdate),
     enddate,
     givendate,
@@ -68,6 +70,7 @@ const formatStudyright = studyright => {
     studentnumber: student.studentnumber,
     code: resolveStudyRightCode(studyright_elements),
     studyrightElements: studyright_elements,
+    cancelled,
     name:
       studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
         ? studyright_elements[0].element_detail.name
@@ -154,8 +157,8 @@ const isNonMajorCredit = (studyrights, attainment_date) =>
 const isSpecialGroupCredit = (studyrights, attainment_date, transfers) =>
   studyrights.some(studyright => {
     if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
-    if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
-    if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+    if (studyright.studystartdate && new Date(studyright.studystartdate) > new Date(attainment_date)) return true // Credits before the studyright started are not major student credits
+    if (studyright.enddate && new Date(attainment_date) > new Date(studyright.enddate)) return true // Credits after studyright are not major student credits
     if ([7, 9, 16, 34, 22, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
     if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
     return false
@@ -169,11 +172,6 @@ const getMedian = values => {
   return (values[half - 1] + values[half]) / 2.0
 }
 
-const getMean = values => {
-  if (values.length === 0) return 0
-  return Math.round(mean(values))
-}
-
 const defineYear = (date, isAcademicYear) => {
   if (!date) return ''
   const year = date.getFullYear()
@@ -224,8 +222,12 @@ const getPercentage = (value, total) => {
 const getEmptyArray = length => new Array(length).fill(0)
 
 const getBachelorCreditGraphStats = years => ({
+  lte15: {
+    name: 'Less than 15 credits',
+    data: getEmptyArray(years.length),
+  },
   lte30: {
-    name: 'Less than 30 credits',
+    name: '15-29 credits',
     data: getEmptyArray(years.length),
   },
   lte60: {
@@ -249,7 +251,7 @@ const getBachelorCreditGraphStats = years => ({
     data: getEmptyArray(years.length),
   },
   mte180: {
-    name: 'More than 180 credits',
+    name: '180 or more credits',
     data: getEmptyArray(years.length),
   },
 })
@@ -280,61 +282,125 @@ const getMasterCreditGraphStats = years => ({
     data: getEmptyArray(years.length),
   },
   mte300: {
-    name: 'More than 300 credits',
+    name: '300 or more credits',
+    data: getEmptyArray(years.length),
+  },
+})
+const getVetenaryCreditGraphStats = years => ({
+  lte210: {
+    name: 'Less than 210 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '210-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte270: {
+    name: '240-269 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte300: {
+    name: '270-299 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte330: {
+    name: '300-329 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte360: {
+    name: '330-359 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte360: {
+    name: '360 or more credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getOnlyMasterCreditGraphStats = years => ({
+  lte15: {
+    name: 'Less than 15 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte30: {
+    name: '15-29 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte60: {
+    name: '30-59 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte90: {
+    name: '60-89 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte120: {
+    name: '90-119 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte120: {
+    name: '120 or more credits',
     data: getEmptyArray(years.length),
   },
 })
 
 const getDoctoralCreditGraphStats = years => ({
-  lte50: {
-    name: 'Less than 50 credits',
+  lte10: {
+    name: 'Less than 10 credits',
     data: getEmptyArray(years.length),
   },
-  lte100: {
-    name: '50-99 credits',
+  lte20: {
+    name: '10-19 credits',
     data: getEmptyArray(years.length),
   },
-  lte150: {
-    name: '100-149 credits',
+  lte30: {
+    name: '20-29 credits',
     data: getEmptyArray(years.length),
   },
-  lte200: {
-    name: '150-199 credits',
+  lte40: {
+    name: '30-39 credits',
     data: getEmptyArray(years.length),
   },
-  lte250: {
-    name: '200-249 credits',
-    data: getEmptyArray(years.length),
-  },
-  lte300: {
-    name: '250-299 credits',
-    data: getEmptyArray(years.length),
-  },
-  mte300: {
-    name: 'More than 300 credits',
+  mte40: {
+    name: '40 or more credits',
     data: getEmptyArray(years.length),
   },
 })
 
-const getCreditGraphStats = (studyprogramme, years) => {
+const getCreditGraphStats = (studyprogramme, years, studyprog = false) => {
   if (studyprogramme.includes('KH')) return getBachelorCreditGraphStats(years)
-  if (studyprogramme.includes('MH')) return getMasterCreditGraphStats(years)
+  if (studyprogramme === 'MH90_001')
+    return studyprog ? getBachelorCreditGraphStats(years) : getVetenaryCreditGraphStats(years)
+  if (studyprogramme.includes('MH')) return getOnlyMasterCreditGraphStats(years)
   return getDoctoralCreditGraphStats(years)
 }
 
-const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
+const bachelorCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
 const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
-const doctoralCreditThresholds = ['lte50', 'lte100', 'lte150', 'lte200', 'lte250', 'lte300', 'mte300']
-const bachelorCreditAmounts = [30, 60, 90, 120, 150, 180, 180]
+const onlyMasterCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'mte120']
+const doctoralCreditThresholds = ['lte10', 'lte20', 'lte30', 'lte40', 'mte40']
+const bachelorCreditAmounts = [15, 30, 60, 90, 120, 150, 180, 180]
 const masterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
-const doctoralCreditAmounts = [50, 100, 150, 200, 250, 300, 300]
-
-const getCreditThresholds = studyprogramme => {
+const onlyMasterCreditAmounts = [15, 30, 60, 90, 120, 120]
+const doctoralCreditAmounts = [10, 20, 30, 40, 40]
+const vetenaryCreditThresholds = ['lte210', 'lte240', 'lte270', 'lte300', 'lte330', 'lte360', 'mte360']
+const vetenaryCreditAmounts = [210, 240, 270, 300, 330, 360, 360]
+const getBcMsThresholds = programmeCode => {
+  if (programmeCode === 'MH90_001')
+    return { creditThresholdKeysBcMs: vetenaryCreditThresholds, creditThresholdAmountsBcMs: vetenaryCreditAmounts }
+  return { creditThresholdKeysBcMs: masterCreditThresholds, creditThresholdAmountsBcMs: masterCreditAmounts }
+}
+const getCreditThresholds = (studyprogramme, studyProg = false) => {
+  if (studyprogramme === 'MH90_001') {
+    return studyProg
+      ? { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
+      : { creditThresholdKeys: vetenaryCreditThresholds, creditThresholdAmounts: vetenaryCreditAmounts }
+  }
   if (studyprogramme.includes('KH')) {
     return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
   }
   if (studyprogramme.includes('MH')) {
-    return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+    return { creditThresholdKeys: onlyMasterCreditThresholds, creditThresholdAmounts: onlyMasterCreditAmounts }
   }
   return { creditThresholdKeys: doctoralCreditThresholds, creditThresholdAmounts: doctoralCreditAmounts }
 }
@@ -359,13 +425,14 @@ const tableTitles = {
     bachelor: [
       '',
       'All',
-      '< 30 credits',
+      '< 15 credits',
+      '15-29 credits',
       '30-59 credits',
       '60-89 credits',
       '90-119 credits',
       '120-149 credits',
       '150-179 credits',
-      '> 180 credits',
+      '180 ≤ credits',
     ],
     master: [
       '',
@@ -376,19 +443,19 @@ const tableTitles = {
       '240-259 credits',
       '260-279 credits',
       '280-299 credits',
-      '> 300 credits',
+      '300 ≤ credits',
     ],
-    doctoral: [
+    masterOnly: [
       '',
       'All',
-      '< 50 credits',
-      '50-99 credits',
-      '100-149 credits',
-      '150-199 credits',
-      '200-249 credits',
-      '250-299 credits',
-      '> 300 credits',
+      '< 15 credits',
+      '15-29 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '120 ≤ credits',
     ],
+    doctoral: ['', 'All', '< 10 credits', '10-19 credits', '20-29 credits', '30-39 credits', '40 ≤ credits'],
   },
   studytracks: [
     '',
@@ -400,16 +467,71 @@ const tableTitles = {
     'Graduated',
     'Men',
     'Women',
-    'Finnish',
+    'Finland',
+  ],
+  programmes: [
+    '',
+    'All',
+    'Started studying',
+    'Currently enrolled',
+    'Absent',
+    'Inactive',
+    'Graduated',
+    'Men',
+    'Women',
+    'Other/Unknown',
+    'Finland',
+    'Other',
   ],
 }
 
 const getCreditProgressTableTitles = studyprogramme => {
   if (studyprogramme.includes('KH')) return tableTitles.creditProgress.bachelor
-  if (studyprogramme.includes('MH')) return tableTitles.creditProgress.master
+  if (studyprogramme.includes('MH')) return tableTitles.creditProgress.masterOnly
   return tableTitles.creditProgress.doctoral
 }
 
+const mapCodesToIds = data => {
+  // Add programme id e.g. TKT
+  const keys = Object.keys(codes)
+  const progs = Object.keys(data)
+
+  for (const prog of progs) {
+    if (keys.includes(prog)) {
+      data[prog].progId = codes[prog].toUpperCase()
+    }
+  }
+}
+
+const getId = code => {
+  if (Object.keys(codes).includes(code)) {
+    return codes[code].toUpperCase()
+  }
+  return ''
+}
+
+const getGoal = programme => {
+  if (programme.startsWith('KH') || programme.endsWith('-ba')) {
+    return 36
+  }
+  if (programme.startsWith('MH') || programme.endsWith('-ma')) {
+    if (['MH30_004', '420420-ma'].includes(programme)) {
+      return 24 + 6
+    }
+    if (['MH30_001', 'MH30_003', '320011-ma', '320001-ma', '320002-ma'].includes(programme)) {
+      return 36 + 24 + 12 // medical, no separate bachelor
+    }
+    return 24
+  }
+  if (programme.includes('T')) {
+    return 48
+  }
+  if (programme.startsWith('LI')) {
+    return 78
+  }
+  return 48 //unknown, likely old doctor or licentiate
+}
+
 module.exports = {
   getCorrectStudentnumbers,
   formatStudyright,
@@ -422,7 +544,6 @@ module.exports = {
   isMajorStudentCredit,
   isSpecialGroupCredit,
   getMedian,
-  getMean,
   defineYear,
   getStartDate,
   alltimeStartDate,
@@ -431,10 +552,16 @@ module.exports = {
   getPercentage,
   getBachelorCreditGraphStats,
   getMasterCreditGraphStats,
+  getVetenaryCreditGraphStats,
   getDoctoralCreditGraphStats,
+  getOnlyMasterCreditGraphStats,
   getCreditGraphStats,
   getCreditThresholds,
   tableTitles,
   getCreditProgressTableTitles,
+  getBcMsThresholds,
   isNonMajorCredit,
+  mapCodesToIds,
+  getId,
+  getGoal,
 }

services/backend/src/services/studyprogrammeUpdates.js

@@ -30,7 +30,6 @@ const updateBasicView = async code => {
   }
 
   const options = [specialCalendar, specialAcademic, specialExcludedCalendar, specialExcludedAcademic]
-
   for (const option of options) {
     try {
       const basicStats = await getBasicStatsForStudytrack({
@@ -62,8 +61,8 @@ const updateBasicView = async code => {
     } catch (e) {
       logger.error(e)
     }
-    return 'OK'
   }
+  return 'OK'
 }
 
 const updateStudytrackView = async code => {

services/backend/src/services/studytrackStats.js

@@ -4,7 +4,6 @@ const moment = require('moment')
 const { getAssociations } = require('./studyrights')
 const {
   getMedian,
-  getMean,
   getStartDate,
   getYearsArray,
   getPercentage,
@@ -14,6 +13,7 @@ const {
   getCreditGraphStats,
   tableTitles,
   getCreditProgressTableTitles,
+  getGoal,
 } = require('./studyprogrammeHelpers')
 const {
   studytrackStudents,
@@ -25,11 +25,11 @@ const {
   graduatedStudyRights,
 } = require('./studyprogramme')
 const { getAcademicYearDates } = require('../util/semester')
+const { countTimeCategories } = require('./graduationHelpers')
 
 const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts) => {
   let data = { female: 0, male: 0, finnish: 0 }
   thresholdKeys.forEach(t => (data[t] = 0))
-
   students.forEach(({ gender_code, home_country_en, credits }) => {
     const creditcount = credits
       .filter(credit => moment(credit.attainment_date).isAfter(startDate))
@@ -42,43 +42,67 @@ const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts) =>
     data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
     data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
     data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
-    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
-    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
-    data[thresholdKeys[6]] += creditcount >= thresholdAmounts[6] ? 1 : 0
+    if (thresholdKeys.length > 7) {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] && creditcount < thresholdAmounts[6] ? 1 : 0
+      data[thresholdKeys[7]] += creditcount >= thresholdAmounts[6] ? 1 : 0
+    } else if (thresholdKeys.length > 6) {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] ? 1 : 0
+    } else if (thresholdKeys.length > 5) {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] ? 1 : 0
+    } else {
+      data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] ? 1 : 0
+    }
   })
   return data
 }
 
-const getGraduationTimeStats = async ({
-  year,
-  graduated,
-  studyprogramme,
-  track,
-  graduationMeanTime,
-  graduationMedianTime,
-  graduationAmounts,
-}) => {
+const getGraduationTimeStats = async ({ year, graduated, track, graduationAmounts, graduationTimes, classSize }) => {
+  if (year === 'Total') {
+    return
+  }
+
   // Count how long each student took to graduate
-  let graduationTimes = []
-  graduated.forEach(({ enddate, studystartdate }) => {
-    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+  let times = []
+  graduated.forEach(({ enddate, startdate }) => {
+    const timeToGraduation = moment(enddate).diff(moment(startdate), 'months')
     graduationAmounts[track][year] += 1
-    graduationTimes = [...graduationTimes, timeToGraduation]
+    times = [...times, timeToGraduation]
   })
 
-  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
-  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+  const median = getMedian(times)
+  const statistics = countTimeCategories(times, graduationTimes.goal)
 
-  // HighCharts graph requires the data to have this format (ie. actual value, "empty value")
-  const median = getMedian(graduationTimes)
-  const mean = getMean(graduationTimes)
-  graduationMedianTime[track][year] = [
-    ['', median],
-    ['', comparisonValue - median],
+  graduationTimes[track].medians = [
+    ...graduationTimes[track].medians,
+    { y: median, amount: graduationAmounts[track][year], name: year, statistics, classSize },
   ]
-  graduationMeanTime[track][year] = [
-    ['', mean],
-    ['', comparisonValue - mean],
+}
+
+const getStats = (all, started, enrolled, absent, inactive, graduated, studentData) => {
+  return [
+    all.length,
+    getPercentage(all.length, all.length),
+    started.length,
+    getPercentage(started.length, all.length),
+    enrolled.length,
+    getPercentage(enrolled.length, all.length),
+    absent.length,
+    getPercentage(absent.length, all.length),
+    inactive.length,
+    getPercentage(inactive.length, all.length),
+    graduated.length,
+    getPercentage(graduated.length, all.length),
+    studentData.male,
+    getPercentage(studentData.male, all.length),
+    studentData.female,
+    getPercentage(studentData.female, all.length),
+    studentData.finnish,
+    getPercentage(studentData.finnish, all.length),
   ]
 }
 
@@ -103,17 +127,16 @@ const getStudytrackDataForTheYear = async ({
     mainStatsByTrack,
     creditGraphStats,
     creditTableStats,
-    graduationMeanTime,
-    graduationMedianTime,
     graduationAmounts,
+    graduationTimes,
     totalAmounts,
     emptyTracks,
+    totals,
   } = data
 
   await Promise.all(
     studytracks.map(async track => {
       const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
-
       const studentnumbers = await getCorrectStudentnumbers({
         codes,
         startDate,
@@ -121,16 +144,42 @@ const getStudytrackDataForTheYear = async ({
         includeAllSpecials,
         includeGraduated,
       })
-
-      // Get all the studyrights and students for the calculations
-      const all = await allStudyrights(track, studentnumbers)
       const students = await studytrackStudents(studentnumbers)
-      const studentData = getStudentData(startDate, students, creditThresholdKeys, creditThresholdAmounts)
-      const started = await startedStudyrights(track, startDate, studentnumbers)
-      const enrolled = await enrolledStudents(track, startDate, studentnumbers)
-      const absent = await absentStudents(track, studentnumbers)
-      const inactive = await inactiveStudyrights(track, studentnumbers)
-      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+
+      let all = []
+      let studentData = {}
+      let started = []
+      let enrolled = []
+      let absent = []
+      let inactive = []
+      let graduated = []
+      // Get all the studyrights and students for the calculations
+      if (year !== 'Total') {
+        all = await allStudyrights(track, studentnumbers)
+        studentData = getStudentData(startDate, students, creditThresholdKeys, creditThresholdAmounts)
+        started = await startedStudyrights(track, startDate, studentnumbers)
+        enrolled = await enrolledStudents(track, startDate, studentnumbers)
+        absent = await absentStudents(track, studentnumbers)
+        inactive = await inactiveStudyrights(track, studentnumbers)
+        graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+        totals[track].all = [...totals[track].all, ...all]
+        totals[track].studentData.male += studentData.male
+        totals[track].studentData.female += studentData.female
+        totals[track].studentData.finnish += studentData.finnish
+        totals[track].started = [...totals[track].started, ...started]
+        totals[track].enrolled = [...totals[track].enrolled, ...enrolled]
+        totals[track].absent = [...totals[track].absent, ...absent]
+        totals[track].inactive = [...totals[track].inactive, ...inactive]
+        totals[track].graduated = [...totals[track].graduated, ...graduated]
+      } else {
+        all = totals[track].all
+        studentData = totals[track].studentData
+        started = totals[track].started
+        enrolled = totals[track].enrolled
+        absent = totals[track].absent
+        inactive = totals[track].inactive
+        graduated = totals[track].graduated
+      }
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
       if (all.length === 0) {
@@ -138,82 +187,63 @@ const getStudytrackDataForTheYear = async ({
         return
       }
 
-      // Count stats for creditgraph for the year per track
-      creditThresholdKeys.forEach(
-        threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
-      )
-
-      // Count stats for the credit progress table for the year per track
-      creditTableStats[track] = [
-        ...creditTableStats[track],
-        [year, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
-      ]
+      if (year !== 'Total') {
+        // Count stats for creditgraph for the year per track
+        creditThresholdKeys.forEach(
+          threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] += studentData[threshold])
+        )
+        creditThresholdKeys.forEach(
+          threshold => (creditGraphStats[track][threshold].data[indexOf(years, 'Total')] += studentData[threshold])
+        )
+        // Count stats for the credit progress table for the year per track
+        creditTableStats[track] = [
+          [year, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
+          ...creditTableStats[track],
+        ]
+      } else {
+        const totalCredits = creditTableStats[track].reduce(
+          (a, b) => a.map((colVal, idx) => colVal + (b[idx] || 0)),
+          Array(creditThresholdKeys.length + 2).fill(0)
+        )
+        creditTableStats[track] = [[year, ...totalCredits.slice(1)], ...creditTableStats[track]]
+      }
 
       // Count stats for the main studytrack table grouped by tracks
       mainStatsByTrack[track] = [
+        [year, ...getStats(all, started, enrolled, absent, inactive, graduated, studentData)],
         ...mainStatsByTrack[track],
-        [
-          year,
-          all.length,
-          getPercentage(all.length, all.length),
-          started.length,
-          getPercentage(started.length, all.length),
-          enrolled.length,
-          getPercentage(enrolled.length, all.length),
-          absent.length,
-          getPercentage(absent.length, all.length),
-          inactive.length,
-          getPercentage(inactive.length, all.length),
-          graduated.length,
-          getPercentage(graduated.length, all.length),
-          studentData.male,
-          getPercentage(studentData.male, all.length),
-          studentData.female,
-          getPercentage(studentData.female, all.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, all.length),
-        ],
       ]
 
       // Count stats for the main studytrack table grouped by year
       mainStatsByYear[year] = [
-        ...mainStatsByYear[year],
         [
           studytrackNames[track]?.name['fi'] ? `${studytrackNames[track]?.name['fi']}, ${track}` : year,
-          all.length,
-          getPercentage(all.length, all.length),
-          started.length,
-          getPercentage(started.length, all.length),
-          enrolled.length,
-          getPercentage(enrolled.length, all.length),
-          absent.length,
-          getPercentage(absent.length, all.length),
-          inactive.length,
-          getPercentage(inactive.length, all.length),
-          graduated.length,
-          getPercentage(graduated.length, all.length),
-          studentData.male,
-          getPercentage(studentData.male, all.length),
-          studentData.female,
-          getPercentage(studentData.female, all.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, all.length),
+          ...getStats(all, started, enrolled, absent, inactive, graduated, studentData),
         ],
+        ...mainStatsByYear[year],
       ]
 
       // Count stats for the graduation time charts grouped by year
-      totalAmounts[track][year] = await all.length
+      if (!(track in graduationTimes)) {
+        graduationTimes[track] = { medians: [] }
+      }
+      totalAmounts[track][year] = all.length
       await getGraduationTimeStats({
         year,
         graduated,
-        studyprogramme,
         track,
-        graduationMedianTime,
-        graduationMeanTime,
         graduationAmounts,
+        graduationTimes,
+        classSize: totalAmounts[track][year],
       })
     })
   )
+  Object.keys(graduationTimes).forEach(track => {
+    if (track !== 'goal') {
+      const sortedMedians = graduationTimes[track].medians.sort((a, b) => b.name.localeCompare(a.name))
+      graduationTimes[track].medians = sortedMedians
+    }
+  })
 }
 
 // Defines the studytrack names for the studytrack selector
@@ -235,20 +265,26 @@ const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
   const mainStatsByTrack = {}
   const creditGraphStats = {}
   const creditTableStats = {}
-  const graduationMedianTime = {}
-  const graduationMeanTime = {}
   const graduationAmounts = {}
+  const graduationTimes = { goal: getGoal(studyprogramme) }
   const totalAmounts = {}
   const emptyTracks = new Map()
-
+  const totals = {}
   studytracks.forEach(async track => {
     mainStatsByTrack[track] = []
-    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years, true)
     creditTableStats[track] = []
-    graduationMedianTime[track] = getYearsObject({ years, emptyArrays: true })
-    graduationMeanTime[track] = getYearsObject({ years, emptyArrays: true })
     graduationAmounts[track] = getYearsObject({ years })
     totalAmounts[track] = getYearsObject({ years })
+    totals[track] = {
+      all: [],
+      started: [],
+      enrolled: [],
+      absent: [],
+      inactive: [],
+      graduated: [],
+      studentData: { male: 0, female: 0, finnish: 0 },
+    }
   })
 
   return {
@@ -256,11 +292,11 @@ const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
     mainStatsByTrack,
     creditGraphStats,
     creditTableStats,
-    graduationMedianTime,
-    graduationMeanTime,
     graduationAmounts,
+    graduationTimes,
     totalAmounts,
     emptyTracks,
+    totals,
   }
 }
 
@@ -279,35 +315,37 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, settings })
   const studytrackNames = associations.studyTracks
 
   const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
-  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme)
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme, true)
 
-  await Promise.all(
-    years.map(async year => {
-      return await getStudytrackDataForTheYear({
-        studyprogramme,
-        since,
-        settings,
-        studytracks,
-        studytrackNames,
-        year,
-        years,
-        data,
-        creditThresholdKeys,
-        creditThresholdAmounts,
-      })
+  const yearsReversed = [...years].reverse()
+
+  for (const year of yearsReversed) {
+    await getStudytrackDataForTheYear({
+      studyprogramme,
+      since,
+      settings,
+      studytracks,
+      studytrackNames,
+      year,
+      years,
+      data,
+      creditThresholdKeys,
+      creditThresholdAmounts,
     })
-  )
+  }
 
   const studytrackOptions = getStudytrackOptions(studyprogramme, studytrackNames, studytracks, data.emptyTracks, years)
-
   return {
     id: studyprogramme,
-    years: getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined),
+    years: years,
     ...data,
     studytrackOptions,
     includeGraduated: settings.graduated,
     populationTitles: tableTitles['studytracks'],
-    creditTableTitles: getCreditProgressTableTitles(studyprogramme),
+    creditTableTitles:
+      studyprogramme === 'MH90_001'
+        ? tableTitles['creditProgress']['bachelor']
+        : getCreditProgressTableTitles(studyprogramme),
   }
 }
 

services/backend/src/services/userService.js

@@ -8,11 +8,10 @@ const { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } = requir
 const _ = require('lodash')
 const { User, UserElementDetails, AccessGroup, UserFaculties, sequelizeUser } = require('../models/models_user')
 const { getOrganizationAccess } = require('../util/organizationAccess')
+const { getUserIams, getAllUserAccess } = require('../util/jami')
 
 const courseStatisticsGroup = 'grp-oodikone-basic-users'
 const facultyStatisticsGroup = 'grp-oodikone-users'
-const hyOneGroup = 'hy-one'
-const toskaGroup = 'grp-toska'
 
 // Max 25 users can be stored in the cache
 const userDataCache = new LRU({
@@ -165,34 +164,85 @@ const updateUser = async (username, fields) => {
 
 const getAccessGroups = async () => await AccessGroup.findAll()
 
+const updateAccessGroups = async (username, iamGroups = [], specialGroup = {}, sisId) => {
+  const { jory, hyOne, superAdmin, openUni } = specialGroup
+
+  const userFromDb = await byUsername(username)
+  const formattedUser = await formatUser(userFromDb, [], Boolean(sisId))
+
+  const { roles: currentAccessGroups } = formattedUser
+
+  // Modify accessgroups based on current groups and iamGroups
+  let newAccessGroups = []
+  if (currentAccessGroups.includes('studyGuidanceGroups') || (sisId && (await checkStudyGuidanceGroupsAccess(sisId))))
+    newAccessGroups.push('studyGuidanceGroups')
+  if (iamGroups.includes(courseStatisticsGroup)) newAccessGroups.push('courseStatistics')
+  if (jory || iamGroups.includes(facultyStatisticsGroup)) newAccessGroups.push('facultyStatistics')
+  if (hyOne || currentAccessGroups.includes('teachers')) newAccessGroups.push('teachers')
+  if (superAdmin || currentAccessGroups.includes('admin')) newAccessGroups.push('admin')
+  if (openUni) newAccessGroups.push('openUniSearch')
+
+  const accessGroups = await AccessGroup.findAll({
+    where: {
+      group_code: {
+        [Op.in]: newAccessGroups,
+      },
+    },
+  })
+
+  // In the difference method "the order and references of result values are determined by the first array." https://lodash.com/docs/4.17.15#difference
+  // Both directions needs to be checked in order to update roles when the access should no longer exists.
+  if (
+    _.difference(newAccessGroups, currentAccessGroups).length > 0 ||
+    _.difference(currentAccessGroups, newAccessGroups).length > 0
+  ) {
+    userFromDb.setAccessgroup(accessGroups.map(({ id }) => id))
+  }
+
+  return { newAccessGroups, accessGroups }
+}
+
 // newer logic
 const enrichProgrammesFromFaculties = faculties =>
   facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
 
-const findAll = async () =>
-  (
-    await User.findAll({
-      include: userIncludes,
-    })
-  ).map(user => ({
-    ...user.get(),
-    is_enabled: true,
-    elementdetails: _.uniqBy([
-      ...user.programme.map(p => p.elementDetailCode),
-      ...enrichProgrammesFromFaculties(user.faculty.map(f => f.faculty_code)),
-    ]),
-  }))
+const findAll = async () => {
+  const allUsers = await User.findAll({
+    include: userIncludes,
+  })
 
-const formatUser = async userFromDb => {
+  const userAccess = await getAllUserAccess()
+
+  const formattedUsers = await Promise.all(
+    allUsers.map(async user => {
+      const { iamGroups, specialGroup } = userAccess.find(({ id }) => id === user.sisu_person_id) || {}
+
+      const { accessGroups } = await updateAccessGroups(user.username, iamGroups, specialGroup)
+
+      return {
+        ...user.get(),
+        is_enabled: true,
+        iam_groups: iamGroups || [],
+        elementdetails: _.uniqBy([
+          ...user.programme.map(p => p.elementDetailCode),
+          ...enrichProgrammesFromFaculties(user.faculty.map(f => f.faculty_code)),
+        ]),
+        accessgroup: accessGroups,
+      }
+    })
+  )
+
+  return formattedUsers
+}
+
+const formatUser = async (userFromDb, extraRights, getStudentAccess = true) => {
   const [accessGroups, programmes, faculties] = [
     ['accessgroup', 'group_code'],
     ['programme', 'elementDetailCode'],
     ['faculty', 'faculty_code'],
   ].map(([field, code]) => userFromDb[field].map(entry => entry[code]))
 
-  const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)])
-
-  const iamRights = Object.keys(await getOrganizationAccess(userFromDb))
+  const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)]).concat(extraRights)
 
   const {
     dataValues: {
@@ -206,9 +256,13 @@ const formatUser = async userFromDb => {
     },
   } = userFromDb
 
-  const studentsUserCanAccess = _.uniqBy(
-    (await Promise.all([getStudentnumbersByElementdetails(rights), getAllStudentsUserHasInGroups(sisPersonId)])).flat()
-  )
+  const studentsUserCanAccess = getStudentAccess
+    ? _.uniqBy(
+        (
+          await Promise.all([getStudentnumbersByElementdetails(rights), getAllStudentsUserHasInGroups(sisPersonId)])
+        ).flat()
+      )
+    : []
 
   // attribute naming is a bit confusing, but it's used so widely in oodikone, that it's not probably worth changing
   return {
@@ -221,7 +275,6 @@ const formatUser = async userFromDb => {
     email,
     is_enabled: true, // this is probably not needed: is here mainly because old userservice created users for every logged in user, even if they hadn't correct iamgroups
     rights,
-    iamRights,
     roles: accessGroups,
     studentsUserCanAccess, // studentnumbers used in various parts of backend. For admin this is usually empty, since no programmes / faculties are set.
     isAdmin: accessGroups.includes('admin'),
@@ -232,15 +285,28 @@ const getMockedUser = async ({ userToMock, mockedBy }) => {
   if (userDataCache.has(userToMock)) {
     return userDataCache.get(userToMock)
   }
+
+  const userFromDb = await byUsername(userToMock)
+  userFromDb.iamGroups = await getUserIams(userFromDb.sisu_person_id)
+
+  const { access = {}, specialGroup = {} } = await getOrganizationAccess(userFromDb)
+
+  const iamRights = Object.keys(access)
+
+  const formattedUser = await formatUser(userFromDb, specialGroup.kosu ? iamRights : [])
+
   const toReturn = {
-    ...(await formatUser(await byUsername(userToMock))),
+    ...formattedUser,
+    iamRights,
+    specialGroup,
     mockedBy,
   }
   userDataCache.set(userToMock, toReturn)
+
   return toReturn
 }
 
-const getUser = async ({ username, name, email, iamGroups, sisId }) => {
+const getUser = async ({ username, name, email, iamGroups, iamRights, specialGroup, sisId }) => {
   if (userDataCache.has(username)) {
     return userDataCache.get(username)
   }
@@ -254,40 +320,19 @@ const getUser = async ({ username, name, email, iamGroups, sisId }) => {
     email,
     language,
     sisu_person_id: sisId,
-    iam_groups: iamGroups,
     last_login: new Date(),
   })
 
   const userFromDb = await byUsername(username)
-  const formattedUser = await formatUser(userFromDb)
-  const { roles: currentAccessGroups } = formattedUser
+  const formattedUser = await formatUser(userFromDb, specialGroup.kosu ? iamRights : [])
 
-  // Modify accessgroups based on current groups and iamGroups
-  let newAccessGroups = []
-  if (await checkStudyGuidanceGroupsAccess(sisId)) newAccessGroups.push('studyGuidanceGroups')
-  if (iamGroups.includes(courseStatisticsGroup)) newAccessGroups.push('courseStatistics')
-  if (iamGroups.includes(facultyStatisticsGroup) || iamGroups.some(element => /-jory$/.test(element)))
-    newAccessGroups.push('facultyStatistics')
-  if (iamGroups.includes(hyOneGroup) || currentAccessGroups.includes('teachers')) newAccessGroups.push('teachers')
-  if (iamGroups.includes(toskaGroup) || currentAccessGroups.includes('admin')) newAccessGroups.push('admin')
-  if (_.difference(newAccessGroups, currentAccessGroups).length > 0) {
-    userFromDb.setAccessgroup(
-      (
-        await AccessGroup.findAll({
-          where: {
-            group_code: {
-              [Op.in]: newAccessGroups,
-            },
-          },
-        })
-      ).map(({ id }) => id)
-    )
-  }
+  const { newAccessGroups } = await updateAccessGroups(username, iamGroups, specialGroup, sisId)
 
   if (isNewUser) await sendNotificationAboutNewUser({ userId: username, userFullName: name })
 
   const toReturn = {
     ...formattedUser,
+    iamRights,
     roles: newAccessGroups,
   }
   userDataCache.set(username, toReturn)

services/backend/src/util/IAMRights.js

@@ -1,239 +0,0 @@
-const {
-  isSuperAdminIam,
-  isAdminIam,
-  isUniversityWideIam,
-  isDoctoralIam,
-  getStudyLeaderGroup,
-  iamToOrganisationCode,
-  isEmployeeIam,
-  iamToDoctoralSchool,
-  kosuIamToFaculties,
-  dekaaniIamToFaculty,
-  opetusVaradekaani,
-} = require('../../config/IAMConfig')
-const { data } = require('../../config/data')
-const { mapToDegreeCode } = require('../../config/common')
-
-/**
- * Return given access to all programmes where predicate is true
- * (all if no predicate defined)
- * @param {object} accessLevel
- * @param {(program: object) => boolean} where
- * @returns {object} access to programmes
- */
-const getAllProgrammeAccess = (accessLevel, where) => {
-  const access = {}
-  data.forEach(faculty => {
-    faculty.programmes.forEach(program => {
-      if (where?.(program) === false) return
-      access[program.key] = { ...accessLevel }
-    })
-  })
-  return access
-}
-
-/**
- * NOT USED
- * Grant super-admin rights if the user has correct iams (eg. grp-toska)
- * @param {string[]} hyGroups
- * @returns superAdmin special group
- */
-const getSuperAdmin = hyGroups => {
-  const isToska = hyGroups.some(isSuperAdminIam)
-  if (isToska) {
-    return { specialGroup: { superAdmin: true } }
-  }
-  return {}
-}
-
-/**
- * NOT USED
- * Grant admin rights if the user has correct iams (eg. grp-ospa)
- * @param {string[]} hyGroups
- * @returns admin special group
- */
-const getAdmin = hyGroups => {
-  const isOspa = hyGroups.some(isAdminIam)
-  if (isOspa) {
-    return { specialGroup: { admin: true } }
-  }
-  return {}
-}
-
-/**
- * Grant reading rights to all programmes if user has uni wide IAM (eg. hy-rehtoraatti)
- * @param {string[]} hyGroups
- * @returns read access to ALL programmes
- */
-const getUniversityReadingRights = hyGroups => {
-  const hasUniversityReadingRights = hyGroups.some(isUniversityWideIam)
-  if (!hasUniversityReadingRights) {
-    return {}
-  }
-
-  const access = getAllProgrammeAccess({ read: true })
-  const specialGroup = { allProgrammes: true }
-
-  return { access, specialGroup }
-}
-
-/**
- * Grant reading rights to programmes of faculties if user is kosu or dekaanaatti of some faculties
- * @param {string[]} hyGroups
- */
-const getFacultyReadingRights = hyGroups => {
-  // faculty codes from kosu iam
-  const facultyCodes = hyGroups
-    .flatMap(kosuIamToFaculties)
-    // faculty codes from dekanaatti iam
-    .concat(hyGroups.map(dekaaniIamToFaculty))
-    .filter(Boolean)
-  const access = {}
-  facultyCodes.forEach(fc => {
-    const faculty = data.find(faculty => faculty.code === fc)
-    const programmeCodes = faculty.programmes.map(p => p.key)
-    programmeCodes.forEach(code => {
-      access[code] = { read: true }
-    })
-  })
-  return { access, specialGroup: {} }
-}
-
-/**
- * Grant admin rights to faculty programmes if user is opetusvaradekaani of that faculty
- * @param {string[]} hyGroups
- */
-const getFacultyAdminRights = hyGroups => {
-  if (!hyGroups.includes(opetusVaradekaani)) return {}
-  const facultyCodes = hyGroups.map(dekaaniIamToFaculty).filter(Boolean)
-
-  const access = {}
-  facultyCodes.forEach(fc => {
-    const faculty = data.find(faculty => faculty.code === fc)
-    const programmeCodes = faculty.programmes.map(p => p.key)
-    programmeCodes.forEach(code => {
-      access[code] = { read: true, write: true, admin: true }
-    })
-  })
-  return { access, specialGroup: {} }
-}
-
-/**
- * Grant reading rights to all doctoral programmes if the user belongs to doctoral IAM
- * @param {string[]} hyGroups
- * @returns read access to ALL doctoral programs
- */
-const getDoctoralAccess = hyGroups => {
-  const hasDoctoralReadingRights = hyGroups.some(isDoctoralIam)
-  if (!hasDoctoralReadingRights) return {}
-  const access = getAllProgrammeAccess({ read: true }, program => program.level === 'doctoral')
-  const specialGroup = { doctoral: true }
-
-  return { access, specialGroup }
-}
-
-/**
- * Grants reading rights to all doctoral programmes that belong to user's
- * doctoral school IAMs
- * @param {string[]} hyGroups
- * @returns read access to doctoral programs
- */
-const getDoctoralSchoolAccess = hyGroups => {
-  const doctoralProgrammeCodes = hyGroups.flatMap(iamToDoctoralSchool)
-  const access = {}
-  doctoralProgrammeCodes.forEach(code => {
-    if (!code) return
-    access[code] = { read: true }
-  })
-  return { access }
-}
-
-/**
- * Grant admin access if the user belongs to studyprogramme's manager group and is a study program leader
- * @param {string[]} hyGroups
- */
-const getProgrammeAdminAccess = hyGroups => {
-  const orgCodes = hyGroups
-    .filter(iam => hyGroups.includes(getStudyLeaderGroup(iam)))
-    .map(iam => iamToOrganisationCode(iam))
-    .filter(Boolean)
-
-  const degreeCodes = orgCodes.flatMap(codes => codes.map(mapToDegreeCode))
-
-  if (!degreeCodes?.length > 0) {
-    return {}
-  }
-
-  const access = {}
-  degreeCodes.forEach(code => {
-    access[code] = { read: true, write: true, admin: true }
-  })
-  return { access }
-}
-
-/**
- * Grant write and read access if the user belongs to employees group and studyprogramme's manager group
- * @param {string[]} hyGroups
- */
-const getProgrammeWriteAccess = hyGroups => {
-  if (!hyGroups.some(isEmployeeIam)) return {}
-  const orgCodes = hyGroups.map(iam => iamToOrganisationCode(iam)).filter(Boolean)
-  const degreeCodes = orgCodes.flatMap(codes => codes.map(mapToDegreeCode))
-  const access = {}
-  degreeCodes.forEach(code => {
-    if (!code) return
-    access[code] = { read: true, write: true }
-  })
-
-  return { access }
-}
-
-/**
- * Grant read access if the user belongs to studyprogramme's manager group
- * @param {string[]} hyGroups
- */
-const getProgrammeReadAccess = hyGroups => {
-  const orgCodes = hyGroups.map(iam => iamToOrganisationCode(iam)).filter(Boolean)
-  const degreeCodes = orgCodes.flatMap(codes => codes.map(mapToDegreeCode))
-  const access = {}
-  degreeCodes.forEach(code => {
-    if (!code) return
-    access[code] = { read: true }
-  })
-
-  return { access }
-}
-
-/**
- * Gets access rights and special groups,
- * based on IAM-groups in IAM header string
- * @param {string} hyGroupsHeader
- */
-const getIAMRights = hyGroups => {
-  let access = {}
-  let specialGroup = {}
-
-  ;[
-    getUniversityReadingRights,
-    getFacultyReadingRights,
-    getDoctoralAccess,
-    getDoctoralSchoolAccess,
-    getProgrammeReadAccess,
-    getProgrammeWriteAccess,
-    getProgrammeAdminAccess,
-    getFacultyAdminRights,
-    getAdmin,
-    getSuperAdmin,
-  ]
-    .map(f => f(hyGroups))
-    .forEach(({ access: newAccess, specialGroup: newSpecialGroup }) => {
-      access = { ...access, ...newAccess }
-      specialGroup = { ...specialGroup, ...newSpecialGroup }
-    })
-
-  return { access, specialGroup }
-}
-
-module.exports = {
-  getIAMRights,
-}

services/backend/src/util/avoinPopulation.js

@@ -0,0 +1,190 @@
+/* eslint-disable */
+
+const { Credit, Enrollment, Studyright } = require('../models')
+const moment = require('moment')
+const { Op } = require('sequelize')
+
+const hasValidStudyright = (studyrights, date) => {
+  const within = (right, date) => {
+    const first = moment(right.startdate)
+    const last = moment(right.enddate)
+    return first.isSameOrBefore(date) && last.isSameOrAfter(date)
+  }
+
+  return studyrights.find(r => within(r, date))
+}
+
+const studyrightsOf = async student => {
+  const rights = await Studyright.findAll({
+    where: {
+      student_studentnumber: student,
+    },
+  })
+
+  return rights
+}
+
+const uniq = objects => [...new Set(objects)]
+
+const courseInfo = async course_code => {
+  const credits = await Credit.findAll({
+    where: {
+      course_code,
+      is_open: true,
+      grade: {
+        [Op.in]: ['1', '2', '3', '4', '5', 'Hyv.'],
+      },
+      attainment_date: {
+        [Op.gte]: moment('8-1-2020', 'MM-DD-YYYY').toDate(),
+      },
+    },
+  })
+
+  const enrollments = await Enrollment.findAll({
+    where: {
+      course_code,
+      is_open: true,
+      enrollment_date_time: {
+        [Op.lte]: moment('8-1-2022', 'MM-DD-YYYY').toDate(),
+        [Op.gte]: moment('8-1-2020', 'MM-DD-YYYY').toDate(),
+      },
+    },
+  })
+
+  let avoin_credited = []
+
+  for (let credit of credits) {
+    const { attainment_date, student_studentnumber: studentnumber } = credit
+    const rights = await studyrightsOf(studentnumber)
+    if (!hasValidStudyright(rights, attainment_date)) {
+      avoin_credited.push(studentnumber)
+    }
+  }
+
+  let avoin_enrolled = []
+
+  for (let enrollment of enrollments) {
+    const { enrollment_date_time, studentnumber } = enrollment
+    const rights = await studyrightsOf(studentnumber)
+    if (!hasValidStudyright(rights, enrollment_date_time)) {
+      avoin_enrolled.push(studentnumber)
+    }
+  }
+
+  avoin_credited = uniq(avoin_credited)
+  avoin_enrolled = uniq(avoin_enrolled)
+
+  return {
+    credits,
+    enrollments,
+    avoin_credited,
+    avoin_enrolled,
+  }
+}
+
+const unifiedCourseInfo = async course_code => {
+  const a = await courseInfo(course_code)
+  const b = await courseInfo('AY' + course_code)
+
+  avoin_credited = uniq(a.avoin_credited.concat(b.avoin_credited))
+  avoin_enrolled = uniq(a.avoin_enrolled.concat(b.avoin_enrolled))
+
+  return {
+    credits: uniq(a.credits.concat(b.credits)),
+    enrollments: uniq(a.enrollments.concat(b.enrollments)),
+    avoin_credited,
+    avoin_enrolled,
+    only_enrolled: avoin_enrolled.filter(e => !avoin_credited.includes(e)),
+  }
+}
+
+const statsOf = async courses => {
+  console.log(Object.keys(courses).join('\n'))
+  console.log()
+  let students = []
+  for (let course_code of Object.keys(courses)) {
+    const { credits, enrollments, avoin_enrolled, avoin_credited, only_enrolled } = await unifiedCourseInfo(course_code)
+    courses[course_code] = { credits, enrollments, avoin_enrolled, avoin_credited, only_enrolled }
+    students = uniq(students.concat(avoin_enrolled).concat(avoin_credited))
+  }
+
+  const rows = []
+
+  for (let s of students) {
+    let string = s + ' '
+    let score = 0
+    for (let course_code of Object.keys(courses)) {
+      const { enrollments, avoin_credited, only_enrolled } = courses[course_code]
+      let course_info = '-'
+      if (avoin_credited.includes(s)) {
+        course_info = 'X'
+        score += 10
+      }
+
+      if (only_enrolled.includes(s)) {
+        const studentsEnrolments = enrollments.filter(e => e.studentnumber === s)
+        course_info = '' + studentsEnrolments.length
+
+        score += 1
+      }
+
+      string += course_info + ' '
+    }
+
+    const rights = await studyrightsOf(s)
+    const stRight = hasValidStudyright(rights, new Date('2022-8-1')) ? 'yes' : 'no'
+
+    string += stRight
+
+    rows.push({ string, score })
+  }
+
+  const byScore = (a, b) => b.score - a.score
+
+  console.log(
+    rows
+      .sort(byScore)
+      .map(r => r.string)
+      .join('\n')
+  )
+}
+
+const main = async () => {
+  const courses_psyk_a = {
+    'PSYK-111': null,
+    'PSYK-121': null,
+    'PSYK-131': null,
+    'PSYK-141': null,
+    'PSYK-181': null,
+  }
+
+  const courses_psyk_p = {
+    'PSYK-251': null,
+    'PSYK-221': null,
+    'PSYK-232': null,
+    'PSYK-226': null,
+    'PSYK-241': null,
+    'PSYK-212': null,
+  }
+
+  const courses_hist = {
+    'HISK-110': null,
+    'HISK-120': null,
+    'HISK-131A': null,
+    'HISK-131': null,
+    'HISK-141': null,
+    'HISK-142': null,
+  }
+
+  await statsOf(courses_psyk_a)
+  console.log()
+  console.log()
+  await statsOf(courses_psyk_p)
+  console.log()
+  console.log()
+  await statsOf(courses_hist)
+
+  process.exit()
+}
+
+main()

services/backend/src/util/jami.js

@@ -0,0 +1,79 @@
+const axios = require('axios')
+const Sentry = require('@sentry/node')
+
+const { importerToken, jamiUrl } = require('../conf-backend')
+const logger = require('./logger')
+
+const jamiClient = axios.create({
+  baseURL: jamiUrl,
+  params: {
+    token: importerToken,
+  },
+})
+
+const getUserIamAccess = async (user, attempt = 1) => {
+  if (user.iamGroups.length === 0) return {}
+
+  const { id, iamGroups } = user
+
+  try {
+    const { data: iamAccess } = await jamiClient.post('/', {
+      userId: id,
+      iamGroups,
+    })
+
+    const { specialGroup } = iamAccess
+    delete iamAccess.specialGroup
+
+    return { iamAccess, specialGroup }
+  } catch (error) {
+    if (attempt > 3) {
+      logger.error('[Jami] error: ', error)
+      Sentry.captureException(error)
+
+      return {}
+    }
+
+    return getUserIamAccess(user, attempt + 1)
+  }
+}
+
+const getUserIams = async userId => {
+  try {
+    const { data } = await jamiClient.get(`/${userId}`)
+
+    return data.iamGroups
+  } catch (error) {
+    if (error.response.status !== 404) {
+      logger.error('[Jami] error: ', error)
+      Sentry.captureException(error)
+    }
+
+    return []
+  }
+}
+
+const getAllUserAccess = async () => {
+  const { data } = await jamiClient.get('/all-access')
+
+  return data
+}
+
+const testJami = async () => {
+  try {
+    await jamiClient.get('/ping', { timeout: 4000 })
+    logger.info('JAMI connected')
+  } catch (error) {
+    logger.error(error)
+    logger.warn('JAMI not responding :(')
+    logger.info('Are you sure you are using the latest JAMI image?')
+  }
+}
+
+testJami()
+
+module.exports = {
+  getUserIamAccess,
+  getUserIams,
+  getAllUserAccess,
+}

services/backend/src/util/organizationAccess.js

@@ -1,92 +1,27 @@
 /* eslint-disable no-unused-vars */
 const _ = require('lodash')
-const Organization = require('../models/organization')
-const { getIAMRights } = require('./IAMRights')
 
-const isNumber = value => !Number.isNaN(parseInt(value, 10))
+const { getUserIamAccess } = require('./jami')
 
-const normalizeOrganizationCode = r => {
-  if (!r.includes('_')) {
-    return r
+const getAccessFromIAMs = async user => {
+  if (user.iamGroups.length === 0) return {}
+
+  const { iamAccess, specialGroup } = await getUserIamAccess(user)
+
+  const access = {}
+  if (_.isObject(iamAccess)) {
+    Object.keys(iamAccess).forEach(code => {
+      access[code] = iamAccess[code]
+    })
   }
 
-  const [left, right] = r.split('_')
-  const prefix = [...left].filter(isNumber).join('')
-  const suffix = `${left[0]}${right}`
-  const providercode = `${prefix}0-${suffix}`
-  return providercode
-}
-
-const RELEVANT_ORGANIZATION_CODES = [
-  'H906', // Kielikeskus
-  'H930', // Avoin yliopisto
-]
-
-const ORGANIZATION_ACCESS_BY_IAM_GROUP = {
-  'grp-kielikeskus-esihenkilot': {
-    // Kielikeskus
-    H906: {
-      read: true,
-      write: true,
-      admin: true,
-    },
-  },
-  'grp-avoin-johto': {
-    // Avoin yliopisto
-    H930: {
-      read: true,
-    },
-  },
-}
-
-const getOrganizationAccessFromIamGroups = user => {
-  const access = {}
-
-  const iamAccess = (user.iamGroups ?? []).reduce(
-    (access, group) => _.merge(access, ORGANIZATION_ACCESS_BY_IAM_GROUP[group] ?? {}),
-    {}
-  )
-
-  Object.keys(iamAccess).forEach(code => {
-    access[code] = iamAccess[code]
-  })
-
-  return access
-}
-
-const organizationIsRelevant = organization => {
-  const { code } = organization
-
-  return code.includes('-') || RELEVANT_ORGANIZATION_CODES.includes(code)
-}
-
-const getAccessToAll = async accessLevel => {
-  const access = {}
-  const allOrganizations = await Organization.findAll({ attributes: ['code'] })
-  allOrganizations.filter(organizationIsRelevant).forEach(({ code }) => {
-    access[code] = accessLevel
-  })
-  return access
-}
-
-const getLomakeAccess = async user => {
-  if (!user.iamGroups) user.iamGroups = user.iam_groups
-
-  const access = {}
-  const { access: iamAccess } = getIAMRights(user.iamGroups)
-
-  if (!_.isObject(iamAccess)) return access
-  Object.keys(iamAccess).forEach(code => {
-    access[code] = iamAccess[code]
-  })
-  return access
+  return { access, specialGroup }
 }
 
 const getOrganizationAccess = async user => {
-  const access = {
-    ...(await getLomakeAccess(user)),
-    ...getOrganizationAccessFromIamGroups(user),
-  }
+  if (!user.iamGroups) user.iamGroups = user.iam_groups
+
+  const access = await getAccessFromIAMs(user)
 
   return access
 }

services/backend/src/util/semester.js

@@ -8,7 +8,7 @@ const getSemester = date => (isSpring(date) ? 'SPRING' : 'FALL')
 
 const semesterStart = {
   SPRING: '01-01',
-  FALL: '07-31',
+  FALL: '08-01',
 }
 
 const semesterEnd = {

services/frontend/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "name": "frontend",
       "dependencies": {
-        "@reduxjs/toolkit": "^1.8.5",
+        "@reduxjs/toolkit": "^1.8.6",
         "@sentry/browser": "^6.18.2",
         "@sentry/tracing": "^6.18.2",
         "axios": "^0.21.4",
@@ -36,7 +36,7 @@
         "react-scripts": "4.0.3",
         "react-scroll-to-component": "^1.0.2",
         "scheduler": "^0.23.0",
-        "semantic-ui-css": "^2.4.1",
+        "semantic-ui-css": "^2.5.0",
         "semantic-ui-react": "^2.1.2",
         "unfuck-spa-shibboleth-session": "^3.1.4",
         "use-context-selector": "^1.3.9",
@@ -2841,9 +2841,9 @@
       }
     },
     "node_modules/@reduxjs/toolkit": {
-      "version": "1.8.5",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.5.tgz",
-      "integrity": "sha512-f4D5EXO7A7Xq35T0zRbWq5kJQyXzzscnHKmjnu2+37B3rwHU6mX9PYlbfXdnxcY6P/7zfmjhgan0Z+yuOfeBmA==",
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.6.tgz",
+      "integrity": "sha512-4Ia/Loc6WLmdSOzi7k5ff7dLK8CgG2b8aqpLsCAJhazAzGdp//YBUSaj0ceW6a3kDBDNRrq5CRwyCS0wBiL1ig==",
       "dependencies": {
         "immer": "^9.0.7",
         "redux": "^4.1.2",
@@ -17914,9 +17914,9 @@
       }
     },
     "node_modules/semantic-ui-css": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.4.1.tgz",
-      "integrity": "sha512-Pkp0p9oWOxlH0kODx7qFpIRYpK1T4WJOO4lNnpNPOoWKCrYsfHqYSKgk5fHfQtnWnsAKy7nLJMW02bgDWWFZFg==",
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.5.0.tgz",
+      "integrity": "sha512-jIWn3WXXE2uSaWCcB+gVJVRG3masIKtTMNEP2X8Aw909H2rHpXGneYOxzO3hT8TpyvB5/dEEo9mBFCitGwoj1A==",
       "dependencies": {
         "jquery": "x.*"
       }
@@ -24283,9 +24283,9 @@
       "integrity": "sha512-9X2obfABZuDVLCgPK9aX0a/x4jaOEweTTWE2+9sr0Qqqevj2Uv5XorvusThmc9XGYpS9yI+fhh8RTafBtGposw=="
     },
     "@reduxjs/toolkit": {
-      "version": "1.8.5",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.5.tgz",
-      "integrity": "sha512-f4D5EXO7A7Xq35T0zRbWq5kJQyXzzscnHKmjnu2+37B3rwHU6mX9PYlbfXdnxcY6P/7zfmjhgan0Z+yuOfeBmA==",
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.6.tgz",
+      "integrity": "sha512-4Ia/Loc6WLmdSOzi7k5ff7dLK8CgG2b8aqpLsCAJhazAzGdp//YBUSaj0ceW6a3kDBDNRrq5CRwyCS0wBiL1ig==",
       "requires": {
         "immer": "^9.0.7",
         "redux": "^4.1.2",
@@ -35833,9 +35833,9 @@
       }
     },
     "semantic-ui-css": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.4.1.tgz",
-      "integrity": "sha512-Pkp0p9oWOxlH0kODx7qFpIRYpK1T4WJOO4lNnpNPOoWKCrYsfHqYSKgk5fHfQtnWnsAKy7nLJMW02bgDWWFZFg==",
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.5.0.tgz",
+      "integrity": "sha512-jIWn3WXXE2uSaWCcB+gVJVRG3masIKtTMNEP2X8Aw909H2rHpXGneYOxzO3hT8TpyvB5/dEEo9mBFCitGwoj1A==",
       "requires": {
         "jquery": "x.*"
       }

services/frontend/package.json

@@ -6,7 +6,7 @@
     "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
   },
   "dependencies": {
-    "@reduxjs/toolkit": "^1.8.5",
+    "@reduxjs/toolkit": "^1.8.6",
     "@sentry/browser": "^6.18.2",
     "@sentry/tracing": "^6.18.2",
     "axios": "^0.21.4",
@@ -36,7 +36,7 @@
     "react-scripts": "4.0.3",
     "react-scroll-to-component": "^1.0.2",
     "scheduler": "^0.23.0",
-    "semantic-ui-css": "^2.4.1",
+    "semantic-ui-css": "^2.5.0",
     "semantic-ui-react": "^2.1.2",
     "unfuck-spa-shibboleth-session": "^3.1.4",
     "use-context-selector": "^1.3.9",

services/frontend/src/common/InfoToolTips/faculty.js

@@ -34,9 +34,79 @@ export default {
   **Non-degree student credits**: Sisältää tiedekuntaan suoritetut opintopisteet, joiden suorittajilla ei ole tutkinto-oikeutta Helsingin yliopistossa.
   `,
   GraduatedOfTheFaculty: `Sisältää kyseisenä vuonna tiedekunnasta valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.`,
-  ThesisWritersOfTheFaculty: `Sisältää kyseisenä vuonna kandidaatintutkielman, pro gradu -tutkielman, väitöskirjan tai lisensiaatintyön hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.`,
+  ThesisWritersOfTheFaculty: `Sisältää kyseisenä vuonna kandidaatintutkielman tai pro gradu -tutkielman suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.`,
   StudentToggle: `
   All studyrights: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat ja ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.\n
   Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
   `,
+
+  // Graduation times
+  GroupByToggle: `Graduation year: Valmistuneet opiskelijat on ryhmitelty opintojen valmistumisvuoden perusteella.\n
+    Group by: Starting year: Valmistuneet opiskelijat on jaoteltu opinto-oikeuden alkuvuoden mukaan.
+  `,
+  AverageGraduationTimes: `
+  Opiskelijoiden keskimääräiset valmistumisajat tutkintotasoittain. 
+
+  - Bachelor: Kandidaatin tutkinnon suoritusajat
+  - Bachelor + Master: Kandi+maisteriopinto-oikeudella opiskelleiden ja siitä maisteriksi valmistuneiden suoritusajat
+  - Master: Pelkällä maisterintutkinnon opinto-oikeudella opiskelleiden suoritusajat
+  - Doctor: Tohtorin tutkinnon suoritusajat
+  - Licentiate: Lisensiaatin tutkinnon suoritusajat
+
+  Valmistumisajoista on **vähennetty lakisääteiset poissaolot**. 
+
+  Palkissa oleva lukumäärä kertoo **montako opiskelijaa** kuhunkin valmistuneiden ryhmään kuuluu.
+
+  Vihreä **vaakapalkki** kuvaa tavoiteajassa (tai alle) valmistuneista, keltainen palkki korkeintaan vuodella tavoiteajan ylittäneitä
+  ja punainen yli vuodella tavoiteajan ylittäneitä.
+
+  Säädöt:
+
+  **Breakdown**: Näyttää, kuinka moni opiskelija valmistui ajallaan, korkeintaan vuoden myöhässä tavoiteajasta tai tätä myöhemmin.
+
+  **Median time**: Näyttää kyseisenä vuonna/lukuvuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina.
+
+  **Graduation year**: Ryhmittelee tulokset valmistumisvuosittain. Näyttää siis vuonna xxxx valmistuneiden opiskelijoiden keskimääräiset valmistumisajat. 
+  Luvut ovat lopullisia (kuluva vuosi poislukien), sillä edeltäville vuosille ei tule uusia valmistumisia.
+
+  **Starting year**. Ryhmittelee tulokset opiskelijoiden opinto-oikeuden alkamisvuosittain. Näyttää siis vuonna xxxx aloittaneiden opiskelijoiden keskimääräiset valmistumisajat. Luvut voivat elää, koska osa vuonna xxxx 
+  aloittaneiden opiskelijoiden opinnoista voi olla edelleen kesken.
+
+  Mediaaninäkymässä vihreä **katkoviiva** kuvastaa tavoiteaikaa. Keltainen katkoviiva on tavoiteaika + yksi vuosi.\n
+
+  Tarkastellessa valmistumisaikoja aloitusvuosittain palkissa näkyy myös **kuinka suuri osa** kyseisenä vuonna tiedekuntaan/opinto-ohjelmaan opinto-oikeuden saaneista on valmistunut.
+ `,
+
+  // Faculty Student population
+  GraduatedToggle: `
+  Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
+
+  Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
+  `,
+  StudentsStatsOfTheFaculty: `
+  Kuvaa tiedekunnan ohjelmissa opiskelevien opiskelijoden tilastoja. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  
+  HUOM!
+  - **All** Sisältää opiskelijat, jotka siirtyneet ohjelmaan tai siitä ulos sekä aloittaneet ohjelmassa
+  - **Currently enrolled** Sisältää kaikki tällä hetkellä läsnäolevat olevat opiskelija.
+  - **Absent** Sisältää kaikki lakisääteisistä syistä poissaolevat opiskelijat.
+  - **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  - **Inactive** Sisältää kaikki opinto-oikeudet, jotka ovat alkaneet kyseisenä vuonna, mutta jotka ovat sittemmin vanhentuneet ilman, että opiskelija on valmistunut ohjelmasta. Sisältää myös opinto-oikeudet,
+  joissa opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle.
+  
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, ohjelmaan ja siitä pois siirtyneet opiskelijat sekä erillisopinto-oikeudella opiskelevat.
+  Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. `,
+  StudentProgress: `
+  Kuvaa tiedekuntaan kuuluvien eri ohjemien opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2021-2022 aloittaneiden opintopistejakaumaa.
+  
+  - Bachelor: Kandidaatin opinto-oikeudella varustettujen opiskelijoiden eteneminen
+  - Bachelor + Master: Kandi+maisteriopinto-oikeudella opiskelleiden ja siitä maisteriksi ohjelmiin siirtyneiden etenenminen. Aloitusvuodeksi on merkitty kandiohjelman aloitusvuosi. 
+    Opinto-ohjelmaksi on merkitty maisteriohjelma, johon opiskelija on siirtynyt kandiksi valmistumisen jälkeen.
+  - Master: Pelkällä maisterintutkinnon opinto-oikeudella varustettujen opiskelijoiden eteneminen 
+  - Doctor: Tohtorin opinto-oikeudella varustettujen opiskelijoiden eteneminen
+  
+  HUOM! Mukana ovat myös muuhun kuin valittuun koulutusohjelmaan tai opintosuuntaan tehdyt opintopisteet. Opintopisteet on laskettu opinto-oikeuden alkamisesta, eikä tällä hetkellä ota huomioon mahdollisia hyväksilukuja.
+
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, erillisopinto-oikeudella opiskelevat ja ohjelmaan siirtyneet.   
+  `,
 }

services/frontend/src/common/InfoToolTips/filters.js

@@ -1,8 +1,8 @@
 export default {
   enrollmentStatus: {
-    label: 'Filter students present or absent',
+    label: 'Filter students present, absent or passive',
     short:
-      'Suodata opiskelijoita sen mukaan ovatko he ilmoittautuneet läsnä- tai poissaoleviksi kyseiselle lukukaudelle',
+      'Suodata opiskelijoita sen mukaan ovatko he joko ilmoittautuneet läsnä- tai poissaoleviksi tai jääneet passiivisiksi kyseiselle lukukaudelle. Huom. osa passiivisista opiskelijoista on jo valmistunut ko. ohjelmasta.',
   },
   studyrightStatus: {
     label: 'Rajaa opiskelijoita opinto-oikeuden statuksen mukaan',
@@ -18,9 +18,9 @@ export default {
     label: null,
     short: 'Rajaa kurssisuorituksia tietyn ajanjakson mukaan. Tämä rajaus ei muuta opiskelijoiden määrää',
   },
-  cherrypicker: {
+  studentNumber: {
     label: null,
     short:
-      'Rajaa yksittäisiä opiskelijoita pois tilastoista (blocklist) tai näytä vain määritellyt opiskelijat (allowlist). Vain täsmälliset opiskelijanumerot toimivat.',
+      '(ent. cherry-picker) Rajaa yksittäisiä opiskelijoita pois tilastoista (blocklist) tai näytä vain määritellyt opiskelijat (allowlist). Vain täsmälliset opiskelijanumerot toimivat.',
   },
 }

services/frontend/src/common/InfoToolTips/populationStatistics.js

@@ -1,7 +1,8 @@
 export default {
-  Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella ilmoittautuneita, haluttu ajanjakso valitaan tästä.
+  Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella aloittaneita, haluttu ajanjakso valitaan tästä.
 
-  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
+  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois ohjelmasta poissiirtyneet opiskelijat. Heidät voi sisällyttää mukaan lisäämällä ruksin ja noutamalla tulokset uudestaan (Fetch class with new settings).`,
+  // Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.
   QueryCard: `**Valitun koulutusohjelmavuosikurssin perustiedot.**
   
   **Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
@@ -16,6 +17,7 @@ export default {
   Opiskelijoiden kumulatiiviset opintopistekertymät opiskelijakohtaisina kuvaajina.
   Kun hiiren vie kuvaajien päälle, saa näkyviin jokaista kuvaajaa vastaavan opiskelijanumeron.
   Opiskelijanumeron yhteydessä olevaa ympyrää klikkaamalla siirtyy ”student statistics” puolelle ja näkee valitun opiskelijan opintotiedot.
+  Timantin muotoinen ikoni kertoo opiskelijan valmistuneen valitusta koulutusohjelmasta kyseisellä hetkellä.
 
   X-akselin skaalaa voi säätää harmaista painikkeista ”1m”, ”3m”, ”6m”, ”YTD”, ”1y” ja ”ALL” tai kuvaajan alla olevan sinisen näkymän harmaita ”kahvoja” siirtämällä.
   Y-akselin skaalaa voi säätää harmaista painikkeista ”small” ”medium” ja "large”`,
@@ -48,27 +50,47 @@ export default {
 
   **Cumulative**-valinnan ollessa käytössä kunkin opiskelijan kaikki aiemmat opintopisteet lasketaan mukaan opintopistemäärään. Muutoin vain pylvästä vastaavalla aikavälillä kirjatut opintopisteet otetaan huomioon.
   `,
-  CoursesOfPopulation: `**Courses of population**
+  CoursesOfClass: `**Courses of class**
 
   Tämä taulukko sisältää näkyvissä olevien opiskelijoiden kaikki kurssisuoritukset.
   
   Opiskelijat on mahdollista suodattaa kurssin perusteella käyttämällä kurssirivin vasemmassa laidassa olevaa suodatinpainiketta.
   Kurssisuodatin aukeaa ikkunan vasempaan laitaan ja tarjoaa lisää vaihtoehtoja suodatukseen. Voit myös valita monta kurssia kerralla suodattimeen.
   
-  Klikkaamalla kurssin nimen vieressä olevaa sinistä nuolta, siirryt ko. kurssin *Course Statistics* -näkymään.
+  Klikkaamalla kurssin nimen vieressä olevaa sinistä nuolta, siirryt ko. kurssin **Course Statistics** -näkymään.
   
   Taulukon tietoja voi suodattaa antamalla opiskelijoiden vähimmäismäärän kurssilla kenttään *Limit to courses where student number at least*.
-  Lisäksi kurssin nimi- ja koodisarakkeissa on hakukentät *("filter here")* suodattamista varten.
-  Suodattimet saa tyhjennettyä painamalla kentän vieressä olevaa ruksia.
+  Lisäksi kurssin nimi **name** ja koodisarakkeissa **code** suodatinkuvaketta  klikkaamalla tulee esiin hakukentät suodattamista varten.
+  Suodattimet saa tyhjennettyä painamalla kentän vieressä olevaa ruksia tai valitsemall valikon alaosasta **Reset column filter**.
   
   **Huom! Nämä suodattimet vaikuttavat vain tähän taulukkoon, eivät koko populatioon.**
   
   **Taulukot**
-  
-  - **pass/fail** - oletusnäkymä, joka sisältää kurssisuoritukset, hylätyt sekä yritykset. Taulukon tiedot voi järjestää sarakkeita (n, after retry, percentage, many times, per student, passed ja attempted) klikkaamalla.
-  - **pass/fail w/ enrollments** - näkymä, joka sisältää suoritustiedot ilmoittautumisten kanssa. Huom.! ilmoittautumistiedot ovat saatavilla vain Sisun käyttöönoton jälkeiseltä ajalta.
+  - **pass/fail** - näkymä sisältää kurssisuoritukset, hylätyt, yritykset sekä ilmoittautumistiedot. Huom.! ilmoittautumistiedot ovat saatavilla vain Sisun käyttöönoton jälkeiseltä ajalta.
+    - **Total students** - sarake näyttää kaikkien opiskelijoiden määrän, mukaanlukien ilmoittautuneet ilman arvosanaa.
+    - **Enrolled, no grade** - sarake näyttää niiden opiskelijoiden määrän, joilla on ilmoittautuminen muttei arvosanaa eikä hylättyä suoritusta.
   - **grades** - arvosanajakauma.
   - **when passed** - kurssisuorituksen ajankohta. Huom.! *Passed*-sarakkeen kokonaissumma ei välttämättä täsmää puolivuosittaisten lukumäärien summan kanssa johtuen tuplakirjauksista Oodissa.
+  - **students** - opiskelijan arvosanat kursseittain.
+  `,
+  CoursesOfPopulation: `**Courses of Population**
+
+  Tämä taulukko sisältää näkyvissä olevien opiskelijoiden kaikki kurssisuoritukset.
+  
+  Opiskelijat on mahdollista suodattaa kurssin perusteella käyttämällä kurssirivin vasemmassa laidassa olevaa suodatinpainiketta.
+  Kurssisuodatin aukeaa ikkunan vasempaan laitaan ja tarjoaa lisää vaihtoehtoja suodatukseen. Voit myös valita monta kurssia kerralla suodattimeen.
+  
+  Klikkaamalla kurssin nimen vieressä olevaa sinistä nuolta, siirryt ko. kurssin **Course Statistics** -näkymään.
+  
+  Taulukon tietoja voi suodattaa antamalla opiskelijoiden vähimmäismäärän kurssilla kenttään *Limit to courses where student number at least*.
+  Lisäksi kurssin nimi **name** ja koodisarakkeissa **code** suodatinkuvaketta  klikkaamalla tulee esiin hakukentät suodattamista varten.
+  Suodattimet saa tyhjennettyä painamalla kentän vieressä olevaa ruksia tai valitsemall valikon alaosasta **Reset column filter**.
+  
+  **Huom! Nämä suodattimet vaikuttavat vain tähän taulukkoon, eivät koko populatioon.**
+  
+  **Taulukot**
+  - **pass/fail** - näkymä sisältää kurssisuoritukset, hylätyt, yritykset sekä ilmoittautumistiedot. Huom.! ilmoittautumistiedot ovat saatavilla vain Sisun käyttöönoton jälkeiseltä ajalta.
+  - **grades** - arvosanajakauma.
   `,
   Search: `Tässä osiossa voi tarkastella koulutusohjelmakohtaisia populaatioita sisäänotto-lukuvuoden mukaisesti,
   vuosikurssi kerrallaan. Uudet koulutusohjelmat ovat pääsääntöisesti alkaneet 1.8.2017,
@@ -82,31 +104,74 @@ export default {
   Tägien ja opintosuuntien valinnat ovat siirtyneet tästä omiksi filtereikseen.
   Voit tehdä valinnat siirryttyäsi tarkastelemaan ohjelman tilastoja.
   `,
-  Students: `**Students**
+  StudentsGuidanceGroups: `**Students**
 
   Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
   
+  Klikkaamalla opiskelijanumeron vieressä olevaa sinistä henkilösymbolia, siirryt kyseisen opiskelijan "student statistic" -näkymään.
+  
+  Klikkaamalla "Sisu"-nuolta, siirryt tarkastelemaan opiskelijan tietoja Sisu-järjestelmässä.
+
   **General**
   
-  Klikkaamalla opiskelijanumeron vieressä olevaa sinistä nuolta, siirtyy "student statistic" näkymään valitun opiskelijan osalta.
+  Valikoituja selvennyksiä: 
+  - **Credits**
+    - **All**  listaa opiskelijan suoritetut opintopisteet valitun tarkastelujakson  ajalta.
+    - **Since ... ** listaa opiskelijan suoritetut opintopisteet määritellystä ajankohdasta alkaen
+  - **Start of studyright**: Opiskelijan valittuun ohjelmaan liittyvän opinto-oikeuden alkupäivä
+  - **Started in programme**: Opiskelijan valitussa ohjelmassa aloituspäivämäärä. Kandi+maisteriopiskelijoille kandiksi valmistumispäivämäärä +1.
   
-  - **credits since start of studyright:**  listaa opiskelijan suoritetut opintopisteet valitun tarkastelujakson / populaation ajalta.
-  - **all credits:** listaa opiskelijan kaikki opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
-  - **transferred from:** opiskelijan vanha koulutusohjelma, josta opiskelija on siirtynyt uuteen.
-  - **studytracks:** opintosuunta
+  `,
+  StudentsClass: `**Students**
+
+  Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
   
-  **Mandatory Courses**
+  Klikkaamalla opiskelijanumeron vieressä olevaa sinistä henkilösymbolia, siirryt kyseisen opiskelijan "student statistic" -näkymään.
+  
+  Klikkaamalla "Sisu"-nuolta, siirryt tarkastelemaan opiskelijan tietoja Sisu-järjestelmässä.
+
+  **General**
+  
+  Valikoituja selvennyksiä: 
+  - **Credits**
+    - **All**  listaa opiskelijan suoritetut opintopisteet valitun tarkastelujakson  ajalta
+    - **HOPS** listaa opiskelijan kaikki HOPSiin sijoitut opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
+    - **Since start in programme** listaa opiskelijan ohjelmassa aloittamisen jälkeen suoritut opintopisteet
+  - **Transferred from:** opiskelijan vanha koulutusohjelma, josta opiskelija on siirtynyt uuteen.
+  - **Start of studyright**: Opiskelijan valittuun ohjelmaan liittyvän opinto-oikeuden alkupäivä
+  - **Started in programme**: Opiskelijan ko. ohjelmassa aloituspäivämäärä. Kandi+maisteriopiskelijoille kandiksi valmistumispäivämäärä +1
+  - **Semesters**: Läsnäololukukausien lukumäärä
+  
+  **Courses**
   
   Tällä välilehdellä näkyy, mitä pakollisiksi määriteltyjä kursseja opiskelija on suorittanut.
-  Kurssit voi määrittää kohdassa "Study program" > "Overview" > haluttu koulutusohjelma. Scrollatessa kurssitaulukkoa alaspäin, kurssin nimen saa näkyviin viemällä kursorin halutun solun kohdalle.
+
   
   **Tags**
   
   Opiskelijoille voi luoda tag:eja ryhmittelyä varten kohdassa "Study Program" > "Overview" > haluttu koulutusohjelma.
+
+  **Progress**
+
+  Tämä näkymä on tällä hetkellä käytössä vain kandivaiheen opinto-ohjelmissa.
+  Tällä välilehdellä näkyy opintojen kehitys annettujen kriteerien mukaan akateemista vuotta kohti. Kurssi merkataan tehdyksi, jos sillä on suoritusmerkintä tai hyväksiluku 
+  minä tahansa lukuvuonna. Opintopistekriteerin täyttämiseen vaaditaan, että opintopisteet on suoritettu kyseisen akateemisen vuoden aikana.
+
+
+  **Huomaa**, että klikkaamalla taulukon yläreunassa olevaa kolme pistettä sisältävää kuvaketta, voit ladata itsellesi Excel tiedoston. Tiedosto sisältää lisäksi muita sarakkeita, kuten läsnäolon lukukausittain sekä yhteystiedot.
+  Yhteystietojen käyttö on kuitenkin sallittua vain opintoneuvontaan liittyvissä asiossa. 
   
-  **Download**
+  `,
+  StudentsCustom: `**Students**
+
+  Taulukko näyttää oletusarvoisesti vain opiskelijanumerot, mutta harmaata liukukytkintä klikkaamalla saa näkyviin opiskelijoiden nimet.
   
-  Klikkaamalla tätä harmaata kuvaketta, saa ladattua .xlsx tiedoston, joka sisältää kaikki "General" ja "Mandatory Courses" välilehtien tiedot.
+  Klikkaamalla opiskelijanumeron vieressä olevaa sinistä henkilösymbolia, siirryt kyseisen opiskelijan "student statistic" -näkymään.
+  
+  Klikkaamalla "Sisu"-nuolta, siirryt tarkastelemaan opiskelijan tietoja Sisu-järjestelmässä.
+  
+  **All credits** listaa opiskelijan kaikki opintopisteet (myös aiemmat suoritukset, eli ne jotka tehty ennen valittuun tarkastelujaksoon ja populaatioon kuulumista)
+
   `,
   Filters: {
     Add: `Harmaata ”add” painiketta klikkaamalla aukeaa filters- eli suodatinvalikko, 

services/frontend/src/common/InfoToolTips/studyprogramme.js

@@ -34,6 +34,21 @@ export default {
   esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat. Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
   
   **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet.
+  **Non-degree students**: Opiskelijat, joilla ei ole tutkinto-oikeutta Helsingin yliopistossa.
+    
+  **Transferred credits**: Sisältää opintopisteet, jotka ovat suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet
+  `,
+  StudentsOfProgrammeCourses: `
+  Sisältää kurssien opiskelijat.
+
+  **Not completed**: Sisältää opiskelijat, jotka ovat ilmoittautuneet kurssille, mutta eivät ole suorittaneet sitä ja opiskelijat, jotka ovat saaneet hylätyn arvosanan.
+
+  **Major students**: Sisältää opiskelijat, joilla on ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan.
+
+  **Non-major students**: Sisältää kurssin opiskelijat, joilla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan. Sisältää
+  esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat.
+
+  **Transferred students**: Sisältää opiskelijat, jotka ovat suorittaneet kyseisenä vuonna hyväksiluettuja opintopisteitä tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet
   `,
   GraduatedAndThesisWritersOfTheProgramme: `
   **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman tai pro gradu-tutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
@@ -41,11 +56,30 @@ export default {
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
   `,
   AverageGraduationTimes: `
-  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
+  Yksittäinen palkki kertoo, kuinka moni opiskelija on valmistunut kyseisenä vuonna/lukuvuonna.
 
-  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
+  Vihreä **vaakapalkki** kuvaa tavoiteajassa (tai alle) valmistuneita, keltainen palkki korkeintaan vuoden tavoiteajasta myöhässä valmistuneita
+  ja punainen yli vuoden tavoiteajan ylittäneitä.
+
+  **Breakdown**: Näyttää, kuinka moni kyseisenä kalenteri- tai lukuvuotena valmistuneista opiskelijoista valmistui ajallaan, korkeintaan vuoden myöhässä tavoiteajasta tai tätä myöhemmin.
+
+  **Median time**: Näyttää kyseisenä kalenteri- tai lukuvuotena valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina.
+  Vihreä **katkoviiva** kuvastaa tavoiteaikaa. Keltainen katkoviiva on tavoiteaika + yksi vuosi.\n
+
+  Valmistumisajoista on vähennetty lakisääteiset poissaolot.`,
+  AverageGraduationTimesStudytracks: `
+  Yksittäinen palkki kertoo, kuinka moni kyseisenä **lukuvuotena ALOITTANEISTA** opiskelijoista on valmistunut.
+
+  Vihreä **vaakapalkki** kuvaa tavoiteajassa (tai alle) valmistuneita, keltainen palkki korkeintaan vuoden tavoiteajasta myöhässä valmistuneita
+  ja punainen yli vuoden tavoiteajan ylittäneitä.
+
+  **Breakdown**: Näyttää, kuinka moni kyseisenä lukuvuotena aloittaneista, jo valmistuneista, opiskelijoista on valmistui ajallaan, korkeintaan vuoden myöhässä tavoiteajasta tai tätä myöhemmin.
+
+  **Median time**: Näyttää valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina sekä heidän prosentuaalisen osuutensa koko vuosikurssista.
+  Vihreä **katkoviiva** kuvastaa tavoiteaikaa. Keltainen katkoviiva on tavoiteaika + yksi vuosi.
   
-  n = **valmistuneet opiskelijat / kaikki ohjelmassa kyseisenä vuonna alkaneet opinto-oikeudet**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
+  
+  Valmistumisajoista on vähennetty lakisääteiset poissaolot.`,
   ProgrammesBeforeOrAfter: `
   **Mikäli valittuna on kandiohjelma:** Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
 
@@ -77,13 +111,10 @@ export default {
 
   Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.    
   `,
-  NonDegree: `
-  Opiskelijat, joilla ei ole tutkinto-oikeutta Helsingin yliopistossa.
+  Name: `
+  Pitkät kurssinimet on lyhennetty. Näet ne kokonaan laittamalla hiiren nimen yläpuolelle.
   `,
-  TransferredCourses: `
-  Sisältää opiskelijat, jotka ovat suorittaneet kyseisenä vuonna hyväksiluettuja opintopisteitä tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet
-  `,
-  TransferredCredits: `
-  Sisältää opintopisteet, jotka ovat suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet
+  NotCompleted: `
+  Sisältää opiskelijat, jotka ovat ilmoittautuneet kurssille, mutta eivät ole suorittaneet sitä ja opiskelijat, jotka ovat saaneet hylätyn arvosanan.
   `,
 }

services/frontend/src/common/index.js

@@ -284,6 +284,8 @@ export const getTargetCreditsForProgramme = code => {
   if (code === 'MH90_001') return 180
   if (code.includes('MH')) return 120
   return 180
+  // There are also doctoral degrees of 40, but probably not necessary
+  // Those codes begin with 'LIS' or 'T'
 }
 
 const getMonthsForDegree = code => getTargetCreditsForProgramme(code) / (60 / 12)
@@ -352,7 +354,7 @@ export const resolveStudyPlan = (studyPlans, studyRight) => {
       .filter(e => e.element_detail.type === 20)
       .sort((a, b) => new Date(b.startdate) - new Date(a.startdate))[0] || {}
   if (!code) return null
-  return studyPlans.find(p => p.programme_code === code)
+  return studyPlans.find(p => p.programme_code === code && p.studyrightid === studyRight.studyrightid)
 }
 
 // These are the new Bachelor's programmes in Matlu, that have BH possibility

services/frontend/src/components/CompletedCoursesSearch/CompletedCoursesSearch.jsx

@@ -0,0 +1,235 @@
+import React, { useState, useEffect } from 'react'
+import { Modal, Form, Button, TextArea, Loader } from 'semantic-ui-react'
+import qs from 'query-string'
+import {
+  useGetSavedCourseListsQuery,
+  useCreateCourseListMutation,
+  useUpdateCourseListMutation,
+  useDeleteCourseListMutation,
+} from 'redux/completedCoursesSearch'
+import SearchHistory from 'components/SearchHistory'
+
+const CompletedCoursesSearch = ({ setValues, history, location }) => {
+  const [modal, setModal] = useState(false)
+  const [courseInput, setCourseInput] = useState('')
+  const [studentInput, setStudentInput] = useState('')
+  const [name, setName] = useState('')
+  const courseLists = useGetSavedCourseListsQuery()
+  const [searchList, setSearches] = useState(null)
+  const [selectedSearchId, setSelectedSearchId] = useState('')
+  const isFetchingOrLoading = courseLists.isLoading || courseLists.isFetching
+  const isError = courseLists.isError || (courseLists.isSuccess && !courseLists.data)
+  const [updateCourseList, { isLoading: updateIsLoading, data: updatedData }] = useUpdateCourseListMutation()
+  const [createCourseList, { isLoading: createIsLoading, data: createdData }] = useCreateCourseListMutation()
+  const [deleteCourseList, { isLoading: deleteIsLoading, data: deletedData }] = useDeleteCourseListMutation()
+
+  const parseQueryFromUrl = () => {
+    const query = qs.parse(location.search)
+    if (!Array.isArray(query.courseList)) query.courseList = [query.courseList]
+    if (!Array.isArray(query.studentList)) query.studentList = [query.studentList]
+    return query
+  }
+
+  useEffect(() => {
+    if (courseLists.isSuccess && courseLists.data) {
+      setSearches(courseLists.data)
+    }
+  }, [courseLists])
+
+  useEffect(() => {
+    if (updatedData) {
+      const updatedSearches = searchList.map(s => (s.id === updatedData.id ? updatedData : s))
+      setSearches(updatedSearches)
+    }
+  }, [updatedData])
+
+  useEffect(() => {
+    if (createdData && !createdData.error) {
+      const newList = searchList.concat(createdData)
+      setSearches(newList)
+    }
+  }, [createdData])
+
+  useEffect(() => {
+    if (deletedData) {
+      const filteredsearches = searchList.filter(s => s.id !== deletedData)
+      setSearches(filteredsearches)
+    }
+  }, [deletedData])
+
+  useEffect(() => {
+    setImmediate(() => {
+      if (location.search) {
+        const query = parseQueryFromUrl()
+        setValues(query)
+      }
+    })
+  }, [location.search])
+
+  const clearForm = () => {
+    setCourseInput('')
+    setStudentInput('')
+    setSelectedSearchId('')
+  }
+
+  const pushQueryToUrl = query => {
+    setImmediate(() => {
+      const searchString = qs.stringify(query)
+      history.push({ search: searchString })
+    })
+  }
+
+  const handleClose = () => {
+    setModal(false)
+    clearForm()
+  }
+
+  const onDelete = () => {
+    deleteCourseList({ id: selectedSearchId })
+    clearForm()
+  }
+
+  const onSave = () => {
+    const courseList = courseInput
+      .split(/[\s,]+/)
+      .map(code => code.trim().toUpperCase())
+      .filter(code => code !== '')
+    if (selectedSearchId !== '') {
+      updateCourseList({ id: selectedSearchId, courseList })
+    } else {
+      createCourseList({ courseList, name })
+    }
+  }
+  const onSelectSearch = selectedId => {
+    if (!selectedId) {
+      clearForm()
+      return
+    }
+    const { id: selectedSearchId } = selectedId
+    const selectedSearch = searchList?.find(search => search.id === selectedSearchId)
+    if (selectedSearch) {
+      setCourseInput(selectedSearch.courseList.join(', '))
+      setName(selectedSearch.name)
+      setSelectedSearchId(selectedSearch.id)
+    }
+  }
+
+  const onClicker = e => {
+    e.preventDefault()
+
+    const courseList = courseInput
+      .split(/[\s,]+/)
+      .map(code => code.trim().toUpperCase())
+      .filter(c => c !== '')
+    const studentList = studentInput
+      .split(/[\s,]+/)
+      .map(code => code.trim())
+      .filter(s => s !== '')
+      .map(s => (s.length === 8 ? `0${s}` : s))
+
+    if (courseList.length === 0 || studentList.length === 0 || studentList[0] === null) {
+      return
+    }
+
+    setValues({
+      studentList,
+      courseList,
+    })
+
+    pushQueryToUrl({ courseList, studentList })
+
+    handleClose()
+  }
+
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+  if (isFetchingOrLoading) return <Loader active style={{ marginTop: '15em' }} />
+
+  return (
+    <Modal
+      trigger={
+        <Button size="small" color="blue" onClick={() => setModal(true)} data-cy="open-completed-courses-modal-button">
+          Search completed courses of students
+        </Button>
+      }
+      open={modal}
+      onClose={handleClose}
+      size="small"
+    >
+      <Modal.Content>
+        <Form>
+          <h2>Search completed courses of students</h2>
+          <Form.Field>
+            <em>Insert one or more student numbers, separated by a space, a newline, or a comma.</em>
+            <TextArea
+              value={studentInput}
+              placeholder="012345678, 12345678"
+              onChange={e => setStudentInput(e.target.value)}
+              data-cy="student-no-input"
+            />
+          </Form.Field>
+          <Form.Field>
+            <em>Insert one or more courses, separated by a space, a newline, or a comma.</em>
+            <TextArea
+              value={courseInput}
+              placeholder="TKT12345, PSYK-123"
+              onChange={e => setCourseInput(e.target.value)}
+              data-cy="course-list-input"
+            />
+          </Form.Field>
+          <Form.Field>
+            <Form.Field data-cy="search-name">
+              <em> Insert name for this course list if you wish to save it </em>
+              <Form.Input
+                disabled={selectedSearchId !== ''}
+                value={name}
+                placeholder="name"
+                onChange={e => setName(e.target.value)}
+              />
+            </Form.Field>
+          </Form.Field>
+          <SearchHistory
+            header="Saved courselists"
+            items={
+              searchList
+                ? searchList.map(list => ({
+                    ...list,
+                    text: list.name,
+                    timestamp: list.updatedAt,
+                    params: { id: list.id },
+                  }))
+                : []
+            }
+            updateItem={() => null}
+            handleSearch={onSelectSearch}
+          />
+          <Modal.Actions>
+            <Button
+              data-cy="save-courselist"
+              disabled={!name || updateIsLoading || createIsLoading}
+              loading={updateIsLoading || createIsLoading}
+              floated="left"
+              icon="save"
+              onClick={onSave}
+              content="Save"
+            />
+            <Button
+              disabled={!selectedSearchId || deleteIsLoading}
+              negative
+              floated="left"
+              icon="trash"
+              onClick={onDelete}
+              content="Delete"
+            />
+
+            <Button onClick={handleClose}>Cancel</Button>
+            <Button positive onClick={e => onClicker(e)} data-cy="completed-courses-search-button">
+              Search
+            </Button>
+          </Modal.Actions>
+        </Form>
+      </Modal.Content>
+    </Modal>
+  )
+}
+
+export default CompletedCoursesSearch

services/frontend/src/components/CompletedCoursesSearch/CompletedCoursesSearchResults.jsx

@@ -0,0 +1,252 @@
+import SortableTable from 'components/SortableTable'
+import React, { useEffect, useState } from 'react'
+import { useGetCompletedCoursesQuery } from 'redux/completedCoursesSearch'
+import { Icon, Loader } from 'semantic-ui-react'
+import moment from 'moment'
+import StudentNameVisibilityToggle, { useStudentNameVisibility } from 'components/StudentNameVisibilityToggle'
+import useLanguage from 'components/LanguagePicker/useLanguage'
+import RightsNotification from 'components/RightsNotification'
+
+const getColumns = (courses, showStudentNames, getTextIn) => {
+  const isPassed = credit => [4, 7, 9].includes(credit)
+  // 4=completed, 7=improved, 9=transferred, 10=failed
+
+  const getCompletion = (student, courseCode, { icon }) => {
+    const completion = student.credits.find(c => c.courseCode === courseCode && isPassed(c.creditType))
+    const enrollment = student.enrollments.find(e => e.courseCode === courseCode)
+    if (completion === undefined) {
+      if (!enrollment) {
+        return icon ? null : ''
+      }
+      if (icon) {
+        if (moment(enrollment.date) > moment().subtract(6, 'months')) {
+          return <Icon fitted name="minus" color="yellow" />
+        }
+        return <Icon fitted name="minus" color="grey" />
+      }
+
+      return `Latest enrollment: ${moment(enrollment.date).format('YYYY-MM-DD')}`
+    }
+
+    const substitutionString = completion.substitution ? ` as ${completion.substitution}` : ''
+
+    return icon ? <Icon fitted name="check" color="green" /> : `Passed${substitutionString}`
+  }
+
+  const getTotalPassed = s => s.credits.filter(c => isPassed(c.creditType)).length
+
+  const getTotalUnfinished = student => student.enrollments.length
+
+  const style = {
+    verticalAlign: 'middle',
+    textAlign: 'center',
+  }
+
+  const studentNbrColumn = [
+    {
+      key: 'studentnumber',
+      title: 'Student Number',
+      cellProps: { style },
+      getRowVal: s => s.studentNumber,
+      getRowContent: s => s.studentNumber,
+    },
+  ]
+
+  const getCellProps = (student, courseCode) => {
+    const creditDate = student.credits.find(credit => credit.courseCode === courseCode)?.date
+    const enrollmentDate = student.enrollments.find(enrollment => enrollment.courseCode === courseCode)?.date
+    if (!creditDate && !enrollmentDate) {
+      return { style }
+    }
+    const title = creditDate
+      ? `Passed on ${moment(creditDate).format('YYYY-MM-DD')} `
+      : `Last enrollment on ${moment(enrollmentDate).format('YYYY-MM-DD')} `
+    return { style, title }
+  }
+
+  const completionStatusColumns = courses.map(course => {
+    return {
+      key: course.code,
+      title: (
+        <div key={course.code} style={{ marginLeft: '-6px', width: '8em', whiteSpace: 'normal' }}>
+          <div key={`key1${course.code}`}>{course.code}</div>
+          <div key={`key2${course.code}`} style={{ color: 'gray', fontWeight: 'normal' }}>
+            {getTextIn(course.name)}
+          </div>
+        </div>
+      ),
+      textTitle: course.code,
+      cellProps: s => getCellProps(s, course.code),
+      headerProps: { title: course.code },
+      getRowVal: s => getCompletion(s, course.code, { icon: false }),
+      getRowContent: s => getCompletion(s, course.code, { icon: true }),
+    }
+  })
+
+  const statisticColumns = [
+    {
+      key: 'passed',
+      title: 'Passed',
+      cellProps: { style },
+      headerProps: { title: 'Passed' },
+      getRowVal: s => getTotalPassed(s),
+      getRowContent: s => getTotalPassed(s),
+      child: true,
+    },
+    {
+      key: 'unfinished',
+      title: 'Unfinished',
+      headerProps: { title: 'Unfinished' },
+      getRowVal: s => getTotalUnfinished(s),
+      getRowContent: s => getTotalUnfinished(s),
+      cellProps: { style },
+      child: true,
+    },
+  ]
+
+  const statisticHeader = {
+    key: 'statistics',
+    title: <b>Totals:</b>,
+    textTitle: null,
+    parent: true,
+    children: statisticColumns,
+  }
+
+  const infoColumnsForCsv = [
+    {
+      key: 'hidden-lastname',
+      title: 'Last name',
+      forceToolsMode: 'none',
+      getRowVal: s => s.lastName,
+      headerProps: { style: { display: 'none' } },
+      cellProps: { style: { display: 'none' } },
+      export: true,
+    },
+    {
+      key: 'hidden-firstnames',
+      title: 'First names',
+      getRowVal: s => s.firstNames,
+      forceToolsMode: 'none',
+      headerProps: { style: { display: 'none' } },
+      cellProps: { style: { display: 'none' } },
+      export: true,
+    },
+    {
+      key: 'hidden-email',
+      title: 'E-mail',
+      getRowVal: s => (s.email ? s.email : ''),
+      forceToolsMode: 'none',
+      headerProps: { style: { display: 'none' } },
+      cellProps: { style: { display: 'none' } },
+      export: true,
+    },
+  ]
+
+  const emailColumn = showStudentNames
+    ? [
+        {
+          key: 'email',
+          title: 'Email',
+          cellProps: { style },
+          headerProps: { title: 'Email' },
+          getRowVal: s => (s.email ? s.email : ''),
+          getRowContent: s => (s.email ? s.email : ''),
+          child: true,
+          export: false,
+        },
+      ]
+    : []
+
+  const nameColumns = showStudentNames
+    ? [
+        {
+          key: 'lastname',
+          title: 'Last name',
+          cellProps: { style },
+          getRowVal: s => s.lastName,
+          getRowContent: s => s.lastName,
+          export: false,
+        },
+        {
+          key: 'firstnames',
+          title: 'First names',
+          cellProps: { style },
+          getRowVal: s => s.firstNames,
+          getRowContent: s => s.firstNames,
+          export: false,
+        },
+      ]
+    : []
+
+  const hiddenParentColumn = {
+    key: 'hiddenFiles',
+    title: '',
+    mergeHeader: true,
+    textTitle: null,
+    parent: true,
+    children: infoColumnsForCsv,
+  }
+
+  return [
+    hiddenParentColumn,
+    ...nameColumns,
+    ...studentNbrColumn,
+    ...completionStatusColumns,
+    statisticHeader,
+    ...emailColumn,
+  ]
+}
+
+const CompletedCoursesSearchResults = ({ searchValues }) => {
+  const { courseList, studentList } = searchValues
+  const [data, setData] = useState(null)
+  const showStudentNames = useStudentNameVisibility()
+  const completedCoursesTable = useGetCompletedCoursesQuery({ courseList, studentList })
+  const isFetchingOrLoading = completedCoursesTable.isLoading || completedCoursesTable.isFetching
+  const isError = completedCoursesTable.isError || (completedCoursesTable.isSuccess && !completedCoursesTable.data)
+  const { getTextIn } = useLanguage()
+
+  useEffect(() => {
+    if (!isError && !isFetchingOrLoading) {
+      setImmediate(() => {
+        const data = completedCoursesTable?.data
+        setData(data)
+      })
+    }
+  }, [completedCoursesTable])
+
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+  if (isFetchingOrLoading || data === null) return <Loader active style={{ marginTop: '15em' }} />
+
+  return (
+    <div>
+      <StudentNameVisibilityToggle />
+      {data.discardedStudentNumbers?.length > 0 && (
+        <RightsNotification discardedStudentNumbers={data.discardedStudentNumbers} />
+      )}
+      <div
+        style={{ maxWidth: '100vh', overflowX: 'auto', paddingBottom: '50px' }}
+        data-cy="completed-courses-table-div"
+      >
+        <SortableTable
+          title="Completed courses search"
+          getRowKey={s => s.studentNumber}
+          tableProps={{
+            celled: true,
+            compact: 'very',
+            padded: false,
+            collapsing: true,
+            basic: true,
+            striped: true,
+            singleLine: true,
+            textAlign: 'center',
+          }}
+          columns={getColumns(data.courses, showStudentNames.visible, getTextIn)}
+          data={data.students}
+        />
+      </div>
+    </div>
+  )
+}
+
+export default CompletedCoursesSearchResults

services/frontend/src/components/CompletedCoursesSearch/index.jsx

@@ -0,0 +1,56 @@
+import React, { useState, useEffect } from 'react'
+import { withRouter, useLocation, useHistory } from 'react-router-dom'
+import { Message, Icon } from 'semantic-ui-react'
+import useLanguage from 'components/LanguagePicker/useLanguage'
+import { useTitle } from '../../common/hooks'
+import CompletedCoursesSearch from './CompletedCoursesSearch'
+import CompletedCoursesSearchResults from './CompletedCoursesSearchResults'
+import TSA from '../../common/tsa'
+
+const CompletedCourses = () => {
+  useTitle('Search completed courses')
+  const { language } = useLanguage()
+  const [searchValues, setValues] = useState({})
+  const location = useLocation()
+  const history = useHistory()
+
+  useEffect(() => {
+    if (searchValues && searchValues.courseList?.length > 0) {
+      TSA.Influx.sendEvent({
+        group: 'Completed Courses Tool Usage',
+        name: 'completed courses tool',
+        label: 'completedCoursesTool',
+        value: 1,
+      })
+    }
+  }, [searchValues])
+  return (
+    <div className="segmentContainer">
+      <Message style={{ maxWidth: '800px', fontSize: '16px' }}>
+        <Message.Header>Search completed courses</Message.Header>
+        <p>
+          Here you can search by a list of student and course numbers to see whether students have completed certain
+          courses yet. The tool will also show if students have enrolled on the course, if they have not yet completed
+          it. Course substitutions are taken into account.
+        </p>
+        <p>
+          <Icon fitted name="check" color="green" />: Student has completed the course with a passing grade. <br />
+          <Icon fitted name="minus" color="yellow" />: Student has not completed the course, but has an active
+          enrollment from less than 6 months ago. <br />
+          <Icon fitted name="minus" color="grey" />: Student has not completed the course, but has an enrollment from
+          more than 6 months ago. <br />
+          <b>Empty cell</b>: Student has no completion or enrollment for the course.
+        </p>
+      </Message>
+      <CompletedCoursesSearch setValues={setValues} history={history} location={location} />
+
+      <div style={{ paddingTop: '25px' }}>
+        {searchValues && searchValues.courseList?.length > 0 && searchValues.studentList?.length > 0 && (
+          <CompletedCoursesSearchResults searchValues={searchValues} language={language} />
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default withRouter(CompletedCourses)

services/frontend/src/components/CoolDataScience/ProtoC.jsx

@@ -628,7 +628,7 @@ const ProtoC = ({
         </Menu.Item>
         {sorterNames.map(sorterName => (
           <Menu.Item
-            basic={sorter !== sorterName}
+            basic={(sorter !== sorterName).toString()}
             color={sorter === sorterName ? 'blue' : 'black'}
             key={sorterName}
             active={sorter === sorterName}
@@ -727,7 +727,7 @@ const ProtoC = ({
       </Segment>
       <RenderBelowGraph />
       <Message>
-        <ReactMarkdown children={CoolDataScience.protoC} escapeHtml={false} />
+        <ReactMarkdown children={CoolDataScience.protoC} />
       </Message>
     </Segment>
   )

services/frontend/src/components/CoolDataScience/ProtoC2.jsx

@@ -1,3 +1,4 @@
+/* eslint-disable babel/no-invalid-this */
 import React, { useState, useCallback, useMemo } from 'react'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
@@ -449,7 +450,7 @@ const ProtoC = ({ programme = 'KH50_005' }) => {
             </Menu.Item>
             {sorterNames.map(sorterName => (
               <Menu.Item
-                basic={sorter !== sorterName}
+                basic={(sorter !== sorterName).toString()}
                 color={sorter === sorterName ? 'blue' : 'black'}
                 key={sorterName}
                 active={sorter === sorterName}
@@ -485,7 +486,7 @@ const ProtoC = ({ programme = 'KH50_005' }) => {
           <Message>
             {
               // eslint-disable-next-line react/no-children-prop
-              <ReactMarkdown children={CoolDataScience.protoC2} escapeHtml={false} />
+              <ReactMarkdown children={CoolDataScience.protoC2} />
             }
           </Message>
         </Segment>
@@ -541,7 +542,7 @@ const ProtoC = ({ programme = 'KH50_005' }) => {
           </Menu.Item>
           {sorterNames.map(sorterName => (
             <Menu.Item
-              basic={sorter !== sorterName}
+              basic={(sorter !== sorterName).toString()}
               color={sorter === sorterName ? 'blue' : 'black'}
               key={sorterName}
               active={sorter === sorterName}
@@ -594,7 +595,7 @@ const ProtoC = ({ programme = 'KH50_005' }) => {
         <Message>
           {
             // eslint-disable-next-line react/no-children-prop
-            <ReactMarkdown children={CoolDataScience.protoC2} escapeHtml={false} />
+            <ReactMarkdown children={CoolDataScience.protoC2} />
           }
         </Message>
       </Segment>

services/frontend/src/components/CoolDataScience/ProtoG.jsx

@@ -100,7 +100,7 @@ const Chart = React.memo(({ tickDates, snapshots }) => (
             return `${d.getDate()}.${d.getMonth()}.${d
               .getFullYear()
               .toString()
-              .substr(-2)}`
+              .substring(-2)}`
           }
         },
         tickPositions: tickDates.map(date => date.getTime()),
@@ -292,7 +292,7 @@ const ProtoG = ({ uberdata, years, isLoading, loadingYears, getUberDispatch, get
         )}
         <Message>
           {//eslint-disable-next-line react/no-children-prop
-            <ReactMarkdown children={CoolDataScience.protoC} escapeHtml={false} />
+            <ReactMarkdown children={CoolDataScience.protoC} />
           }
         </Message>
       </Segment>

services/frontend/src/components/CoolDataScience/Toolbar.jsx

@@ -185,7 +185,7 @@ export default ({ value, changeSetting, settings, generalHelp }) => {
         <div style={{ padding: '1em' }}>
           {
             // eslint-disable-next-line react/no-children-prop
-            <ReactMarkdown children={unindent(generalHelp)} escapeHtml={false} />
+            <ReactMarkdown children={unindent(generalHelp)} />
           }
         </div>
         {settings.map(({ key, label, long }) => (
@@ -196,7 +196,7 @@ export default ({ value, changeSetting, settings, generalHelp }) => {
                 Valinta "<i>{label}</i>"
               </b>
               <div style={{ margin: '0.5em', fontSize: '0.9em' }}>
-                <ReactMarkdown escapeHtml={false}>{unindent(long ?? '')}</ReactMarkdown>
+                <ReactMarkdown>{unindent(long ?? '')}</ReactMarkdown>
               </div>
             </div>
           </div>

services/frontend/src/components/CoursePopulation/index.jsx

@@ -31,7 +31,7 @@ import {
   startYearAtUniFilter,
   programmeFilter,
   studyTrackFilter,
-  cherrypickFilter,
+  studentNumberFilter,
 } from '../FilterView/filters'
 import FilterView from '../FilterView'
 import useLanguage from '../LanguagePicker/useLanguage'
@@ -242,7 +242,7 @@ const CoursePopulation = ({
       content: {
         content: (
           <div ref={programmeRef}>
-            <CustomPopulationCoursesWrapper filteredStudents={filtered} showFilter={false} />
+            <CustomPopulationCoursesWrapper filteredStudents={filtered} showFilter />
           </div>
         ),
       },
@@ -319,7 +319,7 @@ const CoursePopulation = ({
       name="CoursePopulation"
       filters={[
         genderFilter,
-        cherrypickFilter,
+        studentNumberFilter,
         ageFilter,
         courseFilter({ courses: courseStatistics?.coursestatistics }),
         creditsEarnedFilter,
@@ -372,7 +372,12 @@ const CustomPopulationCoursesWrapper = props => {
 
   if (isLoading) return null
 
-  return <CustomPopulationCourses {...props} courses={courseStatistics} />
+  return (
+    <>
+      <InfoBox content={infotooltips.PopulationStatistics.CoursesOfPopulation} />
+      <CustomPopulationCourses {...props} courses={courseStatistics} />
+    </>
+  )
 }
 
 const mapStateToProps = ({ singleCourseStats, populations, courseSearch }) => {

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/student.jsx

@@ -71,7 +71,7 @@ const getColumns = (stats, showDetails, showGrades, userHasAccessToAllStats, alt
     },
     {
       key: 'TOTAL',
-      title: 'Total Students',
+      title: 'Total\nStudents',
       helpText: 'Total count of students, including enrolled students with no grade.',
       cellProps: s => ({
         style: {
@@ -113,7 +113,7 @@ const getColumns = (stats, showDetails, showGrades, userHasAccessToAllStats, alt
     ...getGradeColumns(resolveGrades(stats)),
     {
       key: 'ENROLLMENTS_MISSING_GRADE',
-      title: 'Enrolled no grade',
+      title: 'Enrolled,\nno grade',
       filterType: 'range',
       helpText: 'Total count of students with a valid enrollment and no passing or failing grade.',
       getRowVal: s => (s.rowObfuscated ? 5 : s.students.enrolledStudentsWithNoGrade),

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/util.js

@@ -107,11 +107,11 @@ export const getThesisGradeSpread = (series, isRelative) => {
     3: [],
     4: [],
     5: [],
+    'Hyv.': [],
   }
   const newSeries = series.reduce(
     (acc, cur, i) => {
       const currentEntries = Object.entries(cur)
-
       currentEntries.forEach(([k, v]) => {
         const merged = k === 'LA' ? 'LUB' : k
         acc[merged].push(v)