[Open uni] Add small modifications

Enrollments were retrieved by enrollment date instead of semester code
which corresponds to when the course is acually organized.

.github/workflows/release-oodikone.yaml

@@ -21,7 +21,7 @@ jobs:
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
       - name: 'Build and push'
         uses: docker/build-push-action@v3
         with:
@@ -50,13 +50,13 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: 'Get current time'
         id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
       - name: 'Extract release version'
         # Github context (e.g. github.ref) only has ref formatted as /refs/tags/vX.X.X
         # This steps extracts release version from GITHUB_REF env var
         # See: https://github.community/t/how-to-get-just-the-tag-name/16241/10
         id: extract_version
-        run: echo ::set-output name=VERSION::"${GITHUB_REF#refs/tags/}"
+        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
       - name: 'Build and push'
         uses: docker/build-push-action@v3
         with:

.github/workflows/test-oodikone.yaml

@@ -83,7 +83,7 @@ jobs:
       - uses: docker/setup-buildx-action@v2
       - name: 'Get current time'
         id: get_current_time
-        run: echo ::set-output name=NOW::"$(date --iso-8601=seconds)"
+        run: echo "NOW=$(date --iso-8601=seconds)" >> $GITHUB_OUTPUT
       - name: 'Build image and export it'
         uses: docker/build-push-action@v3
         with:

cypress/e2e/Faculty_statistics.js

@@ -13,58 +13,211 @@ describe('Faculty overview', () => {
     })
   })
 
-  describe('Faculty basic information', () => {
+  describe('Faculty basic information: admin user', () => {
     beforeEach(() => {
       cy.init('/faculties', 'admin')
       cy.contains('td', 'H90').click()
     })
     it('Credits produced by faculty are shown', () => {
-      cy.get(['data-cy=Section-CreditsProducedByTheFaculty'])
-      cy.get(['data-cy=Graph-CreditsProducedByTheFaculty'])
+      cy.get('[data-cy="Section-CreditsProducedByTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Graph-CreditsProducedByTheFaculty"]').should('be.visible')
+    })
+
+    it('Update statistics tab is shown', () => {
+      cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Update statistics')
     })
   })
 
-  describe('Study programme information', () => {
+  describe('Faculty basic information: basic user', () => {
     beforeEach(() => {
-      cy.init('/faculties', 'admin')
+      cy.init('/faculties')
+      cy.contains('td', 'H80').click()
+    })
+    it('Basic information tab show all graphs and tables', () => {
+      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Graph-CreditsProducedByTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Table-ThesisWritersOfTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Graph-ThesisWritersOfTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Table-StudentsOfTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Graph-StudentsOfTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Table-GraduatedOfTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Graph-GraduatedOfTheFaculty"]').should('be.visible')
+    })
+
+    it('Correct tabs are shown', () => {
+      cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Basic information')
+      cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Graduation times')
+      cy.get('[data-cy="FacultySegmentContainer"]').should('contain', 'Programmes and student populations')
+      cy.get('[data-cy="FacultySegmentContainer"]').should('not.contain', 'Update statistics')
+    })
+
+    it('Toggle years works', () => {
+      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('contain', '2022')
+      cy.get('[data-cy="Table-ThesisWritersOfTheFaculty"]').should('contain', '2022')
+      cy.get('[data-cy="Table-StudentsOfTheFaculty"]').should('contain', '2022')
+      cy.get('[data-cy="Table-GraduatedOfTheFaculty"]').should('contain', '2022')
+      cy.get('[data-cy="YearToggle"]').click()
+      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('contain', '2022 - 2023')
+      cy.get('[data-cy="Table-ThesisWritersOfTheFaculty"]').should('contain', '2022 - 2023')
+      cy.get('[data-cy="Table-StudentsOfTheFaculty"]').should('contain', '2022 - 2023')
+      cy.get('[data-cy="Table-GraduatedOfTheFaculty"]').should('contain', '2022 - 2023')
+    })
+
+    it('Toggle programmes works', () => {
+      cy.get('[data-cy="FacultyProgrammesShownInfo"]').should('not.exist')
+      cy.get('[data-cy="ProgrammeToggle"]').click()
+      cy.get('[data-cy="FacultyProgrammesShownInfo"]').should('be.visible')
+    })
+
+    it('Students of the faculty infobox works', () => {
+      cy.get('[data-cy="StudentsOfTheFaculty-info-content"]').should('not.exist')
+      cy.get('[data-cy="StudentsOfTheFaculty-open-info"]').click()
+      cy.get('[data-cy="StudentsOfTheFaculty-info-content"]').should('be.visible')
+      cy.get('[data-cy="StudentsOfTheFaculty-close-info"]').click()
+      cy.get('[data-cy="StudentsOfTheFaculty-info-content"]').should('not.exist')
+    })
+  })
+
+  describe('Study programme information: users with access', () => {
+    beforeEach(() => {
+      cy.init('/faculties')
       cy.contains('td', 'H80').click()
     })
 
     it('Study programme credit information is not visible in the beginning', () => {
       cy.get('table[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
 
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
     })
 
     it('Study programme credit information can be toggled', () => {
-      cy.get('table[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
+      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
 
-      cy.get('div[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
-      cy.get('div[data-cy="Button-Show-CreditsProducedByTheFaculty-3"]').click()
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
-      cy.get('div[data-cy="Button-Hide-CreditsProducedByTheFaculty-0"]').click()
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
-      cy.get('td[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+      // I tried to fix this without cy.wait() method without effort. Sorting the programmes
+      // or something else causes rerendering of the view and yield to failing tests.
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
+      cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+      cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-3"]').click()
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+      cy.get('[data-cy="Button-Hide-CreditsProducedByTheFaculty-0"]').click()
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-1"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-2"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-3"]').should('be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-4"]').should('not.be.visible')
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-5"]').should('not.be.visible')
+    })
+
+    it('Graph stays open when sorted', () => {
+      cy.get('[data-cy="Table-CreditsProducedByTheFaculty"]').should('be.visible')
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
+      cy.get('[data-cy="Button-Show-CreditsProducedByTheFaculty-0"]').click()
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
+      cy.get('[data-cy="Menu-CreditsProducedByTheFaculty-Total"]').click()
+      cy.get('[data-cy="Cell-CreditsProducedByTheFaculty-0"]').should('be.visible')
+    })
+  })
+
+  describe('Average graduation times', () => {
+    beforeEach(() => {
+      cy.init('/faculties')
+      cy.contains('td', 'H60').click()
+      cy.contains('Graduation times').click()
+    })
+
+    it('User can view graduation graphs', () => {
+      cy.get('[data-cy="Section-AverageGraduationTimes"]').should('be.visible')
+      cy.get('[data-cy="Section-bachelor"]').should('be.visible')
+      cy.get('[data-cy="Section-master"]').should('be.visible')
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.contains('Graduation year').should('be.visible')
+        cy.contains('.message', "Click a bar to view that year's programme level breakdown").should('be.visible')
+      })
+    })
+
+    it('Graphs display data', () => {
+      cy.get('[data-cy="Section-bachelor"]').within(() => {
+        cy.get('div[class="faculty-graph"]')
+        cy.contains('1 graduated').should('have.length', 1)
+        cy.contains('1 graduated').trigger('mouseover')
+        cy.contains('1 students graduated in year 2019')
+        cy.contains('median study time: 40 months')
+        cy.contains('0 graduated on time')
+        cy.contains('1 graduated max year overtime')
+
+        cy.contains('1 graduated').click()
+        cy.contains('Year 2019 by graduation year')
+        cy.get('div[class="programmes-graph"]').should('be.visible')
+        cy.get('div[class="programmes-graph"]').within(() => {
+          cy.contains('EDUK')
+          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('Kasvatustieteiden kandiohjelma')
+          cy.contains('KH60_001')
+        })
+      })
+
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.get('div[class="faculty-graph"]')
+        cy.contains('1 graduated').should('have.length', 1)
+        cy.contains('1 graduated').trigger('mouseover')
+        cy.contains('1 students graduated in year 2020')
+        cy.contains('median study time: 25 months')
+        cy.contains('0 graduated over year late')
+        cy.contains('1 graduated max year overtime')
+
+        cy.contains('1 graduated').click()
+        cy.contains('Year 2020 by graduation year')
+        cy.get('div[class="programmes-graph"]').should('be.visible')
+        cy.get('div[class="programmes-graph"]').within(() => {
+          cy.contains('EDUM')
+          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('Kasvatustieteiden maisteriohjelma')
+          cy.contains('MH60_001')
+        })
+      })
+    })
+
+    it('Graduation times grouping and time types can be toggled', () => {
+      cy.get('[data-cy="GroupByToggle"]').click()
+      cy.get('[data-cy="Section-bachelor"]').should('not.exist')
+      cy.get('[data-cy="Section-master"]').should('be.visible')
+
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.get('div[class="faculty-graph"]')
+        cy.contains('1 graduated (100 % of class)').should('have.length', 1)
+        cy.contains('1 graduated').trigger('mouseover')
+        cy.contains('From class of 2018, 1/undefined students have graduated')
+      })
+
+      cy.get('[data-cy="GraduationTimeToggle"]').click()
+      cy.get('[data-cy="Section-master"]').within(() => {
+        cy.contains('1 graduated (100 % of class)').click()
+        cy.get('div[class="programmes-graph"]').should('be.visible')
+        cy.get('div[class="programmes-graph"]').within(() => {
+          cy.contains('Year 2018 by start year')
+          cy.contains('1 graduated').trigger('mouseover')
+          cy.contains('From class of 2018, 1/1 students have graduated')
+          cy.contains('mean study time: 25 months')
+        })
+      })
     })
   })
 })

cypress/e2e/Filters.js

@@ -87,21 +87,6 @@ describe('Population Statistics', () => {
     })
   })
 
-  it('Programme defaults to "Active Study Right" mode', () => {
-    runTestStepWithPreAndPostParts('Programme', () => {
-      cy.cs('Programme-filter-mode-selector').children().eq(0).should('contain', 'Active Study Right')
-    })
-  })
-
-  it('Programme filter works', () => {
-    runTestStepWithPreAndPostParts('Programme', () => {
-      const card = cy.cs('Programme-filter-card')
-      const programmeDropdown = card.cs('Programme-filter-dropdown').selectFromDropdown(0)
-      checkFilteringResult(5)
-      programmeDropdown.get('i.delete').click()
-    })
-  })
-
   // The test does not work since it tries to filter by "Ravitsemustiede"-studytrack when checking
   // CS Bachelor students. Now that the irrelevant studytracks have been removed, the test fails.
   // So fix this

cypress/e2e/Population_statistics.js

@@ -1,4 +1,5 @@
 /// <reference types="Cypress" />
+// Now "Class statistics" in UI
 
 const moment = require('moment')
 const _ = require('lodash')
@@ -24,8 +25,8 @@ describe('Population Statistics tests', () => {
     })
 
     it('Population statistics search form is usable', () => {
-      cy.contains('See population').should('be.disabled')
-      cy.contains('Search for population')
+      cy.contains('See class').should('be.disabled')
+      cy.contains('Search for class')
       cy.contains('Class of')
         .parent()
         .within(() => {
@@ -47,7 +48,7 @@ describe('Population Statistics tests', () => {
         .children()
         .contains('Tietojenkäsittelytieteen maisteriohjelma')
         .click()
-      cy.contains('See population').should('be.enabled')
+      cy.contains('See class').should('be.enabled')
     })
 
     it('Searching for population really shows population', () => {
@@ -57,8 +58,8 @@ describe('Population Statistics tests', () => {
         .children()
         .contains('Tietojenkäsittelytieteen kandiohjelma')
         .click()
-      cy.contains('See population').click()
-      cy.contains('Population statistics')
+      cy.contains('See class').click()
+      cy.contains('class size')
       cy.contains('Tietojenkäsittelytieteen kandiohjelma')
     })
 
@@ -68,16 +69,17 @@ describe('Population Statistics tests', () => {
       // This advanced settings does not work correctly
       // see https://github.com/UniversityOfHelsinkiCS/oodikone/issues/3552
       // setPopStatsUntil('toukokuu 2020')
-
-      cy.get('.card').within(() => {
+      cy.get('.header').within(() => {
         cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 170 students')
+        cy.contains('class size 170 students')
+      })
+      cy.get('.card').within(() => {
         cy.contains('Excludes exchange students')
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')
       })
       cy.cs('filtered-students')
-      cy.contains('Courses of population').click()
+      cy.contains('Courses of class').click()
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
@@ -106,9 +108,10 @@ describe('Population Statistics tests', () => {
       cy.contains('Statistics until')
       // only spring
       cy.cs('toggle-fall').click()
-      cy.contains('Fetch population').click()
+      cy.contains('Fetch class').click()
 
       // moar waiting hack
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
       cy.wait(10000)
 
       cy.contains('No statistics found for the given query.')
@@ -118,14 +121,14 @@ describe('Population Statistics tests', () => {
       cy.cs('toggle-fall').click()
       cy.cs('toggle-spring').click()
 
-      cy.contains('Fetch population').click()
+      cy.contains('Fetch class').click()
 
       cy.contains('Credit accumulation (for 170 students)')
 
       // spring + fall
       cy.get('[data-cy=advanced-toggle]').click()
       cy.cs('toggle-spring').click()
-      cy.contains('Fetch population').click()
+      cy.contains('Fetch class').click()
 
       cy.contains('Credit accumulation (for 170 students)')
     })
@@ -134,10 +137,9 @@ describe('Population Statistics tests', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics').click()
       cy.contains('Credits Gained').click()
-      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the population')
-
-      const months = Math.ceil(moment.duration(moment().diff(moment('2017-08-1'))).asMonths())
-
+      cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the class')
+      const today = moment().endOf('month')
+      const months = Math.round(moment.duration(moment(today).diff(moment('2017-08-01'))).asMonths())
       const limits = [1, ..._.range(1, 4).map(p => Math.ceil(months * ((p * 15) / 12))), null]
       const ranges = _.range(1, limits.length).map(i => _.slice(limits, i - 1, i + 1))
 
@@ -185,7 +187,7 @@ describe('Population Statistics tests', () => {
       cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
 
       cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 170 Students')
-      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.75')
+      cy.get("[data-cy='credit-stats-mean']").should('contain', '114.74')
       cy.get("[data-cy='credit-stats-stdev']").should('contain', '65.53')
       cy.get("[data-cy='credit-stats-min']").should('contain', '5')
       cy.get("[data-cy='credit-stats-max']").should('contain', '314')
@@ -226,9 +228,10 @@ describe('Population Statistics tests', () => {
     it('Population statistics is visible', () => {
       cy.reload()
 
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+      cy.contains('class size 170 students')
+
       cy.get('.card').within(() => {
-        cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-        cy.contains('Sample size: 170 students')
         cy.contains('Excludes exchange students')
         cy.contains('Excludes students with non-degree study right')
         cy.contains('Excludes students who have transferred out of this programme')

cypress/e2e/Student_statistics.js

@@ -93,8 +93,8 @@ describe('Student Statistics tests', () => {
     cy.get('.prompt').type(student.studentnumber)
     cy.contains('td a', student.studentnumber).click()
     cy.get('i.level.up.alternate.icon').eq(0).click()
-    cy.contains('Population statistics')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma')
+    cy.contains('class size')
   })
 })
 

cypress/e2e/Studyprogramme_overview.js

@@ -25,12 +25,14 @@ describe('Studyprogramme overview', () => {
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
-      cy.viewport(1536, 960)
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click({ force: true })
+      // cy.viewport(1536, 960)
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
     it('Basic information -tab loads', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
       cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
       cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
@@ -39,7 +41,6 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Basic information contains correct students', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Started, Graduated, Transferred Away, Transferred to]
@@ -55,23 +56,21 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Basic information contains correct credits', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Total, Major credits, Non-major credits, Non-degree credits, Transferred credits]
         ...years.map(year => [year, 0, 0, 0, 0, 0]),
         [2021, 0, 0, 0, 0, 0],
         [2020, 5860, 5591, 102, 0, 167],
-        [2019, 8696, 7737, 587, 23, 349],
-        [2018, 7818, 6584, 857, 105, 272],
-        [2017, 3085, 2492, 158, 47, 388],
+        [2019, 8696, 7737, 577, 33, 349],
+        [2018, 7818, 6584, 852, 110, 272],
+        [2017, 3085, 2492, 153, 52, 388],
       ]
 
       cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
     })
 
     it('Basic information contains correct thesis writers and graduates', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
       const years = getEmptyYears()
       const tableContents = [
         // [Year, Graduated, Wrote thesis]
@@ -87,7 +86,6 @@ describe('Studyprogramme overview', () => {
     })
 
     it.skip('Special studyrights can be excluded and basic data changes accordingly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
       cy.get('[data-cy=StudentToggle]').click()
       const years = getEmptyYears()
       const studentTableContents = [
@@ -129,39 +127,39 @@ describe('Studyprogramme overview', () => {
     })
 
     it('Year can be changed to academic year, and data changes accordingly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen maisteriohjelma').click()
-      cy.get('[data-cy=YearToggle]').click()
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.get('[data-cy=YearToggle]').click({ force: true })
       const isAcademicYear = true
       const years = getEmptyYears(isAcademicYear)
       const studentTableContents = [
         // [Year, Started, Graduated, Transferred away, Transferred to]
         ...years.map(year => [year, 0, 0, 0, 0]),
         ['2021 - 2022', 0, 0, 0, 0],
-        ['2020 - 2021', 5, 0, 0, 0],
-        ['2019 - 2020', 27, 0, 0, 0],
-        ['2018 - 2019', 4, 0, 0, 0],
-        ['2017 - 2018', 0, 0, 0, 0],
+        ['2020 - 2021', 12, 6, 3, 2],
+        ['2019 - 2020', 86, 41, 1, 0],
+        ['2018 - 2019', 162, 5, 0, 1],
+        ['2017 - 2018', 171, 1, 0, 0],
       ]
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
 
       const creditTableContents = [
         // [Year, Total, Major students credits, Non-major students credits, Non-degree credits, Transferred credits]
-        ...years.map(year => [year, 0, 0, 0, 0]),
-        ['2021 - 2022', 0, 0, 0, 0],
-        ['2020 - 2021', 0, 0, 0, 0],
-        ['2019 - 2020', 338.5, 177, 154, 0, 7.5],
-        ['2018 - 2019', 173, 0, 173, 0, 0],
-        ['2017 - 2018', 42, 0, 42, 0, 0],
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0, 0],
+        ['2020 - 2021', 566, 486, 20, 0, 60],
+        ['2019 - 2020', 8595, 8271, 106, 0, 218],
+        ['2018 - 2019', 9992, 8855, 781, 53, 303],
+        ['2017 - 2018', 6055, 4792, 777, 142, 344],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+      //cy.wait(10000)
       cy.get('[data-cy=YearToggle]').click()
     })
 
     it('Basic information graphs render', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-
       cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
         .should('contain', 'Started')
         .should('contain', 'Graduated')
@@ -171,8 +169,8 @@ describe('Studyprogramme overview', () => {
         .should('contain', 'Major students credits')
         .should('contain', 'Non-major students credits')
         .should('contain', 'Transferred credits')
-        .should('contain', 857)
-        .should('contain', 587)
+        .should('contain', 852)
+        .should('contain', 577)
 
       cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
         .should('contain', 'Graduated students')
@@ -202,157 +200,50 @@ describe('Studyprogramme overview', () => {
   describe('Studytrack overview works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
     })
 
     // If the backend breaks for one of the sections, the section header is not rendered and this will fail
     it('Studytracks and student populations -tab loads', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
       cy.get('[data-cy=Section-StudytrackOverview]')
       cy.get('[data-cy=Section-StudytrackProgress]')
       cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
     it('Students of the studyprogramme are shown correctly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
       const tableContents = [
         // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
-        [
-          '2020 - 2021',
-          12,
-          '100.0 %',
-          10,
-          '83.3 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          10,
-          '83.3 %',
-          2,
-          '16.7 %',
-          12,
-          '100.0 %',
-        ],
-        [
-          '2019 - 2020',
-          86,
-          '100.0 %',
-          86,
-          '100.0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          1,
-          '1.2 %',
-          1,
-          '1.2 %',
-          65,
-          '75.6 %',
-          21,
-          '24.4 %',
-          86,
-          '100.0 %',
-        ],
-        [
-          '2018 - 2019',
-          162,
-          '100.0 %',
-          161,
-          '99.4 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          11,
-          '6.8 %',
-          118,
-          '72.8 %',
-          44,
-          '27.2 %',
-          161,
-          '99.4 %',
-        ],
-        [
-          '2017 - 2018',
-          171,
-          '100.0 %',
-          171,
-          '100.0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          4,
-          '2.3 %',
-          42,
-          '24.6 %',
-          135,
-          '78.9 %',
-          36,
-          '21.1 %',
-          166,
-          '97.1 %',
-        ],
-        [
-          'Total',
-          431,
-          '100.0 %',
-          431,
-          '100.0 %',
-          0,
-          '0 %',
-          0,
-          '0 %',
-          5,
-          '1.2 %',
-          54,
-          '12.5 %',
-          328,
-          '76.1 %',
-          103,
-          '23.9 %',
-          425,
-          '98.6 %',
-        ],
+        ['2020 - 2021', 12, 10, 0, 0, 0, 0, 10, 2, 12],
+        ['2019 - 2020', 86, 86, 0, 0, 1, 1, 65, 21, 86],
+        ['2018 - 2019', 162, 161, 0, 0, 0, 11, 118, 44, 161],
+        ['2017 - 2018', 171, 171, 0, 0, 4, 42, 135, 36, 166],
+        ['Total', 431, 431, 0, 0, 5, 54, 328, 103, 425],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackOverview')
     })
 
     it('Student progress data is shown correctly', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
       const tableContents = [
-        ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0],
-        ['2019 - 2020', 86, 26, 34, 21, 5, 0, 0, 0],
-        ['2018 - 2019', 162, 10, 27, 47, 36, 26, 12, 4],
-        ['2017 - 2018', 171, 20, 21, 26, 22, 22, 33, 27],
-        ['Total', 431, 51, 64, 97, 79, 55, 51, 34],
+        ['2020 - 2021', 12, 12, 0, 0, 0, 0, 0, 0, 0],
+        ['2019 - 2020', 86, 11, 15, 35, 20, 5, 0, 0, 0],
+        ['2018 - 2019', 162, 3, 9, 28, 50, 33, 25, 11, 3],
+        ['2017 - 2018', 171, 7, 13, 22, 32, 18, 23, 32, 24],
+        ['Total', 431, 16, 35, 65, 103, 75, 56, 50, 31],
       ]
 
       cy.checkTableStats(tableContents, 'StudytrackProgress')
     })
 
     it('Studytrack overview graphs render', () => {
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Studytracks and student populations').click()
-
       cy.get('[data-cy=Graph-StudytrackProgress]')
-        .should('contain', 'Less than 30 credits')
+        .should('contain', 'Less than 15 credits')
         .should('contain', '30-59 credits')
-        .should('contain', 'More than 180 credits')
-        .should('contain', '11.7%') // The percentage for less than 30 credits in 2017-2018, to check that the graph renders
-        .should('contain', '6.2%') // The percentage for less than 30 credits in 2018-2019
-        .should('contain', '30.2%') // The percentage for less than 30 credits in 2019-2020
+        .should('contain', '180 or more credits')
+        .should('contain', '4.1%') // The percentage for less than 15 credits in 2017-2018, to check that the graph renders
+        .should('contain', '5.6%') // The percentage for 15-29 credits in 2018-2019
+        .should('contain', '12.8%') // The percentage for less than 15 credits in 2019-2020
 
       cy.get('[data-cy=Graph-2018-AverageGraduationTimes]')
         .should('contain', '2018 - 2019')
@@ -385,16 +276,21 @@ describe('Studyprogramme overview', () => {
     })
 
     it('calendar year -> academic year toggle works', () => {
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       cy.get('[data-cy=fromYear]').click().contains('2018').click()
 
       cy.get('[data-cy=toYear').click().contains('2019').click()
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(20000)
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('684')
       })
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(20000)
       cy.get('[data-cy=calendarAcademicYearToggle]').first().click()
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(20000)
       cy.get('[data-cy=CoursesSortableTable]').within(() => {
         cy.get('tr').eq(1).contains('772')
       })
@@ -436,12 +332,12 @@ describe('Studyprogramme overview', () => {
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
+      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Tags').click()
     })
 
     it('can create and delete tags for population', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
       cy.contains('Create new tag').click()
@@ -452,8 +348,6 @@ describe('Studyprogramme overview', () => {
 
     it('can create personal tags', () => {
       const name = `tag-${new Date().getTime()}`
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
 
@@ -467,9 +361,6 @@ describe('Studyprogramme overview', () => {
       const name = `tag-${new Date().getTime()}`
 
       const student = '010113437'
-
-      cy.contains('a', 'Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.get('.attached').contains('Tags').click()
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
       cy.contains('Create new tag').click()
@@ -490,7 +381,8 @@ describe('Studyprogramme overview', () => {
 
       cy.go('back')
       cy.go('back')
-
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(10000)
       deleteTag(name)
 
       cy.contains('Student statistics').click()

cypress/support/commands.js

@@ -118,7 +118,7 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('navbar-studyProgramme').click()
   cy.cs('navbar-class').click()
   cy.cs('select-study-programme').click().children().contains(name).click()
-  cy.contains('See population').click()
+  cy.contains('See class').click()
 })
 
 Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {

package.json

@@ -29,12 +29,12 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "@babel/core": "^7.18.10",
-    "@babel/eslint-parser": "^7.17.0",
+    "@babel/core": "^7.19.3",
+    "@babel/eslint-parser": "^7.19.1",
     "@babel/plugin-syntax-jsx": "^7.16.7",
-    "concurrently": "^7.3.0",
-    "cypress": "^10.4.0",
-    "eslint": "^8.22.0",
+    "concurrently": "^7.4.0",
+    "cypress": "^10.9.0",
+    "eslint": "^8.25.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-babel": "^5.3.1",
@@ -43,13 +43,13 @@
     "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^4.2.1",
-    "eslint-plugin-react": "^7.29.3",
+    "eslint-plugin-react": "^7.31.9",
     "eslint-plugin-react-hooks": "^4.3.0",
     "husky": "^8.0.1",
     "lint-staged": "^13.0.3",
     "node-actionlint": "^1.2.2",
     "prettier": "^2.5.1",
-    "stylelint": "^14.10.0",
+    "stylelint": "^14.13.0",
     "stylelint-config-prettier": "^9.0.3",
     "stylelint-config-standard": "^26.0.0",
     "stylelint-prettier": "^2.0.0"

services/backend/config/IAMConfig.js

@@ -339,6 +339,8 @@ const doctoralSchoolMap = {
 
 const opetusVaradekaani = 'hy-varadekaanit-opetus'
 
+const universityServices = 'hy-ypa-opa-dojo'
+
 const dekaaniFacultyMap = {
   'hy-ttdk-dekanaatti': 'teologinen',
   'hy-oiktdk-dekanaatti': 'oikeustieteellinen',
@@ -411,6 +413,7 @@ const relevantIAMs = []
   .concat(Object.keys(kosuFacultyMap))
   .concat(Object.keys(dekaaniFacultyMap))
   .concat(opetusVaradekaani)
+  .concat(universityServices)
   .concat(doctoralIams)
   .concat(Object.keys(doctoralSchoolMap))
   .concat(universityWideGroups)

services/backend/config/programmeCodes.js

@@ -0,0 +1,139 @@
+// Map programme codes to "letter indentifier"
+
+const codes = {
+  KH10_001: 'tuk',
+  MH10_001: 'tum',
+  KH20_001: 'on',
+  MH20_001: 'otm',
+  MH20_002: 'ibl',
+  MH20_003: 'ggl',
+  KH30_001: 'psyk',
+  MH30_004: 'psym',
+  KH30_002: 'logk',
+  MH30_005: 'logm',
+  MH30_001: 'll',
+  MH30_002: 'tmed',
+  MH30_003: 'hll',
+  KH40_001: 'filk',
+  KH40_002: 'ttk',
+  KH40_003: 'kik',
+  KH40_004: 'kok',
+  KH40_005: 'kuka',
+  KH40_006: 'hisk',
+  MH40_001: 'ttm',
+  MH40_002: 'kim',
+  MH40_003: 'eng',
+  MH40_004: 'rus',
+  MH40_005: 'lingdig',
+  MH40_006: 'tra',
+  MH40_007: 'suku',
+  MH40_008: 'nor',
+  MH40_009: 'kir',
+  MH40_010: 'kuma',
+  MH40_011: 'ice',
+  MH40_012: 'alku',
+  MH40_014: 'spt',
+  MH40_015: 'hism',
+  KH50_001: 'mat',
+  KH50_002: 'fys',
+  KH50_003: 'kemk',
+  KH50_004: 'mfkk',
+  KH50_005: 'tkt',
+  KH50_006: 'geok',
+  KH50_007: 'maa',
+  KH50_008: 'bsc',
+  MH50_001: 'mast',
+  MH50_002: 'lsi',
+  MH50_003: 'tcm',
+  MH50_004: 'paras',
+  MH50_005: 'matres',
+  MH50_006: 'atm',
+  MH50_007: 'kem',
+  MH50_008: 'mfkm',
+  MH50_009: 'csm',
+  MH50_010: 'data',
+  MH50_011: 'geom',
+  MH50_012: 'geog',
+  MH50_013: 'usp',
+  KH55_001: 'farm',
+  MH55_001: 'prov',
+  MH55_002: 'mpharm',
+  KH57_001: 'bio',
+  KH57_002: 'mole',
+  KH57_003: 'env',
+  MH57_001: 'eeb',
+  MH57_002: 'ips',
+  MH57_003: 'gmb',
+  MH57_004: 'neuro',
+  MH57_005: 'ecgs',
+  KH60_001: 'eduk',
+  MH60_001: 'edum',
+  MH60_002: 'ce',
+  KH70_001: 'pvk',
+  KH70_002: 'yk',
+  KH70_003: 'sosk',
+  KH70_004: 'ecok',
+  MH70_001: 'film',
+  MH70_002: 'pvm',
+  MH70_003: 'gpc',
+  MH70_004: 'ym',
+  MH70_005: 'cos',
+  MH70_006: 'ens',
+  MH70_007: 'msv',
+  MH70_008: 'sosm',
+  MH70_009: 'econ',
+  MH70_011: 'sote',
+  KH74_001: 'ksv',
+  KH80_001: 'maatk',
+  KH80_002: 'metsak',
+  KH80_003: 'etk',
+  KH80_004: 'yet',
+  MH80_001: 'agri',
+  MH80_002: 'agere',
+  MH80_003: 'for',
+  MH80_004: 'food',
+  MH80_005: 'hnfb',
+  MH80_006: 'ekm',
+  MH80_007: 'mmb',
+  KH90_001: 'elk',
+  MH90_001: 'ell',
+  T920101: 'dptheol',
+  T920102: 'dplaw',
+  T920111: 'philartsoc',
+  T920103: 'dphistcult',
+  T920104: 'dplang',
+  T920105: 'sky',
+  T920106: 'dpsocs',
+  T920107: 'pyam',
+  T920108: 'dpe',
+  T920109: 'seduce',
+  T920110: 'clic',
+  T921101: 'dpbm',
+  T921102: 'klto',
+  T921103: 'docpop',
+  T921104: 'findos',
+  T921105: 'dpdr',
+  T921106: 'ils',
+  T921107: 'bandm',
+  T921108: 'cvm',
+  T921109: 'dphub',
+  T922101: 'luova',
+  T922102: 'dpps',
+  T922103: 'denvi',
+  T922104: 'agforee',
+  T922105: 'mbdp',
+  T922106: 'foodhealth',
+  T923101: 'papu',
+  T923102: 'geodoc',
+  T923103: 'atm',
+  T923104: 'chems',
+  T923105: 'domast',
+  T923106: 'matrena',
+  T923107: 'docs',
+  // manually added programs with no IAM, either abolished, or special co-operation programmes
+  MH50_014: 'ech',
+  MH40_013: 'mkk',
+  MH70_010: 'imes',
+}
+
+module.exports = { codes }

services/backend/package-lock.json

@@ -14,20 +14,20 @@
         "bluebird": "^3.7.2",
         "cors": "^2.8.5",
         "cron": "^1.8.2",
-        "express": "^4.17.3",
+        "express": "^4.18.2",
         "express-async-errors": "^3.1.1",
         "lodash": "^4.17.21",
         "lru-cache": "^5.1.1",
         "moment": "^2.29.1",
-        "moment-timezone": "^0.5.34",
+        "moment-timezone": "^0.5.37",
         "morgan": "^1.10.0",
-        "pg": "^8.7.3",
+        "pg": "^8.8.0",
         "redis": "^3.1.2",
-        "sequelize": "^6.17.0",
+        "sequelize": "^6.24.0",
         "umzug": "^2.3.0",
         "unfuck-utf8-headers-middleware": "^1.0.1",
         "uuid": "^3.4.0",
-        "winston": "^3.6.0"
+        "winston": "^3.8.2"
       },
       "devDependencies": {
         "nodemon": "^1.19.4"
@@ -495,9 +495,9 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "node_modules/body-parser": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
-      "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
       "dependencies": {
         "bytes": "3.1.2",
         "content-type": "~1.0.4",
@@ -507,7 +507,7 @@
         "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "on-finished": "2.4.1",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "raw-body": "2.5.1",
         "type-is": "~1.6.18",
         "unpipe": "1.0.0"
@@ -1178,13 +1178,13 @@
       }
     },
     "node_modules/express": {
-      "version": "4.18.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz",
-      "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==",
+      "version": "4.18.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.0",
+        "body-parser": "1.20.1",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
         "cookie": "0.5.0",
@@ -1203,7 +1203,7 @@
         "parseurl": "~1.3.3",
         "path-to-regexp": "0.1.7",
         "proxy-addr": "~2.0.7",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "range-parser": "~1.2.1",
         "safe-buffer": "5.2.1",
         "send": "0.18.0",
@@ -1407,9 +1407,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "node_modules/get-intrinsic": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz",
-      "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
+      "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
       "dependencies": {
         "function-bind": "^1.1.1",
         "has": "^1.0.3",
@@ -2209,9 +2209,9 @@
       }
     },
     "node_modules/moment-timezone": {
-      "version": "0.5.34",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
-      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
+      "version": "0.5.37",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.37.tgz",
+      "integrity": "sha512-uEDzDNFhfaywRl+vwXxffjjq1q0Vzr+fcQpQ1bU0kbzorfS7zVtZnCnGc8mhWmF39d4g4YriF6kwA75mJKE/Zg==",
       "dependencies": {
         "moment": ">= 2.9.0"
       },
@@ -2619,14 +2619,14 @@
       "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
     },
     "node_modules/pg": {
-      "version": "8.7.3",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.3.tgz",
-      "integrity": "sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==",
+      "version": "8.8.0",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz",
+      "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==",
       "dependencies": {
         "buffer-writer": "2.0.0",
         "packet-reader": "1.0.0",
         "pg-connection-string": "^2.5.0",
-        "pg-pool": "^3.5.1",
+        "pg-pool": "^3.5.2",
         "pg-protocol": "^1.5.0",
         "pg-types": "^2.1.0",
         "pgpass": "1.x"
@@ -2635,7 +2635,7 @@
         "node": ">= 8.0.0"
       },
       "peerDependencies": {
-        "pg-native": ">=2.0.0"
+        "pg-native": ">=3.0.1"
       },
       "peerDependenciesMeta": {
         "pg-native": {
@@ -2657,9 +2657,9 @@
       }
     },
     "node_modules/pg-pool": {
-      "version": "3.5.1",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.1.tgz",
-      "integrity": "sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ==",
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz",
+      "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==",
       "peerDependencies": {
         "pg": ">=8.0"
       }
@@ -2785,9 +2785,9 @@
       "dev": true
     },
     "node_modules/qs": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
-      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "dependencies": {
         "side-channel": "^1.0.4"
       },
@@ -3013,9 +3013,9 @@
       }
     },
     "node_modules/retry-as-promised": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
-      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-6.1.0.tgz",
+      "integrity": "sha512-Hj/jY+wFC+SB9SDlIIFWiGOHnNG0swYbGYsOj2BJ8u2HKUaobNKab0OIC0zOLYzDy0mb7A4xA5BMo4LMz5YtEA=="
     },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
@@ -3108,9 +3108,9 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/sequelize": {
-      "version": "6.21.3",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.21.3.tgz",
-      "integrity": "sha512-cJPrTTVCofUxaaNKoIETiXCYh2xJ+OFq5jMHJQqftp34M4kNoLpTfUMPSwYtRUeTcSh1/5HodfJXIBi7troIFA==",
+      "version": "6.24.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.24.0.tgz",
+      "integrity": "sha512-mPo7Q7gWkrsstjR2aw8ahkrj8RUJCozz3kedAz2B5ZIdLoQEH1z2tvaeJONI8R6RqeuyRQosx9Yn3s9yQyo6lQ==",
       "funding": [
         {
           "type": "opencollective",
@@ -3127,7 +3127,7 @@
         "moment": "^2.29.1",
         "moment-timezone": "^0.5.34",
         "pg-connection-string": "^2.5.0",
-        "retry-as-promised": "^5.0.0",
+        "retry-as-promised": "^6.1.0",
         "semver": "^7.3.5",
         "sequelize-pool": "^7.1.0",
         "toposort-class": "^1.0.1",
@@ -3148,6 +3148,9 @@
         "mysql2": {
           "optional": true
         },
+        "oracledb": {
+          "optional": true
+        },
         "pg": {
           "optional": true
         },
@@ -4035,10 +4038,11 @@
       }
     },
     "node_modules/winston": {
-      "version": "3.8.1",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.1.tgz",
-      "integrity": "sha512-r+6YAiCR4uI3N8eQNOg8k3P3PqwAm20cLKlzVD9E66Ch39+LZC+VH1UKf9JemQj2B3QoUHfKD7Poewn0Pr3Y1w==",
+      "version": "3.8.2",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.2.tgz",
+      "integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==",
       "dependencies": {
+        "@colors/colors": "1.5.0",
         "@dabh/diagnostics": "^2.0.2",
         "async": "^3.2.3",
         "is-stream": "^2.0.0",
@@ -4526,9 +4530,9 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "body-parser": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
-      "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
       "requires": {
         "bytes": "3.1.2",
         "content-type": "~1.0.4",
@@ -4538,7 +4542,7 @@
         "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "on-finished": "2.4.1",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "raw-body": "2.5.1",
         "type-is": "~1.6.18",
         "unpipe": "1.0.0"
@@ -5085,13 +5089,13 @@
       }
     },
     "express": {
-      "version": "4.18.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz",
-      "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==",
+      "version": "4.18.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
       "requires": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.0",
+        "body-parser": "1.20.1",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
         "cookie": "0.5.0",
@@ -5110,7 +5114,7 @@
         "parseurl": "~1.3.3",
         "path-to-regexp": "0.1.7",
         "proxy-addr": "~2.0.7",
-        "qs": "6.10.3",
+        "qs": "6.11.0",
         "range-parser": "~1.2.1",
         "safe-buffer": "5.2.1",
         "send": "0.18.0",
@@ -5261,9 +5265,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "get-intrinsic": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz",
-      "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
+      "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
       "requires": {
         "function-bind": "^1.1.1",
         "has": "^1.0.3",
@@ -5884,9 +5888,9 @@
       "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
     },
     "moment-timezone": {
-      "version": "0.5.34",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.34.tgz",
-      "integrity": "sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg==",
+      "version": "0.5.37",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.37.tgz",
+      "integrity": "sha512-uEDzDNFhfaywRl+vwXxffjjq1q0Vzr+fcQpQ1bU0kbzorfS7zVtZnCnGc8mhWmF39d4g4YriF6kwA75mJKE/Zg==",
       "requires": {
         "moment": ">= 2.9.0"
       }
@@ -6207,14 +6211,14 @@
       "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
     },
     "pg": {
-      "version": "8.7.3",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-8.7.3.tgz",
-      "integrity": "sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==",
+      "version": "8.8.0",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz",
+      "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==",
       "requires": {
         "buffer-writer": "2.0.0",
         "packet-reader": "1.0.0",
         "pg-connection-string": "^2.5.0",
-        "pg-pool": "^3.5.1",
+        "pg-pool": "^3.5.2",
         "pg-protocol": "^1.5.0",
         "pg-types": "^2.1.0",
         "pgpass": "1.x"
@@ -6231,9 +6235,9 @@
       "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
     },
     "pg-pool": {
-      "version": "3.5.1",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.1.tgz",
-      "integrity": "sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ==",
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz",
+      "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==",
       "requires": {}
     },
     "pg-protocol": {
@@ -6330,9 +6334,9 @@
       "dev": true
     },
     "qs": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
-      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "requires": {
         "side-channel": "^1.0.4"
       }
@@ -6509,9 +6513,9 @@
       "dev": true
     },
     "retry-as-promised": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-5.0.0.tgz",
-      "integrity": "sha512-6S+5LvtTl2ggBumk04hBo/4Uf6fRJUwIgunGZ7CYEBCeufGFW1Pu6ucUf/UskHeWOIsUcLOGLFXPig5tR5V1nA=="
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-6.1.0.tgz",
+      "integrity": "sha512-Hj/jY+wFC+SB9SDlIIFWiGOHnNG0swYbGYsOj2BJ8u2HKUaobNKab0OIC0zOLYzDy0mb7A4xA5BMo4LMz5YtEA=="
     },
     "safe-buffer": {
       "version": "5.2.1",
@@ -6580,9 +6584,9 @@
       }
     },
     "sequelize": {
-      "version": "6.21.3",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.21.3.tgz",
-      "integrity": "sha512-cJPrTTVCofUxaaNKoIETiXCYh2xJ+OFq5jMHJQqftp34M4kNoLpTfUMPSwYtRUeTcSh1/5HodfJXIBi7troIFA==",
+      "version": "6.24.0",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.24.0.tgz",
+      "integrity": "sha512-mPo7Q7gWkrsstjR2aw8ahkrj8RUJCozz3kedAz2B5ZIdLoQEH1z2tvaeJONI8R6RqeuyRQosx9Yn3s9yQyo6lQ==",
       "requires": {
         "@types/debug": "^4.1.7",
         "@types/validator": "^13.7.1",
@@ -6593,7 +6597,7 @@
         "moment": "^2.29.1",
         "moment-timezone": "^0.5.34",
         "pg-connection-string": "^2.5.0",
-        "retry-as-promised": "^5.0.0",
+        "retry-as-promised": "^6.1.0",
         "semver": "^7.3.5",
         "sequelize-pool": "^7.1.0",
         "toposort-class": "^1.0.1",
@@ -7283,10 +7287,11 @@
       }
     },
     "winston": {
-      "version": "3.8.1",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.1.tgz",
-      "integrity": "sha512-r+6YAiCR4uI3N8eQNOg8k3P3PqwAm20cLKlzVD9E66Ch39+LZC+VH1UKf9JemQj2B3QoUHfKD7Poewn0Pr3Y1w==",
+      "version": "3.8.2",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.2.tgz",
+      "integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==",
       "requires": {
+        "@colors/colors": "1.5.0",
         "@dabh/diagnostics": "^2.0.2",
         "async": "^3.2.3",
         "is-stream": "^2.0.0",

services/backend/package.json

@@ -15,20 +15,20 @@
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
-    "express": "^4.17.3",
+    "express": "^4.18.2",
     "express-async-errors": "^3.1.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.34",
+    "moment-timezone": "^0.5.37",
     "morgan": "^1.10.0",
-    "pg": "^8.7.3",
+    "pg": "^8.8.0",
     "redis": "^3.1.2",
-    "sequelize": "^6.17.0",
+    "sequelize": "^6.24.0",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "uuid": "^3.4.0",
-    "winston": "^3.6.0"
+    "winston": "^3.8.2"
   },
   "devDependencies": {
     "nodemon": "^1.19.4"

services/backend/src/database/migrations_kone/20221111_00_create_table_open_uni_population_search.js

@@ -0,0 +1,26 @@
+const { BIGINT, STRING, DATE, ARRAY } = require('sequelize')
+
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.createTable('open_uni_population_searches', {
+      id: {
+        type: BIGINT,
+        primaryKey: true,
+        autoIncrement: true,
+      },
+      user_id: {
+        type: BIGINT,
+      },
+      name: {
+        type: STRING,
+        allowNull: false,
+      },
+      course_codes: {
+        type: ARRAY(STRING),
+      },
+      created_at: DATE,
+      updated_at: DATE,
+    })
+  },
+  down: () => {},
+}

services/backend/src/database/migrations_user/20220831_00_add_facultystats_access.js

@@ -0,0 +1,17 @@
+module.exports = {
+  up: queryInterface => {
+    return queryInterface.bulkInsert('access_groups', [
+      {
+        group_code: 'facultyStatistics',
+        group_info: 'grants access to faculty statistics',
+      },
+    ])
+  },
+  down: queryInterface => {
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: 'facultyStatistics',
+      },
+    ])
+  },
+}

services/backend/src/database/migrations_user/20221026_00_add_open_uni_dojo_access.js

@@ -0,0 +1,17 @@
+module.exports = {
+  up: queryInterface => {
+    return queryInterface.bulkInsert('access_groups', [
+      {
+        group_code: 'openUniSearch',
+        group_info: 'grants access to custom open uni course searches',
+      },
+    ])
+  },
+  down: queryInterface => {
+    return queryInterface.bulkDelete('access_groups', [
+      {
+        group_code: 'openUniSearch',
+      },
+    ])
+  },
+}

services/backend/src/events.js

@@ -5,6 +5,8 @@ const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
 const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
+const { faculties } = require('./services/organisations')
+const { updateFacultyOverview, updateFacultyProgressOverview } = require('./services/faculty/facultyUpdates')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -16,6 +18,24 @@ const refreshStudyrightAssociations = async () => {
   logger.info('Studyright associations refreshed!')
 }
 
+const refreshFaculties = async () => {
+  logger.info('Refreshing faculty overview statistics for all faculties')
+  const facultyList = (await faculties()).filter(f => !['Y', 'H99', 'Y01', 'H92', 'H930'].includes(f.code))
+  let ready = 0
+
+  for (const faculty of facultyList) {
+    try {
+      await updateFacultyOverview(faculty.code, 'ALL')
+      await updateFacultyProgressOverview(faculty.code)
+      ready += 1
+    } catch (e) {
+      logger.error({ message: `Failed to update stats for faculty ${faculty?.code} with type ALL`, meta: e })
+    }
+    logger.info(`${ready}/${facultyList.length} faculties done`)
+  }
+  logger.info('Faculty stats refreshed!')
+}
+
 const refreshNewOverviews = async () => {
   logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
   const codes = (await getAllProgrammes())
@@ -100,7 +120,7 @@ const startCron = () => {
     logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
     // refresh 3am every day
     schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews]) {
+      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews, refreshFaculties]) {
         await func()
       }
     })
@@ -111,4 +131,5 @@ module.exports = {
   startCron,
   refreshStatistics,
   refreshTrends,
+  refreshFaculties,
 }

services/backend/src/middleware/accessLogger.js

@@ -27,11 +27,16 @@ const accessLogger = morgan((tokens, req, res) => {
     'ms',
   ].join(' ')
 
+  const usingIamRights = user.iamRights.some(programmeCode => tokens['url'](req, res).includes(programmeCode))
+  const onlyIamRights = !user.isAdmin && user.rights.length === 0
+
   logger.info(message, {
     // don't log student list which might be huge
     ..._.omit(meta, ['studentsUserCanAccess']),
     // pass this as a custom field so we can filter by it in graylog
     isUsageStats: true,
+    // needed for Grafana IAM users panel
+    isIamRights: onlyIamRights || usingIamRights,
   })
 })
 

services/backend/src/models/models_kone.js

@@ -121,6 +121,28 @@ const CustomPopulationSearch = sequelizeKone.define('custom_population_searches'
   },
 })
 
+const OpenUniPopulationSearch = sequelizeKone.define(
+  'open_uni_population_searches',
+  {
+    id: {
+      primaryKey: true,
+      type: Sequelize.BIGINT,
+      autoIncrement: true,
+    },
+    userId: {
+      type: Sequelize.BIGINT,
+    },
+    name: {
+      type: Sequelize.STRING,
+      allowNull: false,
+    },
+    courseCodes: {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+    },
+  },
+  { underscored: true, timestamps: true }
+)
+
 const StudyGuidanceGroupTag = sequelizeKone.define(
   'study_guidance_group_tags',
   {
@@ -162,6 +184,7 @@ module.exports = {
   ThesisTypeEnums,
   MandatoryCourseLabels,
   CustomPopulationSearch,
+  OpenUniPopulationSearch,
   sequelizeKone,
   ExcludedCourse,
   StudyGuidanceGroupTag,

services/backend/src/models/student.js

@@ -1,4 +1,4 @@
-const { Model, STRING, DATE, INTEGER } = require('sequelize')
+const { Model, STRING, DATE, INTEGER, BOOLEAN } = require('sequelize')
 const { dbConnections } = require('../database/connection')
 
 class Student extends Model {}
@@ -16,6 +16,7 @@ Student.init(
     creditcount: { type: INTEGER },
     dateofuniversityenrollment: { type: DATE },
     email: { type: STRING },
+    secondary_email: { type: STRING },
     national_student_number: { type: STRING },
     country_fi: { type: STRING },
     country_sv: { type: STRING },
@@ -25,6 +26,7 @@ Student.init(
     home_country_en: { type: STRING },
     gender_code: { type: INTEGER },
     sis_person_id: { type: STRING },
+    dissemination_info_allowed: { type: BOOLEAN },
     createdAt: {
       type: DATE,
     },

services/backend/src/routes.js

@@ -30,6 +30,7 @@ const trends = require('./routes/trends')
 const programmeModules = require('./routes/programmeModules')
 const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 const studyProgramme = require('./routes/studyProgramme')
+const customOpenUniSearch = require('./routes/customOpenUniSearch')
 const initializeSentry = require('./util/sentry')
 
 const errorMiddleware = require('./middleware/errorMiddleware')
@@ -57,7 +58,8 @@ module.exports = (app, url) => {
   app.use(url, tags)
   app.use(url, programmeModules)
   app.use(url, studyProgramme)
-  app.use(url, faculties)
+  app.use(`${url}/openunisearch`, auth.roles(['openUniSearch']), customOpenUniSearch)
+  app.use(`${url}/faculties`, auth.roles(['facultyStatistics']), faculties)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['admin']), users)

services/backend/src/routes/courses.js

@@ -1,6 +1,7 @@
 const router = require('express').Router()
 const crypto = require('crypto')
 const Course = require('../services/courses')
+const { faculties } = require('../services/organisations')
 const { validateParamLength } = require('../util')
 
 router.get('/v2/coursesmulti', async (req, res) => {
@@ -43,4 +44,9 @@ router.get('/v3/courseyearlystats', async (req, res) => {
   }
 })
 
+router.get('/faculties-only', async (_req, res) => {
+  const facultyList = await faculties()
+  res.json(facultyList)
+})
+
 module.exports = router

services/backend/src/routes/customOpenUniSearch.js

@@ -0,0 +1,74 @@
+const router = require('express').Router()
+var moment = require('moment-timezone')
+const { getCustomOpenUniCourses } = require('../services/openUni/openUniStats')
+const {
+  getOpenUniSearches,
+  createNewSearch,
+  deleteSearch,
+  updateSearch,
+} = require('../services/openUni/openUniManageSearches')
+
+router.get('/', async (req, res) => {
+  const courseCodes = JSON.parse(req.query?.courselist) || []
+  const startdate = req.query?.startdate ? req.query?.startdate : moment('01-08-2017 00:00:00', 'DD-MM-YYYY')
+  const enddate = req.query?.enddate ? req.query?.enddate : moment().endOf('day')
+
+  if (!Array.isArray(courseCodes)) return res.status(400).json({ error: 'Courses must be of type array' })
+  const customOpenUniSearches = await getCustomOpenUniCourses(courseCodes, startdate, enddate)
+  return res.json(customOpenUniSearches)
+})
+
+router.get('/searches', async (req, res) => {
+  const userId = req.user.id
+  const foundSearches = await getOpenUniSearches(userId)
+  return res.json(foundSearches)
+})
+
+router.post('/searches', async (req, res) => {
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  const name = req.body?.name
+
+  if (!name) return res.status(400).json({ error: 'Name missing' })
+  if (courseCodes && !Array.isArray(courseCodes))
+    return res.status(400).json({ error: 'Course codes must be type of array' })
+
+  const createdSearch = await createNewSearch(userId, name, courseCodes)
+  if (!createdSearch) return res.status(400).json({ error: '' })
+  return res.status(201).json({
+    id: createdSearch.id,
+    userId: createdSearch.userId,
+    name: createdSearch.name,
+    courseList: createdSearch.courseCodes,
+    updatedAt: createdSearch.updatedAt,
+  })
+})
+
+router.put('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const courseCodes = req.body?.courselist || []
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+  const updatedSearch = await updateSearch(userId, id, courseCodes)
+
+  if (!updatedSearch) return res.status(404).json({ error: `Open uni search could not be found` })
+  return res.json({
+    id: updatedSearch.id,
+    userId: updatedSearch.userId,
+    name: updatedSearch.name,
+    courseList: updatedSearch.courseCodes,
+    updatedAt: updatedSearch.updatedAt,
+  })
+})
+
+router.delete('/searches/:id', async (req, res) => {
+  const id = req.params?.id
+  const userId = req.user.id
+  if (!id || !userId) return res.status(422).end()
+
+  const deletedSearch = await deleteSearch(userId, id)
+  if (!deletedSearch) return res.status(404).json({ error: `Open uni search could not be found` })
+  return res.json(id)
+})
+
+module.exports = router

services/backend/src/routes/faculties.js

@@ -1,131 +1,206 @@
 const router = require('express').Router()
 const { faculties } = require('../services/organisations')
-const { combineFacultyBasics } = require('../services/facultyBasics')
-const { combineFacultyCredits } = require('../services/facultyCredits')
-const { degreeProgrammesOfFaculty, findFacultyProgrammeCodes } = require('../services/faculty')
-const { combineFacultyThesisWriters } = require('../services/facultyThesisWriters')
+const { combineFacultyBasics } = require('../services/faculty/facultyBasics')
+const { combineFacultyCredits } = require('../services/faculty/facultyCredits')
+const { findFacultyProgrammeCodes } = require('../services/faculty/faculty')
+const { combineFacultyThesisWriters } = require('../services/faculty/facultyThesisWriters')
+const { countGraduationTimes } = require('../services/faculty/facultyGraduationTimes')
+const { updateFacultyOverview, updateFacultyProgressOverview } = require('../services/faculty/facultyUpdates')
+const { combineFacultyStudentProgress } = require('../services/faculty/facultyStudentProgress')
+const { combineFacultyStudents } = require('../services/faculty/facultyStudents')
+
+const {
+  getFacultyProgrammes,
+  setFacultyProgrammes,
+  getBasicStats,
+  setBasicStats,
+  getCreditStats,
+  setCreditStats,
+  getThesisWritersStats,
+  setThesisWritersStats,
+  getGraduationStats,
+  setGraduationStats,
+  getFacultyStudentStats,
+  setFacultyStudentStats,
+  getFacultyProgressStats,
+  setFacultyProgressStats,
+} = require('../services/faculty/facultyService')
+const logger = require('../util/logger')
 
 // Faculty uses a lot of tools designed for Study programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
 
-router.get('/faculties', async (req, res) => {
-  const facultyList = await faculties()
+const getProgrammes = async (code, programmeFilter) => {
+  const programmes = await getFacultyProgrammes(code, programmeFilter)
+  if (programmes) return programmes
+  let updatedProgrammes = await findFacultyProgrammeCodes(code, programmeFilter)
+  if (updatedProgrammes) updatedProgrammes = await setFacultyProgrammes(code, updatedProgrammes, programmeFilter)
+
+  return updatedProgrammes
+}
+
+router.get('/', async (req, res) => {
+  const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
+  const facultyList = (await faculties()).filter(f => !ignore.includes(f.code))
+  facultyList.sort((a, b) => (a.name.fi > b.name.fi ? 1 : -1))
   res.json(facultyList)
 })
 
-router.get('/faculties/:id/basicstats', async (req, res) => {
+router.get('/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const programmeFilter = req.query?.programme_filter
+  const specialGroups = req.query?.special_groups
 
   if (!code) return res.status(422).end()
+  const data = await getBasicStats(code, yearType, programmeFilter, specialGroups)
+  if (data) return res.json(data)
+  const wantedProgrammes = await getProgrammes(code, programmeFilter)
+  if (!wantedProgrammes) return res.status(422).end()
 
-  let allBasics = {
-    id: code,
-    years: [],
-    programmeNames: {},
-    studentInfo: {
-      tableStats: [],
-      graphStats: [],
-      titles: ['', 'Started studying', 'Graduated', 'Transferred inside', 'Transferred away', 'Transferred to'],
-      programmeTableStats: {},
-    },
-    graduationInfo: {
-      tableStats: [],
-      graphStats: [],
-      titles: ['', 'All graduations', 'Bachelors', 'Masters', 'Doctors', 'Others'],
-      programmeTableStats: {},
-    },
-    status: 'DONE',
-    lastUpdated: '',
+  //all programmes are required for correct sorting of transfers
+  let allProgrammeCodes = []
+  if (programmeFilter === 'NEW_STUDY_PROGRAMMES') {
+    const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
+    allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+  } else {
+    wantedProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
   }
 
-  const programmes = await findFacultyProgrammeCodes(code)
-
-  if (programmes) {
-    await combineFacultyBasics(allBasics, code, programmes, yearType)
+  let updatedStats = await combineFacultyBasics(
+    code,
+    wantedProgrammes.data,
+    yearType,
+    allProgrammeCodes,
+    programmeFilter,
+    specialGroups
+  )
+  if (updatedStats) {
+    updatedStats = await setBasicStats(updatedStats, yearType, programmeFilter, specialGroups)
   }
-  return res.json(allBasics)
+  return res.json(updatedStats)
 })
 
-router.get('/faculties/:id/creditstats', async (req, res) => {
+router.get('/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
-  const specialGroups = 'SPECIAL_INCLUDED' // req.query?.special_groups
+  const specialGroups = req.query?.special_groups
+  const programmeFilter = req.query?.programme_filter
 
   if (!code) return res.status(422).end()
+  const data = await getCreditStats(code, yearType, programmeFilter, specialGroups)
+  if (data) return res.json(data)
 
-  let counts = {}
-  let years = []
-  let allCredits = {
-    id: code,
-    years: [],
-    tableStats: [],
-    graphStats: [],
-    programmeTableStats: {},
-    programmeNames: {},
-    titles: [
-      '',
-      'Total',
-      'Major students credits',
-      'Non-major faculty students credits',
-      'Non-major other faculty students credits',
-      'Non-degree student credits',
-    ],
-    status: 'DONE',
-    lastUpdated: '',
+  const programmes = await getProgrammes(code, programmeFilter)
+  if (!programmes) return res.status(422).end()
+
+  // list of all programmes is also needed in classification of credits
+  const allProgrammes = programmeFilter === 'ALL_PROGRAMMES' ? programmes : await getProgrammes(code, 'ALL_PROGRAMMES')
+
+  let updatedStats = await combineFacultyCredits(code, programmes.data, allProgrammes.data, yearType, specialGroups)
+  if (updatedStats) {
+    updatedStats = await setCreditStats(updatedStats, yearType, programmeFilter, specialGroups)
   }
 
-  const programmes = await degreeProgrammesOfFaculty(code)
-
-  if (programmes) {
-    await combineFacultyCredits(allCredits, programmes, yearType, specialGroups, counts, years)
-  }
-  let majors = []
-  let facultyNonMajor = []
-  let otherNonMajor = []
-  let nonDegree = []
-
-  years.forEach(year => {
-    majors.push(counts[year][1])
-    facultyNonMajor.push(counts[year][2])
-    otherNonMajor.push(counts[year][3])
-    nonDegree.push(counts[year][4])
-  })
-
-  allCredits.graphStats = [
-    { name: 'Major students credits', data: majors },
-    { name: 'Non-major faculty students credits', data: facultyNonMajor },
-    { name: 'Non-major other faculty students credits', data: otherNonMajor },
-    { name: 'Non-degree credits', data: nonDegree },
-  ]
-
-  return res.json(allCredits)
+  return res.json(updatedStats)
 })
 
-router.get('/faculties/:id/thesisstats', async (req, res) => {
+router.get('/:id/thesisstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
-  //const specialGroups = 'SPECIAL_INCLUDED'
+  const specialGroups = req.query?.special_groups
+  const programmeFilter = req.query?.programme_filter
+
   if (!code) return res.status(422).end()
-
-  let allThesisWriters = {
-    id: code,
-    years: [],
-    tableStats: [],
-    graphStats: [],
-    programmeTableStats: {},
-    titles: ['', 'All thesis writers', 'Bachelors', 'Masters', 'Doctors', 'Others'],
-    programmeNames: {},
-    status: 'Done',
-    lastUpdated: '',
+  const data = await getThesisWritersStats(code, yearType, programmeFilter, specialGroups)
+  if (data) return res.json(data)
+  const programmes = await getProgrammes(code, programmeFilter)
+  if (!programmes) return res.status(422).end()
+  let updateStats = await combineFacultyThesisWriters(code, programmes.data, yearType, specialGroups)
+  if (updateStats) {
+    updateStats = await setThesisWritersStats(updateStats, yearType, programmeFilter, specialGroups)
   }
-
-  const programmes = await degreeProgrammesOfFaculty(code)
-  if (programmes) {
-    await combineFacultyThesisWriters(allThesisWriters, programmes, code, yearType)
-  }
-
-  return res.json(allThesisWriters)
+  return res.json(updateStats)
 })
 
+router.get('/:id/graduationtimes', async (req, res) => {
+  const code = req.params.id
+  const programmeFilter = req.query?.programme_filter
+
+  if (!code) return res.status(422).end()
+  const data = await getGraduationStats(code, programmeFilter)
+  if (data) {
+    return res.json(data)
+  }
+  let updatedStats = await countGraduationTimes(code, programmeFilter)
+  if (updatedStats) {
+    updatedStats = await setGraduationStats(updatedStats, programmeFilter)
+  }
+  return res.json(updatedStats)
+})
+
+router.get('/:id/progressstats', async (req, res) => {
+  const code = req.params.id
+  const programmeFilter = req.query.programme_filter
+  const specialGroups = req.query?.special_groups
+  const graduated = req.query?.graduated
+
+  if (!code) return res.status(422).end()
+  const data = await getFacultyProgressStats(code, specialGroups, graduated)
+  if (data) return res.json(data)
+  const programmes = await getProgrammes(code, programmeFilter)
+  if (!programmes) return res.status(422).end()
+  let updateStats = await combineFacultyStudentProgress(code, programmes.data, specialGroups, graduated)
+  if (updateStats) {
+    updateStats = await setFacultyProgressStats(updateStats, specialGroups, graduated)
+  }
+  return res.json(updateStats)
+})
+
+router.get('/:id/studentstats', async (req, res) => {
+  const code = req.params.id
+  const programmeFilter = req.query.programme_filter
+  const specialGroups = req.query?.special_groups
+  const graduated = req.query?.graduated
+
+  if (!code) return res.status(422).end()
+  const data = await getFacultyStudentStats(code, specialGroups, graduated)
+  if (data) return res.json(data)
+  const programmes = await getProgrammes(code, programmeFilter)
+  if (!programmes) return res.status(422).end()
+  let updateStats = await combineFacultyStudents(code, programmes.data, specialGroups, graduated)
+  if (updateStats) {
+    updateStats = await setFacultyStudentStats(updateStats, specialGroups, graduated)
+  }
+  return res.json(updateStats)
+})
+
+router.get('/:id/update_basicview', async (req, res) => {
+  const code = req.params.id
+  const statsType = req.query?.stats_type
+  if (code) {
+    let result = null
+    try {
+      result = await updateFacultyOverview(code, statsType)
+    } catch (e) {
+      logger.error(`Failed to update faculty ${code} basic tab stats for ${statsType}: ${e}`)
+    }
+    return res.json(result)
+  }
+  return res.status(422).end()
+})
+
+router.get('/:id/update_progressview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateFacultyProgressOverview(code)
+    } catch (e) {
+      logger.error(`Failed to update faculty ${code} progress tab stats: ${e}`)
+    }
+    return res.json(result)
+  }
+  return res.status(422).end()
+})
 module.exports = router

services/backend/src/routes/population.js

@@ -10,6 +10,7 @@ const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
 const { mapToProviders } = require('../util/utils')
 const { encrypt, decrypt } = require('../services/encrypt')
+const { mapCodesToIds } = require('../services/studyprogrammeHelpers')
 
 const { ApplicationError } = require('../util/customErrors')
 
@@ -48,7 +49,11 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
         if (req.body.selectedStudentsByYear) {
           req.body.selectedStudents = encrypted
             ? req.body.selectedStudents
-                .filter(({ encryptedData }) => req.body.selectedStudentsByYear[year].includes(encryptedData))
+                .filter(
+                  ({ encryptedData }) =>
+                    req.body.selectedStudentsByYear[year] &&
+                    req.body.selectedStudentsByYear[year].includes(encryptedData)
+                )
                 .map(decrypt)
             : req.body.selectedStudentsByYear[year]
         }
@@ -77,6 +82,20 @@ router.post('/v2/populationstatistics/courses', async (req, res) => {
       return
     }
 
+    //These cause problems in front (where they were originally calculated)
+    // Do them here for the moment
+    for (const course of result.coursestatistics) {
+      // course.students.notParticipated = {}
+      course.students.didNotPass = {}
+
+      for (const student of Object.keys(course.students.all)) {
+        if (
+          !Object.keys(course.students.passed).includes(student) &&
+          !Object.keys(course.students.retryPassed).includes(student)
+        )
+          course.students.didNotPass[student] = true
+      }
+    }
     res.json(result)
   }
 })
@@ -411,9 +430,11 @@ router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
   } = req
   if (roles?.includes('admin')) {
     const studyrights = await StudyrightService.getAssociations()
+    mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   } else {
     const studyrights = await StudyrightService.getFilteredAssociations(rights.concat(iamRights))
+    mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   }
 })

services/backend/src/services/courses.js

@@ -450,7 +450,7 @@ const codeLikeTerm = code =>
     : {
         code: {
           // Starts with code or has AY/A in front of the code
-          [Op.regexp]: `^(AY|ay|A|a)?${code}`,
+          [Op.iRegexp]: `^(AY|ay|A|a)?${code}`,
         },
       }
 

services/backend/src/services/faculty/faculty.js

@@ -4,22 +4,40 @@ const {
   ElementDetail,
   Organization,
   ProgrammeModule,
+  Semester,
+  SemesterEnrollment,
   Studyright,
   StudyrightElement,
   Student,
   Transfer,
   Credit,
   Course,
-} = require('../models')
+} = require('../../models')
 const {
   facultyFormatStudyright,
   facultyFormatProgramme,
   formatFacultyTransfer,
   formatFacultyThesisWriter,
   formatOrganization,
+  formatAbsence,
+  isNewProgramme,
+  mapCodesToIds,
 } = require('./facultyHelpers')
 
-const startedStudyrights = async (faculty, since) =>
+const getTransferredToAndAway = async (programmeCodes, allProgrammeCodes, since) => {
+  const awayTransfers = (await transferredAway(programmeCodes, allProgrammeCodes, since)).map(t => t.studyrightid)
+  const toTransfers = (await transferredTo(programmeCodes, allProgrammeCodes, since)).map(t => t.studyrightid)
+  const transferredToOrAway = [...toTransfers, ...awayTransfers]
+  return transferredToOrAway
+}
+
+const getTransferredInside = async (programmeCodes, allProgrammeCodes, since) => {
+  const transfersInside = (await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)).map(
+    t => t.studyrightid
+  )
+  return transfersInside
+}
+const startedStudyrights = async (faculty, since, studyRightWhere) =>
   (
     await Studyright.findAll({
       include: [
@@ -43,11 +61,12 @@ const startedStudyrights = async (faculty, since) =>
           [Op.gte]: since,
         },
         student_studentnumber: { [Op.not]: null },
+        ...studyRightWhere,
       },
     })
   ).map(facultyFormatStudyright)
 
-const graduatedStudyrights = async (faculty, since) =>
+const graduatedStudyrights = async (faculty, since, studyrightWhere) =>
   (
     await Studyright.findAll({
       include: [
@@ -72,24 +91,73 @@ const graduatedStudyrights = async (faculty, since) =>
         },
         graduated: 1,
         student_studentnumber: { [Op.not]: null },
+        ...studyrightWhere,
       },
     })
   ).map(facultyFormatStudyright)
 
-const transferredInsideFaculty = async (programmes, since) =>
+const studyrightsByRightStartYear = async (faculty, since, graduated = 1) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+          },
+        },
+      ],
+      where: {
+        faculty_code: faculty,
+        startdate: {
+          [Op.gte]: since,
+        },
+        graduated: graduated,
+        student_studentnumber: { [Op.not]: null },
+      },
+    })
+  ).map(facultyFormatStudyright)
+
+const bachelorStudyright = async id => {
+  return await Studyright.findOne({
+    attributes: ['studystartdate'],
+    where: {
+      studyrightid: id,
+      extentcode: 1,
+    },
+  })
+}
+
+const hasMasterRight = async id => {
+  return await Studyright.findOne({
+    where: {
+      studyrightid: id,
+      extentcode: 2,
+    },
+  })
+}
+
+const transferredInsideFaculty = async (programmes, allProgrammeCodes, since) =>
   (
     await Transfer.findAll({
       where: {
         transferdate: {
           [Op.gte]: since,
         },
-        sourcecode: programmes,
+        sourcecode: allProgrammeCodes,
         targetcode: programmes,
       },
     })
   ).map(formatFacultyTransfer)
 
-const transferredAway = async (programmes, since) =>
+const transferredAway = async (programmes, allProgrammeCodes, since) =>
   (
     await Transfer.findAll({
       where: {
@@ -98,13 +166,13 @@ const transferredAway = async (programmes, since) =>
         },
         sourcecode: programmes,
         targetcode: {
-          [Op.notIn]: programmes,
+          [Op.notIn]: allProgrammeCodes,
         },
       },
     })
   ).map(formatFacultyTransfer)
 
-const transferredTo = async (programmes, since) =>
+const transferredTo = async (programmes, allProgrammeCodes, since) =>
   (
     await Transfer.findAll({
       where: {
@@ -112,7 +180,7 @@ const transferredTo = async (programmes, since) =>
           [Op.gte]: since,
         },
         sourcecode: {
-          [Op.notIn]: programmes,
+          [Op.notIn]: allProgrammeCodes,
         },
         targetcode: programmes,
       },
@@ -126,7 +194,9 @@ const degreeProgrammesOfFaculty = async facultyCode =>
       include: {
         model: Organization,
         where: {
-          code: facultyCode,
+          code: {
+            [Op.startsWith]: facultyCode,
+          },
         },
       },
     })
@@ -151,13 +221,13 @@ const getChildOrganizations = async facultyId =>
     })
   ).map(formatOrganization)
 
-const thesisWriters = async (providers, since, thesisTypes) =>
+const thesisWriters = async (provider, since, thesisTypes, students) =>
   (
     await Credit.findAll({
-      attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
+      attributes: ['id', 'course_code', 'attainment_date', 'student_studentnumber'],
       include: {
         model: Course,
-        attributes: ['code', 'course_unit_type'],
+        attributes: ['course_unit_type'],
         required: true,
         where: {
           course_unit_type: {
@@ -166,33 +236,52 @@ const thesisWriters = async (providers, since, thesisTypes) =>
         },
         include: {
           model: Organization,
-          attributes: ['code'],
+          attributes: [],
           required: true,
           where: {
-            code: {
-              [Op.in]: providers,
-            },
+            code: provider,
           },
         },
       },
       where: {
-        credittypecode: {
-          [Op.notIn]: [10, 9, 7],
-        },
+        credittypecode: 4,
         isStudyModule: {
           [Op.not]: true,
         },
         attainment_date: {
           [Op.gte]: since,
         },
-        student_studentnumber: { [Op.not]: null },
+        student_studentnumber: students.length > 0 ? students : { [Op.not]: null },
       },
     })
   ).map(formatFacultyThesisWriter)
 
+const statutoryAbsences = async (studentnumber, startdate, enddate) =>
+  (
+    await SemesterEnrollment.findAll({
+      attributes: ['semestercode'],
+      include: {
+        model: Semester,
+        attributes: ['startdate', 'enddate'],
+        where: {
+          startdate: {
+            [Op.gte]: startdate,
+          },
+          enddate: {
+            [Op.lte]: enddate,
+          },
+        },
+      },
+      where: {
+        studentnumber: studentnumber,
+        statutory_absence: true,
+      },
+    })
+  ).map(formatAbsence)
+
 // Some programme modules are not directly associated to a faculty (organization).
-// Some have intermediate organizations, such as department, so the connection must be diggeg up
-const findFacultyProgrammeCodes = async faculty => {
+// Some have intermediate organizations, such as department, so the connection must be digged up
+const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
   let allProgrammes = []
   let allProgrammeCodes = []
 
@@ -205,7 +294,7 @@ const findFacultyProgrammeCodes = async faculty => {
   const { id } = await facultyOgranizationId(faculty)
   const facultyChildOrganizations = await getChildOrganizations(id)
 
-  // get programme modules that have a found child as organization(_id)
+  // get programme modules that have a faculty child as organization(_id)
   for (const org of facultyChildOrganizations) {
     const childAssociationProgrammes = await degreeProgrammesOfFaculty(org.code)
     if (childAssociationProgrammes.length > 0) {
@@ -215,19 +304,48 @@ const findFacultyProgrammeCodes = async faculty => {
           allProgrammeCodes = allProgrammeCodes.concat([prog.code])
         }
       })
+    } else {
+      // dig deeper
+      const grandChildren = await getChildOrganizations(org.id)
+      if (grandChildren.length > 0) {
+        for (const gcOrg of grandChildren) {
+          const associatedProgrammes = await degreeProgrammesOfFaculty(gcOrg.code)
+          if (associatedProgrammes.length > 0) {
+            associatedProgrammes.forEach(prog => {
+              if (!(prog.code in allProgrammeCodes)) {
+                allProgrammes = allProgrammes.concat([prog])
+                allProgrammeCodes = allProgrammeCodes.concat([prog.code])
+              }
+            })
+          }
+        }
+      }
     }
   }
 
+  if (programmeFilter === 'NEW_STUDY_PROGRAMMES') {
+    allProgrammes = allProgrammes.filter(prog => isNewProgramme(prog.code))
+  }
+
+  mapCodesToIds(allProgrammes)
+
   return allProgrammes
 }
 
 module.exports = {
   startedStudyrights,
   graduatedStudyrights,
+  studyrightsByRightStartYear,
+  bachelorStudyright,
+  hasMasterRight,
   transferredInsideFaculty,
   transferredAway,
   transferredTo,
   degreeProgrammesOfFaculty,
   thesisWriters,
   findFacultyProgrammeCodes,
+  facultyOgranizationId,
+  statutoryAbsences,
+  getTransferredToAndAway,
+  getTransferredInside,
 }

services/backend/src/services/faculty/facultyBasics.js

@@ -0,0 +1,442 @@
+const { indexOf } = require('lodash')
+const {
+  startedStudyrights,
+  graduatedStudyrights,
+  transferredInsideFaculty,
+  transferredAway,
+  transferredTo,
+  getTransferredToAndAway,
+  getTransferredInside,
+} = require('./faculty')
+const { getStatsBasis, getYearsArray, defineYear } = require('../studyprogrammeHelpers')
+const { findRightProgramme, isNewProgramme, checkTransfers, getExtentFilter } = require('./facultyHelpers')
+const { codes } = require('../../../config/programmeCodes')
+
+const filterDuplicateStudyrights = studyrights => {
+  // bachelor+master students have two studyrights (separated by two last digits in studyrightid)
+  // choose only the earlier started one, so we don't count start of masters as starting in faculty
+  let rightsToCount = {}
+
+  studyrights.forEach(right => {
+    const id = right.studyrightid.slice(0, -2)
+    const start = new Date(right.studystartdate)
+    if (id in rightsToCount) {
+      if (new Date(rightsToCount[id].studystartdate) > start) {
+        rightsToCount[id] = right
+      }
+    } else {
+      rightsToCount[id] = right
+    }
+  })
+  return Object.values(rightsToCount)
+}
+
+const addProgramme = async (
+  stats,
+  programme,
+  tableStats,
+  transferYear,
+  index,
+  allBasics,
+  facultyProgrammes,
+  includeAllSpecials
+) => {
+  let { progId, name, code } = facultyProgrammes.filter(prog => prog.code === programme)[0]
+  if (!(progId in stats)) {
+    stats[progId] = {}
+    if (includeAllSpecials) {
+      Object.keys(tableStats).forEach(year => (stats[progId][year] = [0, 0, 0]))
+    } else {
+      Object.keys(tableStats).forEach(year => (stats[progId][year] = [0]))
+    }
+  }
+  stats[progId][transferYear][index] += 1
+  if (!(progId in allBasics.programmeNames)) {
+    allBasics.programmeNames[progId] = { ...name, code: code }
+  }
+}
+
+const getFacultyStarters = async (
+  faculty,
+  transfersToOrAwayStudyrights,
+  insideTransfersStudyrights,
+  since,
+  isAcademicYear,
+  yearsArray,
+  graphStats,
+  tableStats,
+  counts,
+  allBasics,
+  programmeData,
+  programmeFilter,
+  includeAllSpecials
+) => {
+  const startedGraphStats = [...graphStats]
+  const startedTableStats = { ...tableStats }
+  const studyrightWhere = getExtentFilter(includeAllSpecials)
+  const studyrights = await startedStudyrights(faculty, since, studyrightWhere)
+  let filteredStudyrights = filterDuplicateStudyrights(studyrights)
+  const keys = Object.keys(codes)
+  const programmeTableStats = {}
+  if (!includeAllSpecials) {
+    filteredStudyrights = filteredStudyrights.filter(
+      s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
+    )
+  }
+
+  filteredStudyrights.forEach(({ studystartdate, studyrightElements }) => {
+    const { programme, programmeName } = findRightProgramme(studyrightElements, 'started')
+    let programmeId = programme
+
+    if (keys.includes(programme)) {
+      programmeId = codes[programme].toUpperCase()
+    }
+
+    if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
+      const startYear = defineYear(studystartdate, isAcademicYear)
+
+      startedGraphStats[indexOf(yearsArray, startYear)] += 1
+      startedTableStats[startYear] += 1
+
+      if (!(programmeId in programmeTableStats)) {
+        programmeTableStats[programmeId] = { ...tableStats }
+      }
+      programmeTableStats[programmeId][startYear] += 1
+
+      if (!(programmeId in allBasics.programmeNames)) {
+        allBasics.programmeNames[programmeId] = { ...programmeName, code: programme }
+      }
+    }
+  })
+
+  allBasics.studentInfo.graphStats.push({ name: 'Started studying', data: startedGraphStats })
+  programmeData['started'] = programmeTableStats
+
+  Object.keys(startedTableStats).forEach(year => {
+    counts[year] = [startedTableStats[year]]
+  })
+}
+
+const getFacultyGraduates = async (
+  faculty,
+  transfersToOrAwayStudyrights,
+  insideTransfersStudyrights,
+  since,
+  isAcademicYear,
+  yearsArray,
+  graphStats,
+  tableStats,
+  allBasics,
+  counts,
+  countsGraduations,
+  programmeData,
+  programmeFilter,
+  includeAllSpecials
+) => {
+  const graduatedGraphStats = [[...graphStats], [...graphStats], [...graphStats], [...graphStats], [...graphStats]]
+  const graduatedTableStats = {}
+  const keys = Object.keys(codes)
+  let studyrightWhere = getExtentFilter(includeAllSpecials)
+  let graduatedRights = await graduatedStudyrights(faculty, since, studyrightWhere)
+  const programmeTableStats = {}
+
+  Object.keys(tableStats).forEach(year => (graduatedTableStats[year] = [0, 0, 0, 0, 0]))
+  if (!includeAllSpecials) {
+    graduatedRights = graduatedRights.filter(
+      s => !checkTransfers(s, insideTransfersStudyrights, transfersToOrAwayStudyrights)
+    )
+  }
+  graduatedRights.forEach(({ enddate, extentcode, studyrightElements }) => {
+    const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
+    let programmeId = programme
+
+    if (keys.includes(programme)) {
+      programmeId = codes[programme].toUpperCase()
+    }
+    if (programmeFilter === 'ALL_PROGRAMMES' || isNewProgramme(programme)) {
+      const endYear = defineYear(enddate, isAcademicYear)
+      graduatedGraphStats[0][indexOf(yearsArray, endYear)] += 1
+      graduatedTableStats[endYear][0] += 1
+
+      if (!(programmeId in programmeTableStats)) {
+        programmeTableStats[programmeId] = {}
+        Object.keys(tableStats).forEach(year => (programmeTableStats[programmeId][year] = [0, 0, 0, 0, 0]))
+      }
+      programmeTableStats[programmeId][endYear][0] += 1
+
+      if (extentcode === 1) {
+        graduatedGraphStats[1][indexOf(yearsArray, endYear)] += 1
+        graduatedTableStats[endYear][1] += 1
+        programmeTableStats[programmeId][endYear][1] += 1
+      } else if (extentcode === 2) {
+        graduatedGraphStats[2][indexOf(yearsArray, endYear)] += 1
+        graduatedTableStats[endYear][2] += 1
+        programmeTableStats[programmeId][endYear][2] += 1
+      } else if (extentcode === 4) {
+        graduatedGraphStats[3][indexOf(yearsArray, endYear)] += 1
+        graduatedTableStats[endYear][3] += 1
+        programmeTableStats[programmeId][endYear][3] += 1
+      } else {
+        graduatedGraphStats[4][indexOf(yearsArray, endYear)] += 1
+        graduatedTableStats[endYear][4] += 1
+        programmeTableStats[programmeId][endYear][4] += 1
+      }
+
+      if (!(programmeId in allBasics.programmeNames)) {
+        allBasics.programmeNames[programmeId] = { code: programme, ...programmeName }
+      }
+    }
+  })
+
+  allBasics.studentInfo.graphStats.push({ name: 'Graduated', data: graduatedGraphStats[0] })
+
+  allBasics.graduationInfo.graphStats.push({ name: 'Bachelors', data: graduatedGraphStats[1] })
+  allBasics.graduationInfo.graphStats.push({ name: 'Masters', data: graduatedGraphStats[2] })
+  allBasics.graduationInfo.graphStats.push({ name: 'Doctors', data: graduatedGraphStats[3] })
+  allBasics.graduationInfo.graphStats.push({ name: 'Others', data: graduatedGraphStats[4] })
+
+  programmeData['graduated'] = programmeTableStats
+
+  Object.keys(graduatedTableStats).forEach(year => {
+    counts[year] = counts[year].concat(graduatedTableStats[year][0])
+    countsGraduations[year] = graduatedTableStats[year]
+  })
+}
+
+const getFacultyTransfers = async (
+  programmes,
+  programmeCodes,
+  allProgrammeCodes,
+  since,
+  isAcademicYear,
+  yearsArray,
+  graphStats,
+  tableStats,
+  allBasics,
+  counts,
+  programmeData,
+  includeAllSpecials
+) => {
+  const insideTransfers = await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
+  // Filter still away those  that have started studying before 1.8.2017, when include all specials false.
+  let transferGraphStats = []
+  let transferTableStats = {}
+  if (includeAllSpecials) {
+    transferGraphStats = [[...graphStats], [...graphStats], [...graphStats]]
+    Object.keys(tableStats).forEach(year => (transferTableStats[year] = [0, 0, 0]))
+  } else {
+    transferGraphStats = [[...graphStats]]
+    Object.keys(tableStats).forEach(year => (transferTableStats[year] = [0]))
+  }
+
+  const programmeTableStats = {}
+  // inside transfer is counted towards target
+  for (const { transferdate, targetcode } of insideTransfers) {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    transferGraphStats[0][indexOf(yearsArray, transferYear)] += 1
+    transferTableStats[transferYear][0] += 1
+    await addProgramme(
+      programmeTableStats,
+      targetcode,
+      tableStats,
+      transferYear,
+      0,
+      allBasics,
+      programmes,
+      includeAllSpecials
+    )
+  }
+  if (includeAllSpecials) {
+    const awayTransfers = await transferredAway(programmeCodes, allProgrammeCodes, since)
+    const toTransfers = await transferredTo(programmeCodes, allProgrammeCodes, since)
+
+    for (const { transferdate, sourcecode } of awayTransfers) {
+      const transferYear = defineYear(transferdate, isAcademicYear)
+      transferGraphStats[1][indexOf(yearsArray, transferYear)] += 1
+      transferTableStats[transferYear][1] += 1
+      await addProgramme(
+        programmeTableStats,
+        sourcecode,
+        tableStats,
+        transferYear,
+        1,
+        allBasics,
+        programmes,
+        includeAllSpecials
+      )
+    }
+
+    for (const { transferdate, targetcode } of toTransfers) {
+      const transferYear = defineYear(transferdate, isAcademicYear)
+      transferGraphStats[2][indexOf(yearsArray, transferYear)] += 1
+      transferTableStats[transferYear][2] += 1
+      await addProgramme(
+        programmeTableStats,
+        targetcode,
+        tableStats,
+        transferYear,
+        2,
+        allBasics,
+        programmes,
+        includeAllSpecials
+      )
+    }
+  }
+
+  allBasics.studentInfo.graphStats.push({ name: 'Transferred inside', data: transferGraphStats[0] })
+  if (includeAllSpecials) {
+    allBasics.studentInfo.graphStats.push({ name: 'Transferred away', data: transferGraphStats[1] })
+    allBasics.studentInfo.graphStats.push({ name: 'Transferred to', data: transferGraphStats[2] })
+  }
+  programmeData['transferred'] = programmeTableStats
+
+  Object.keys(transferTableStats).forEach(year => {
+    counts[year] = counts[year].concat(transferTableStats[year])
+  })
+}
+
+const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeCodes, programmeFilter, special) => {
+  let counts = {}
+  let countsGraduations = {}
+  let years = []
+  let programmeData = {}
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = special === 'SPECIAL_INCLUDED'
+  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
+  const yearsArray = getYearsArray(since.getFullYear(), isAcademicYear)
+  const { graphStats, tableStats } = getStatsBasis(yearsArray)
+  Object.keys(tableStats).forEach(year => years.push(year))
+  const wantedProgrammeCodes = programmes.map(prog => prog.code)
+
+  let allBasics = {
+    id: faculty,
+    years: [],
+    programmeNames: {},
+    studentInfo: {
+      tableStats: [],
+      graphStats: [],
+      titles: includeAllSpecials
+        ? ['', 'Started studying', 'Graduated', 'Transferred inside', 'Transferred away', 'Transferred to']
+        : ['', 'Started studying', 'Graduated', 'Transferred inside'],
+      programmeTableStats: {},
+    },
+    graduationInfo: {
+      tableStats: [],
+      graphStats: [],
+      titles: ['', 'All', 'Bachelors', 'Masters', 'Doctors', 'Others'],
+      programmeTableStats: {},
+    },
+  }
+  const transfersToAwayStudyrights = await getTransferredToAndAway(wantedProgrammeCodes, allProgrammeCodes, since)
+  const transferInsideStudyrights = await getTransferredInside(wantedProgrammeCodes, allProgrammeCodes, since)
+  // Started studying in faculty
+  await getFacultyStarters(
+    faculty,
+    transfersToAwayStudyrights,
+    transferInsideStudyrights,
+    since,
+    isAcademicYear,
+    yearsArray,
+    graphStats,
+    tableStats,
+    counts,
+    allBasics,
+    programmeData,
+    programmeFilter,
+    includeAllSpecials
+  )
+
+  // Graduated in faculty
+  await getFacultyGraduates(
+    faculty,
+    transfersToAwayStudyrights,
+    transferInsideStudyrights,
+    since,
+    isAcademicYear,
+    yearsArray,
+    graphStats,
+    tableStats,
+    allBasics,
+    counts,
+    countsGraduations,
+    programmeData,
+    programmeFilter,
+    includeAllSpecials
+  )
+
+  // Transfers
+  await getFacultyTransfers(
+    programmes,
+    wantedProgrammeCodes,
+    allProgrammeCodes,
+    since,
+    isAcademicYear,
+    yearsArray,
+    graphStats,
+    tableStats,
+    allBasics,
+    counts,
+    programmeData,
+    includeAllSpecials
+  )
+
+  // combine tableStats from all categories
+  allBasics.years = years
+  years.forEach(year => {
+    allBasics.studentInfo.tableStats.push([year, ...counts[year]])
+    allBasics.graduationInfo.tableStats.push([year, ...countsGraduations[year]])
+  })
+  allBasics.studentInfo.tableStats.reverse()
+  allBasics.graduationInfo.tableStats.reverse()
+
+  //combine programme level tablestats
+  // all programmes are not present in all data types, check all found programmes and patch missing details with 0
+
+  let allCodes = Object.keys(programmeData.started).concat(
+    Object.keys(programmeData.graduated),
+    Object.keys(programmeData.transferred)
+  )
+  allCodes = [...new Set(allCodes)]
+
+  let studentInfo = {}
+  let graduationInfo = {}
+
+  allCodes.forEach(code => {
+    if (!(code in studentInfo)) studentInfo[code] = {}
+
+    for (const year of years) {
+      // started
+      if (code in programmeData.started) {
+        studentInfo[code][year] = [year, programmeData.started[code][year]]
+      } else {
+        studentInfo[code][year] = [year, 0]
+      }
+      // graduated total
+      if (code in programmeData.graduated) {
+        studentInfo[code][year] = studentInfo[code][year].concat([programmeData.graduated[code][year][0]])
+        if (!(code in graduationInfo)) graduationInfo[code] = {}
+        graduationInfo[code][year] = [year, ...programmeData.graduated[code][year]]
+      } else {
+        studentInfo[code][year] = studentInfo[code][year].concat([0])
+      }
+      // trasferred
+      if (code in programmeData.transferred) {
+        studentInfo[code][year] = studentInfo[code][year].concat(programmeData.transferred[code][year])
+      } else {
+        studentInfo[code][year] = studentInfo[code][year].concat([0, 0, 0])
+      }
+    }
+  })
+
+  Object.keys(studentInfo).forEach(prog => {
+    allBasics.studentInfo.programmeTableStats[prog] = [...Object.values(studentInfo[prog])].reverse()
+  })
+
+  Object.keys(graduationInfo).forEach(prog => {
+    allBasics.graduationInfo.programmeTableStats[prog] = Object.values(graduationInfo[prog]).reverse()
+  })
+
+  return allBasics
+}
+
+module.exports = { combineFacultyBasics }

services/backend/src/services/faculty/facultyCredits.js

@@ -1,5 +1,5 @@
 const { indexOf } = require('lodash')
-const { mapToProviders } = require('../util/utils')
+const { mapToProviders } = require('../../util/utils')
 const {
   getStatsBasis,
   defineYear,
@@ -8,14 +8,13 @@ const {
   getYearsArray,
   getStartDate,
   isSpecialGroupCredit,
-  tableTitles,
-} = require('./studyprogrammeHelpers')
+} = require('../studyprogrammeHelpers')
 const {
   getStudyRights,
   getCreditsForStudyProgramme,
   allTransfers,
   getCourseCodesForStudyProgramme,
-} = require('./studyprogramme')
+} = require('../studyprogramme')
 
 const isFacultyNonMajorCredit = (studyrights, attainment_date, facultyProgrammes) => {
   let right = ''
@@ -56,8 +55,8 @@ const getFacultyRegularCreditStats = async ({
   const credits = await getCreditsForStudyProgramme(courses, since)
   const students = [...new Set(credits.map(({ student_studentnumber }) => student_studentnumber))]
 
-  let studyrights = await getStudyRights(students, since)
-  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+  let studyrights = await getStudyRights(students)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid).filter(s => s !== 'undefined')
   if (!includeAllSpecials) {
     studyrights = studyrights.filter(s => !transfers.includes(s.studyrightid))
   }
@@ -126,7 +125,6 @@ const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgr
   const { majors, facultyNonMajors, otherNonMajors, nonDegree } = await getFacultyRegularCreditStats(queryParameters)
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const titles = tableTitles['credits'][includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED']
   const dataFound = [majors, facultyNonMajors, otherNonMajors].some(d => d.graphStats.length) // , transferred
 
   if (!dataFound) return null
@@ -144,48 +142,44 @@ const getFacultyCreditStatsForStudytrack = async ({ studyprogramme, facultyProgr
           otherNonMajors.tableStats[year],
           nonDegree.tableStats[year],
         ]
-      : [
-          year,
-          majors.tableStats[year] +
-            facultyNonMajors.tableStats[year] +
-            otherNonMajors.tableStats[year] +
-            nonDegree.tableStats[year],
-          majors.tableStats[year],
-        ]
+      : [year, majors.tableStats[year], majors.tableStats[year]]
   )
 
-  const graphStats = [
-    {
-      name: 'Major students credits',
-      data: majors.graphStats,
-    },
-    {
-      name: 'Non-major faculty students credits',
-      data: facultyNonMajors.graphStats,
-    },
-    {
-      name: 'Non-major other faculty students credits 2',
-      data: otherNonMajors.graphStats,
-    },
-    {
-      name: 'Non-degree credits',
-      data: nonDegree.graphStats,
-    },
-  ]
+  const graphStats = includeAllSpecials
+    ? [
+        {
+          name: 'Major students credits',
+          data: majors.graphStats,
+        },
+        {
+          name: 'Non-major faculty students credits',
+          data: facultyNonMajors.graphStats,
+        },
+        {
+          name: 'Non-major other faculty students credits 2',
+          data: otherNonMajors.graphStats,
+        },
+        {
+          name: 'Non-degree credits',
+          data: nonDegree.graphStats,
+        },
+      ]
+    : [
+        {
+          name: 'Major students credits',
+          data: majors.graphStats,
+        },
+      ]
 
   return {
     id: studyprogramme,
     years,
     tableStats,
     graphStats,
-    titles,
   }
 }
 
 const getProgrammeCredits = async (code, yearType, specialGroups, facultyProgrammes) => {
-  // redis for faculty? Create faculty versions for out-commented functions
-  // const data = await getCreditStats(code, yearType, specialGroups)
-  // if (data) return data
   const updatedStats = await getFacultyCreditStatsForStudytrack({
     studyprogramme: code,
     facultyProgrammes,
@@ -195,17 +189,40 @@ const getProgrammeCredits = async (code, yearType, specialGroups, facultyProgram
     },
   })
 
-  // if (updatedStats) await setCreditStats(updatedStats, yearType, specialGroups)
   return updatedStats
 }
 
-const combineFacultyCredits = async (allCredits, programmes, yearType, specialGroups, counts, years) => {
-  const progCodes = programmes.map(p => p.code)
+const combineFacultyCredits = async (faculty, programmes, allProgrammes, yearType, specialGroups) => {
+  let counts = {}
+  let years = []
+  const titles =
+    specialGroups === 'SPECIAL_INCLUDED'
+      ? [
+          '',
+          'Total',
+          'Major students credits',
+          'Non-major faculty students credits',
+          'Non-major other faculty students credits',
+          'Non-degree student credits',
+        ]
+      : ['', 'Total', 'Major students credits']
+
+  let allCredits = {
+    id: faculty,
+    years: [],
+    tableStats: [],
+    graphStats: [],
+    programmeTableStats: {},
+    programmeNames: {},
+    titles,
+  }
+
+  const progCodes = allProgrammes.map(p => p.code)
   for (const prog of programmes) {
     const data = await getProgrammeCredits(prog.code, yearType, specialGroups, progCodes)
     if (data) {
-      allCredits.programmeNames[prog.code] = prog.name
-      allCredits.programmeTableStats[prog.code] = data.tableStats
+      allCredits.programmeNames[prog.progId] = { code: prog.code, ...prog.name }
+      allCredits.programmeTableStats[prog.progId] = data.tableStats
       if (!allCredits.lastUpdated || new Date(data.lastUpdated) > new Date(allCredits.lastUpdated))
         allCredits.lastUpdated = data.lastUpdated
 
@@ -221,6 +238,7 @@ const combineFacultyCredits = async (allCredits, programmes, yearType, specialGr
       })
     }
   }
+
   // save table stats and graph stats
   years.forEach(year => {
     allCredits.tableStats.push([year, ...counts[year]])
@@ -228,6 +246,30 @@ const combineFacultyCredits = async (allCredits, programmes, yearType, specialGr
 
   years.sort()
   allCredits.years = years
+
+  let majors = []
+  let facultyNonMajor = []
+  let otherNonMajor = []
+  let nonDegree = []
+
+  years.forEach(year => {
+    majors.push(counts[year][1])
+    facultyNonMajor.push(counts[year][2])
+    otherNonMajor.push(counts[year][3])
+    nonDegree.push(counts[year][4])
+  })
+
+  allCredits.graphStats =
+    specialGroups === 'SPECIAL_INCLUDED'
+      ? [
+          { name: 'Major students credits', data: majors },
+          { name: 'Non-major faculty students credits', data: facultyNonMajor },
+          { name: 'Non-major other faculty students credits', data: otherNonMajor },
+          { name: 'Non-degree credits', data: nonDegree },
+        ]
+      : [{ name: 'Major students credits', data: majors }]
+
+  return allCredits
 }
 
 module.exports = { combineFacultyCredits, getFacultyCreditStatsForStudytrack }

services/backend/src/services/faculty/facultyGraduationTimes.js

@@ -0,0 +1,389 @@
+const moment = require('moment')
+const {
+  graduatedStudyrights,
+  studyrightsByRightStartYear,
+  bachelorStudyright,
+  hasMasterRight,
+  statutoryAbsences,
+} = require('./faculty')
+const { findRightProgramme, isNewProgramme } = require('./facultyHelpers')
+const { getYearsArray, getYearsObject, getMean, getMedian, defineYear } = require('../studyprogrammeHelpers')
+const { codes } = require('../../../config/programmeCodes')
+
+const sortProgrammes = data => {
+  const check = name => {
+    if (isNaN(name[0])) return -1
+    return 1
+  }
+  data.sort((a, b) => {
+    if (check(a.name) === check(b.name)) return a.name.localeCompare(b.name)
+    return check(a.name) - check(b.name)
+  })
+}
+
+const getSortedProgIds = progs => {
+  let programmes = []
+
+  for (const prog of progs) {
+    if (Object.keys(codes).includes(prog)) {
+      programmes = [...programmes, { name: codes[prog].toUpperCase(), code: prog }]
+    } else {
+      programmes = [...programmes, { name: prog, code: prog }]
+    }
+  }
+  sortProgrammes(programmes)
+  return programmes
+}
+
+const findBachelorStartdate = async id => {
+  const studyright = await bachelorStudyright(id)
+  if (studyright) return studyright.studystartdate
+  return null
+}
+
+const getProgrammeObjectBasis = (years, levels, emptyObject = true) => {
+  return levels.reduce(
+    (acc, level) => ({ ...acc, [level]: years.reduce((acc, year) => ({ ...acc, [year]: emptyObject ? {} : 0 }), {}) }),
+    {}
+  )
+}
+
+const getStatutoryAbsences = async (studentnumber, startdate, enddate) => {
+  const absences = await statutoryAbsences(studentnumber, startdate, enddate)
+  if (absences.length) {
+    const absentMonths = absences.reduce((sum, ab) => sum + moment(ab.end).diff(moment(ab.start), 'months'), 0)
+    return absentMonths
+  }
+  return 0
+}
+
+const countTimeCategories = (times, goal) => {
+  const statistics = { onTime: 0, yearOver: 0, wayOver: 0 }
+  times.forEach(time => {
+    if (time <= goal) statistics.onTime += 1
+    else if (time <= goal + 12) statistics.yearOver += 1
+    else statistics.wayOver += 1
+  })
+  return statistics
+}
+
+const hasMS = async (programme, elements, studyrightid) => {
+  // Only bachelor studyright: socCom, kasvatustiede - varhaiskasvatus track, farmasia - applied only to Bc = farmaseutti,
+  if (['KH74_001', '620050-ba', '620030-ba', '520091-ba'].includes(programme)) {
+    return false
+  }
+  if (programme === 'KH60_001') {
+    return !elements.some(el => ['EDUK-VO', '0371', '620050-ba', '620030-ba'].includes(el.dataValues.code))
+  }
+  if (programme === 'KH55_001') {
+    const result = await hasMasterRight(studyrightid.replace(/-1$/, '-2'))
+    if (!result) {
+      return false
+    }
+  }
+  return true
+}
+
+const addGraduation = async (
+  extentcode,
+  startdate,
+  studyrightid,
+  enddate,
+  studentnumber,
+  graduationAmounts,
+  graduationTimes,
+  year,
+  programmes,
+  programme
+) => {
+  let actualStartdate = startdate
+  let level = null
+
+  if (programme === 'MH30_001' || programme === 'MH30_003') {
+    level = 'bcMsCombo'
+  } else if (extentcode === 1) {
+    level = 'bachelor'
+  } else if (extentcode === 2) {
+    if (studyrightid.slice(-2) === '-2') {
+      level = 'bcMsCombo'
+      actualStartdate = await findBachelorStartdate(studyrightid.replace(/-2$/, '-1'))
+    } else {
+      level = 'master'
+    }
+  } else if (extentcode === 4) {
+    level = 'doctor'
+  } else if (extentcode === 3) {
+    level = 'licentiate'
+  } else {
+    return
+  }
+
+  const absoluteTimeToGraduation = moment(enddate).diff(moment(actualStartdate), 'months')
+  const statutoryAbsences = await getStatutoryAbsences(studentnumber, actualStartdate, enddate)
+  const timeToGraduation = absoluteTimeToGraduation - statutoryAbsences
+
+  graduationAmounts[level][year] += 1
+  graduationTimes[level][year] = [...graduationTimes[level][year], timeToGraduation]
+
+  if (!(programme in programmes[level][year]))
+    programmes[level][year][programme] = { graduationTimes: [], graduationAmounts: 0 }
+
+  programmes[level][year][programme].graduationAmounts += 1
+  programmes[level][year][programme].graduationTimes = [
+    ...programmes[level][year][programme].graduationTimes,
+    timeToGraduation,
+  ]
+}
+
+const getClassSizes = async (faculty, since, classSizes, programmeFilter, years) => {
+  const studyrights = await studyrightsByRightStartYear(faculty, since, [0, 1])
+
+  // get all received studyrights for each year
+  // a transferred student is counted into new programmes class size i.e.
+  // students who received studyright on year X and graduated/will be graduating from programme Y
+  // if we only counted those who started fresh in the new programme we could get a bigger number of graduates
+  // than the class size, as the graduatation time count only looks at a degree as whole studyright
+  for (const right of studyrights) {
+    const { startdate, studyrightid, extentcode, studyrightElements } = right
+    const { programme } = findRightProgramme(studyrightElements, 'graduated')
+    if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
+
+    let actualStartdate = startdate
+    let level = null
+
+    if (programme === 'MH30_001' || programme === 'MH30_003') {
+      level = 'bcMsCombo'
+    } else if (extentcode === 1) {
+      level = 'bachelor'
+    } else if (extentcode === 2) {
+      if (studyrightid.slice(-2) === '-2') {
+        level = 'bcMsCombo'
+        actualStartdate = await findBachelorStartdate(studyrightid.replace(/-2$/, '-1'))
+      } else {
+        level = 'master'
+      }
+    } else if (extentcode === 4) {
+      level = 'doctor'
+    } else if (extentcode === 3) {
+      level = 'licentiate'
+    } else {
+      continue
+    }
+
+    const startYear = defineYear(actualStartdate, false)
+
+    // On faculty level, count started bachelor rights to BcMsCombo as well  --> useful class size even if master studies have yet not started
+    // On programme level BcMsCombo sizes are iffy as many students haven't started masters yet --> no Ms programme element to count
+    if (level !== 'bcMsCombo') {
+      classSizes[level][startYear] += 1
+      if (level === 'bachelor' && (await hasMS(programme, studyrightElements, studyrightid))) {
+        classSizes['bcMsCombo'][startYear] += 1
+      }
+    }
+
+    if (!(programme in classSizes.programmes)) {
+      if (extentcode === 2) {
+        classSizes.programmes[programme] = {}
+        classSizes.programmes[programme]['bcMsCombo'] = getYearsObject({ years, emptyArrays: false })
+        classSizes.programmes[programme]['master'] = getYearsObject({ years, emptyArrays: false })
+      } else {
+        classSizes.programmes[programme] = getYearsObject({ years, emptyArrays: false })
+      }
+    }
+
+    if (extentcode === 2) {
+      classSizes.programmes[programme][level][startYear] += 1
+    } else {
+      classSizes.programmes[programme][startYear] += 1
+    }
+  }
+}
+
+const count = async (faculty, since, years, yearsList, levels, programmeFilter, programmeNames, goals, mode) => {
+  let graduationAmounts = {}
+  let graduationTimes = {}
+  let programmes = getProgrammeObjectBasis(yearsList, levels)
+  let data = {
+    means: {},
+    medians: {},
+    programmes: {
+      medians: getProgrammeObjectBasis(yearsList, levels),
+      means: getProgrammeObjectBasis(yearsList, levels),
+    },
+  }
+
+  levels.forEach(level => {
+    graduationAmounts[level] = getYearsObject({ years: yearsList })
+    graduationTimes[level] = getYearsObject({ years: yearsList, emptyArrays: true })
+    data.medians[level] = []
+    data.means[level] = []
+  })
+
+  let studyrights = null
+  if (mode === 'gradYear') {
+    studyrights = await graduatedStudyrights(faculty, since)
+  } else {
+    studyrights = await studyrightsByRightStartYear(faculty, since)
+  }
+
+  for (const right of studyrights) {
+    const { enddate, startdate, studyrightid, extentcode, studyrightElements, studentnumber } = right
+    const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
+    if (programmeFilter === 'NEW_STUDY_PROGRAMMES' && !isNewProgramme(programme)) continue
+
+    if (!(programme in programmeNames)) {
+      programmeNames[programme] = programmeName
+    }
+
+    let year = null
+    if (mode === 'gradYear') {
+      year = defineYear(enddate, false)
+    } else {
+      year = defineYear(startdate, false)
+    }
+
+    await addGraduation(
+      extentcode,
+      startdate,
+      studyrightid,
+      enddate,
+      studentnumber,
+      graduationAmounts,
+      graduationTimes,
+      year,
+      programmes,
+      programme
+    )
+  }
+
+  years.forEach(year => {
+    levels.forEach(level => {
+      const median = getMedian(graduationTimes[level][year])
+      const mean = getMean(graduationTimes[level][year])
+      const statistics = { onTime: 0, yearOver: 0, wayOver: 0 }
+
+      // Programme level breakdown
+      data.programmes.medians[level][year] = { programmes: [], data: [] }
+      data.programmes.means[level][year] = { programmes: [], data: [] }
+      const programmeCodes = Object.keys(programmes[level][year])
+      const programmeIds = getSortedProgIds(programmeCodes)
+
+      if (programmeIds.length > 0) {
+        for (const prog of programmeIds) {
+          const progMedian = getMedian(programmes[level][year][prog.code].graduationTimes)
+          const progMean = getMean(programmes[level][year][prog.code].graduationTimes)
+
+          let goal = goals[level]
+          if (faculty === 'H30' && Object.keys(goals.exceptions).includes(prog.code)) {
+            goal += goals.exceptions[prog.code]
+          }
+
+          const progStatistics = countTimeCategories(programmes[level][year][prog.code].graduationTimes, goal)
+          Object.keys(progStatistics).forEach(key => (statistics[key] += progStatistics[key]))
+
+          data.programmes.medians[level][year].programmes = [
+            ...data.programmes.medians[level][year].programmes,
+            prog.code,
+          ]
+          data.programmes.medians[level][year].data = [
+            ...data.programmes.medians[level][year].data,
+            {
+              y: progMedian,
+              amount: programmes[level][year][prog.code].graduationAmounts,
+              statistics: progStatistics,
+              name: prog.name,
+              code: prog.code,
+            },
+          ]
+
+          data.programmes.means[level][year].programmes = [...data.programmes.means[level][year].programmes, prog.code]
+          data.programmes.means[level][year].data = [
+            ...data.programmes.means[level][year].data,
+            {
+              y: progMean,
+              amount: programmes[level][year][prog.code].graduationAmounts,
+              statistics: progStatistics,
+              name: prog.name,
+              code: prog.code,
+            },
+          ]
+        }
+      }
+      data.medians[level] = [
+        ...data.medians[level],
+        { y: median, amount: graduationAmounts[level][year], statistics, name: year },
+      ]
+      data.means[level] = [
+        ...data.means[level],
+        { y: mean, amount: graduationAmounts[level][year], statistics, name: year },
+      ]
+    })
+  })
+
+  return data
+}
+
+const countGraduationTimes = async (faculty, programmeFilter) => {
+  const isAcademicYear = false // = yearType === 'ACADEMIC_YEAR'
+  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
+  const yearsList = getYearsArray(since.getFullYear(), isAcademicYear)
+  const levels = ['bachelor', 'bcMsCombo', 'master', 'doctor', 'licentiate']
+  const programmeNames = {}
+
+  const years = [...yearsList].reverse()
+  const goals = {
+    bachelor: 36,
+    bcMsCombo: 60,
+    master: 24,
+    doctor: 48,
+    licentiate: 78,
+    exceptions: {
+      MH30_004: 6, //months more
+      '420420-ma': 6,
+      MH30_001: 12,
+      '320011-ma': 12,
+      '320001-ma': 12,
+      MH30_003: 12,
+      '320002-ma': 12,
+    },
+  }
+
+  if (faculty === 'H90') {
+    goals.bcMsCombo += 12
+  }
+
+  // We count studyrights (vs. studyright_elements)
+  // This way we get the whole time for a degree, even if the student was transferred to a new programme
+  // E.g. started 8/2016 in old Bc, transferred to new 10/2020, graduated from new 1/2021 --> total 53 months (not 3)
+
+  const byGradYear = await count(
+    faculty,
+    since,
+    years,
+    yearsList,
+    levels,
+    programmeFilter,
+    programmeNames,
+    goals,
+    'gradYear'
+  )
+  const byStartYear = await count(
+    faculty,
+    since,
+    years,
+    yearsList,
+    levels,
+    programmeFilter,
+    programmeNames,
+    goals,
+    'startYear'
+  )
+
+  const classSizes = getProgrammeObjectBasis(yearsList, levels, false)
+  classSizes['programmes'] = {}
+  await getClassSizes(faculty, since, classSizes, programmeFilter, years)
+
+  return { id: faculty, goals, byGradYear, byStartYear, programmeNames, classSizes }
+}
+
+module.exports = { countGraduationTimes }

services/backend/src/services/faculty/facultyHelpers.js

@@ -1,3 +1,7 @@
+const Sequelize = require('sequelize')
+const { Op } = Sequelize
+const { codes } = require('../../../config/programmeCodes')
+
 const findRightProgramme = (studyrightElements, mode) => {
   let programme = ''
   let programmeName = ''
@@ -35,11 +39,13 @@ const facultyFormatStudyright = studyright => {
     extentcode,
     student,
     studyright_elements,
+    startdate,
   } = studyright
 
   return {
     studyrightid,
     studystartdate,
+    startdate,
     enddate,
     givendate,
     graduated,
@@ -74,7 +80,6 @@ const formatFacultyThesisWriter = credit => {
     attainment_date,
     student_studentnumber,
     courseUnitType: course.course_unit_type,
-    organizations: course.organizations.map(organization => organization.code),
   }
 }
 
@@ -83,6 +88,54 @@ const formatOrganization = org => {
   return { id, name, code, parentId: parent_id }
 }
 
+const formatAbsence = absence => {
+  const { semestercode, semester } = absence
+  return { semestercode, start: semester.startdate, end: semester.enddate }
+}
+
+const newProgrammes = [/^KH/, /^MH/, /^T/, /^LI/, /^K-/, /^FI/, /^00901$/, /^00910$/]
+
+const isNewProgramme = code => {
+  for (let i = 0; i < newProgrammes.length; i++) {
+    if (newProgrammes[i].test(code)) {
+      return true
+    }
+  }
+  return false
+}
+
+const checkTransfers = (s, insideTransfersStudyrights, transfersToOrAwayStudyrights) => {
+  if (transfersToOrAwayStudyrights.includes(s.studyrightid)) return true
+  if (insideTransfersStudyrights.includes(s.studyrightid) && new Date(s.studystartdate) < new Date('2017-08-01'))
+    return true
+  return false
+}
+
+const getExtentFilter = includeAllSpecials => {
+  const filteredExtents = [16] // always filter out secondary subject students
+  if (!includeAllSpecials) {
+    filteredExtents.push(7, 34, 22, 99, 14, 13, 9)
+  }
+  let studyrightWhere = {
+    extentcode: {
+      [Op.notIn]: filteredExtents,
+    },
+  }
+  return studyrightWhere
+}
+
+const mapCodesToIds = data => {
+  // Add programme id e.g. TKT
+  const keys = Object.keys(codes)
+  for (const prog of data) {
+    if (keys.includes(prog.code)) {
+      prog.progId = codes[prog.code].toUpperCase()
+    } else {
+      prog.progId = prog.code
+    }
+  }
+}
+
 module.exports = {
   findRightProgramme,
   facultyFormatStudyright,
@@ -90,4 +143,9 @@ module.exports = {
   formatFacultyTransfer,
   formatFacultyThesisWriter,
   formatOrganization,
+  formatAbsence,
+  isNewProgramme,
+  checkTransfers,
+  getExtentFilter,
+  mapCodesToIds,
 }

services/backend/src/services/faculty/facultyService.js

@@ -0,0 +1,171 @@
+const { redisClient } = require('../redis')
+const moment = require('moment')
+
+const createRedisKeyForFacultyProgrammes = (id, programmeFilter) => `FACULTY_PROGRAMMES_${id}_${programmeFilter}`
+const createRedisKeyForBasicStats = (id, yearType, programmeFilter, specialGroups) =>
+  `FACULTY_BASIC_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, programmeFilter, specialGroups) =>
+  `FACULTY_CREDIT_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
+const createRedisKeyForThesiswriters = (id, yearType, programmeFilter, specialGroups) =>
+  `FACULTY_THESIS_WRITERS_STATS_${id}_${yearType}_${programmeFilter}_${specialGroups}`
+const createRedisKeyForGraduationTimeStats = (id, programmeFilter) =>
+  `FACULTY_GRADUATION_TIME_STATS_${id}_${programmeFilter}`
+const createRediskeyForFacultyProgress = (id, special_groups, graduated) =>
+  `FACULTY_PROGRESS_STATS_${id}_${special_groups}_${graduated}`
+const createRediskeyForFacultyStudents = (id, special_groups, graduated) =>
+  `FACULTY_STUDENTS_STATS_${id}_${special_groups}_${graduated}`
+
+const setFacultyProgrammes = async (id, data, programmeFilter) => {
+  const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
+  const dataToRedis = {
+    data: data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getFacultyProgrammes = async (id, programmeFilter) => {
+  const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setBasicStats = async (data, yearType, programmeFilter, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getBasicStats = async (id, yearType, programmeFilter, specialGroups) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setCreditStats = async (data, yearType, programmeFilter, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getCreditStats = async (id, yearType, programmeFilter, specialGroups) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType, programmeFilter, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setThesisWritersStats = async (data, yearType, programmeFilter, specialGroups) => {
+  const { id } = data
+  const redisKey = createRedisKeyForThesiswriters(id, yearType, programmeFilter, specialGroups)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+const getThesisWritersStats = async (id, yearType, programmeFilter, specialGroups) => {
+  const redisKey = createRedisKeyForThesiswriters(id, yearType, programmeFilter, specialGroups)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setGraduationStats = async (data, programmeFilter) => {
+  const { id } = data
+  const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getGraduationStats = async (id, programmeFilter) => {
+  const redisKey = createRedisKeyForGraduationTimeStats(id, programmeFilter)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const getFacultyProgressStats = async (id, specialGroups, graduated) => {
+  const redisKey = createRediskeyForFacultyProgress(id, specialGroups, graduated)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setFacultyProgressStats = async (data, specialGroups, graduated) => {
+  const { id } = data
+  const redisKey = createRediskeyForFacultyProgress(id, specialGroups, graduated)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+const getFacultyStudentStats = async (id, specialGroups, graduated) => {
+  const redisKey = createRediskeyForFacultyStudents(id, specialGroups, graduated)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
+const setFacultyStudentStats = async (data, specialGroups, graduated) => {
+  const { id } = data
+  const redisKey = createRediskeyForFacultyStudents(id, specialGroups, graduated)
+  const dataToRedis = {
+    ...data,
+    status: 'DONE',
+    lastUpdated: moment().format(),
+  }
+  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
+  if (setOperationStatus !== 'OK') return null
+  return dataToRedis
+}
+
+module.exports = {
+  setFacultyProgrammes,
+  getFacultyProgrammes,
+  setBasicStats,
+  getBasicStats,
+  setCreditStats,
+  getCreditStats,
+  setThesisWritersStats,
+  getThesisWritersStats,
+  setGraduationStats,
+  getGraduationStats,
+  getFacultyProgressStats,
+  setFacultyProgressStats,
+  getFacultyStudentStats,
+  setFacultyStudentStats,
+}

services/backend/src/services/faculty/facultyStudentProgress.js

@@ -0,0 +1,323 @@
+const { indexOf } = require('lodash')
+const moment = require('moment')
+const { getAcademicYearDates } = require('../../util/semester')
+const {
+  getCreditThresholds,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
+  getDoctoralCreditGraphStats,
+  getYearsArray,
+  getCorrectStudentnumbers,
+  tableTitles,
+  getOnlyMasterCreditGraphStats,
+  getOnlyMasterThresholds,
+} = require('../studyprogrammeHelpers')
+const { studytrackStudents, allStudyrights } = require('../studyprogramme')
+
+const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts, limits, limitKeys, prog) => {
+  let data = {}
+  let programmeData = {}
+  thresholdKeys.forEach(t => (data[t] = 0))
+  limitKeys.forEach(t => (programmeData[t] = 0))
+  students.forEach(({ credits }) => {
+    const creditcount = credits
+      .filter(credit => moment(credit.attainment_date).isAfter(startDate))
+      .reduce((prev, curr) => (prev += curr.credits), 0)
+
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+    if (thresholdKeys.length === 8) {
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] && creditcount < thresholdAmounts[6] ? 1 : 0
+      data[thresholdKeys[7]] += creditcount >= thresholdAmounts[6] ? 1 : 0
+    } else if (thresholdKeys.length === 7) {
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] ? 1 : 0
+    } else {
+      data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] ? 1 : 0
+    }
+    if (['KH', 'MH'].includes(prog)) {
+      programmeData[limitKeys[0]] += creditcount <= limits[5][0] ? 1 : 0
+      programmeData[limitKeys[1]] += limits[4][0] <= creditcount && limits[4][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[2]] += limits[3][0] <= creditcount && limits[3][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[3]] += limits[2][0] <= creditcount && limits[2][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[4]] += limits[1][0] <= creditcount && limits[1][1] > creditcount ? 1 : 0
+      programmeData[limitKeys[5]] += creditcount >= limits[0][0] ? 1 : 0
+    }
+  })
+
+  return [data, programmeData]
+}
+
+const combineFacultyStudentProgress = async (faculty, programmes, specialGroups, graduated) => {
+  const since = new Date('2017-08-01')
+  const isAcademicYear = true
+  const includeYearsCombined = true
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
+  const yearsArray = getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined)
+
+  let allBachelorGraphStats = getBachelorCreditGraphStats(yearsArray)
+  let allBsMsGraphstats = getMasterCreditGraphStats(yearsArray)
+  let allMastersGraphStats = getOnlyMasterCreditGraphStats(yearsArray)
+  let allDoctoralGraphStats = getDoctoralCreditGraphStats(yearsArray)
+
+  let bachelorsProgrammeStats = {}
+  let bcMsProgrammeStats = {}
+  let mastersProgrammeStats = {}
+  let doctoralProgrammeStats = {}
+
+  let bachelorsTableStats = new Array(yearsArray.length)
+  let bachelorMastersTableStats = new Array(yearsArray.length)
+  let mastersTableStats = new Array(yearsArray.length)
+  let doctoralTableStats = new Array(yearsArray.length)
+
+  let yearlyBachelorTitles = []
+  let yearlyBcMsTitles = []
+  let yearlyMasterTitles = []
+
+  const reversedYears = [...yearsArray].reverse()
+  reversedYears.forEach(year => {
+    bachelorsTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    bachelorMastersTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0, 0]
+    mastersTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0]
+    doctoralTableStats[indexOf(reversedYears, year)] = [year, 0, 0, 0, 0, 0, 0, 0, 0]
+  })
+  const limitKeys = ['zero', 't5', 't4', 't3', 't2', 't1']
+  let bachelorlimits = []
+  let masterlimits = []
+  let bcmslimits = []
+  for (const year of reversedYears) {
+    const { startDate, endDate } = getAcademicYearDates(year, since)
+    const today = moment()
+    const lastDayOfMonth = moment(today).endOf('month')
+    let months = Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(startDate))).asMonths())
+
+    if (months >= 36) {
+      bachelorlimits = [
+        [Math.ceil(36 * (60 / 12)), null],
+        [Math.ceil(36 * (45 / 12)), Math.ceil(36 * (60 / 12))],
+        [Math.ceil(36 * (30 / 12)), Math.ceil(36 * (45 / 12))],
+        [Math.ceil(36 * (15 / 12)), Math.ceil(36 * (30 / 12))],
+        [1, Math.ceil(36 * (15 / 12))],
+        [0, null],
+      ]
+    } else {
+      bachelorlimits = [
+        [Math.ceil(months * (60 / 12)), null],
+        [Math.ceil(months * (45 / 12)), Math.ceil(months * (60 / 12))],
+        [Math.ceil(months * (30 / 12)), Math.ceil(months * (45 / 12))],
+        [Math.ceil(months * (15 / 12)), Math.ceil(months * (30 / 12))],
+        [1, Math.ceil(months * (15 / 12))],
+        [0, null],
+      ]
+    }
+    if (months >= 24) {
+      masterlimits = [
+        [Math.ceil(24 * (60 / 12)), null],
+        [Math.ceil(24 * (45 / 12)), Math.ceil(24 * (60 / 12))],
+        [Math.ceil(24 * (30 / 12)), Math.ceil(24 * (45 / 12))],
+        [Math.ceil(24 * (15 / 12)), Math.ceil(24 * (30 / 12))],
+        [1, Math.ceil(20 * (15 / 12))],
+        [0, null],
+      ]
+    } else {
+      masterlimits = [
+        [Math.ceil(months * (60 / 12)), null],
+        [Math.ceil(months * (45 / 12)), Math.ceil(months * (60 / 12))],
+        [Math.ceil(months * (30 / 12)), Math.ceil(months * (45 / 12))],
+        [Math.ceil(months * (15 / 12)), Math.ceil(months * (30 / 12))],
+        [1, Math.ceil(months * (15 / 12))],
+        [0, null],
+      ]
+    }
+    if (months >= 60) {
+      bcmslimits = [
+        [Math.ceil(60 * (60 / 12)) + 180, null],
+        [Math.ceil(60 * (45 / 12)) + 180, Math.ceil(60 * (60 / 12)) + 180],
+        [Math.ceil(60 * (30 / 12)) + 180, Math.ceil(60 * (45 / 12)) + 180],
+        [Math.ceil(60 * (15 / 12)) + 180, Math.ceil(60 * (30 / 12)) + 180],
+        [181, Math.ceil(60 * (15 / 12)) + 180],
+        [180, null],
+      ]
+    } else {
+      bcmslimits = [
+        [Math.ceil(months * (60 / 12)) + 180, null],
+        [Math.ceil(months * (45 / 12)) + 180, Math.ceil(months * (60 / 12)) + 180],
+        [Math.ceil(months * (30 / 12)) + 180, Math.ceil(months * (45 / 12)) + 180],
+        [Math.ceil(months * (15 / 12)) + 180, Math.ceil(months * (30 / 12)) + 180],
+        [181, Math.ceil(months * (15 / 12)) + 180],
+        [180, null],
+      ]
+    }
+
+    yearlyBachelorTitles.push([
+      '0 Credits',
+      `${bachelorlimits[4][0]} ≤ Credits < ${bachelorlimits[4][1]}`,
+      `${bachelorlimits[3][0]} ≤ Credits < ${bachelorlimits[3][1]}`,
+      `${bachelorlimits[2][0]} ≤ Credits < ${bachelorlimits[2][1]}`,
+      `${bachelorlimits[1][0]} ≤ Credits < ${bachelorlimits[1][1]}`,
+      `${bachelorlimits[0][0]} ≤ Credits`,
+    ])
+    yearlyMasterTitles.push([
+      '0 Credits',
+      `${masterlimits[4][0]} ≤ Credits < ${masterlimits[4][1]}`,
+      `${masterlimits[3][0]} ≤ Credits < ${masterlimits[3][1]}`,
+      `${masterlimits[2][0]} ≤ Credits < ${masterlimits[2][1]}`,
+      `${masterlimits[1][0]} ≤ Credits < ${masterlimits[1][1]}`,
+      `${masterlimits[0][0]} ≤ Credits`,
+    ])
+    yearlyBcMsTitles.push([
+      '180 Credits',
+      `${bcmslimits[4][0]} ≤ Credits < ${bcmslimits[4][1]}`,
+      `${bcmslimits[3][0]} ≤ Credits < ${bcmslimits[3][1]}`,
+      `${bcmslimits[2][0]} ≤ Credits < ${bcmslimits[2][1]}`,
+      `${bcmslimits[1][0]} ≤ Credits < ${bcmslimits[1][1]}`,
+      `${bcmslimits[0][0]} ≤ Credits`,
+    ])
+    for (const { progId, code } of programmes) {
+      let { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(code)
+      if (!creditThresholdKeys) return
+      const studentnumbers = await getCorrectStudentnumbers({
+        codes: [code],
+        startDate,
+        endDate,
+        includeAllSpecials,
+        includeGraduated,
+      })
+
+      const all = await allStudyrights(code, studentnumbers)
+      const students = await studytrackStudents(studentnumbers)
+
+      if (code.includes('KH')) {
+        const data = getStudentData(
+          startDate,
+          students,
+          creditThresholdKeys,
+          creditThresholdAmounts,
+          bachelorlimits,
+          limitKeys,
+          'KH'
+        )
+        const studentData = data[0]
+        const programmeData = data[1]
+        if (!(progId in bachelorsProgrammeStats)) {
+          bachelorsProgrammeStats[progId] = new Array(reversedYears.length - 1)
+        }
+        bachelorsTableStats[indexOf(reversedYears, year)][1] += all.length || 0
+        if (year !== 'Total') {
+          bachelorsProgrammeStats[progId][indexOf(reversedYears, year)] = limitKeys.map(key => programmeData[key])
+        }
+        for (const key of Object.keys(studentData)) {
+          bachelorsTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeys, key) + 2] +=
+            studentData[key] || 0
+          allBachelorGraphStats[key].data[indexOf(yearsArray, year)] += studentData[key] || 0
+        }
+      } else if (code.includes('MH')) {
+        const all_BcMs = all.filter(sr => sr.studyrightid.slice(-2) === '-2')
+        const all_Ms = all.filter(sr => sr.studyrightid.slice(-2) !== '-2')
+        const all_MsStudentnumbers = all_Ms.map(sr => sr.studentnumber)
+        const all_BcMsStudentnumbers = all_BcMs.map(sr => sr.studentnumber)
+        mastersTableStats[indexOf(reversedYears, year)][1] += all_Ms.length || 0
+        bachelorMastersTableStats[indexOf(reversedYears, year)][1] += all_BcMs.length || 0
+
+        const msStudents = students.filter(student => all_MsStudentnumbers.includes(student.studentnumber))
+        const bcMsStudents = students.filter(student => all_BcMsStudentnumbers.includes(student.studentnumber))
+
+        const data = getStudentData(
+          startDate,
+          bcMsStudents,
+          creditThresholdKeys,
+          creditThresholdAmounts,
+          bcmslimits,
+          limitKeys,
+          'MH'
+        )
+        const bcMsStudentdata = data[0]
+        const bcMsProgrammedata = data[1]
+        const { msOnlyCreditThresholdKeys, msOnlyCreditThresholdAmount } = getOnlyMasterThresholds()
+        const data2 = getStudentData(
+          startDate,
+          msStudents,
+          msOnlyCreditThresholdKeys,
+          msOnlyCreditThresholdAmount,
+          masterlimits,
+          limitKeys,
+          'MH'
+        )
+        const msStudentdata = data2[0]
+        const msProgrammedata = data2[1]
+        if (!(progId in mastersProgrammeStats)) {
+          mastersProgrammeStats[progId] = new Array(reversedYears.length - 1)
+        }
+        if (!(progId in bcMsProgrammeStats)) {
+          bcMsProgrammeStats[progId] = new Array(reversedYears.length - 1)
+        }
+        if (year !== 'Total') {
+          mastersProgrammeStats[progId][indexOf(reversedYears, year)] = limitKeys.map(key => msProgrammedata[key])
+          bcMsProgrammeStats[progId][indexOf(reversedYears, year)] = limitKeys.map(key => bcMsProgrammedata[key])
+        }
+        for (const key of Object.keys(msStudentdata)) {
+          mastersTableStats[indexOf(reversedYears, year)][indexOf(msOnlyCreditThresholdKeys, key) + 2] +=
+            msStudentdata[key] || 0
+          allMastersGraphStats[key].data[indexOf(yearsArray, year)] += msStudentdata[key] || 0
+        }
+        for (const key of Object.keys(bcMsStudentdata)) {
+          bachelorMastersTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeys, key) + 2] +=
+            bcMsStudentdata[key] || 0
+          allBsMsGraphstats[key].data[indexOf(yearsArray, year)] += bcMsStudentdata[key] || 0
+        }
+      } else {
+        const data = getStudentData(startDate, students, creditThresholdKeys, creditThresholdAmounts, [], [], 'T')
+        const studentData = data[0]
+        doctoralTableStats[indexOf(reversedYears, year)][1] += all.length || 0
+        if (!(progId in doctoralProgrammeStats)) {
+          doctoralProgrammeStats[progId] = new Array(reversedYears.length - 1)
+        }
+        if (year !== 'Total') {
+          doctoralProgrammeStats[progId][indexOf(reversedYears, year)] = creditThresholdKeys.map(
+            key => studentData[key]
+          )
+        }
+        for (const key of Object.keys(studentData)) {
+          doctoralTableStats[indexOf(reversedYears, year)][indexOf(creditThresholdKeys, key) + 2] +=
+            studentData[key] || 0
+          allDoctoralGraphStats[key].data[indexOf(yearsArray, year)] += studentData[key] || 0
+        }
+      }
+    }
+  }
+
+  const facultyProgressStats = {
+    id: faculty,
+    years: yearsArray,
+    bachelorsTableStats: bachelorsTableStats,
+    bcMsTableStats: bachelorMastersTableStats,
+    mastersTableStats: mastersTableStats,
+    doctoralTableStats: doctoralTableStats,
+    bachelorsGraphStats: allBachelorGraphStats,
+    bcMsGraphStats: allBsMsGraphstats,
+    mastersGraphStats: allMastersGraphStats,
+    doctoralGraphStats: allDoctoralGraphStats,
+    bachelorTitles: tableTitles.creditProgress.bachelor,
+    bcMsTitles: tableTitles.creditProgress.master,
+    mastersTitles: tableTitles.creditProgress.masterOnly,
+    doctoralTitles: tableTitles.creditProgress.doctoral,
+    yearlyBachelorTitles: yearlyBachelorTitles,
+    yearlyBcMsTitles: yearlyBcMsTitles,
+    yearlyMasterTitles: yearlyMasterTitles,
+    programmeNames: programmes.reduce(
+      (obj, dataItem) => ({ ...obj, [dataItem.progId]: { code: dataItem.code, ...dataItem.name } }),
+      {}
+    ),
+    bachelorsProgrammeStats: bachelorsProgrammeStats,
+    bcMsProgrammeStats: bcMsProgrammeStats,
+    mastersProgrammeStats: mastersProgrammeStats,
+    doctoralProgrammeStats: doctoralProgrammeStats,
+  }
+  return facultyProgressStats
+}
+module.exports = { combineFacultyStudentProgress }

services/backend/src/services/faculty/facultyStudents.js

@@ -0,0 +1,160 @@
+const {
+  getStartDate,
+  getYearsArray,
+  getPercentage,
+  getYearsObject,
+  getCorrectStudentnumbers,
+  tableTitles,
+} = require('../studyprogrammeHelpers')
+const {
+  studytrackStudents,
+  allStudyrights,
+  startedStudyrights,
+  inactiveStudyrights,
+  enrolledStudents,
+  absentStudents,
+  graduatedStudyRights,
+} = require('../studyprogramme')
+const { getAcademicYearDates } = require('../../util/semester')
+
+const getStudentData = students => {
+  let data = { female: 0, male: 0, finnish: 0 }
+
+  students.forEach(({ gender_code, home_country_en }) => {
+    data.male += gender_code === '1' ? 1 : 0
+    data.female += gender_code === '2' ? 1 : 0
+    data.finnish += home_country_en === 'Finland' ? 1 : 0
+  })
+  return data
+}
+
+// Goes through the faculty and its study programmes for the year
+const getFacultyDataForYear = async ({ programmes, since, settings, year, years, programmeTableStats }) => {
+  const { includeAllSpecials, includeGraduated } = settings
+  const { startDate, endDate } = getAcademicYearDates(year, since)
+  // Totals for the year into table stats
+  const totals = {
+    total: 0,
+    allStarted: 0,
+    allEnrolled: 0,
+    allAbsent: 0,
+    allInactive: 0,
+    allGraduated: 0,
+    allFinnish: 0,
+    allFemale: 0,
+    allMale: 0,
+  }
+
+  for (const { progId, code } of programmes) {
+    const studentnumbers = await getCorrectStudentnumbers({
+      codes: [code],
+      startDate,
+      endDate,
+      includeAllSpecials,
+      includeGraduated,
+    })
+
+    // Get all the studyrights and students for the calculations
+    let all = await allStudyrights(code, studentnumbers)
+    const students = await studytrackStudents(studentnumbers)
+    const studentData = getStudentData(students)
+    const started = await startedStudyrights(code, startDate, studentnumbers)
+    const enrolled = await enrolledStudents(code, startDate, studentnumbers)
+    const absent = await absentStudents(code, studentnumbers)
+    const inactive = await inactiveStudyrights(code, studentnumbers)
+    const graduated = await graduatedStudyRights(code, startDate, studentnumbers)
+
+    // Count stats for the programme
+    if (!(progId in programmeTableStats)) {
+      programmeTableStats[progId] = years.reduce((resultObj, yearKey) => {
+        return { ...resultObj, [yearKey]: [] }
+      }, {})
+    }
+
+    programmeTableStats[progId][year] = [
+      all.length,
+      started.length,
+      getPercentage(started.length, all.length),
+      enrolled.length,
+      getPercentage(enrolled.length, all.length),
+      absent.length,
+      getPercentage(absent.length, all.length),
+      inactive.length,
+      getPercentage(inactive.length, all.length),
+      graduated.length,
+      getPercentage(graduated.length, all.length),
+      studentData.male,
+      getPercentage(studentData.male, all.length),
+      studentData.female,
+      getPercentage(studentData.female, all.length),
+      studentData.finnish,
+      getPercentage(studentData.finnish, all.length),
+    ]
+
+    totals.total += all.length
+    totals.allStarted += started.length
+    totals.allEnrolled += enrolled.length
+    totals.allAbsent += absent.length
+    totals.allInactive += inactive.length
+    totals.allGraduated += graduated.length
+    totals.allFemale += studentData.female
+    totals.allMale += studentData.male
+    totals.allFinnish += studentData.finnish
+  }
+
+  return totals
+}
+
+// Combines all the data for faculty students table
+const combineFacultyStudents = async (code, programmes, specialGroups, graduated) => {
+  // Only academic years are considered
+  const isAcademicYear = true
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
+  const includeYearsCombined = true
+  const since = getStartDate(code, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear, includeYearsCombined)
+  const settings = { includeAllSpecials, includeGraduated }
+  let facultyTableStats = getYearsObject({ years, emptyArrays: true })
+  let programmeTableStats = {}
+
+  for (const year of years.reverse()) {
+    const queryParams = { programmes, since, settings, year, years, programmeTableStats }
+    const totals = await getFacultyDataForYear(queryParams)
+    facultyTableStats[year] = [
+      year,
+      totals.total,
+      totals.allStarted,
+      getPercentage(totals.allStarted, totals.total),
+      totals.allEnrolled,
+      getPercentage(totals.allEnrolled, totals.total),
+      totals.allAbsent,
+      getPercentage(totals.allAbsent, totals.total),
+      totals.allInactive,
+      getPercentage(totals.allInactive, totals.total),
+      totals.allGraduated,
+      getPercentage(totals.allGraduated, totals.total),
+      totals.allMale,
+      getPercentage(totals.allMale, totals.total),
+      totals.allFemale,
+      getPercentage(totals.allFemale, totals.total),
+      totals.allFinnish,
+      getPercentage(totals.allFinnish, totals.total),
+    ]
+  }
+
+  const studentsData = {
+    id: code,
+    years: years,
+    facultyTableStats: facultyTableStats,
+    programmeStats: programmeTableStats,
+    titles: tableTitles['studytracks'],
+    programmeNames: programmes.reduce(
+      (obj, dataItem) => ({ ...obj, [dataItem.progId]: { code: dataItem.code, ...dataItem.name } }),
+      {}
+    ),
+  }
+  return studentsData
+}
+
+module.exports = { combineFacultyStudents }

services/backend/src/services/faculty/facultyThesisWriters.js

@@ -0,0 +1,134 @@
+const { indexOf } = require('lodash')
+const { mapToProviders } = require('../../util/utils')
+const { thesisWriters } = require('./faculty')
+const {
+  getStatsBasis,
+  getYearsArray,
+  defineYear,
+  getCorrectStudentnumbers,
+  alltimeEndDate,
+  alltimeStartDate,
+} = require('../studyprogrammeHelpers')
+
+const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }) => {
+  const thesisTypes = [
+    'urn:code:course-unit-type:bachelors-thesis',
+    'urn:code:course-unit-type:masters-thesis',
+    'urn:code:course-unit-type:doctors-thesis',
+    'urn:code:course-unit-type:licentiate-thesis',
+  ]
+
+  let bachelors = getStatsBasis(years)
+  let masters = getStatsBasis(years)
+  let doctors = getStatsBasis(years)
+  let licentiates = getStatsBasis(years)
+  let programmeCounts = {}
+  let programmeNames = {}
+
+  for (const { progId, code, name } of facultyProgrammes) {
+    const provider = mapToProviders([code])[0]
+    const students = await getCorrectStudentnumbers({
+      codes: [code],
+      startDate: alltimeStartDate,
+      endDate: alltimeEndDate,
+      includeAllSpecials,
+    })
+    const thesisCourseCodes = await thesisWriters(provider, since, thesisTypes, students)
+    thesisCourseCodes?.forEach(({ attainment_date, courseUnitType }) => {
+      const thesisYear = defineYear(attainment_date, isAcademicYear)
+
+      if (!(progId in programmeCounts)) {
+        programmeCounts[progId] = {}
+
+        Object.keys(bachelors.tableStats).forEach(year => (programmeCounts[progId][year] = [0, 0, 0, 0, 0]))
+        programmeNames[progId] = { ...name, code: code }
+      }
+      programmeCounts[progId][thesisYear][0] += 1
+
+      if (courseUnitType === thesisTypes[0]) {
+        bachelors.graphStats[indexOf(years, thesisYear)] += 1
+        bachelors.tableStats[thesisYear] += 1
+        programmeCounts[progId][thesisYear][1] += 1
+      } else if (courseUnitType === thesisTypes[1]) {
+        masters.graphStats[indexOf(years, thesisYear)] += 1
+        masters.tableStats[thesisYear] += 1
+        programmeCounts[progId][thesisYear][2] += 1
+      } else if (courseUnitType === thesisTypes[2]) {
+        doctors.graphStats[indexOf(years, thesisYear)] += 1
+        doctors.tableStats[thesisYear] += 1
+        programmeCounts[progId][thesisYear][3] += 1
+      } else if (courseUnitType === thesisTypes[3]) {
+        licentiates.graphStats[indexOf(years, thesisYear)] += 1
+        licentiates.tableStats[thesisYear] += 1
+        programmeCounts[progId][thesisYear][4] += 1
+      }
+    })
+  }
+
+  return { bachelors, masters, doctors, licentiates, programmeCounts, programmeNames }
+}
+
+const getFacultyThesisWritersForStudyTrack = async (
+  allThesisWriters,
+  facultyProgrammes,
+  isAcademicYear,
+  includeAllSpecials
+) => {
+  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+  const queryParameters = { since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }
+  const { bachelors, masters, doctors, licentiates, programmeCounts, programmeNames } = await getFacultyThesisWriters(
+    queryParameters
+  )
+
+  allThesisWriters.years = years.map(year => year.toString())
+  const reversedYears = years.reverse()
+  allThesisWriters.tableStats = reversedYears.map(year => [
+    year,
+    bachelors.tableStats[year] + masters.tableStats[year] + doctors.tableStats[year] + licentiates.tableStats[year],
+    bachelors.tableStats[year],
+    masters.tableStats[year],
+    doctors.tableStats[year],
+    licentiates.tableStats[year],
+  ])
+  allThesisWriters.graphStats.push({ name: 'Bachelors', data: bachelors.graphStats })
+  allThesisWriters.graphStats.push({ name: 'Masters', data: masters.graphStats })
+  allThesisWriters.graphStats.push({ name: 'Doctors', data: doctors.graphStats })
+  allThesisWriters.graphStats.push({ name: 'Licentiates', data: licentiates.graphStats })
+
+  const programmes = programmeNames ? Object.keys(programmeNames) : facultyProgrammes.map(programme => programme.code)
+  programmes.forEach(programmeId => {
+    reversedYears.forEach(year => {
+      if (programmeCounts && programmeId in programmeCounts) {
+        if (!(programmeId in allThesisWriters.programmeTableStats)) {
+          allThesisWriters.programmeTableStats[programmeId] = []
+        }
+        allThesisWriters.programmeTableStats[programmeId].push([year, ...programmeCounts[programmeId][year]])
+      }
+    })
+  })
+
+  allThesisWriters.programmeNames = programmeNames
+}
+
+const combineFacultyThesisWriters = async (faculty, facultyProgrammes, yearType, specialGroups) => {
+  let allThesisWriters = {
+    id: faculty,
+    years: [],
+    tableStats: [],
+    graphStats: [],
+    programmeTableStats: {},
+    titles: ['', 'All', 'Bachelors', 'Masters', 'Doctors', 'Licentiates'],
+    programmeNames: {},
+    status: 'Done',
+    lastUpdated: '',
+  }
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+
+  await getFacultyThesisWritersForStudyTrack(allThesisWriters, facultyProgrammes, isAcademicYear, includeAllSpecials)
+
+  return allThesisWriters
+}
+
+module.exports = { combineFacultyThesisWriters }

services/backend/src/services/faculty/facultyUpdates.js

@@ -0,0 +1,188 @@
+const logger = require('../../util/logger')
+const { findFacultyProgrammeCodes } = require('./faculty')
+const { combineFacultyBasics } = require('./facultyBasics')
+const { combineFacultyCredits } = require('./facultyCredits')
+const { combineFacultyThesisWriters } = require('./facultyThesisWriters')
+const { countGraduationTimes } = require('./facultyGraduationTimes')
+const {
+  setFacultyProgrammes,
+  setBasicStats,
+  setCreditStats,
+  setThesisWritersStats,
+  setGraduationStats,
+  setFacultyProgressStats,
+  setFacultyStudentStats,
+} = require('./facultyService')
+const { combineFacultyStudentProgress } = require('./facultyStudentProgress')
+const { combineFacultyStudents } = require('./facultyStudents')
+
+const updateFacultyOverview = async (faculty, statsType) => {
+  const calendarNewSpecial = {
+    yearType: 'CALENDAR_YEAR',
+    programmeFilter: 'NEW_STUDY_PROGRAMMES',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const calendarNewSpecialOut = {
+    yearType: 'CALENDAR_YEAR',
+    programmeFilter: 'NEW_STUDY_PROGRAMMES',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const calendarAllSpecial = {
+    yearType: 'CALENDAR_YEAR',
+    programmeFilter: 'ALL_PROGRAMMES',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+  const calendarAllSpecialOut = {
+    yearType: 'CALENDAR_YEAR',
+    programmeFilter: 'ALL_PROGRAMMES',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+  const academicNewSpecial = {
+    yearType: 'ACADEMIC_YEAR',
+    programmeFilter: 'NEW_STUDY_PROGRAMMES',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+
+  const academicAllSpecial = {
+    yearType: 'ACADEMIC_YEAR',
+    programmeFilter: 'ALL_PROGRAMMES',
+    specialGroups: 'SPECIAL_INCLUDED',
+  }
+
+  const academicNewSpecialOut = {
+    yearType: 'ACADEMIC_YEAR',
+    programmeFilter: 'NEW_STUDY_PROGRAMMES',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const academicAllSpecialOut = {
+    yearType: 'ACADEMIC_YEAR',
+    programmeFilter: 'ALL_PROGRAMMES',
+    specialGroups: 'SPECIAL_EXCLUDED',
+  }
+
+  const options = [
+    calendarNewSpecial,
+    calendarAllSpecial,
+    academicNewSpecial,
+    academicAllSpecial,
+    calendarNewSpecialOut,
+    calendarAllSpecialOut,
+    academicNewSpecialOut,
+    academicAllSpecialOut,
+  ]
+  let allProgrammes = []
+  let newProgrammes = []
+  let allProgrammeCodes = []
+
+  try {
+    const all = await findFacultyProgrammeCodes(faculty, 'ALL_PROGRAMMES')
+    const onlyNew = await findFacultyProgrammeCodes(faculty, 'NEW_STUDY_PROGRAMMES')
+    allProgrammes = await setFacultyProgrammes(faculty, all, 'ALL_PROGRAMMES')
+    newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
+    allProgrammes?.data.forEach(prog => allProgrammeCodes.push(prog.code))
+  } catch (e) {
+    logger.error(e)
+  }
+
+  for (const option of options) {
+    const { yearType, programmeFilter, specialGroups } = option
+    try {
+      if (statsType === 'ALL' || statsType === 'STUDENT') {
+        const updatedStudentInfo = await combineFacultyBasics(
+          faculty,
+          programmeFilter === 'ALL_PROGRAMMES' ? allProgrammes.data : newProgrammes.data,
+          yearType,
+          allProgrammeCodes,
+          programmeFilter,
+          specialGroups
+        )
+        await setBasicStats(updatedStudentInfo, option.yearType, option.programmeFilter, option.specialGroups)
+      }
+      if (statsType === 'ALL' || statsType === 'CREDITS') {
+        const updatedCredits = await combineFacultyCredits(
+          faculty,
+          programmeFilter === 'ALL_PROGRAMMES' ? allProgrammes.data : newProgrammes.data,
+          yearType,
+          specialGroups
+        )
+        await setCreditStats(updatedCredits, yearType, programmeFilter, specialGroups)
+      }
+      if (statsType === 'ALL' || statsType === 'THESIS') {
+        const updateThesisWriters = await combineFacultyThesisWriters(
+          faculty,
+          programmeFilter === 'ALL_PROGRAMMES' ? allProgrammes.data : newProgrammes.data,
+          yearType,
+          specialGroups
+        )
+        await setThesisWritersStats(updateThesisWriters, yearType, programmeFilter, specialGroups)
+      }
+    } catch (e) {
+      logger.error(e)
+    }
+  }
+
+  try {
+    const updatedTimesAll = await countGraduationTimes(faculty, 'ALL_PROGRAMMES')
+    await setGraduationStats(updatedTimesAll, 'ALL_PROGRAMMES')
+    const updatedTimesNew = await countGraduationTimes(faculty, 'NEW_STUDY_PROGRAMMES')
+    await setGraduationStats(updatedTimesNew, 'NEW_STUDY_PROGRAMMES')
+  } catch (e) {
+    logger.error(e)
+  }
+  return 'OK'
+}
+
+const updateFacultyProgressOverview = async faculty => {
+  const specialGraduated = {
+    specialGroups: 'SPECIAL_INCLUDED',
+    graduated: 'GRADUATED_INCLUDED',
+  }
+  const specialNotGraduated = {
+    specialGroups: 'SPECIAL_INCLUDED',
+    graduated: 'GRADUATED_EXCLUDED',
+  }
+  const notSpecialGraduated = {
+    specialGroups: 'SPECIAL_EXCLUDED',
+    graduated: 'GRADUATED_INCLUDED',
+  }
+  const notSpecialNotGraduated = {
+    specialGroups: 'SPECIAL_EXCLUDED',
+    graduated: 'GRADUATED_EXCLUDED',
+  }
+  const options = [specialGraduated, specialNotGraduated, notSpecialGraduated, notSpecialNotGraduated]
+
+  let newProgrammes = []
+  try {
+    const onlyNew = await findFacultyProgrammeCodes(faculty, 'NEW_STUDY_PROGRAMMES')
+    newProgrammes = await setFacultyProgrammes(faculty, onlyNew, 'NEW_STUDY_PROGRAMMES')
+  } catch (e) {
+    logger.error(e)
+  }
+
+  for (const option of options) {
+    const { specialGroups, graduated } = option
+    try {
+      const updateFacultyStudentStats = await combineFacultyStudents(
+        faculty,
+        newProgrammes.data,
+        specialGroups,
+        graduated
+      )
+      await setFacultyStudentStats(updateFacultyStudentStats, specialGroups, graduated)
+      const updateFacultyProgressStats = await combineFacultyStudentProgress(
+        faculty,
+        newProgrammes.data,
+        specialGroups,
+        graduated
+      )
+      await setFacultyProgressStats(updateFacultyProgressStats, specialGroups, graduated)
+    } catch (e) {
+      logger.error(e)
+    }
+  }
+  return 'OK'
+}
+
+module.exports = { updateFacultyOverview, updateFacultyProgressOverview }

services/backend/src/services/facultyBasics.js

@@ -1,316 +0,0 @@
-const { indexOf } = require('lodash')
-const {
-  startedStudyrights,
-  graduatedStudyrights,
-  transferredInsideFaculty,
-  transferredAway,
-  transferredTo,
-} = require('./faculty')
-const { getStatsBasis, getYearsArray, defineYear } = require('./studyprogrammeHelpers')
-const { findRightProgramme } = require('./facultyHelpers')
-
-const filterDuplicateStudyrights = studyrights => {
-  // bachelor+master students have two studyrights (separated by two last digits in studyrightid)
-  // choose only the earlier started one, so we don't count start of masters as starting in faculty
-  let rightsToCount = {}
-
-  studyrights.forEach(right => {
-    const id = right.studyrightid.slice(0, -2)
-    const start = new Date(right.studystartdate)
-    if (id in rightsToCount) {
-      if (new Date(rightsToCount[id].studystartdate) > start) {
-        rightsToCount[id] = right
-      }
-    } else {
-      rightsToCount[id] = right
-    }
-  })
-  return Object.values(rightsToCount)
-}
-
-const addProgramme = async (stats, programme, tableStats, transferYear, index, allBasics, facultyProgrammes) => {
-  if (!(programme in stats)) {
-    stats[programme] = {}
-    Object.keys(tableStats).forEach(year => (stats[programme][year] = [0, 0, 0]))
-  }
-  stats[programme][transferYear][index] += 1
-
-  if (!(programme in allBasics.programmeNames)) {
-    const name = facultyProgrammes.filter(prog => prog.code === programme)[0]?.name
-    allBasics.programmeNames[programme] = name
-  }
-}
-
-const getFacultyStarters = async (
-  faculty,
-  since,
-  isAcademicYear,
-  yearsArray,
-  graphStats,
-  tableStats,
-  counts,
-  allBasics,
-  programmeData
-) => {
-  const startedGraphStats = [...graphStats]
-  const startedTableStats = { ...tableStats }
-  const studyrights = await startedStudyrights(faculty, since)
-  const filteredStudyrights = filterDuplicateStudyrights(studyrights)
-  const programmeTableStats = {}
-
-  filteredStudyrights.forEach(({ studystartdate, studyrightElements }) => {
-    const { programme, programmeName } = findRightProgramme(studyrightElements, 'started')
-    const startYear = defineYear(studystartdate, isAcademicYear)
-    startedGraphStats[indexOf(yearsArray, startYear)] += 1
-    startedTableStats[startYear] += 1
-
-    if (!(programme in programmeTableStats)) {
-      programmeTableStats[programme] = { ...tableStats }
-    }
-    programmeTableStats[programme][startYear] += 1
-
-    if (!(programme in allBasics.programmeNames)) {
-      allBasics.programmeNames[programme] = programmeName
-    }
-  })
-
-  allBasics.studentInfo.graphStats.push({ name: 'Started studying', data: startedGraphStats })
-  programmeData['started'] = programmeTableStats
-
-  Object.keys(startedTableStats).forEach(year => {
-    counts[year] = [startedTableStats[year]]
-  })
-}
-
-const getFacultyGraduates = async (
-  faculty,
-  since,
-  isAcademicYear,
-  yearsArray,
-  graphStats,
-  tableStats,
-  allBasics,
-  counts,
-  countsGraduations,
-  programmeData
-) => {
-  const graduatedGraphStats = [[...graphStats], [...graphStats], [...graphStats], [...graphStats], [...graphStats]]
-  const graduatedTableStats = {}
-  Object.keys(tableStats).forEach(year => (graduatedTableStats[year] = [0, 0, 0, 0, 0]))
-  const programmeTableStats = {}
-
-  const graduatedRights = await graduatedStudyrights(faculty, since)
-
-  graduatedRights.forEach(({ enddate, extentcode, studyrightElements }) => {
-    const { programme, programmeName } = findRightProgramme(studyrightElements, 'graduated')
-    const endYear = defineYear(enddate, isAcademicYear)
-    graduatedGraphStats[0][indexOf(yearsArray, endYear)] += 1
-    graduatedTableStats[endYear][0] += 1
-
-    if (!(programme in programmeTableStats)) {
-      programmeTableStats[programme] = {}
-      Object.keys(tableStats).forEach(year => (programmeTableStats[programme][year] = [0, 0, 0, 0, 0]))
-    }
-    programmeTableStats[programme][endYear][0] += 1
-
-    if (extentcode === 1) {
-      graduatedGraphStats[1][indexOf(yearsArray, endYear)] += 1
-      graduatedTableStats[endYear][1] += 1
-      programmeTableStats[programme][endYear][1] += 1
-    } else if (extentcode === 2) {
-      graduatedGraphStats[2][indexOf(yearsArray, endYear)] += 1
-      graduatedTableStats[endYear][2] += 1
-      programmeTableStats[programme][endYear][2] += 1
-    } else if (extentcode === 4) {
-      graduatedGraphStats[3][indexOf(yearsArray, endYear)] += 1
-      graduatedTableStats[endYear][3] += 1
-      programmeTableStats[programme][endYear][3] += 1
-    } else {
-      graduatedGraphStats[4][indexOf(yearsArray, endYear)] += 1
-      graduatedTableStats[endYear][4] += 1
-      programmeTableStats[programme][endYear][4] += 1
-    }
-
-    if (!(programme in allBasics.programmeNames)) {
-      allBasics.programmeNames[programme] = programmeName
-    }
-  })
-
-  allBasics.studentInfo.graphStats.push({ name: 'Graduated', data: graduatedGraphStats[0] })
-  // Is this needed in graph
-  // allBasics.graduationInfo.graphStats.push({ name: 'All graduations', data: graduatedGraphStats[0] })
-  allBasics.graduationInfo.graphStats.push({ name: 'Bachelors', data: graduatedGraphStats[1] })
-  allBasics.graduationInfo.graphStats.push({ name: 'Masters', data: graduatedGraphStats[2] })
-  allBasics.graduationInfo.graphStats.push({ name: 'Doctors', data: graduatedGraphStats[3] })
-  allBasics.graduationInfo.graphStats.push({ name: 'Others', data: graduatedGraphStats[4] })
-
-  programmeData['graduated'] = programmeTableStats
-
-  Object.keys(graduatedTableStats).forEach(year => {
-    counts[year] = counts[year].concat(graduatedTableStats[year][0])
-    countsGraduations[year] = graduatedTableStats[year]
-  })
-}
-
-const getFacultyTransfers = async (
-  programmes,
-  since,
-  isAcademicYear,
-  yearsArray,
-  graphStats,
-  tableStats,
-  allBasics,
-  counts,
-  programmeData
-) => {
-  const programmeCodes = programmes.map(prog => prog.code)
-  const insideTransfers = await transferredInsideFaculty(programmeCodes, since)
-  const awayTransfers = await transferredAway(programmeCodes, since)
-  const toTransfers = await transferredTo(programmeCodes, since)
-
-  const transferGraphStats = [[...graphStats], [...graphStats], [...graphStats]]
-  const transferTableStats = {}
-  Object.keys(tableStats).forEach(year => (transferTableStats[year] = [0, 0, 0]))
-  const programmeTableStats = {}
-
-  // inside transfer is counted towards target
-  for (const { transferdate, targetcode } of insideTransfers) {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    transferGraphStats[0][indexOf(yearsArray, transferYear)] += 1
-    transferTableStats[transferYear][0] += 1
-    await addProgramme(programmeTableStats, targetcode, tableStats, transferYear, 0, allBasics, programmes)
-  }
-  for (const { transferdate, sourcecode } of awayTransfers) {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    transferGraphStats[1][indexOf(yearsArray, transferYear)] += 1
-    transferTableStats[transferYear][1] += 1
-    await addProgramme(programmeTableStats, sourcecode, tableStats, transferYear, 1, allBasics, programmes)
-  }
-
-  for (const { transferdate, targetcode } of toTransfers) {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    transferGraphStats[2][indexOf(yearsArray, transferYear)] += 1
-    transferTableStats[transferYear][2] += 1
-    await addProgramme(programmeTableStats, targetcode, tableStats, transferYear, 2, allBasics, programmes)
-  }
-
-  allBasics.studentInfo.graphStats.push({ name: 'Transferred inside', data: transferGraphStats[0] })
-  allBasics.studentInfo.graphStats.push({ name: 'Transferred away', data: transferGraphStats[1] })
-  allBasics.studentInfo.graphStats.push({ name: 'Transferred to', data: transferGraphStats[2] })
-
-  programmeData['transferred'] = programmeTableStats
-
-  Object.keys(transferTableStats).forEach(year => {
-    counts[year] = counts[year].concat(transferTableStats[year])
-  })
-}
-
-const combineFacultyBasics = async (allBasics, faculty, programmes, yearType) => {
-  let counts = {}
-  let countsGraduations = {}
-  let years = []
-  let programmeData = {}
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
-  const yearsArray = getYearsArray(since.getFullYear(), isAcademicYear)
-  const { graphStats, tableStats } = getStatsBasis(yearsArray)
-  Object.keys(tableStats).forEach(year => years.push(year))
-
-  // Started studying in faculty
-  await getFacultyStarters(
-    faculty,
-    since,
-    isAcademicYear,
-    yearsArray,
-    graphStats,
-    tableStats,
-    counts,
-    allBasics,
-    programmeData
-  )
-
-  // Graduated in faculty
-  await getFacultyGraduates(
-    faculty,
-    since,
-    isAcademicYear,
-    yearsArray,
-    graphStats,
-    tableStats,
-    allBasics,
-    counts,
-    countsGraduations,
-    programmeData
-  )
-
-  // Transfers
-  await getFacultyTransfers(
-    programmes,
-    since,
-    isAcademicYear,
-    yearsArray,
-    graphStats,
-    tableStats,
-    allBasics,
-    counts,
-    programmeData
-  )
-
-  // combine tableStats from all categories
-  allBasics.years = years
-  years.forEach(year => {
-    allBasics.studentInfo.tableStats.push([year, ...counts[year]])
-    allBasics.graduationInfo.tableStats.push([year, ...countsGraduations[year]])
-  })
-  allBasics.studentInfo.tableStats.reverse()
-  allBasics.graduationInfo.tableStats.reverse()
-
-  //combine programme level tablestats
-  // all programmes are not present in all data types, check all found programmes and patch missing details with 0
-
-  let allCodes = Object.keys(programmeData.started).concat(
-    Object.keys(programmeData.graduated),
-    Object.keys(programmeData.transferred)
-  )
-  allCodes = [...new Set(allCodes)]
-
-  let studentInfo = {}
-  let graduationInfo = {}
-
-  allCodes.forEach(code => {
-    if (!(code in studentInfo)) studentInfo[code] = {}
-
-    for (const year of years) {
-      // started
-      if (code in programmeData.started) {
-        studentInfo[code][year] = [year, programmeData.started[code][year]]
-      } else {
-        studentInfo[code][year] = [year, 0]
-      }
-      // graduated total
-      if (code in programmeData.graduated) {
-        studentInfo[code][year] = studentInfo[code][year].concat([programmeData.graduated[code][year][0]])
-        if (!(code in graduationInfo)) graduationInfo[code] = {}
-        graduationInfo[code][year] = [year, ...programmeData.graduated[code][year]]
-      } else {
-        studentInfo[code][year] = studentInfo[code][year].concat([0])
-      }
-      // trasferred
-      if (code in programmeData.transferred) {
-        studentInfo[code][year] = studentInfo[code][year].concat(programmeData.transferred[code][year])
-      } else {
-        studentInfo[code][year] = studentInfo[code][year].concat([0, 0, 0])
-      }
-    }
-  })
-
-  Object.keys(studentInfo).forEach(prog => {
-    allBasics.studentInfo.programmeTableStats[prog] = [...Object.values(studentInfo[prog])].reverse()
-  })
-
-  Object.keys(graduationInfo).forEach(prog => {
-    allBasics.graduationInfo.programmeTableStats[prog] = Object.values(graduationInfo[prog]).reverse()
-  })
-}
-
-module.exports = { combineFacultyBasics }

services/backend/src/services/facultyThesisWriters.js

@@ -1,124 +0,0 @@
-const { indexOf } = require('lodash')
-const { mapToProviders } = require('../util/utils')
-const { thesisWriters } = require('./faculty')
-const { getStatsBasis, getYearsArray, defineYear } = require('./studyprogrammeHelpers')
-
-const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyProgrammes }) => {
-  const programmeProviders = facultyProgrammes.reduce(
-    (results, studyprogramme) => ({
-      ...results,
-      [mapToProviders([studyprogramme.code])[0]]: { ...studyprogramme },
-    }),
-    {}
-  )
-
-  const thesisTypes = [
-    'urn:code:course-unit-type:bachelors-thesis',
-    'urn:code:course-unit-type:masters-thesis',
-    'urn:code:course-unit-type:doctors-thesis',
-    'urn:code:course-unit-type:licentiate-thesis',
-    'urn:code:course-unit-type:bachelors-thesis-seminar',
-    'urn:code:course-unit-type:masters-thesis-seminar',
-    '',
-  ]
-  const thesisCourseCodes = await thesisWriters(Object.keys(programmeProviders), since, thesisTypes)
-  let bachelors = getStatsBasis(years)
-  let masters = getStatsBasis(years)
-  let doctors = getStatsBasis(years)
-  let licentiates = getStatsBasis(years)
-  let programmeCounts = {}
-  let programmeNames = {}
-
-  thesisCourseCodes.forEach(({ attainment_date, courseUnitType, organizations }) => {
-    const thesisYear = defineYear(attainment_date, isAcademicYear)
-    const programme = programmeProviders[organizations[0]]
-    if (!(programme.code in programmeCounts)) {
-      programmeCounts[programme.code] = {}
-      Object.keys(bachelors.tableStats).forEach(year => (programmeCounts[programme.code][year] = [0, 0, 0, 0, 0]))
-      programmeNames[programme.code] = programme.name
-    }
-    programmeCounts[programme.code][thesisYear][0] += 1
-
-    if (courseUnitType === thesisTypes[0] || courseUnitType == thesisTypes[4]) {
-      bachelors.graphStats[indexOf(years, thesisYear)] += 1
-      bachelors.tableStats[thesisYear] += 1
-      programmeCounts[programme.code][thesisYear][1] += 1
-    } else if (courseUnitType === thesisTypes[1] || courseUnitType == thesisTypes[5]) {
-      masters.graphStats[indexOf(years, thesisYear)] += 1
-      masters.tableStats[thesisYear] += 1
-      programmeCounts[programme.code][thesisYear][2] += 1
-    } else if (courseUnitType === thesisTypes[2]) {
-      doctors.graphStats[indexOf(years, thesisYear)] += 1
-      doctors.tableStats[thesisYear] += 1
-      programmeCounts[programme.code][thesisYear][3] += 1
-    } else if (courseUnitType === thesisTypes[3]) {
-      licentiates.graphStats[indexOf(years, thesisYear)] += 1
-      licentiates.tableStats[thesisYear] += 1
-      programmeCounts[programme.code][thesisYear][4] += 1
-    }
-  })
-
-  if (
-    bachelors.graphStats.every(year => year === 0) &&
-    masters.graphStats.every(year => year === 0) &&
-    doctors.graphStats.every(year => year === 0) &&
-    licentiates.graphStats.every(year => year === 0)
-  ) {
-    return {
-      bachelors: { graphStats: [], tableStats: {} },
-      masters: { graphStats: [], tableStats: {} },
-      doctors: { graphStats: [], tableStats: {} },
-      licentiates: { graphStats: [], tableStats: {} },
-      programmeCounts: {},
-    }
-  }
-  return { bachelors, masters, doctors, licentiates, programmeCounts, programmeNames }
-}
-
-const getFacultyThesisWritersForStudyTrack = async (allThesisWriters, facultyProgrammes, code, isAcademicYear) => {
-  const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-
-  const queryParameters = { since, years, isAcademicYear, facultyProgrammes }
-  const { bachelors, masters, doctors, licentiates, programmeCounts, programmeNames } = await getFacultyThesisWriters(
-    queryParameters
-  )
-
-  allThesisWriters.years = years.map(year => year.toString())
-  const reversedYears = years.reverse()
-
-  allThesisWriters.tableStats = reversedYears.map(year => [
-    year,
-    bachelors.tableStats[year] + masters.tableStats[year] + doctors.tableStats[year],
-    bachelors.tableStats[year],
-    masters.tableStats[year],
-    doctors.tableStats[year],
-    licentiates.tableStats[year],
-  ])
-  allThesisWriters.graphStats.push({ name: 'Bachelors', data: bachelors.graphStats })
-  allThesisWriters.graphStats.push({ name: 'Masters', data: masters.graphStats })
-  allThesisWriters.graphStats.push({ name: 'Doctors', data: doctors.graphStats })
-  allThesisWriters.graphStats.push({ name: 'Others', data: licentiates.graphStats })
-
-  facultyProgrammes.forEach(programme => {
-    reversedYears.forEach(year => {
-      if (programme.code in programmeCounts) {
-        if (!(programme.code in allThesisWriters.programmeTableStats)) {
-          allThesisWriters.programmeTableStats[programme.code] = []
-        }
-        allThesisWriters.programmeTableStats[programme.code].push([year, ...programmeCounts[programme.code][year]])
-      }
-    })
-  })
-
-  allThesisWriters.programmeNames = programmeNames
-}
-
-const combineFacultyThesisWriters = async (allThesisWriters, facultyProgrammes, faculty, yearType) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR' ? true : false
-  await getFacultyThesisWritersForStudyTrack(allThesisWriters, facultyProgrammes, faculty, isAcademicYear)
-
-  return allThesisWriters
-}
-
-module.exports = { combineFacultyThesisWriters }

services/backend/src/services/openUni/openUniHelpers.js

@@ -0,0 +1,47 @@
+const mapOpenCredits = credit => {
+  const { course_code, attainment_date, student_studentnumber } = credit
+  return {
+    course_code,
+    attainment_date,
+    student_studentnumber,
+  }
+}
+
+const mapStundentInfo = student => {
+  const { email, secondary_email, studentnumber, dissemination_info_allowed } = student
+  return {
+    studentnumber,
+    email,
+    secondary_email,
+    dissemination_info_allowed,
+  }
+}
+const mapOpenEnrollments = enrollment => {
+  const { studentnumber, course_code, state, enrollment_date_time } = enrollment
+  return {
+    enrollmentStudentnumber: studentnumber,
+    course_code,
+    state,
+    enrollment_date_time,
+  }
+}
+
+const mapStudyRights = studyright => {
+  const { enddate, startdate, student } = studyright
+
+  return {
+    startdate,
+    enddate,
+    studyrightStudentnumber: student.studentnumber,
+  }
+}
+
+const mapCourseInfo = course => {
+  const { code, name } = course
+  return {
+    label: code,
+    name,
+  }
+}
+
+module.exports = { mapOpenCredits, mapOpenEnrollments, mapStundentInfo, mapStudyRights, mapCourseInfo }

services/backend/src/services/openUni/openUniManageSearches.js

@@ -0,0 +1,65 @@
+const logger = require('../../util/logger')
+const {
+  getOpenUniSearchesByUser,
+  createOpenUniPopulationSearch,
+  updateOpenUniPopulationSearch,
+  deleteOpenUniSearch,
+} = require('./openUniSearches')
+
+const getOpenUniSearches = async userId => {
+  const savedSearches = await getOpenUniSearchesByUser(userId)
+  if (savedSearches === undefined) return []
+  const foundSearches = savedSearches.reduce(
+    (res, search) => [
+      ...res,
+      {
+        id: search.id,
+        userId: search.userId,
+        name: search.name,
+        courseList: search.courseCodes,
+        updatedAt: search.updatedAt,
+      },
+    ],
+    []
+  )
+  return foundSearches
+}
+
+const createNewSearch = async (userId, name, courseCodes) => {
+  const savedSearches = await getOpenUniSearchesByUser(userId)
+  if (savedSearches !== undefined) {
+    const searchNames = savedSearches.map(search => search.name)
+    if (searchNames.includes(name)) return null
+  }
+  try {
+    const created = await createOpenUniPopulationSearch(userId, name, courseCodes)
+    if (!created) return null
+    return created
+  } catch (e) {
+    logger.error(`Couldn't create new open uni search instance: ${e}`)
+    return null
+  }
+}
+
+const deleteSearch = async (userId, id) => {
+  try {
+    const deleted = await deleteOpenUniSearch(userId, id)
+    if (!deleted) return null
+    return deleted
+  } catch (e) {
+    logger.error(`Couldn't delete open uni search instance: ${e}`)
+    return null
+  }
+}
+
+const updateSearch = async (userId, id, courseCodes) => {
+  try {
+    const updated = await updateOpenUniPopulationSearch(userId, id, courseCodes)
+    if (!updated) return null
+    return updated
+  } catch (e) {
+    logger.error(`Couldn't update open uni search instance: ${e}`)
+  }
+}
+
+module.exports = { getOpenUniSearches, createNewSearch, deleteSearch, updateSearch }

services/backend/src/services/openUni/openUniSearches.js

@@ -0,0 +1,137 @@
+const { Credit, Enrollment, Studyright, Student, Course } = require('../../models')
+const { OpenUniPopulationSearch } = require('../../models/models_kone')
+const {
+  mapOpenCredits,
+  mapOpenEnrollments,
+  mapStundentInfo,
+  mapStudyRights,
+  mapCourseInfo,
+} = require('./openUniHelpers')
+const { Op } = require('sequelize')
+
+// 1. iteration: time is hardcoded. Check hyväksytty grades
+const getCredits = async (courseCodes, startdate) =>
+  (
+    await Credit.findAll({
+      where: {
+        course_code: {
+          [Op.in]: courseCodes,
+        },
+        is_open: true,
+        grade: {
+          [Op.in]: ['1', '2', '3', '4', '5', 'Hyv.', 'hyv', 'HT', 'TT'],
+        },
+        attainment_date: {
+          [Op.gte]: startdate,
+        },
+      },
+    })
+  ).map(mapOpenCredits)
+
+const getStudentInfo = async students =>
+  (
+    await Student.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: students,
+        },
+      },
+    })
+  ).map(mapStundentInfo)
+
+const getEnrollments = async (courseCodes, startdate, enddate) =>
+  (
+    await Enrollment.findAll({
+      where: {
+        course_code: {
+          [Op.in]: courseCodes,
+        },
+        is_open: true,
+        enrollment_date_time: {
+          [Op.and]: {
+            [Op.lte]: enddate,
+            [Op.gte]: startdate,
+          },
+        },
+      },
+    })
+  ).map(mapOpenEnrollments)
+
+const getCourseNames = async courseCodes =>
+  (
+    await Course.findAll({
+      attributes: ['code', 'name'],
+      where: {
+        code: {
+          [Op.in]: courseCodes,
+        },
+      },
+    })
+  ).map(mapCourseInfo)
+
+const getStudyRights = async students =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        student_studentnumber: {
+          [Op.in]: students.length > 0 ? students : { [Op.not]: null },
+        },
+      },
+    })
+  ).map(mapStudyRights)
+
+const getOpenUniSearchesByUser = async userId => {
+  return await OpenUniPopulationSearch.findAll({
+    where: {
+      userId,
+    },
+  })
+}
+
+const createOpenUniPopulationSearch = async (userId, name, courseCodes) => {
+  return await OpenUniPopulationSearch.create({
+    userId,
+    name,
+    courseCodes,
+  })
+}
+
+const updateOpenUniPopulationSearch = async (userId, id, courseCodes) => {
+  const searchToUpdate = await OpenUniPopulationSearch.findOne({
+    where: {
+      userId: userId,
+      id: id,
+    },
+  })
+
+  if (!searchToUpdate) return null
+  return await searchToUpdate.update({ courseCodes })
+}
+
+const deleteOpenUniSearch = async (userId, id) => {
+  return await OpenUniPopulationSearch.destroy({
+    where: {
+      userId: userId,
+      id: id,
+    },
+  })
+}
+
+module.exports = {
+  getCredits,
+  getEnrollments,
+  getStudyRights,
+  getStudentInfo,
+  getCourseNames,
+  getOpenUniSearchesByUser,
+  createOpenUniPopulationSearch,
+  updateOpenUniPopulationSearch,
+  deleteOpenUniSearch,
+}

services/backend/src/services/openUni/openUniStats.js

@@ -0,0 +1,79 @@
+const moment = require('moment')
+const { getCredits, getStudyRights, getEnrollments, getStudentInfo, getCourseNames } = require('./openUniSearches')
+
+const uniq = objects => [...new Set(objects)]
+
+const getCustomOpenUniCourses = async (courseCodes, startdate, enddate) => {
+  const ayCourseCodes = courseCodes.map(courseCode => 'AY' + courseCode)
+  const allCourseCodes = courseCodes.concat(ayCourseCodes)
+  const allCredits = await getCredits(allCourseCodes, startdate)
+  const allEnrollments = await getEnrollments(allCourseCodes, startdate, enddate)
+  const students = uniq(allEnrollments.map(enrollment => enrollment.enrollmentStudentnumber))
+  const courses = await getCourseNames(courseCodes)
+
+  const allStudyrights = await getStudyRights(students)
+  const studentInfo = await getStudentInfo(students)
+  // Filter out current studyrights:
+  // Case 1: Both startdate and enddate are outside of the given interval
+  // Case 2: Startdate is inside of the given interval and enddate is outside
+  // Case 3: Startdate is before the interval start and the enddate is within the interval
+  const studentsWithCurrentStudyRight = allStudyrights
+    .filter(
+      right =>
+        moment(right.startdate).isBetween(startdate, moment()) ||
+        (moment(right.startdate).isSameOrBefore(startdate) && moment(right.enddate).isSameOrAfter(moment())) ||
+        moment(right.enddate).isSameOrBefore(moment())
+    )
+    .map(right => right.studyrightStudentnumber)
+  const uniqueStudentsWithCurrentStudyRight = uniq(studentsWithCurrentStudyRight)
+
+  const studentStats = {}
+  const getEmptyCourseInfo = () => {
+    return {
+      enrolledPassed: null,
+      enrolledNotPassed: [],
+    }
+  }
+
+  for (const { studentnumber, email, secondary_email, dissemination_info_allowed } of studentInfo) {
+    // Check if the student has existing studyright: if yes, then stop here
+    if (!uniqueStudentsWithCurrentStudyRight.includes(studentnumber)) {
+      if (!(studentnumber in studentStats)) {
+        studentStats[studentnumber] = {
+          courseInfo: courseCodes.reduce((acc, code) => ({ ...acc, [code]: getEmptyCourseInfo() }), {}),
+          email: email,
+          secondaryEmail: secondary_email,
+          disseminationInfoAllowed: dissemination_info_allowed,
+          passedTotal: 0,
+          enrolledTotal: 0,
+        }
+      }
+      const passedCourses = []
+      const unfinishedCourses = []
+      for (const { course_code, attainment_date, student_studentnumber } of allCredits) {
+        if (student_studentnumber === studentnumber) {
+          const courseCode = course_code.replace('AY', '')
+          studentStats[studentnumber].courseInfo[courseCode].enrolledPassed = attainment_date
+          if (!passedCourses.includes(courseCode)) {
+            passedCourses.push(courseCode)
+          }
+        }
+      }
+      for (const { enrollmentStudentnumber, course_code, enrollment_date_time } of allEnrollments) {
+        if (enrollmentStudentnumber === studentnumber) {
+          const courseCode = course_code.replace('AY', '')
+          if (!studentStats[studentnumber].courseInfo[courseCode].enrolledPassed && studentnumber === studentnumber) {
+            studentStats[studentnumber].courseInfo[courseCode].enrolledNotPassed.push(enrollment_date_time)
+            if (!unfinishedCourses.includes(courseCode)) unfinishedCourses.push(courseCode)
+          }
+        }
+      }
+      studentStats[studentnumber].passedTotal = passedCourses.length
+      studentStats[studentnumber].enrolledTotal = unfinishedCourses.length
+    }
+  }
+  const openUniStats = { students: studentStats, courses: courses }
+  return openUniStats
+}
+
+module.exports = { getCustomOpenUniCourses }

services/backend/src/services/populations.js

@@ -210,7 +210,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       raw: true,
     }),
     Enrollment.findAll({
-      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber'],
+      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber', 'semestercode'],
       where: {
         enrollment_date_time: {
           [Op.between]: [attainmentDateFrom, endDate],
@@ -414,7 +414,7 @@ const studentnumbersWithAllStudyrightElements = async (
   // db startdate is formatted to utc so need to change it when querying
   const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
 
-  const filteredExtents = [10] // always filter out secondary subject students
+  const filteredExtents = [16] // always filter out secondary subject students
   let studyrightWhere = {
     extentcode: {
       [Op.notIn]: filteredExtents,
@@ -424,7 +424,7 @@ const studentnumbersWithAllStudyrightElements = async (
     filteredExtents.push(7, 34)
   }
   if (!nondegreeStudents) {
-    filteredExtents.push(33, 99, 14, 13)
+    filteredExtents.push(22, 99, 14, 13)
   }
 
   let studentWhere = {}

services/backend/src/services/programmeModules.js

@@ -38,8 +38,14 @@ const byProgrammeCode = async code => {
     },
   })
 
+  // filter out possible duplicates
+  const filtered = result.filter(
+    (course, index, array) =>
+      array.findIndex(c => c.id == course.id && c.code == course.code && c.label_code == course.label_code) == index
+  )
+
   // this labels the modules to match the old system in frontend
-  const labeled = result.map(module => {
+  const labeled = filtered.map(module => {
     const label = {
       id: module.label_name.fi,
       label: `${module.label_code}\n${module.label_name.fi}`,

services/backend/src/services/students.js

@@ -121,7 +121,7 @@ const getUnifyStatus = unifyCourses => {
 }
 
 const findByCourseAndSemesters = async (coursecodes, from, to, separate, unifyCourses = 'unifyStats') => {
-  const { startdate } = await Semester.findOne({
+  const { startdate, semestercode: fromSemester } = await Semester.findOne({
     where: {
       [separate ? 'semestercode' : 'yearcode']: from,
     },
@@ -129,7 +129,7 @@ const findByCourseAndSemesters = async (coursecodes, from, to, separate, unifyCo
     limit: 1,
     raw: true,
   })
-  const { enddate } = await Semester.findOne({
+  const { enddate, semestercode: toSemester } = await Semester.findOne({
     where: {
       [separate ? 'semestercode' : 'yearcode']: to,
     },
@@ -164,7 +164,7 @@ const findByCourseAndSemesters = async (coursecodes, from, to, separate, unifyCo
       WHERE
         e.studentnumber = s.studentnumber
         AND e.course_code IN (:coursecodes)
-        AND e.enrollment_date_time BETWEEN '${startdate.toISOString()}' AND '${enddate.toISOString()}'
+        AND e.semestercode BETWEEN ${fromSemester} AND ${toSemester}
         AND e.enrollment_date_time >= '2021-05-31'
         AND e.state IN ('ENROLLED', 'CONFIRMED')
     );

services/backend/src/services/studyprogrammeGraduations.js

@@ -14,6 +14,7 @@ const {
   getThesisType,
   alltimeStartDate,
   alltimeEndDate,
+  getId,
 } = require('./studyprogrammeHelpers')
 const {
   graduatedStudyRights,
@@ -126,7 +127,7 @@ const getTransferStudyrightMap = async (studyprogramme, since) => {
 const formatStats = (stats, years) => {
   const tableStats = Object.values(stats)
     .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
-    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+    .map(p => [p.code, getId(p.code), p.name['fi'], ...years.map(year => p[year])])
 
   const graphStats = Object.values(stats)
     .filter(p => years.map(year => p[year]).find(started => started !== 0)) // Filter out programmes with no-one started between the selected years
@@ -229,7 +230,7 @@ const getGraduationStatsForStudytrack = async ({ studyprogramme, settings }) =>
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
 
   const titles = ['', 'Graduated', 'Wrote thesis']
-  const programmesBeforeOrAfterTitles = ['Code', 'Programme', ...years]
+  const programmesBeforeOrAfterTitles = ['Code', 'Id', 'Programme', ...years]
   const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
 
   return {

services/backend/src/services/studyprogrammeHelpers.js

@@ -1,8 +1,8 @@
 const { mean } = require('lodash')
 const { studentnumbersWithAllStudyrightElements } = require('./populations')
+const { codes } = require('../../config/programmeCodes')
 
 // Helper functions
-
 const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
   let studentnumbers = []
   const exchangeStudents = includeAllSpecials
@@ -151,15 +151,16 @@ const isNonMajorCredit = (studyrights, attainment_date) =>
     )
   })
 
-const isSpecialGroupCredit = (studyrights, attainment_date, transfers) =>
+const isSpecialGroupCredit = (studyrights, attainment_date, transfers) => {
   studyrights.some(studyright => {
     if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
     if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
     if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
-    if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+    if ([7, 9, 16, 34, 22, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
     if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
     return false
   })
+}
 
 const getMedian = values => {
   if (values.length === 0) return 0
@@ -224,8 +225,12 @@ const getPercentage = (value, total) => {
 const getEmptyArray = length => new Array(length).fill(0)
 
 const getBachelorCreditGraphStats = years => ({
+  lte15: {
+    name: 'Less than 15 credits',
+    data: getEmptyArray(years.length),
+  },
   lte30: {
-    name: 'Less than 30 credits',
+    name: '15-29 credits',
     data: getEmptyArray(years.length),
   },
   lte60: {
@@ -249,7 +254,7 @@ const getBachelorCreditGraphStats = years => ({
     data: getEmptyArray(years.length),
   },
   mte180: {
-    name: 'More than 180 credits',
+    name: '180 or more credits',
     data: getEmptyArray(years.length),
   },
 })
@@ -280,7 +285,34 @@ const getMasterCreditGraphStats = years => ({
     data: getEmptyArray(years.length),
   },
   mte300: {
-    name: 'More than 300 credits',
+    name: '300 or more credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getOnlyMasterCreditGraphStats = years => ({
+  lte15: {
+    name: 'Less than 15 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte30: {
+    name: '15-29 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte60: {
+    name: '30-59 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte90: {
+    name: '60-89 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte120: {
+    name: '90-119 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte120: {
+    name: '120 or more credits',
     data: getEmptyArray(years.length),
   },
 })
@@ -311,7 +343,7 @@ const getDoctoralCreditGraphStats = years => ({
     data: getEmptyArray(years.length),
   },
   mte300: {
-    name: 'More than 300 credits',
+    name: '300 or more credits',
     data: getEmptyArray(years.length),
   },
 })
@@ -322,13 +354,18 @@ const getCreditGraphStats = (studyprogramme, years) => {
   return getDoctoralCreditGraphStats(years)
 }
 
-const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
+const bachelorCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'lte180', 'mte180']
 const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'lte300', 'mte300']
+const onlyMasterCreditThresholds = ['lte15', 'lte30', 'lte60', 'lte90', 'lte120', 'mte120']
 const doctoralCreditThresholds = ['lte50', 'lte100', 'lte150', 'lte200', 'lte250', 'lte300', 'mte300']
-const bachelorCreditAmounts = [30, 60, 90, 120, 150, 180, 180]
+const bachelorCreditAmounts = [15, 30, 60, 90, 120, 150, 180, 180]
 const masterCreditAmounts = [200, 220, 240, 260, 280, 300, 300]
+const onlyMasterCreditAmounts = [15, 30, 60, 90, 120, 120]
 const doctoralCreditAmounts = [50, 100, 150, 200, 250, 300, 300]
 
+const getOnlyMasterThresholds = () => {
+  return { msOnlyCreditThresholdKeys: onlyMasterCreditThresholds, msOnlyCreditThresholdAmount: onlyMasterCreditAmounts }
+}
 const getCreditThresholds = studyprogramme => {
   if (studyprogramme.includes('KH')) {
     return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
@@ -350,8 +387,8 @@ const tableTitles = {
       '',
       'Total',
       'Major students credits',
-      'Non major students credits',
-      'Non degree students credits',
+      'Non-major students credits',
+      'Non-degree students credits',
       'Transferred credits',
     ],
   },
@@ -359,13 +396,14 @@ const tableTitles = {
     bachelor: [
       '',
       'All',
-      '< 30 credits',
+      '< 15 credits',
+      '15-29 credits',
       '30-59 credits',
       '60-89 credits',
       '90-119 credits',
       '120-149 credits',
       '150-179 credits',
-      '> 180 credits',
+      '>= 180 credits',
     ],
     master: [
       '',
@@ -376,7 +414,17 @@ const tableTitles = {
       '240-259 credits',
       '260-279 credits',
       '280-299 credits',
-      '> 300 credits',
+      '>= 300 credits',
+    ],
+    masterOnly: [
+      '',
+      'All',
+      '< 15 credits',
+      '15-29 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '>= 120 credits',
     ],
     doctoral: [
       '',
@@ -387,7 +435,7 @@ const tableTitles = {
       '150-199 credits',
       '200-249 credits',
       '250-299 credits',
-      '> 300 credits',
+      '>= 300 credits',
     ],
   },
   studytracks: [
@@ -410,6 +458,25 @@ const getCreditProgressTableTitles = studyprogramme => {
   return tableTitles.creditProgress.doctoral
 }
 
+const mapCodesToIds = data => {
+  // Add programme id e.g. TKT
+  const keys = Object.keys(codes)
+  const progs = Object.keys(data)
+
+  for (const prog of progs) {
+    if (keys.includes(prog)) {
+      data[prog].progId = codes[prog].toUpperCase()
+    }
+  }
+}
+
+const getId = code => {
+  if (Object.keys(codes).includes(code)) {
+    return codes[code].toUpperCase()
+  }
+  return ''
+}
+
 module.exports = {
   getCorrectStudentnumbers,
   formatStudyright,
@@ -432,9 +499,13 @@ module.exports = {
   getBachelorCreditGraphStats,
   getMasterCreditGraphStats,
   getDoctoralCreditGraphStats,
+  getOnlyMasterCreditGraphStats,
   getCreditGraphStats,
   getCreditThresholds,
   tableTitles,
   getCreditProgressTableTitles,
+  getOnlyMasterThresholds,
   isNonMajorCredit,
+  mapCodesToIds,
+  getId,
 }

services/backend/src/services/studyprogrammeUpdates.js

@@ -30,7 +30,6 @@ const updateBasicView = async code => {
   }
 
   const options = [specialCalendar, specialAcademic, specialExcludedCalendar, specialExcludedAcademic]
-
   for (const option of options) {
     try {
       const basicStats = await getBasicStatsForStudytrack({
@@ -62,8 +61,8 @@ const updateBasicView = async code => {
     } catch (e) {
       logger.error(e)
     }
-    return 'OK'
   }
+  return 'OK'
 }
 
 const updateStudytrackView = async code => {

services/backend/src/services/studytrackStats.js

@@ -44,7 +44,13 @@ const getStudentData = (startDate, students, thresholdKeys, thresholdAmounts) =>
     data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
     data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
     data[thresholdKeys[5]] += creditcount >= thresholdAmounts[4] && creditcount < thresholdAmounts[5] ? 1 : 0
-    data[thresholdKeys[6]] += creditcount >= thresholdAmounts[6] ? 1 : 0
+    // bachelor programme has more columns than master and doctoral programmes
+    if (thresholdKeys.length > 7) {
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] && creditcount < thresholdAmounts[6] ? 1 : 0
+      data[thresholdKeys[7]] += creditcount >= thresholdAmounts[6] ? 1 : 0
+    } else {
+      data[thresholdKeys[6]] += creditcount >= thresholdAmounts[5] ? 1 : 0
+    }
   })
   return data
 }

services/backend/src/services/userService.js

@@ -10,8 +10,10 @@ const { User, UserElementDetails, AccessGroup, UserFaculties, sequelizeUser } =
 const { getOrganizationAccess } = require('../util/organizationAccess')
 
 const courseStatisticsGroup = 'grp-oodikone-basic-users'
+const facultyStatisticsGroup = 'grp-oodikone-users'
 const hyOneGroup = 'hy-one'
 const toskaGroup = 'grp-toska'
+const openUniGroup = 'hy-ypa-opa-dojo'
 
 // Max 25 users can be stored in the cache
 const userDataCache = new LRU({
@@ -191,7 +193,9 @@ const formatUser = async userFromDb => {
 
   const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)])
 
-  const iamRights = Object.keys(await getOrganizationAccess(userFromDb))
+  const iamRights = Object.keys(await getOrganizationAccess(userFromDb)).filter(
+    programmeCode => !rights.includes(programmeCode)
+  )
 
   const {
     dataValues: {
@@ -265,9 +269,17 @@ const getUser = async ({ username, name, email, iamGroups, sisId }) => {
   let newAccessGroups = []
   if (await checkStudyGuidanceGroupsAccess(sisId)) newAccessGroups.push('studyGuidanceGroups')
   if (iamGroups.includes(courseStatisticsGroup)) newAccessGroups.push('courseStatistics')
+  if (iamGroups.includes(facultyStatisticsGroup) || iamGroups.some(element => /-jory$/.test(element)))
+    newAccessGroups.push('facultyStatistics')
   if (iamGroups.includes(hyOneGroup) || currentAccessGroups.includes('teachers')) newAccessGroups.push('teachers')
   if (iamGroups.includes(toskaGroup) || currentAccessGroups.includes('admin')) newAccessGroups.push('admin')
-  if (_.difference(newAccessGroups, currentAccessGroups).length > 0) {
+  if (iamGroups.includes(openUniGroup)) newAccessGroups.push('openUniSearch')
+  // In the difference method "the order and references of result values are determined by the first array." https://lodash.com/docs/4.17.15#difference
+  // Both directions needs to be checked in order to update roles when the access should no longer exists.
+  if (
+    _.difference(newAccessGroups, currentAccessGroups).length > 0 ||
+    _.difference(currentAccessGroups, newAccessGroups).length > 0
+  ) {
     userFromDb.setAccessgroup(
       (
         await AccessGroup.findAll({

services/backend/src/util/avoinPopulation.js

@@ -0,0 +1,190 @@
+/* eslint-disable */
+
+const { Credit, Enrollment, Studyright } = require('../models')
+const moment = require('moment')
+const { Op } = require('sequelize')
+
+const hasValidStudyright = (studyrights, date) => {
+  const within = (right, date) => {
+    const first = moment(right.startdate)
+    const last = moment(right.enddate)
+    return first.isSameOrBefore(date) && last.isSameOrAfter(date)
+  }
+
+  return studyrights.find(r => within(r, date))
+}
+
+const studyrightsOf = async student => {
+  const rights = await Studyright.findAll({
+    where: {
+      student_studentnumber: student,
+    },
+  })
+
+  return rights
+}
+
+const uniq = objects => [...new Set(objects)]
+
+const courseInfo = async course_code => {
+  const credits = await Credit.findAll({
+    where: {
+      course_code,
+      is_open: true,
+      grade: {
+        [Op.in]: ['1', '2', '3', '4', '5', 'Hyv.'],
+      },
+      attainment_date: {
+        [Op.gte]: moment('8-1-2020', 'MM-DD-YYYY').toDate(),
+      },
+    },
+  })
+
+  const enrollments = await Enrollment.findAll({
+    where: {
+      course_code,
+      is_open: true,
+      enrollment_date_time: {
+        [Op.lte]: moment('8-1-2022', 'MM-DD-YYYY').toDate(),
+        [Op.gte]: moment('8-1-2020', 'MM-DD-YYYY').toDate(),
+      },
+    },
+  })
+
+  let avoin_credited = []
+
+  for (let credit of credits) {
+    const { attainment_date, student_studentnumber: studentnumber } = credit
+    const rights = await studyrightsOf(studentnumber)
+    if (!hasValidStudyright(rights, attainment_date)) {
+      avoin_credited.push(studentnumber)
+    }
+  }
+
+  let avoin_enrolled = []
+
+  for (let enrollment of enrollments) {
+    const { enrollment_date_time, studentnumber } = enrollment
+    const rights = await studyrightsOf(studentnumber)
+    if (!hasValidStudyright(rights, enrollment_date_time)) {
+      avoin_enrolled.push(studentnumber)
+    }
+  }
+
+  avoin_credited = uniq(avoin_credited)
+  avoin_enrolled = uniq(avoin_enrolled)
+
+  return {
+    credits,
+    enrollments,
+    avoin_credited,
+    avoin_enrolled,
+  }
+}
+
+const unifiedCourseInfo = async course_code => {
+  const a = await courseInfo(course_code)
+  const b = await courseInfo('AY' + course_code)
+
+  avoin_credited = uniq(a.avoin_credited.concat(b.avoin_credited))
+  avoin_enrolled = uniq(a.avoin_enrolled.concat(b.avoin_enrolled))
+
+  return {
+    credits: uniq(a.credits.concat(b.credits)),
+    enrollments: uniq(a.enrollments.concat(b.enrollments)),
+    avoin_credited,
+    avoin_enrolled,
+    only_enrolled: avoin_enrolled.filter(e => !avoin_credited.includes(e)),
+  }
+}
+
+const statsOf = async courses => {
+  console.log(Object.keys(courses).join('\n'))
+  console.log()
+  let students = []
+  for (let course_code of Object.keys(courses)) {
+    const { credits, enrollments, avoin_enrolled, avoin_credited, only_enrolled } = await unifiedCourseInfo(course_code)
+    courses[course_code] = { credits, enrollments, avoin_enrolled, avoin_credited, only_enrolled }
+    students = uniq(students.concat(avoin_enrolled).concat(avoin_credited))
+  }
+
+  const rows = []
+
+  for (let s of students) {
+    let string = s + ' '
+    let score = 0
+    for (let course_code of Object.keys(courses)) {
+      const { enrollments, avoin_credited, only_enrolled } = courses[course_code]
+      let course_info = '-'
+      if (avoin_credited.includes(s)) {
+        course_info = 'X'
+        score += 10
+      }
+
+      if (only_enrolled.includes(s)) {
+        const studentsEnrolments = enrollments.filter(e => e.studentnumber === s)
+        course_info = '' + studentsEnrolments.length
+
+        score += 1
+      }
+
+      string += course_info + ' '
+    }
+
+    const rights = await studyrightsOf(s)
+    const stRight = hasValidStudyright(rights, new Date('2022-8-1')) ? 'yes' : 'no'
+
+    string += stRight
+
+    rows.push({ string, score })
+  }
+
+  const byScore = (a, b) => b.score - a.score
+
+  console.log(
+    rows
+      .sort(byScore)
+      .map(r => r.string)
+      .join('\n')
+  )
+}
+
+const main = async () => {
+  const courses_psyk_a = {
+    'PSYK-111': null,
+    'PSYK-121': null,
+    'PSYK-131': null,
+    'PSYK-141': null,
+    'PSYK-181': null,
+  }
+
+  const courses_psyk_p = {
+    'PSYK-251': null,
+    'PSYK-221': null,
+    'PSYK-232': null,
+    'PSYK-226': null,
+    'PSYK-241': null,
+    'PSYK-212': null,
+  }
+
+  const courses_hist = {
+    'HISK-110': null,
+    'HISK-120': null,
+    'HISK-131A': null,
+    'HISK-131': null,
+    'HISK-141': null,
+    'HISK-142': null,
+  }
+
+  await statsOf(courses_psyk_a)
+  console.log()
+  console.log()
+  await statsOf(courses_psyk_p)
+  console.log()
+  console.log()
+  await statsOf(courses_hist)
+
+  process.exit()
+}
+
+main()

services/backend/src/util/semester.js

@@ -8,7 +8,7 @@ const getSemester = date => (isSpring(date) ? 'SPRING' : 'FALL')
 
 const semesterStart = {
   SPRING: '01-01',
-  FALL: '07-31',
+  FALL: '08-01',
 }
 
 const semesterEnd = {

services/frontend/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "name": "frontend",
       "dependencies": {
-        "@reduxjs/toolkit": "^1.8.4",
+        "@reduxjs/toolkit": "^1.8.6",
         "@sentry/browser": "^6.18.2",
         "@sentry/tracing": "^6.18.2",
         "axios": "^0.21.4",
@@ -36,7 +36,7 @@
         "react-scripts": "4.0.3",
         "react-scroll-to-component": "^1.0.2",
         "scheduler": "^0.23.0",
-        "semantic-ui-css": "^2.4.1",
+        "semantic-ui-css": "^2.5.0",
         "semantic-ui-react": "^2.1.2",
         "unfuck-spa-shibboleth-session": "^3.1.4",
         "use-context-selector": "^1.3.9",
@@ -2841,9 +2841,9 @@
       }
     },
     "node_modules/@reduxjs/toolkit": {
-      "version": "1.8.4",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.4.tgz",
-      "integrity": "sha512-IpFq1WI7sCYeLQpDCGvlcQY9wn70UpAM3cOLq78HRnVn1746RI+l3y5xcuOeVOxORaxABJh3cfJMxycD2IwH5w==",
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.6.tgz",
+      "integrity": "sha512-4Ia/Loc6WLmdSOzi7k5ff7dLK8CgG2b8aqpLsCAJhazAzGdp//YBUSaj0ceW6a3kDBDNRrq5CRwyCS0wBiL1ig==",
       "dependencies": {
         "immer": "^9.0.7",
         "redux": "^4.1.2",
@@ -17914,9 +17914,9 @@
       }
     },
     "node_modules/semantic-ui-css": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.4.1.tgz",
-      "integrity": "sha512-Pkp0p9oWOxlH0kODx7qFpIRYpK1T4WJOO4lNnpNPOoWKCrYsfHqYSKgk5fHfQtnWnsAKy7nLJMW02bgDWWFZFg==",
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.5.0.tgz",
+      "integrity": "sha512-jIWn3WXXE2uSaWCcB+gVJVRG3masIKtTMNEP2X8Aw909H2rHpXGneYOxzO3hT8TpyvB5/dEEo9mBFCitGwoj1A==",
       "dependencies": {
         "jquery": "x.*"
       }
@@ -24283,9 +24283,9 @@
       "integrity": "sha512-9X2obfABZuDVLCgPK9aX0a/x4jaOEweTTWE2+9sr0Qqqevj2Uv5XorvusThmc9XGYpS9yI+fhh8RTafBtGposw=="
     },
     "@reduxjs/toolkit": {
-      "version": "1.8.4",
-      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.4.tgz",
-      "integrity": "sha512-IpFq1WI7sCYeLQpDCGvlcQY9wn70UpAM3cOLq78HRnVn1746RI+l3y5xcuOeVOxORaxABJh3cfJMxycD2IwH5w==",
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.8.6.tgz",
+      "integrity": "sha512-4Ia/Loc6WLmdSOzi7k5ff7dLK8CgG2b8aqpLsCAJhazAzGdp//YBUSaj0ceW6a3kDBDNRrq5CRwyCS0wBiL1ig==",
       "requires": {
         "immer": "^9.0.7",
         "redux": "^4.1.2",
@@ -35833,9 +35833,9 @@
       }
     },
     "semantic-ui-css": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.4.1.tgz",
-      "integrity": "sha512-Pkp0p9oWOxlH0kODx7qFpIRYpK1T4WJOO4lNnpNPOoWKCrYsfHqYSKgk5fHfQtnWnsAKy7nLJMW02bgDWWFZFg==",
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/semantic-ui-css/-/semantic-ui-css-2.5.0.tgz",
+      "integrity": "sha512-jIWn3WXXE2uSaWCcB+gVJVRG3masIKtTMNEP2X8Aw909H2rHpXGneYOxzO3hT8TpyvB5/dEEo9mBFCitGwoj1A==",
       "requires": {
         "jquery": "x.*"
       }

services/frontend/package.json

@@ -6,7 +6,7 @@
     "build": "DISABLE_ESLINT_PLUGIN=true react-scripts build"
   },
   "dependencies": {
-    "@reduxjs/toolkit": "^1.8.4",
+    "@reduxjs/toolkit": "^1.8.6",
     "@sentry/browser": "^6.18.2",
     "@sentry/tracing": "^6.18.2",
     "axios": "^0.21.4",
@@ -36,7 +36,7 @@
     "react-scripts": "4.0.3",
     "react-scroll-to-component": "^1.0.2",
     "scheduler": "^0.23.0",
-    "semantic-ui-css": "^2.4.1",
+    "semantic-ui-css": "^2.5.0",
     "semantic-ui-react": "^2.1.2",
     "unfuck-spa-shibboleth-session": "^3.1.4",
     "use-context-selector": "^1.3.9",

services/frontend/src/common/InfoToolTips/faculty.js

@@ -0,0 +1,117 @@
+export default {
+  ProgrammeToggle: `
+    New study programmes: Näyttää kaikissa luvuissa ja kuvaajissa pelkästään uuden opetussuunnitelman ohjelmat.
+    All programmes: Näyttää kaikissa luvuissa ja kuvaajissa kuvaajissa myös vanhan opetussuunnitelman ohjelmat.`,
+  YearToggle: `
+    Academic year: Näyttää tilastot akateemisille vuosille jaoteltuna, esimerkiksi 1.8.2019-31.7.2020
+    Calendar year: Näyttää tilastot kalenterivuosille jaoteltuna, esimerkiksi 1.1.2019-31.12.2019 
+    `,
+  StudentsOfTheFaculty: `
+  **Started studying**: Sisältää kyseisenä vuonna tiedekunnassa aloittaneet opiskelijat (aloitetut opinto-oikeudet) riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut myöhemmin keskeyttää opintonsa tai siirtyä pois ohjelmasta.
+    Kandi-maisteri-opinto-oikeudellisista opiskelijoista lasketaan vain kandissa ts. tiedekunnassa aloitus, ei toistamiseen enää maisteriopintojen aloitusta.\n
+  **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
+  
+  **Transferred inside**: Sisältää kyseisenä vuonna tiedekunnan sisällä siirtyneet opiskelijat (opinto-oikeudet). Ohjelmakohtaisissa tilastoissa siirto on laskettu kohdeohjelman tilastoihin.
+
+  **Transferred away**: Sisältää kyseisenä vuonna pois tiedekunnasta siirtyneet opiskelijat (opinto-oikeudet). Sama opiskelija voi esiintyä tilastoissa useana vuonna, mikäli hän on siirtynyt tiedekuntaan ja siitä pois useasti.
+  
+  **Transferred to**: Sisältää kyseisenä vuonna tiedekuntaan siirtyneet opiskelijat (opinto-oikeudet).
+
+  HUOM! Siirtyneiden määrä on erityisen suuri vuonna 2020, jolloin vanhoista koulutusohjelmista luovuttiin ja opiskelijat siirtyivät uusiin ohjelmiin.
+  `,
+  CreditsProducedByTheFaculty: `
+  Sisältää opintopisteet suoritusvuosittain. Suoritukset on jaoteltu Sisussa näkyvän kurssin suorituspäivän mukaan.
+
+  **Major students credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus johonkin tiedekunnan koulutusohjelmaan. Sisältää myös tiedekunnan ohjelmaan siirtyneet ensisijaiset opiskelijat.
+
+  **Non-major faculty students credits**: Sisältää tiedekunnan koulutusohjelmien tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan,
+  MUTTA hänellä on sellainen johonkin toiseen tiedekunnan ohjelmaan. Sisältää esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat.
+  Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
+  
+  **Non-major other faculty students credits**: Sisältää tiedekunnan koulutusohjelmien tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan tai mihinkään muuhun kyseisen tiedekunnan koulutusohjelmaan.
+  Sisältää esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat. Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
+  
+  **Non-degree student credits**: Sisältää tiedekuntaan suoritetut opintopisteet, joiden suorittajilla ei ole tutkinto-oikeutta Helsingin yliopistossa.
+  `,
+  GraduatedOfTheFaculty: `Sisältää kyseisenä vuonna tiedekunnasta valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.`,
+  ThesisWritersOfTheFaculty: `Sisältää kyseisenä vuonna kandidaatintutkielman, pro gradu -tutkielman, väitöskirjan tai lisensiaatintyön hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.`,
+  StudentToggle: `
+  All studyrights: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat ja ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.\n
+  Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
+  `,
+
+  // Graduation times
+  GroupByToggle: `Graduation year: Valmistuneet opiskelijat on ryhmitelty opintojen valmistumisvuoden perusteella.\n
+    Group by: Starting year: Valmistuneet opiskelijat on jaoteltu opinto-oikeuden alkuvuoden mukaan.
+  `,
+  AverageGraduationTimes: `
+  Opiskelijoiden keskimääräiset valmistumisajat tutkintotasoittain. 
+
+  - Bachelor: Kandidaatin tutkinnon suoritusajat
+  - Bachelor + Master: Kandi+maisteriopinto-oikeudella opiskelleiden ja siitä maisteriksi valmistuneiden suoritusajat
+  - Master: Pelkällä maisterintutkinnon opinto-oikeudella opiskelleiden suoritusajat
+  - Doctor: Tohtorin tutkinnon suoritusajat
+  - Licentiate: Lisensiaatin tutkinnon suoritusajat
+
+  Valmistumisajoista on **vähennetty lakisääteiset poissaolot**. 
+
+  Vihreä **katkoviiva** kuvastaa tavoiteaikaa. Keltainen katkoviiva on tavoiteaika + yksi vuosi.\n
+  Vihreä **vaakapalkki** merkitsee, että keskimääräinen valmistumisaika on alle tai yhtä kuin tavoiteaika, keltainen palkki merkitsee korkeintaan vuoden ylitystä
+  ja punainen yli vuoden tavoiteajan ylitystä.
+
+  Palkissa oleva lukumäärä kertoo **montako opiskelijaa** kuhunkin valmistuneiden ryhmään kuuluu. 
+  Tarkastellessa valmistumisaikoja aloitusvuosittain palkissa näkyy myös **kuinka suuri osa** kyseisenä vuonna tiedekuntaan/opinto-ohjelmaan opinto-oikeuden saaneista on valmistunut.
+
+
+  Säädöt:
+
+  **Graduation year**: Ryhmittelee tulokset valmistumisvuosittain. Näyttää siis vuonna xxxx valmistuneiden opiskelijoiden keskimääräiset valmistumisajat. 
+  Luvut ovat lopullisia (kuluva vuosi poislukien), sillä edeltäville vuosille ei tule uusia valmistumisia.
+
+  **Starting year**. Ryhmittelee tulokset opiskelijoiden opinto-oikeuden alkamisvuosittain. Näyttää siis vuonna xxxx aloittaneiden opiskelijoiden keskimääräiset valmistumisajat. Luvut voivat elää, koska osa vuonna xxxx 
+  aloittaneiden opiskelijoiden opinnoista voi olla edelleen kesken.
+
+  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina.
+
+  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina.`,
+
+  // Faculty Student population
+  GraduatedToggle: `
+  Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
+
+  Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
+  `,
+  StudentsStatsOfTheFaculty: `
+  Kuvaa tiedekunnan ohjelmissa opiskelevien opiskelijoden tilastoja. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  
+  HUOM!
+  **All** Sisältää kaikki opiskelijat, joiden opinto-oikeudella on aloituspäivä kyseisenä vuonna, riippumatta siitä, onko opiskelija aloittanut opiskelemaan ko. vuonna.
+  **Currently enrolled** Sisältää kaikki tällä hetkellä läsnäolevat olevat opiskelija.
+  **Absent** Sisältää kaikki lakisääteisistä syistä poissaolevat opiskelijat.
+  **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  **Inactive** Sisältää kaikki opinto-oikeudet, jotka ovat alkaneet kyseisenä vuonna, mutta jotka ovat sittemmin vanhentuneet ilman, että opiskelija on valmistunut ohjelmasta. Sisältää myös opinto-oikeudet,
+  joissa opiskelija on laiminlyönyt ilmoittautumisen tälle lukukaudelle.
+  
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, ohjelmaan ja siitä pois siirtyneet opiskelijat sekä erillisopinto-oikeudella opiskelevat.
+  Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
+
+  Yläosan valikosta on mahdollista valita tarkasteluun yhden opinto-ohjelman opiskelijat tietylle vuodelle. Sekä opiskelijoiden yleistiedot että edistyminen opintopisteittäin
+  kuvaavat tällöin kyseisen opintosuunnan opiskelijoita.`,
+  StudentProgress: `
+  Kuvaa tiedekuntaan kuuluvien eri ohjemien opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2021-2022 aloittaneiden opintopistejakaumaa.
+  
+  - Bachelor: Kandidaatin opinto-oikeudella varustettujen opiskelijoiden eteneminen
+  - Bachelor + Master: Kandi+maisteriopinto-oikeudella opiskelleiden ja siitä maisteriksi ohjelmiin siirtyneiden etenenminen
+  - Master: Pelkällä maisterintutkinnon opinto-oikeudella varustettujen opiskelijoiden eteneminen 
+  - Doctor: Tohtorin ja lisensiaatin opinto-oikeudella varustettujen opiskelijoiden eteneminen
+  
+  HUOM! Mukana ovat myös muuhun kuin valittuun koulutusohjelmaan tai opintosuuntaan tehdyt opintopisteet.
+
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, erillisopinto-oikeudella opiskelevat ja ohjelmaan siirtyneet.   
+
+  Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.
+  
+  **Total** Kuvaa opiskelijoinden opintopisteiden määrien kertymää alkaen 1.8.2017. Luvut ovat isompia kuin vuosittain lasketut, sillä nyt kaikki 1.8.2017 jälkeen 
+  suoritetut opintopisteet lasketaan mukaan, vaikka ne olisi suoritettu ennen varsinaista opinto-oikeuden alkua.
+  `,
+}

services/frontend/src/common/InfoToolTips/filters.js

@@ -18,9 +18,9 @@ export default {
     label: null,
     short: 'Rajaa kurssisuorituksia tietyn ajanjakson mukaan. Tämä rajaus ei muuta opiskelijoiden määrää',
   },
-  cherrypicker: {
+  studentNumber: {
     label: null,
     short:
-      'Rajaa yksittäisiä opiskelijoita pois tilastoista (blocklist) tai näytä vain määritellyt opiskelijat (allowlist). Vain täsmälliset opiskelijanumerot toimivat.',
+      '(ent. cherry-picker) Rajaa yksittäisiä opiskelijoita pois tilastoista (blocklist) tai näytä vain määritellyt opiskelijat (allowlist). Vain täsmälliset opiskelijanumerot toimivat.',
   },
 }

services/frontend/src/common/InfoToolTips/index.jsx

@@ -3,6 +3,7 @@ import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
 import CoursePopulation from './coursePopulation'
 import Studyprogramme from './studyprogramme'
+import Faculty from './faculty'
 
 export default {
   PopulationStatistics,
@@ -10,4 +11,5 @@ export default {
   CourseStatistics,
   CoursePopulation,
   Studyprogramme,
+  Faculty,
 }

services/frontend/src/common/InfoToolTips/populationStatistics.js

@@ -1,7 +1,8 @@
 export default {
-  Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella ilmoittautuneita, haluttu ajanjakso valitaan tästä.
+  Advanced: `**Semesters:**  Jos haluaa tarkastella vain syyslukukaudella tai kevätlukukaudella aloittaneita, haluttu ajanjakso valitaan tästä.
 
-  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.`,
+  **Include:** Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois ohjelmasta poissiirtyneet opiskelijat. Heidät voi sisällyttää mukaan lisäämällä ruksin ja noutamalla tulokset uudestaan (Fetch class with new settings).`,
+  // Koulutusohjelmapopulaatiosta on oletusarvoisesti suodatettu pois vaihto-opiskelijat ja erillisopinto-oikeudella opiskelevat. Näitä ryhmiä voi sisällyttää mukaan lisäämällä ruksin haluamaansa kohtaan.
   QueryCard: `**Valitun koulutusohjelmavuosikurssin perustiedot.**
   
   **Sample size** luku sisältää läsnä- ja poissaolevaksi ilmoittautuneet opiskelijat, muttei "excludes"-ryhmiä, jotka voi määritellä "advanced settings" -kohdasta.
@@ -48,7 +49,7 @@ export default {
 
   **Cumulative**-valinnan ollessa käytössä kunkin opiskelijan kaikki aiemmat opintopisteet lasketaan mukaan opintopistemäärään. Muutoin vain pylvästä vastaavalla aikavälillä kirjatut opintopisteet otetaan huomioon.
   `,
-  CoursesOfPopulation: `**Courses of population**
+  CoursesOfPopulation: `**Courses of class**
 
   Tämä taulukko sisältää näkyvissä olevien opiskelijoiden kaikki kurssisuoritukset.
   

services/frontend/src/components/CoursePopulation/index.jsx

@@ -31,7 +31,7 @@ import {
   startYearAtUniFilter,
   programmeFilter,
   studyTrackFilter,
-  cherrypickFilter,
+  studentNumberFilter,
 } from '../FilterView/filters'
 import FilterView from '../FilterView'
 import useLanguage from '../LanguagePicker/useLanguage'
@@ -319,7 +319,7 @@ const CoursePopulation = ({
       name="CoursePopulation"
       filters={[
         genderFilter,
-        cherrypickFilter,
+        studentNumberFilter,
         ageFilter,
         courseFilter({ courses: courseStatistics?.coursestatistics }),
         creditsEarnedFilter,

services/frontend/src/components/CourseStatistics/SingleCourseStats/countTotalStats.js

@@ -65,7 +65,7 @@ const countTotalStats = (formattedStats, userHasAccessToAllStats) => {
           categories: { passed, failed },
           grades: cgrades,
           enrollmentsByState: { ...acc.attempts.enrollmentsByState },
-          totalEnrollments: acc.students.totalEnrollments + (curr.students.totalEnrollments || 0),
+          totalEnrollments: acc.attempts.totalEnrollments + (curr.attempts.totalEnrollments || 0),
         },
         students: {
           totalEnrollments: acc.students.totalEnrollments + (curr.students.totalEnrollments || 0),

services/frontend/src/components/CustomOpenUniPopulation/CustomOpenUniSearch.js

@@ -0,0 +1,230 @@
+import moment from 'moment'
+import React, { useEffect, useState } from 'react'
+import Datetime from 'react-datetime'
+import { Modal, Form, Button, TextArea } from 'semantic-ui-react'
+import qs from 'query-string'
+import SearchHistory from 'components/SearchHistory'
+import {
+  useCreateOpenUniCourseSearchMutation,
+  useDeleteOpenUniCourseSearchMutation,
+  useUpdateOpenUniCourseSearchMutation,
+} from 'redux/openUniPopulations'
+
+const CustomOpenUniSearch = ({ setValues, savedSearches, location, history }) => {
+  const [modal, setModal] = useState(false)
+  const [input, setInput] = useState('')
+  const [searchList, setSearches] = useState(savedSearches)
+  const [name, setName] = useState('')
+  const [startdate, setStartdate] = useState(moment('01-08-2017 00:00:00', 'DD-MM-YYYY'))
+  const [enddate, setEnddate] = useState(moment().endOf('day'))
+  const [selectedSearchId, setSelectedSearchId] = useState('')
+  const [updateOpenUniCourseSearch, { isLoading: updateIsLoading, data: updatedData }] =
+    useUpdateOpenUniCourseSearchMutation()
+  const [createOpenUniCourseSearch, { isLoading: createIsLoading, data: createdData }] =
+    useCreateOpenUniCourseSearchMutation()
+  const [deleteOpenUniCourseSearch, { isLoading: deleteIsLoading, data: deletedData }] =
+    useDeleteOpenUniCourseSearchMutation()
+
+  const parseQueryFromUrl = () => {
+    const { courseCode, startdate, enddate } = qs.parse(location.search)
+    let courseCodes = courseCode
+    if (!Array.isArray(courseCode)) courseCodes = [courseCode]
+    const query = {
+      courseList: courseCodes,
+      startdate: moment(startdate, 'DD-MM-YYYY').toISOString(),
+      enddate: moment(enddate, 'DD-MM-YYYY').toISOString(),
+    }
+    return query
+  }
+
+  useEffect(() => {
+    if (updatedData) {
+      const updatedSearches = searchList.map(s => (s.id === updatedData.id ? updatedData : s))
+      setSearches(updatedSearches)
+    }
+  }, [updatedData])
+
+  useEffect(() => {
+    if (createdData && !createdData.error) {
+      const newList = searchList.concat(createdData)
+      setSearches(newList)
+    }
+  }, [createdData])
+
+  useEffect(() => {
+    if (deletedData) {
+      const filteredsearches = searchList.filter(s => s.id !== deletedData)
+      setSearches(filteredsearches)
+    }
+  }, [deletedData])
+
+  useEffect(() => {
+    if (!location.search) {
+      setValues({})
+    } else {
+      const query = parseQueryFromUrl()
+      setValues(query)
+    }
+  }, [location.search])
+
+  const pushQueryToUrl = query => {
+    setImmediate(() => {
+      const searchString = qs.stringify(query)
+      history.push({ search: searchString })
+    })
+  }
+
+  const clearForm = () => {
+    setInput('')
+    setName('')
+    setStartdate(moment('01-08-2017 00-00-00', 'DD-MM-YYYY'))
+    setEnddate(moment().endOf('day'))
+    setSelectedSearchId('')
+  }
+
+  const handleClose = () => {
+    setModal(false)
+    clearForm()
+  }
+
+  const onClicker = e => {
+    e.preventDefault()
+    const courseList = input
+      .split(',')
+      .map(code => code.trim().toUpperCase())
+      .filter(code => code.length > 0)
+    const query = {
+      courseCode: courseList,
+      startdate: moment(startdate).format('DD-MM-YYYY'),
+      enddate: moment(enddate).format('DD-MM-YYYY'),
+    }
+    pushQueryToUrl(query)
+    handleClose()
+  }
+
+  const onDelete = () => {
+    deleteOpenUniCourseSearch({ id: selectedSearchId })
+    clearForm()
+  }
+
+  const onSelectSearch = selectedId => {
+    if (!selectedId) {
+      clearForm()
+      return
+    }
+    const { id: selectedSearchId } = selectedId
+    const selectedSearch = searchList.find(search => search.id === selectedSearchId)
+    if (selectedSearch) {
+      setInput(selectedSearch.courseList.join(', '))
+      setName(selectedSearch.name)
+      setSelectedSearchId(selectedSearch.id)
+    }
+  }
+
+  const onSave = () => {
+    const courseList = input
+      .split(',')
+      .map(code => code.trim().toUpperCase())
+      .filter(code => code.length > 0)
+    if (selectedSearchId !== '') {
+      updateOpenUniCourseSearch({ id: selectedSearchId, courseList })
+    } else {
+      createOpenUniCourseSearch({ courseList, name })
+    }
+  }
+  return (
+    <Modal
+      trigger={
+        <Button size="small" color="blue" onClick={() => setModal(true)} data-cy="custom-pop-search-button">
+          Fetch Open Uni Students
+        </Button>
+      }
+      open={modal}
+      onClose={handleClose}
+      size="small"
+    >
+      <Modal.Content>
+        <Form>
+          <h2> Fetch open uni course population</h2>
+          <Form.Field>
+            <Form.Field>
+              <em> Insert name for this population if you wish to save it </em>
+              <Form.Input
+                disabled={selectedSearchId !== ''}
+                value={name}
+                placeholder="name"
+                onChange={e => setName(e.target.value)}
+              />
+            </Form.Field>
+            <em>Insert course code(s)</em>
+            <TextArea
+              value={input}
+              placeholder="TKT12345, PSYK-123"
+              onChange={e => setInput(e.target.value)}
+              data-cy="s-no-input"
+            />
+          </Form.Field>
+          <SearchHistory
+            header="Saved populations"
+            items={searchList?.map(s => ({
+              ...s,
+              text: s.name,
+              timestamp: s.updatedAt,
+              params: { id: s.id },
+            }))}
+            updateItem={() => null}
+            handleSearch={onSelectSearch}
+          />
+          <Form.Field>
+            <em>Select beginning</em>
+            <Datetime
+              value={startdate}
+              onChange={value => setStartdate(value)}
+              timeFormat={false}
+              locale="fi"
+              isValidDate={date => date.isBefore(enddate)}
+              closeOnSelect
+            />
+          </Form.Field>
+          <Form.Field>
+            <em>Select ending for enrollments:</em>
+            <br />
+            <em>Attainments are fetched until today.</em>
+            <Datetime
+              value={enddate}
+              onChange={value => setEnddate(value)}
+              timeFormat={false}
+              locale="fi"
+              isValidDate={date => date.isAfter(startdate)}
+              closeOnSelect
+            />
+          </Form.Field>
+        </Form>
+      </Modal.Content>
+      <Modal.Actions>
+        <Button
+          disabled={!name || updateIsLoading || createIsLoading}
+          loading={updateIsLoading || createIsLoading}
+          floated="left"
+          icon="save"
+          onClick={onSave}
+          content="Save"
+        />
+        <Button
+          disabled={!selectedSearchId || deleteIsLoading}
+          negative
+          floated="left"
+          icon="trash"
+          onClick={onDelete}
+          content="Delete"
+        />
+        <Button onClick={handleClose}>Cancel</Button>
+        <Button positive onClick={e => onClicker(e)} data-cy="search-button">
+          Search population
+        </Button>
+      </Modal.Actions>
+    </Modal>
+  )
+}
+
+export default CustomOpenUniSearch

services/frontend/src/components/CustomOpenUniPopulation/OpenUniPopulationResults/index.jsx

@@ -0,0 +1,257 @@
+import React from 'react'
+import moment from 'moment'
+import { Loader, Icon } from 'semantic-ui-react'
+import { useGetOpenUniCourseStudentsQuery } from 'redux/openUniPopulations'
+import SortableTable from 'components/SortableTable'
+
+const OpenUniPopulationResults = ({ fieldValues, language }) => {
+  const { courseList, startdate, enddate } = fieldValues
+  const openUniStudentStats = useGetOpenUniCourseStudentsQuery({ courseList, startdate, enddate })
+  const isFetchingOrLoading = openUniStudentStats.isLoading || openUniStudentStats.isFetching
+  const isError = openUniStudentStats.isError || (openUniStudentStats.isSuccess && !openUniStudentStats.data)
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+  if (isFetchingOrLoading) return <Loader active style={{ marginTop: '15em' }} />
+
+  const studentNbrColumn = [
+    {
+      key: 'studentnumber-parent',
+      mergeHeader: true,
+      // merge: true,
+      title: 'Student Number',
+      children: [
+        {
+          key: 'studentnumber-child',
+          title: 'Student Number',
+          cellProps: {
+            style: {
+              verticalAlign: 'middle',
+              textAlign: 'center',
+            },
+          },
+          getRowVal: s => s.studentnumber,
+          getRowContent: s => s.studentnumber,
+          child: true,
+        },
+      ],
+    },
+  ]
+
+  const statisticColumns = [
+    {
+      key: 'passed',
+      title: 'Passed',
+      cellProps: {
+        style: {
+          verticalAlign: 'middle',
+          textAlign: 'center',
+        },
+      },
+      headerProps: { title: 'Passed' },
+      getRowVal: s => s.passedTotal,
+      getRowContent: s => s.passedTotal,
+      child: true,
+    },
+    {
+      key: 'unfinished',
+      title: 'Unfinished',
+      headerProps: { title: 'Unfinished' },
+      getRowVal: s => s.enrolledTotal,
+      getRowContent: s => s.enrolledTotal,
+      cellProps: {
+        style: {
+          verticalAlign: 'middle',
+          textAlign: 'center',
+        },
+      },
+      child: true,
+    },
+  ]
+
+  const getDisseminationInfoAllowed = s => {
+    if (s.disseminationInfoAllowed === null) return 'unknown'
+    if (s.disseminationInfoAllowed) return 'yes'
+    return 'no'
+  }
+
+  const informationColumns = [
+    {
+      key: 'disseminationInfoAllowed',
+      title: 'Marketing Allowed',
+      getRowVal: s => getDisseminationInfoAllowed(s),
+      getRowContent: s => getDisseminationInfoAllowed(s),
+      headerProps: { title: 'Marketing Allowed' },
+      child: true,
+      cellProps: {
+        style: {
+          verticalAlign: 'middle',
+          textAlign: 'center',
+        },
+      },
+    },
+    {
+      key: 'email-child',
+      title: 'Email',
+      getRowVal: s => (s.email ? s.email : ''),
+      getRowContent: s => (s.email ? s.email : ''),
+      headerProps: { title: 'Email' },
+      child: true,
+    },
+    {
+      key: 'secondary_email-child',
+      title: 'Secondary Email',
+      getRowVal: s => (s.secondaryEmail ? s.secondaryEmail : ''),
+      getRowContent: s => (s.secondaryEmail ? s.secondaryEmail : ''),
+      headerProps: { title: 'Secondary Email' },
+      child: true,
+    },
+  ]
+  const findRowContent = (s, courseCode) => {
+    if (s.courseInfo[courseCode] === undefined) return null
+    if (s.courseInfo[courseCode].notEnrolled) return null
+    if (s.courseInfo[courseCode].enrolledNotPassed.length > 0) return <Icon fitted name="times" color="red" />
+    if (s.courseInfo[courseCode].enrolledPassed !== null) return <Icon fitted name="check" color="green" />
+    return null
+  }
+
+  const findRowValue = (s, courseCode) => {
+    if (s.courseInfo[courseCode] === undefined) return ''
+    if (s.courseInfo[courseCode].enrolledNotPassed.length === 0 && s.courseInfo[courseCode].enrolledPassed === null)
+      return ''
+    if (s.courseInfo[courseCode].enrolledNotPassed.length > 0)
+      return s.courseInfo[courseCode].enrolledNotPassed
+        .map((dat, idx) => `Enrollment ${idx + 1}: ${moment(dat).format('DD-MM-YYYY')}`)
+        .join(' ')
+    if (s.courseInfo[courseCode].enrolledPassed !== null)
+      return `Passed: ${moment(s.courseInfo[courseCode].enrolledPassed).format('DD-MM-YYYY')}`
+    return ''
+  }
+  const findProp = (s, courseCode) => {
+    const propObj = {
+      title: '',
+      style: {
+        verticalAlign: 'middle',
+        textAlign: 'center',
+      },
+    }
+    if (s.courseInfo[courseCode] === undefined) return propObj
+    if (s.courseInfo[courseCode].enrolledNotPassed.length === 0 && s.courseInfo[courseCode].enrolledPassed === null)
+      return propObj
+    if (s.courseInfo[courseCode].enrolledNotPassed.length > 0)
+      return {
+        ...propObj,
+        title: `Enrollments: ${s.courseInfo[courseCode].enrolledNotPassed
+          .map(dat => moment(dat).format('DD-MM-YYYY'))
+          .join(', ')}`,
+      }
+    if (s.courseInfo[courseCode].enrolledPassed !== null)
+      return { ...propObj, title: `Passed: ${moment(s.courseInfo[courseCode].enrolledPassed).format('DD-MM-YYYY')}` }
+    return { ...propObj, title: '' }
+  }
+
+  const unOrderedlabels = openUniStudentStats?.data.courses
+  const labelsToCourses = [...unOrderedlabels].sort((a, b) => a.label.localeCompare(b.label))
+  const columns = []
+  columns.push(
+    {
+      key: 'general',
+      title: <b>Labels:</b>,
+      textTitle: null,
+      parent: true,
+      children: studentNbrColumn,
+    },
+    {
+      key: 'Courses',
+      title: <b>Courses:</b>,
+      textTitle: null,
+      parent: true,
+      children: labelsToCourses.map(course => ({
+        key: `${course.label}-${course.name[language]}`,
+        title: (
+          <div style={{ maxWidth: '15em', whiteSpace: 'normal', overflow: 'hidden', width: 'max-content' }}>
+            <div>{course.label}</div>
+            {/* <div style={{ color: 'gray', fontWeight: 'normal' }}>{course.name[language]}</div> */}
+          </div>
+        ),
+        textTitle: `${course.label}-${course.name[language]}`,
+        vertical: false,
+        forceToolsMode: 'dangling',
+        cellProps: s => findProp(s, course.label),
+        headerProps: { title: `${course.label}-${course.name[language]}` },
+        getRowVal: s => {
+          return findRowValue(s, course.label)
+        },
+        getRowContent: s => {
+          return findRowContent(s, course.label)
+        },
+        child: true,
+        childOf: course.label,
+        code: course.label,
+      })),
+    },
+    {
+      key: 'statistics',
+      title: <b>Total of:</b>,
+      textTitle: null,
+      parent: true,
+      children: statisticColumns,
+    },
+    {
+      key: 'information',
+      title: <b>Information:</b>,
+      textTitle: null,
+      parent: true,
+      children: informationColumns,
+    }
+  )
+
+  const data = Object.keys(openUniStudentStats?.data.students).reduce(
+    (acc, student) => [
+      ...acc,
+      {
+        studentnumber: student,
+        courseInfo: { ...openUniStudentStats?.data.students[student].courseInfo },
+        email: openUniStudentStats?.data.students[student].email,
+        secondaryEmail: openUniStudentStats?.data.students[student].secondaryEmail,
+        disseminationInfoAllowed: openUniStudentStats?.data.students[student].disseminationInfoAllowed,
+        enrolledTotal: openUniStudentStats?.data.students[student].enrolledTotal,
+        passedTotal: openUniStudentStats?.data.students[student].passedTotal,
+      },
+    ],
+    []
+  )
+
+  return (
+    <div style={{ paddingBottom: '50px' }}>
+      {isFetchingOrLoading ? (
+        <Loader active style={{ marginTop: '15em' }} />
+      ) : (
+        <div style={{ display: 'flex', paddingBottom: '10px' }}>
+          <div style={{ maxHeight: '70vh', width: '100%' }}>
+            {courseList.length > 0 && (
+              <SortableTable
+                title={`Open Uni Student Population (${
+                  Object.keys(openUniStudentStats?.data.students).length
+                } students)`}
+                getRowKey={s => s.studentNumber}
+                tableProps={{
+                  celled: true,
+                  compact: 'very',
+                  padded: false,
+                  collapsing: true,
+                  basic: true,
+                  striped: true,
+                  singleLine: true,
+                  textAlign: 'center',
+                }}
+                columns={columns}
+                data={data}
+              />
+            )}
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+export default OpenUniPopulationResults

services/frontend/src/components/CustomOpenUniPopulation/index.js

@@ -0,0 +1,82 @@
+import React, { useEffect, useState } from 'react'
+import { withRouter, useLocation, useHistory } from 'react-router-dom'
+import { Message, Icon, Loader } from 'semantic-ui-react'
+import moment from 'moment'
+import { useGetSavedSearchesQuery } from 'redux/openUniPopulations'
+import OpenUniPopulationResults from './OpenUniPopulationResults'
+import { useTitle } from '../../common/hooks'
+import useLanguage from '../LanguagePicker/useLanguage'
+import CustomOpenUniSearch from './CustomOpenUniSearch'
+import TSA from '../../common/tsa'
+
+const CustomOpenUniPopulation = () => {
+  useTitle('Custom open uni population')
+  const [fieldValues, setValues] = useState({})
+  const { language } = useLanguage()
+  const location = useLocation()
+  const history = useHistory()
+  const savedSearches = useGetSavedSearchesQuery()
+  const isFetchingOrLoading = savedSearches.isLoading || savedSearches.isFetching
+  const isError = savedSearches.isError || (savedSearches.isSuccess && !savedSearches.data)
+
+  useEffect(() => {
+    if (fieldValues && fieldValues.courseList?.length > 0) {
+      TSA.Influx.sendEvent({
+        group: 'Custom Open Uni Usage',
+        name: 'open uni populations',
+        label: 'openUniPopulations',
+        value: 1,
+      })
+    }
+  }, [fieldValues])
+
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+  if (isFetchingOrLoading) return <Loader active style={{ marginTop: '15em' }} />
+  return (
+    <div className="segmentContainer">
+      <Message style={{ maxWidth: '800px', fontSize: '16px' }}>
+        <Message.Header>Open uni student population</Message.Header>
+        <p>
+          Here you can create custom population using a list of courses. Clicking the blue custom population button will
+          open a modal where you can enter a list of courses.
+          <br />
+          <Icon fitted name="check" color="green" />: Student passed the course.
+          <br />
+          <Icon fitted name="times" color="red" />: Student has enrolled but has not passed the course.
+          <br />
+          <b>Empty cell</b>: Student has no enrollments for the course.
+          <br />
+          View under progress.
+        </p>
+      </Message>
+      <CustomOpenUniSearch
+        setValues={setValues}
+        savedSearches={savedSearches.data}
+        location={location}
+        history={history}
+      />
+      <div style={{ paddingTop: '25px', paddingBottom: '10px', fontSize: '20px' }}>
+        {fieldValues && fieldValues.courseList?.length > 0 && (
+          <p>
+            <span style={{ color: '#484848' }}>Beginning of the search for all fields:</span>
+            <span style={{ paddingLeft: '10px', fontWeight: '600' }}>
+              {moment(fieldValues.startdate).format('DD.MM.YYYY')}
+            </span>
+            <br />
+            <span style={{ color: '#484848' }}>End of the search for enrollments:</span>
+            <span style={{ paddingLeft: '30px', fontWeight: '600' }}>
+              {moment(fieldValues.enddate).format('DD.MM.YYYY')}
+            </span>
+          </p>
+        )}
+      </div>
+      <div style={{ paddingTop: '25px' }}>
+        {fieldValues && fieldValues.courseList?.length > 0 && (
+          <OpenUniPopulationResults fieldValues={fieldValues} language={language} />
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default withRouter(CustomOpenUniPopulation)

services/frontend/src/components/FacultyStatistics/BasicOverview/index.jsx

@@ -1,6 +1,6 @@
 import React from 'react'
-import { Divider, Loader } from 'semantic-ui-react'
-
+import { Divider, Loader, Popup, Button, Message } from 'semantic-ui-react'
+import xlsx from 'xlsx'
 import {
   useGetFacultyCreditStatsQuery,
   useGetFacultyBasicStatsQuery,
@@ -12,27 +12,91 @@ import InteractiveDataTable from '../InteractiveDataView'
 import Toggle from '../../StudyProgramme/Toggle'
 import InfoBox from '../../Info/InfoBox'
 import InfotoolTips from '../../../common/InfoToolTips'
+import sortProgrammeKeys from '../facultyHelpers'
 import '../faculty.css'
 
-const Overview = ({ faculty, academicYear, setAcademicYear }) => {
-  const toolTips = InfotoolTips.Studyprogramme
+const Overview = ({
+  faculty,
+  academicYear,
+  setAcademicYear,
+  studyProgrammes,
+  setStudyProgrammes,
+  specialGroups,
+  setSpecialGroups,
+  language,
+}) => {
+  const toolTips = InfotoolTips.Faculty
   const yearType = academicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR'
-  const special = 'SPECIAL_INCLUDED' // specialGroups ? 'SPECIAL_EXCLUDED' : 'SPECIAL_INCLUDED'
-  const credits = useGetFacultyCreditStatsQuery({ id: faculty?.code, yearType, specialGroups: special })
-  const basics = useGetFacultyBasicStatsQuery({ id: faculty?.code, yearType, specialGroups: special })
-  const thesisWriters = useGetFacultyThesisStatsQuery({ id: faculty?.code, yearType })
-  // basic.data.studentsInfo --> data for students of faculty table
-  // basics.data.graduationInfo --> data for graduated and thesis writers table
+  const studyProgrammeFilter = studyProgrammes ? 'ALL_PROGRAMMES' : 'NEW_STUDY_PROGRAMMES'
+  const special = specialGroups ? 'SPECIAL_EXCLUDED' : 'SPECIAL_INCLUDED'
+  const credits = useGetFacultyCreditStatsQuery({
+    id: faculty?.code,
+    yearType,
+    studyProgrammeFilter,
+    specialGroups: special,
+  })
+  const basics = useGetFacultyBasicStatsQuery({
+    id: faculty?.code,
+    yearType,
+    studyProgrammeFilter,
+    specialGroups: special,
+  })
+  const thesisWriters = useGetFacultyThesisStatsQuery({
+    id: faculty?.code,
+    yearType,
+    studyProgrammeFilter,
+    specialGroups: special,
+  })
 
-  const getDivider = (title, toolTipText) => (
+  const downloadCsv = (titles, tableStats, programmeStats, programmeNames, toolTipText) => {
+    const headers = titles.map(title => ({ label: title === '' ? 'Year' : title, key: title === '' ? 'Year' : title }))
+    const csvData = sortProgrammeKeys(Object.keys(programmeStats)).reduce(
+      (results, programme) => [
+        ...results,
+        ...programmeStats[programme].map(yearRow => {
+          return {
+            Programme: programme,
+            Name: programmeNames[programme][language]
+              ? programmeNames[programme][language]
+              : programmeNames[programme].fi,
+            ...yearRow.reduce((result, value, valueIndex) => ({ ...result, [headers[valueIndex].key]: value }), {}),
+          }
+        }),
+      ],
+      []
+    )
+
+    const tableStatsAsCsv = tableStats.map(yearArray =>
+      yearArray.reduce((result, value, yearIndex) => ({ ...result, [headers[yearIndex].key]: value }), {})
+    )
+
+    const book = xlsx.utils.book_new()
+    const tableSheet = xlsx.utils.json_to_sheet(tableStatsAsCsv)
+    xlsx.utils.book_append_sheet(book, tableSheet, 'TableStats')
+    const sheet = xlsx.utils.json_to_sheet(csvData)
+    xlsx.utils.book_append_sheet(book, sheet, 'ProgrammeStats')
+    xlsx.writeFile(book, `${faculty.code}-${toolTipText}.xlsx`)
+  }
+
+  const getDivider = (title, toolTipText, titles, tableStats, programmeStats, programmeNames) => (
     <>
       <div className="divider">
         <Divider data-cy={`Section-${toolTipText}`} horizontal>
           {title}
         </Divider>
       </div>
-      <InfoBox content="Sisältää opintopisteet suoritusvuosittain. Suoritukset on jaoteltu Sisussa näkyvän kurssin suorituspäivän mukaan. Kaikki valmistuneet -lukumäärä sisältää kaikki mahdolliset suoritetuksi merkityt opiskeluoikeudet." />
-      {/* <InfoBox content={toolTips[toolTipText]} /> */}
+      <InfoBox content={toolTips[toolTipText]} cypress={toolTipText} />
+      <Popup
+        content="Download statistics as csv"
+        trigger={
+          <Button
+            icon="download"
+            floated="right"
+            onClick={() => downloadCsv(titles, tableStats, programmeStats, programmeNames, toolTipText)}
+            style={{ backgroundColor: 'white', borderRadius: 0 }}
+          />
+        }
+      />
     </>
   )
   const isFetchingOrLoading =
@@ -54,39 +118,16 @@ const Overview = ({ faculty, academicYear, setAcademicYear }) => {
 
   if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
 
-  /*
-  Order of the programme keys: KH -> MH -> T -> FI -> K- -> Numbers containing letters at end -> Y- -> Numbers
-  */
-  const regexValuesAll = [
-    /^KH/,
-    /^MH/,
-    /^T/,
-    /^LI/,
-    /^K-/,
-    /^FI/,
-    /^00901$/,
-    /^00910$/,
-    /^\d.*a$/,
-    /^Y/,
-    /\d$/,
-    /^\d.*e$/,
-  ]
-  // const regexValuesNewProgrammes = [/^KH/, /^MH/, /^T/, /^LI/, /^K-/, /^FI/, /^00901$/, /^00910$/]
-  // const regexValuesOldProgrammes = [/^\d.*a$/, /\^d.*$/, /^Y/, /^\d.*e$/]
+  let creditShortTitles = []
+  let transferShortTitles = []
+  if (special === 'SPECIAL_INCLUDED') {
+    creditShortTitles = ['Code', 'Total', 'Major', 'Non-major', 'Non-major other', 'Non-degree']
+    transferShortTitles = ['Code', 'Started', 'Graduated', 'Transferred in', 'Transferred away', 'Transferred to']
+  } else {
+    creditShortTitles = ['Code', 'Major']
+    transferShortTitles = ['Code', 'Started', 'Graduated']
+  }
 
-  const testKey = value => {
-    for (let i = 0; i < regexValuesAll.length; i++) {
-      if (regexValuesAll[i].test(value)) {
-        return i
-      }
-    }
-    return 6
-  }
-  const sortProgrammeKeys = programmeKeys => {
-    return programmeKeys.sort((a, b) => {
-      return testKey(a) - testKey(b)
-    })
-  }
   return (
     <div className="faculty-overview">
       <div className="toggle-container">
@@ -98,96 +139,194 @@ const Overview = ({ faculty, academicYear, setAcademicYear }) => {
           value={academicYear}
           setValue={setAcademicYear}
         />
+        <Toggle
+          cypress="ProgrammeToggle"
+          toolTips={toolTips.ProgrammeToggle}
+          firstLabel="New study programmes"
+          secondLabel="All study programmes"
+          value={studyProgrammes}
+          setValue={setStudyProgrammes}
+        />
+        <Toggle
+          cypress="StudentToggle"
+          toolTips={toolTips.StudentToggle}
+          firstLabel="All studyrights"
+          secondLabel="Special studyrights excluded"
+          value={specialGroups}
+          setValue={setSpecialGroups}
+        />
       </div>
 
       {isFetchingOrLoading ? (
-        <Loader active style={{ marginTop: '10em' }} />
+        <Loader active style={{ marginTop: '15em' }} />
       ) : (
         <>
+          {studyProgrammeFilter === 'ALL_PROGRAMMES' && (
+            <Message data-cy="FacultyProgrammesShownInfo">
+              Please note that the data is complete only for current Bachelor, Masters and Doctoral programmes.
+              Especially, credits and thesis writers contain only data for current programmes.
+            </Message>
+          )}
+          {special === 'SPECIAL_EXCLUDED' && (
+            <Message data-cy="FacultyExcludeSpecialsInfo">
+              Please note that the data might be inaccurate in some fields. Checking the values is in progress.
+            </Message>
+          )}
           {basics.isSuccess && basics.data && (
             <>
-              {getDivider('Students of the faculty', 'StudentsOfTheFaculty')}
+              {getDivider(
+                'Students of the faculty',
+                'StudentsOfTheFaculty',
+                basics?.data?.studentInfo.titles,
+                basics?.data?.studentInfo.tableStats,
+                basics?.data?.studentInfo.programmeTableStats,
+                basics?.data?.programmeNames
+              )}
               <div className="section-container">
-                <LineGraph
-                  cypress="StudentsOfTheFaculty"
-                  data={{ ...basics?.data.studentInfo, years: basics.data.years }}
-                />
-                <InteractiveDataTable
-                  cypress="StudentsOfTheFaculty"
-                  dataStats={basics?.data?.studentInfo.tableStats}
-                  dataProgrammeStats={basics?.data?.studentInfo.programmeTableStats}
-                  programmeNames={basics?.data?.programmeNames}
-                  sortedKeys={sortProgrammeKeys(Object.keys(basics?.data?.studentInfo.programmeTableStats))}
-                  titles={basics?.data?.studentInfo.titles}
-                  sliceStart={1}
-                  language="fi"
-                />
+                <div className="graph-container-narrow">
+                  <LineGraph
+                    cypress="StudentsOfTheFaculty"
+                    data={{ ...basics?.data.studentInfo, years: basics.data.years }}
+                  />
+                </div>
+                <div className="table-container-wide">
+                  <InteractiveDataTable
+                    cypress="StudentsOfTheFaculty"
+                    dataStats={basics?.data?.studentInfo.tableStats}
+                    dataProgrammeStats={basics?.data?.studentInfo.programmeTableStats}
+                    programmeNames={basics?.data?.programmeNames}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(basics?.data?.studentInfo.programmeTableStats).map(obj => [
+                        obj,
+                        basics?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    titles={basics?.data?.studentInfo.titles}
+                    sliceStart={1}
+                    language={language}
+                    yearsVisible={Array(basics?.data?.studentInfo.tableStats.length).fill(false)}
+                    shortNames={transferShortTitles}
+                  />
+                </div>
               </div>
             </>
           )}
           {credits.isSuccess && credits.data && (
             <>
-              {getDivider('Credits produced by the faculty', 'CreditsProducedByTheFaculty')}
+              {getDivider(
+                'Credits produced by the faculty',
+                'CreditsProducedByTheFaculty',
+                credits?.data?.titles,
+                credits?.data?.tableStats,
+                credits?.data?.programmeTableStats,
+                credits?.data?.programmeNames
+              )}
               <div className="section-container">
-                <StackedBarChart
-                  cypress="CreditsProducedByTheSFaculty"
-                  data={credits?.data?.graphStats}
-                  labels={credits?.data?.years}
-                />
-                <InteractiveDataTable
-                  cypress="CreditsProducedByTheFaculty"
-                  dataStats={credits?.data?.tableStats}
-                  dataProgrammeStats={credits?.data?.programmeTableStats}
-                  programmeNames={credits?.data?.programmeNames}
-                  sortedKeys={sortProgrammeKeys(Object.keys(credits?.data?.programmeTableStats))}
-                  titles={credits?.data?.titles}
-                  extraHeight="EXTRA HEIGHT"
-                  sliceStart={2}
-                  language="fi"
-                />
+                <div className="graph-container-narrow">
+                  <StackedBarChart
+                    cypress="CreditsProducedByTheFaculty"
+                    data={credits?.data?.graphStats}
+                    labels={credits?.data?.years}
+                    wideTable="narrow"
+                  />
+                </div>
+                <div className="table-container-wide">
+                  <InteractiveDataTable
+                    cypress="CreditsProducedByTheFaculty"
+                    dataStats={credits?.data?.tableStats}
+                    dataProgrammeStats={credits?.data?.programmeTableStats}
+                    programmeNames={credits?.data?.programmeNames}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(credits?.data.programmeTableStats).map(obj => [
+                        obj,
+                        credits?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    titles={credits?.data?.titles}
+                    extraHeight="EXTRA HEIGHT"
+                    sliceStart={2}
+                    language={language}
+                    yearsVisible={Array(credits?.data?.tableStats.length).fill(false)}
+                    shortNames={creditShortTitles}
+                  />
+                </div>
               </div>
             </>
           )}
           {basics.isSuccess && basics.data && (
             <>
-              {getDivider('Graduated of the faculty', 'GraduatedOfTheFaculty')}
+              {getDivider(
+                'Graduated of the faculty',
+                'GraduatedOfTheFaculty',
+                basics?.data?.graduationInfo.titles,
+                basics?.data?.graduationInfo.tableStats,
+                basics?.data?.graduationInfo.programmeTableStats,
+                basics?.data?.programmeNames
+              )}
               <div className="section-container">
-                <LineGraph
-                  cypress="GraduatedOfTheFaculty"
-                  data={{ ...basics?.data.graduationInfo, years: basics.data.years }}
-                />
-                <InteractiveDataTable
-                  cypress="GraduatedOfTheFaculty"
-                  dataStats={basics?.data?.graduationInfo.tableStats}
-                  dataProgrammeStats={basics?.data?.graduationInfo.programmeTableStats}
-                  programmeNames={basics?.data?.programmeNames}
-                  sortedKeys={sortProgrammeKeys(Object.keys(basics?.data?.graduationInfo.programmeTableStats))}
-                  titles={basics?.data?.graduationInfo.titles}
-                  sliceStart={2}
-                  language="fi"
-                />
+                <div className="graph-container-narrow">
+                  <LineGraph
+                    cypress="GraduatedOfTheFaculty"
+                    data={{ ...basics?.data.graduationInfo, years: basics.data.years }}
+                  />
+                </div>
+                <div className="table-container-wide">
+                  <InteractiveDataTable
+                    cypress="GraduatedOfTheFaculty"
+                    dataStats={basics?.data?.graduationInfo.tableStats}
+                    dataProgrammeStats={basics?.data?.graduationInfo.programmeTableStats}
+                    programmeNames={basics?.data?.programmeNames}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(basics?.data?.graduationInfo.programmeTableStats).map(obj => [
+                        obj,
+                        basics?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    titles={basics?.data?.graduationInfo.titles}
+                    sliceStart={2}
+                    language={language}
+                    yearsVisible={Array(basics?.data?.graduationInfo.tableStats.length).fill(false)}
+                  />
+                </div>
               </div>
             </>
           )}
           {thesisWriters.isSuccess && thesisWriters.data && (
             <>
-              {getDivider('Thesis writers of the faculty', 'ThesisWritersOfTheFaculty')}
+              {getDivider(
+                'Thesis writers of the faculty',
+                'ThesisWritersOfTheFaculty',
+                thesisWriters?.data?.titles,
+                thesisWriters?.data.tableStats,
+                thesisWriters?.data.programmeTableStats,
+                thesisWriters?.data?.programmeNames
+              )}
               <div className="section-container">
-                <LineGraph
-                  cypress="ThesisWritersOfTheFaculty"
-                  data={{ ...thesisWriters?.data, years: thesisWriters?.data.years }}
-                />
-                <InteractiveDataTable
-                  cypress="ThesisWritersOfTheFaculty"
-                  dataStats={thesisWriters?.data.tableStats}
-                  dataProgrammeStats={thesisWriters?.data.programmeTableStats}
-                  programmeNames={thesisWriters?.data.programmeNames}
-                  sortedKeys={sortProgrammeKeys(Object.keys(thesisWriters?.data.programmeTableStats))}
-                  titles={thesisWriters?.data?.titles}
-                  sliceStart={2}
-                  extraHeight="EXTRA HEIGHT"
-                  language="fi"
-                />
+                <div className="graph-container-narrow">
+                  <LineGraph
+                    cypress="ThesisWritersOfTheFaculty"
+                    data={{ ...thesisWriters?.data, years: thesisWriters?.data.years }}
+                  />
+                </div>
+                <div className="table-container-wide">
+                  <InteractiveDataTable
+                    cypress="ThesisWritersOfTheFaculty"
+                    dataStats={thesisWriters?.data.tableStats}
+                    dataProgrammeStats={thesisWriters?.data.programmeTableStats}
+                    programmeNames={thesisWriters?.data.programmeNames}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(thesisWriters?.data.programmeTableStats).map(obj => [
+                        obj,
+                        thesisWriters?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    titles={thesisWriters?.data?.titles}
+                    sliceStart={2}
+                    extraHeight="EXTRA HEIGHT"
+                    language={language}
+                    yearsVisible={Array(thesisWriters?.data.tableStats.length).fill(false)}
+                  />
+                </div>
               </div>
             </>
           )}

services/frontend/src/components/FacultyStatistics/FacultyProgrammeOverview/FacultyBarChart.jsx

@@ -0,0 +1,70 @@
+import React from 'react'
+
+const ReactHighcharts = require('react-highcharts')
+
+const getColors = len => {
+  if (len < 8) return ['#f57368', '#fb8c6e', '#fba678', '#dbda7d', '#9ec27c', '#60a866', '#008c59']
+  return ['#e66067', '#f57368', '#fb8c6e', '#fba678', '#dbda7d', '#9ec27c', '#60a866', '#008c59']
+}
+
+const FacultyBarChart = ({ cypress, data }) => {
+  if (!data) return null
+
+  const colors = getColors(Object.keys(data.stats).length)
+  const dataWithColors = Object.values(data?.stats).map((series, index) => ({ ...series, color: colors[index] }))
+  const getFileName = () => {
+    return `oodikone_progress_of_students_in_${data?.id}_by_study_start_year`
+  }
+
+  const defaultConfig = {
+    title: {
+      text: '',
+    },
+    series: dataWithColors,
+    credits: {
+      text: 'oodikone | TOSKA',
+    },
+    xAxis: {
+      categories: data?.years,
+    },
+    chart: {
+      type: 'column',
+      height: '450px',
+    },
+    plotOptions: {
+      column: {
+        stacking: 'percent',
+        dataLabels: {
+          enabled: true,
+          format: '{point.percentage:.1f}%',
+        },
+      },
+    },
+    exporting: {
+      filename: getFileName(),
+      width: 2200,
+      height: 1400,
+      sourceWidth: 1200,
+      sourceHeight: 600,
+      buttons: {
+        contextButton: {
+          menuItems: ['viewFullscreen', 'downloadPNG', 'downloadSVG', 'downloadPDF'],
+        },
+      },
+    },
+    yAxis: {
+      allowDecimals: false,
+      min: 0,
+      reversed: false,
+      title: '',
+    },
+  }
+
+  return (
+    <div className="graph-container" data-cy={`Graph-${cypress}`}>
+      <ReactHighcharts config={defaultConfig} />
+    </div>
+  )
+}
+
+export default FacultyBarChart

services/frontend/src/components/FacultyStatistics/FacultyProgrammeOverview/FacultyProgressTable.jsx

@@ -0,0 +1,80 @@
+import React, { useState } from 'react'
+import { Table } from 'semantic-ui-react'
+import ProgrammeProgressChart from './ProgrammeProgressChart'
+import BasicRow from '../InteractiveDataView/BasicRow'
+
+const FacultyProgressTable = ({
+  data,
+  programmeStats,
+  titles,
+  sortedKeys,
+  progressYearsVisible,
+  programmeNames,
+  language,
+  cypress,
+  progressTitles,
+}) => {
+  const [visible, setVisible] = useState(progressYearsVisible)
+  if (!data) return null
+  const toggleVisibility = yearIndex => {
+    const arrayToModify = [...visible]
+    arrayToModify[yearIndex] = !visible[yearIndex]
+    setVisible(arrayToModify)
+  }
+  const lenOfYearArray = data.length - 1
+  return (
+    <div>
+      <Table data-cy={cypress} compact celled>
+        <Table.Header>
+          <Table.Row key={`${cypress}-${Math.random()}`} textAlign="center">
+            {titles?.map(title => (
+              <Table.HeaderCell key={title}>{title}</Table.HeaderCell>
+            ))}
+          </Table.Row>
+        </Table.Header>
+
+        <Table.Body>
+          {data?.map((yearArray, yearIndex) => (
+            <React.Fragment key={`random-fragment-key-${Math.random()}`}>
+              {yearIndex < lenOfYearArray ? (
+                <BasicRow
+                  icon={visible[yearIndex] ? 'angle down' : 'angle right'}
+                  yearArray={yearArray}
+                  cypress={visible[yearIndex] ? `Hide-${cypress}` : `Show-${cypress}`}
+                  yearIndex={yearIndex}
+                  toggleVisibility={() => toggleVisibility(yearIndex)}
+                />
+              ) : (
+                <Table.Row>
+                  {yearArray?.map(value => (
+                    <Table.Cell key={`last-row-${Math.random()}`}>{value}</Table.Cell>
+                  ))}
+                </Table.Row>
+              )}
+              {yearIndex < lenOfYearArray ? (
+                <Table.Row
+                  key={`${cypress}-row-${Math.random()}`}
+                  style={{ display: visible[yearIndex] ? '' : 'none' }}
+                >
+                  <Table.Cell data-cy={`Cell-${cypress}-${yearIndex}`} colSpan={100}>
+                    <ProgrammeProgressChart
+                      data={sortedKeys?.map(programme => programmeStats[programme][yearIndex])}
+                      longLabels={programmeNames}
+                      labels={sortedKeys}
+                      language={language}
+                      names={progressTitles ? progressTitles[yearIndex] : titles?.slice(2)}
+                    />
+                  </Table.Cell>
+                </Table.Row>
+              ) : (
+                <></>
+              )}
+            </React.Fragment>
+          ))}
+        </Table.Body>
+      </Table>
+    </div>
+  )
+}
+
+export default FacultyProgressTable

services/frontend/src/components/FacultyStatistics/FacultyProgrammeOverview/FacultyStudentDataTable.jsx

@@ -0,0 +1,145 @@
+import PopulationLink from 'components/StudyProgramme/StudytrackOverview/PopulationLink'
+import React, { useState } from 'react'
+import { Table, Button, Icon, Label } from 'semantic-ui-react'
+import Toggle from '../../StudyProgramme/Toggle'
+
+const FacultyStudentDataTable = ({
+  tableStats,
+  programmeStats,
+  programmeNames,
+  titles,
+  years,
+  sortedKeys,
+  language,
+  cypress,
+  requiredRights,
+}) => {
+  const [yearsVisible, setVisible] = useState(new Array(years.length).fill(false))
+  const [showPercentages, setShowPercentages] = useState(false)
+
+  const calendarYears = years.reduce((all, year) => {
+    if (year === 'Total') return all
+    return all.concat(Number(year.slice(0, 4)))
+  }, [])
+
+  const toggleVisibility = yearIndex => {
+    const arrayToModify = [...yearsVisible]
+    arrayToModify[yearIndex] = !yearsVisible[yearIndex]
+    setVisible(arrayToModify)
+  }
+
+  return (
+    <div className="datatable">
+      <Toggle
+        firstLabel="Hide percentages"
+        secondLabel="Show percentages"
+        value={showPercentages}
+        setValue={setShowPercentages}
+      />
+      <Table data-cy="Table-StudytrackOverview" celled>
+        <Table.Header>
+          <Table.Row>
+            {titles.map((title, index) => (
+              <Table.HeaderCell
+                key={title}
+                colSpan={index === 0 || index === 1 || !showPercentages ? 1 : 2}
+                textAlign="left"
+                style={{ fontWeight: 'bold' }}
+              >
+                {title}
+              </Table.HeaderCell>
+            ))}
+          </Table.Row>
+        </Table.Header>
+
+        <Table.Body>
+          {years.map((year, yearIndex) => {
+            return (
+              <>
+                <Table.Row key={`${year}-faculty-row-${Math.random()}`} className="header-row">
+                  {tableStats[year].map((value, valueIdx) => {
+                    if (valueIdx === 0)
+                      return (
+                        <Table.Cell key={`${year}-faculty-cell-${Math.random()}`}>
+                          <Button
+                            as="div"
+                            onClick={() => toggleVisibility(yearIndex)}
+                            labelPosition="right"
+                            style={{ backgroundColor: 'white', borderRadius: 0, padding: 0, margin: 0 }}
+                            data-cy={`Button-${cypress}-${yearIndex}`}
+                          >
+                            <Button icon style={{ backgroundColor: 'white', borderRadius: 0, padding: 0, margin: 0 }}>
+                              <Icon name={yearsVisible[yearIndex] ? 'angle down' : 'angle right'} />
+                            </Button>
+                            <Label as="a" style={{ backgroundColor: 'white', borderRadius: 0, padding: 0, margin: 0 }}>
+                              <b>{value}</b>
+                            </Label>
+                          </Button>
+                        </Table.Cell>
+                      )
+                    if (!showPercentages && typeof value === 'string' && (value.includes('%') || value.includes('NA')))
+                      return null
+                    if ([1, 4, 5, 8, 9, 12, 13, 16, 17].includes(valueIdx))
+                      return (
+                        <Table.Cell style={{ backgroundColor: '#f9f9f9' }} key={`${year}-cell-color-${Math.random()}`}>
+                          {value}
+                        </Table.Cell>
+                      )
+                    return <Table.Cell key={`${year}$-${value}-cell-${Math.random()}`}>{value}</Table.Cell>
+                  })}
+                </Table.Row>
+                {yearsVisible[yearIndex] &&
+                  sortedKeys.map(programme => {
+                    return (
+                      <Table.Row className="regular-row" key={`${year}-regular-row-${Math.random()}`}>
+                        <Table.Cell textAlign="left" style={{ paddingLeft: '50px' }} key={`${year}-${programme}`}>
+                          <b>
+                            {programme} - {programmeNames[programme].code}
+                            <br />
+                            {programmeNames[programme][language]
+                              ? programmeNames[programme][language]
+                              : programmeNames[programme].fi}
+                          </b>
+                          {(requiredRights.rights?.includes(programmeNames[programme].code) ||
+                            requiredRights.isAdmin ||
+                            requiredRights.IAMrights?.includes(programmeNames[programme].code)) && (
+                            <PopulationLink
+                              studyprogramme={programmeNames[programme].code}
+                              year={year}
+                              years={calendarYears}
+                            />
+                          )}
+                        </Table.Cell>
+                        {programmeStats[programme][year].map((value, valIdx) => {
+                          if (
+                            !showPercentages &&
+                            typeof value === 'string' &&
+                            (value.includes('%') || value.includes('NA'))
+                          )
+                            return null
+                          if ([0, 3, 4, 7, 8, 11, 12, 15, 16].includes(valIdx))
+                            return (
+                              <Table.Cell
+                                key={`${year}-${programme}-color-${Math.random()}`}
+                                style={{ backgroundColor: '#f9f9f9' }}
+                              >
+                                {value}
+                              </Table.Cell>
+                            )
+                          return (
+                            <Table.Cell key={`${year}-${programme}-colorless-${Math.random()}`}>{value}</Table.Cell>
+                          )
+                        })}
+                      </Table.Row>
+                    )
+                  })}
+              </>
+            )
+          })}
+        </Table.Body>
+      </Table>
+    </div>
+  )
+}
+
+export default FacultyStudentDataTable

services/frontend/src/components/FacultyStatistics/FacultyProgrammeOverview/ProgrammeProgressChart.jsx

@@ -0,0 +1,106 @@
+/* eslint-disable react/no-this-in-sfc */
+import React from 'react'
+import Highcharts from 'highcharts'
+import ReactHighcharts from 'react-highcharts'
+
+const getColors = len => {
+  if (len < 8) return ['#f57368', '#fb8c6e', '#fba678', '#dbda7d', '#9ec27c', '#60a866', '#008c59']
+  return ['#e66067', '#f57368', '#fb8c6e', '#fba678', '#dbda7d', '#9ec27c', '#60a866', '#008c59']
+}
+
+const ProgrammeProgressChart = ({ data, labels, longLabels, names, language }) => {
+  if (!data || data.length === 0)
+    return (
+      <>
+        <b>No data available</b>
+      </>
+    )
+  const transpose = matrix => {
+    return matrix.reduce((prev, next) => next.map((_item, i) => (prev[i] || []).concat(next[i])), [])
+  }
+
+  const colors = getColors(data[0]?.length || 6)
+  const dataTranspose = transpose(data)
+    .map((obj, idx) => ({ name: names[idx], data: obj, color: colors[idx] }))
+    .reverse()
+
+  // Tooltip formatter function: this.<some value> needs to be used in order to obtain access to the values.
+  const defaultConfig = {
+    chart: {
+      type: 'bar',
+      marginTop: 60,
+    },
+    credits: {
+      text: 'oodikone | TOSKA',
+    },
+    title: {
+      text: '',
+      style: {
+        display: 'none',
+      },
+    },
+    series: dataTranspose,
+    xAxis: {
+      categories: labels,
+      fontSize: '24px',
+    },
+    yAxis: {
+      allowDecimals: false,
+      min: 0,
+      reversed: true,
+      title: '',
+    },
+    legend: {
+      layout: 'horizontal',
+      align: 'left',
+      fontSize: '24px',
+      x: 10,
+      verticalAlign: 'top',
+      y: -10,
+      floating: true,
+      backgroundColor: Highcharts.defaultOptions.legend.backgroundColor || 'white',
+      borderColor: '#CCC',
+      borderWidth: 1,
+      shadow: false,
+    },
+    tooltip: {
+      shared: true,
+      backgroundColor: 'white',
+      fontSize: '25px',
+      formatter() {
+        let tooltipString = `<b>${
+          longLabels[this.x] && longLabels[this.x][language] ? longLabels[this.x][language] : longLabels[this.x].fi
+        } </b><br /><p>${this.x} - ${longLabels[this.x]?.code}</p><br />`
+        this.points.forEach(point => {
+          tooltipString += `<span style="color:${point.color}">●</span> <b>${point.series.name}: ${point.y}</b>
+          </span><br />`
+        })
+        return tooltipString
+      },
+    },
+    plotOptions: {
+      series: {
+        stacking: 'percent',
+        pointWidth: 20,
+        dataLabels: {
+          enabled: true,
+          format: '{point.percentage:.1f}%',
+          color: '#000000',
+          align: 'left',
+          style: {
+            textOutline: 'none',
+          },
+          filter: {
+            property: 'percentage',
+            operator: '>',
+            value: 3,
+          },
+        },
+      },
+    },
+  }
+
+  return <ReactHighcharts config={defaultConfig} />
+}
+
+export default ProgrammeProgressChart

services/frontend/src/components/FacultyStatistics/FacultyProgrammeOverview/index.jsx

@@ -1,7 +1,250 @@
 import React from 'react'
+import { Divider, Loader } from 'semantic-ui-react'
+import { useGetFacultyProgressStatsQuery, useGetFacultyStudentStatsQuery } from 'redux/facultyStats'
+import FacultyProgressTable from './FacultyProgressTable'
+import FacultyBarChart from './FacultyBarChart'
+import Toggle from '../../StudyProgramme/Toggle'
+import InfoBox from '../../Info/InfoBox'
+import InfotoolTips from '../../../common/InfoToolTips'
+import '../faculty.css'
+import FacultyStudentDataTable from './FacultyStudentDataTable'
+import sortProgrammeKeys from '../facultyHelpers'
 
-const FacultyProgrammeOverview = () => {
-  return <div className="programmes-overview">Faculty programmes</div>
+const getDivider = (title, toolTipText, content) => (
+  <>
+    <div className="divider">
+      <Divider data-cy={`Section-${toolTipText}`} horizontal>
+        {title}
+      </Divider>
+    </div>
+    {content === 'no-infobox' ? null : <InfoBox content={content} />}
+  </>
+)
+
+const FacultyProgrammeOverview = ({
+  faculty,
+  language,
+  graduatedGroup,
+  setGraduatedGroup,
+  specialGroups,
+  setSpecialGroups,
+  requiredRights,
+}) => {
+  const toolTips = InfotoolTips.Faculty
+  const studyProgrammeFilter = 'NEW_STUDY_PROGRAMMES'
+  const specials = specialGroups ? 'SPECIAL_EXCLUDED' : 'SPECIAL_INCLUDED'
+  const graduated = graduatedGroup ? 'GRADUATED_EXCLUDED' : 'GRADUATED_INCLUDED'
+  const progressStats = useGetFacultyProgressStatsQuery({
+    id: faculty?.code,
+    studyProgrammeFilter,
+    specialGroups: specials,
+    graduated,
+  })
+  const studentStats = useGetFacultyStudentStatsQuery({
+    id: faculty.code,
+    studyProgrammeFilter,
+    specialGroups: specials,
+    graduated,
+  })
+
+  const isFetchingOrLoading =
+    progressStats.isLoading || progressStats.isFetching || studentStats.isLoading || studentStats.isFetching
+
+  const isError =
+    progressStats.isError ||
+    studentStats.isError ||
+    (progressStats.isSuccess && !progressStats.data) ||
+    (studentStats.isSuccess && !studentStats.data)
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+
+  return (
+    <div className="faculty-overview">
+      <div className="toggle-container">
+        <Toggle
+          cypress="StudentToggle"
+          toolTips={toolTips.StudentToggle}
+          firstLabel="All studyrights"
+          secondLabel="Special studyrights excluded"
+          value={specialGroups}
+          setValue={setSpecialGroups}
+        />
+        <Toggle
+          cypress="GraduatedToggle"
+          toolTips={toolTips.GraduatedToggle}
+          firstLabel="Graduated included"
+          secondLabel="Graduated excluded"
+          value={graduatedGroup}
+          setValue={setGraduatedGroup}
+        />
+      </div>
+      {isFetchingOrLoading ? (
+        <Loader active style={{ marginTop: '15em' }} />
+      ) : (
+        <div className="programmes-overview">
+          {studentStats && studentStats.data && (
+            <>
+              {getDivider(
+                'Students of the faculty By Starting year',
+                'StudentsOfTheFacultyByStartingYear',
+                toolTips.StudentsStatsOfTheFaculty
+              )}
+              <div>
+                <FacultyStudentDataTable
+                  tableStats={studentStats?.data.facultyTableStats}
+                  programmeStats={studentStats?.data.programmeStats}
+                  programmeNames={studentStats?.data.programmeNames}
+                  titles={studentStats?.data.titles}
+                  years={studentStats?.data.years}
+                  sortedKeys={sortProgrammeKeys(
+                    Object.keys(studentStats?.data.programmeStats).map(obj => [
+                      obj,
+                      studentStats?.data?.programmeNames[obj].code,
+                    ])
+                  ).map(listObj => listObj[0])}
+                  language={language}
+                  cypress="FacultyStudentStatsTable"
+                  requiredRights={requiredRights}
+                />
+              </div>
+            </>
+          )}
+
+          {progressStats.isSuccess && progressStats.data && (
+            <>
+              {getDivider(
+                'Progress of students of the faculty ',
+                'BachelorStudentsOfTheFacultyByStartingYear',
+                toolTips.StudentProgress
+              )}
+              {getDivider('Bachelor', 'BachelorStudentsOfTheFacultyByStartingYear', 'no-infobox')}
+              <div className="section-container">
+                <div className="graph-container">
+                  <FacultyBarChart
+                    cypress="FacultyBachelorsProgress"
+                    data={{
+                      id: faculty.code,
+                      stats: progressStats?.data.bachelorsGraphStats,
+                      years: progressStats?.data.years,
+                    }}
+                  />
+                </div>
+                <div className="table-container">
+                  <FacultyProgressTable
+                    data={progressStats?.data.bachelorsTableStats}
+                    programmeStats={progressStats?.data.bachelorsProgrammeStats}
+                    titles={progressStats?.data.bachelorTitles}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(progressStats?.data.bachelorsProgrammeStats).map(obj => [
+                        obj,
+                        progressStats?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    progressYearsVisible={Array(progressStats?.data.years.slice(1).length).fill(false)}
+                    programmeNames={progressStats?.data.programmeNames}
+                    language={language}
+                    cypress="FacultyBachelorsProgressTable"
+                    progressTitles={progressStats?.data.yearlyBachelorTitles}
+                  />
+                </div>
+              </div>
+              {getDivider('Bachelor + Master', 'ProgressOfBachelorMaster', 'no-infobox')}
+              <div className="section-container">
+                <div className="graph-container">
+                  <FacultyBarChart
+                    cypress="FacultyBachelorMastersProgress"
+                    data={{
+                      id: faculty.code,
+                      stats: progressStats?.data.bcMsGraphStats,
+                      years: progressStats?.data.years,
+                    }}
+                  />
+                </div>
+                <div className="table-container">
+                  <FacultyProgressTable
+                    data={progressStats?.data.bcMsTableStats}
+                    programmeStats={progressStats?.data.bcMsProgrammeStats}
+                    titles={progressStats?.data.bcMsTitles}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(progressStats?.data.bcMsProgrammeStats).map(obj => [
+                        obj,
+                        progressStats?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    progressYearsVisible={Array(progressStats?.data.years.slice(1).length).fill(false)}
+                    programmeNames={progressStats?.data.programmeNames}
+                    language={language}
+                    cypress="FacultyBachelorMasterProgressTable"
+                    progressTitles={progressStats?.data.yearlyBcMsTitles}
+                  />
+                </div>
+              </div>
+              {getDivider('Master', 'MasterStudentsOfTheFacultyByStartingYear', 'no-infobox')}
+              <div className="section-container">
+                <div className="graph-container">
+                  <FacultyBarChart
+                    cypress="FacultyMastersProgress"
+                    data={{
+                      id: faculty.code,
+                      stats: progressStats?.data.mastersGraphStats,
+                      years: progressStats?.data.years,
+                    }}
+                  />
+                </div>
+                <div className="table-container">
+                  <FacultyProgressTable
+                    data={progressStats?.data.mastersTableStats}
+                    programmeStats={progressStats?.data.mastersProgrammeStats}
+                    titles={progressStats?.data.mastersTitles}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(progressStats?.data.mastersProgrammeStats).map(obj => [
+                        obj,
+                        progressStats?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    progressYearsVisible={Array(progressStats?.data.years.slice(1).length).fill(false)}
+                    programmeNames={progressStats?.data.programmeNames}
+                    language={language}
+                    cypress="FacultyBachelorsProgressTable"
+                    progressTitles={progressStats?.data.yearlyMasterTitles}
+                  />
+                </div>
+              </div>
+              {getDivider('Doctor and licentiate', 'DoctoralStudentsOfTheFacultyByStartingYear', 'no-infobox')}
+              <div className="section-container">
+                <div className="graph-container">
+                  <FacultyBarChart
+                    cypress="FacultyDoctoralProgress"
+                    data={{
+                      id: faculty.code,
+                      stats: progressStats?.data.doctoralGraphStats,
+                      years: progressStats?.data.years,
+                    }}
+                  />
+                </div>
+                <div className="table-container">
+                  <FacultyProgressTable
+                    data={progressStats?.data.doctoralTableStats}
+                    programmeStats={progressStats?.data.doctoralProgrammeStats}
+                    titles={progressStats?.data.doctoralTitles}
+                    sortedKeys={sortProgrammeKeys(
+                      Object.keys(progressStats?.data.doctoralProgrammeStats).map(obj => [
+                        obj,
+                        progressStats?.data?.programmeNames[obj].code,
+                      ])
+                    ).map(listObj => listObj[0])}
+                    progressYearsVisible={Array(progressStats?.data.years.slice(1).length).fill(false)}
+                    programmeNames={progressStats?.data.programmeNames}
+                    language={language}
+                    cypress="FacultyBachelorsProgressTable"
+                  />
+                </div>
+              </div>
+            </>
+          )}
+        </div>
+      )}
+    </div>
+  )
 }
 
 export default FacultyProgrammeOverview

services/frontend/src/components/FacultyStatistics/InteractiveDataView/BasicRow.jsx

@@ -0,0 +1,37 @@
+import React from 'react'
+import { Button, Icon, Table, Label } from 'semantic-ui-react'
+
+const roundValue = value => {
+  return Number.isInteger(value) ? value : value.toFixed(1)
+}
+
+const BasicRow = ({ icon, yearArray, cypress, yearIndex, toggleVisibility, styles }) => {
+  return (
+    <Table.Row style={styles} key={`Basic-row-${cypress}-${yearIndex}`}>
+      {yearArray?.map((value, idx) => (
+        <Table.Cell key={`random-button-cell-key-${Math.random()}`}>
+          {idx === 0 ? (
+            <Button
+              as="div"
+              onClick={toggleVisibility}
+              labelPosition="right"
+              style={{ backgroundColor: 'white', borderRadius: 0, padding: 0, margin: 0 }}
+              data-cy={`Button-${cypress}-${yearIndex}`}
+            >
+              <Button icon style={{ backgroundColor: 'white', borderRadius: 0, padding: 0, margin: 0 }}>
+                <Icon name={icon} />
+              </Button>
+              <Label as="a" style={{ backgroundColor: 'white', borderRadius: 0, padding: 0, margin: 0 }}>
+                {value}
+              </Label>
+            </Button>
+          ) : (
+            roundValue(value)
+          )}
+        </Table.Cell>
+      ))}
+    </Table.Row>
+  )
+}
+
+export default BasicRow

services/frontend/src/components/FacultyStatistics/InteractiveDataView/CollapsedStackedBar.jsx

@@ -3,7 +3,7 @@ import React from 'react'
 import Highcharts from 'highcharts'
 import ReactHighcharts from 'react-highcharts'
 
-const colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#2b908f', '#f45b5b', '#91e8e1']
+const colors = ['#7cb5ec', '#90ed7d', '#434348', '#f7a35c', '#FFF000', '#2b908f', '#f45b5b', '#91e8e1']
 
 const CollapsedStackedBar = ({ data, labels, longLabels, names, language, differenceData, extraHeight }) => {
   const transpose = matrix => {
@@ -11,13 +11,6 @@ const CollapsedStackedBar = ({ data, labels, longLabels, names, language, differ
   }
 
   const needsExtra = extraHeight === 'EXTRA HEIGHT'
-  const manyProgrammes =
-    labels.includes('KH57_001') ||
-    labels.includes('KH50_001') ||
-    labels.includes('KH70_001') ||
-    labels.includes('KH40_001') ||
-    labels.includes('KH30_001') ||
-    labels.includes('KH80_001')
   const dataTranspose = transpose(data)
     .map((obj, idx) => ({ name: names[idx], data: obj, color: colors[idx] }))
     .reverse()
@@ -41,11 +34,15 @@ const CollapsedStackedBar = ({ data, labels, longLabels, names, language, differ
   }
 
   // Point width is 24 px different multipliers adjusts the height.
-  const getFlexHeight = (len, needsExtra, manyProgrammes, labels) => {
-    if (labels.includes('KH74_001')) return `${len * 24 * 5}px`
-    if (needsExtra && manyProgrammes) return `${len * 24 * 1.5}px`
-    if (needsExtra) return `${len * 24 * 2.5}px`
-    return `${len * 24 * 1.5}px`
+  const getFlexHeight = (len, needsExtra) => {
+    if (len > 7 && needsExtra) return `${len * 24 * 1.5}px`
+    if (len > 5 && !needsExtra) return `${len * 24 * 1.5}px`
+    if (needsExtra && len <= 2) return `${len * 24 * 6}px`
+    if (needsExtra && len <= 4) return `${len * 24 * 3}px`
+    if (needsExtra) return `${len * 24 * 2}px`
+    if (len <= 2) return `${len * 24 * 5}px`
+    if (len <= 4) return `${len * 24 * 3}px`
+    return `${len * 24}px`
   }
 
   const getColor = change => {
@@ -59,7 +56,7 @@ const CollapsedStackedBar = ({ data, labels, longLabels, names, language, differ
     chart: {
       type: 'bar',
       marginTop: 60,
-      height: getFlexHeight(labels.length, needsExtra, manyProgrammes, labels),
+      height: getFlexHeight(labels.length, needsExtra),
     },
     credits: {
       text: 'oodikone | TOSKA',
@@ -109,9 +106,9 @@ const CollapsedStackedBar = ({ data, labels, longLabels, names, language, differ
       backgroundColor: 'white',
       fontSize: '25px',
       formatter() {
-        let tooltipString = `<b>${longLabels[this.x] ? longLabels[this.x][language] : ''}</b><br /><p>${
-          this.x
-        }</p><br />`
+        let tooltipString = `<b>${
+          longLabels[this.x] && longLabels[this.x][language] ? longLabels[this.x][language] : longLabels[this.x].fi
+        }</b><br /><p>${this.x} - ${longLabels[this.x]?.code}</p><br />`
         const diffArray = differenceArray[this.x]
         this.points.forEach(point => {
           tooltipString += `<span style="color:${point.color}">●</span> <b>${point.series.name}: ${point.y}</b>
@@ -119,15 +116,21 @@ const CollapsedStackedBar = ({ data, labels, longLabels, names, language, differ
             diffArray[point.series.name]
           )}</span>)<br />`
         })
+        tooltipString += `<b>Total: ${this.points.reduce((prev, current) => prev + current.y, 0)}</b>`
         return tooltipString
       },
     },
     plotOptions: {
       series: {
         stacking: 'normal',
-        pointWidth: 24,
+        pointWidth: 20,
         dataLabels: {
           enabled: true,
+          align: 'left',
+          formatter() {
+            if (Number.isInteger(this.y)) return `${this.y}`
+            return `${this.y.toFixed(1)}`
+          },
           fontSize: '24px',
           filter: {
             property: 'y',

services/frontend/src/components/FacultyStatistics/InteractiveDataView/ToggleTableView.jsx

@@ -1,75 +0,0 @@
-import React, { useState, useImperativeHandle } from 'react'
-import { Button, Icon, Table, Label } from 'semantic-ui-react'
-
-const BasicRow = ({ icon, yearArray, cypress, yearIndex, toggleVisibility, styles }) => {
-  return (
-    <Table.Row style={styles} key={`Basic-row-${cypress}-${yearIndex}`}>
-      {yearArray?.map((value, idx) => (
-        <Table.Cell key={`random-button-cell-key-${Math.random()}`}>
-          {idx === 0 ? (
-            <Button
-              as="div"
-              onClick={toggleVisibility}
-              labelPosition="right"
-              style={{ backgroundColor: 'white', borderRadius: 0 }}
-              data-cy={`Button-${cypress}-${yearIndex}`}
-            >
-              <Button icon style={{ backgroundColor: 'white', borderRadius: 0 }}>
-                <Icon name={icon} />
-              </Button>
-              <Label as="a" style={{ backgroundColor: 'white', borderRadius: 0 }}>
-                {value}
-              </Label>
-            </Button>
-          ) : (
-            value
-          )}
-        </Table.Cell>
-      ))}
-    </Table.Row>
-  )
-}
-
-const ToggleTableView = React.forwardRef((props, ref) => {
-  const [visible, setVisible] = useState(false)
-
-  const toggleVisibility = () => {
-    setVisible(!visible)
-  }
-  useImperativeHandle(ref, () => {
-    return toggleVisibility
-  })
-  const chartRowStyles = {
-    display: visible ? '' : 'none',
-  }
-  const basicRowStyles = {
-    display: visible ? 'none' : '',
-  }
-  return (
-    <React.Fragment key={`random-fragment-key-${Math.random()}`}>
-      <BasicRow
-        icon="chevron right"
-        yearArray={props.yearArray}
-        cypress={`Show-${props.cypress}`}
-        yearIndex={props.yearIndex}
-        toggleVisibility={toggleVisibility}
-        styles={basicRowStyles}
-      />
-      <BasicRow
-        icon="chevron down"
-        yearArray={props.yearArray}
-        cypress={`Hide-${props.cypress}`}
-        yearIndex={props.yearIndex}
-        toggleVisibility={toggleVisibility}
-        styles={chartRowStyles}
-      />
-      <Table.Row key={`stack-row-key-${Math.random()}`} style={chartRowStyles}>
-        {props.children}
-      </Table.Row>
-    </React.Fragment>
-  )
-})
-
-ToggleTableView.displayName = 'ToggleTableView'
-
-export default ToggleTableView

services/frontend/src/components/FacultyStatistics/InteractiveDataView/index.jsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useState } from 'react'
-import { Table, Dropdown, Header } from 'semantic-ui-react'
+import { Table, Menu } from 'semantic-ui-react'
 import CollapsedStackedBar from './CollapsedStackedBar'
-import ToggleTableView from './ToggleTableView'
+import BasicRow from './BasicRow'
 
 const InteractiveDataTable = ({
   cypress,
@@ -10,32 +10,39 @@ const InteractiveDataTable = ({
   programmeNames,
   sortedKeys,
   titles,
-  wideTable,
   language,
   sliceStart,
   extraHeight,
+  yearsVisible,
+  shortNames,
+  studentsTable,
 }) => {
   const [keyOrder, setkeyOrder] = useState({})
+  const [sorter, setSorter] = useState('Code')
+  const [sortDir, setSortDir] = useState(1)
   const [sortbyColumn, setSortByColumn] = useState(0)
-  const [columnNumber, sortValuesByColumn] = useState(0)
+  const [columnIndex, setSelectedIndex] = useState(0)
+  const [visible, setVisible] = useState(yearsVisible)
 
-  useEffect(() => {
+  const sortBySelectedColumn = () => {
     const keys = {}
-    if (!(columnNumber === 0)) {
+    if (!(columnIndex === 0)) {
       const numbersOfYears = dataStats.length
       const data = Object.entries(dataProgrammeStats)
       for (let yearIdx = 0; yearIdx < numbersOfYears; yearIdx++) {
         const yearlySortedKeys = data
           .sort((a, b) => {
-            return b[1][yearIdx][columnNumber] - a[1][yearIdx][columnNumber]
+            if (sortDir === -1) return a[1][yearIdx][columnIndex] - b[1][yearIdx][columnIndex]
+            return b[1][yearIdx][columnIndex] - a[1][yearIdx][columnIndex]
           })
           .map(yearlyProgrammes => yearlyProgrammes[0])
         keys[yearIdx] = yearlySortedKeys
       }
     }
     setkeyOrder(keys)
-    setSortByColumn(columnNumber)
-  }, [columnNumber, dataStats])
+    setSortByColumn(columnIndex)
+  }
+  useEffect(sortBySelectedColumn, [columnIndex, dataStats, sortDir])
 
   if (!dataStats || !titles || !dataProgrammeStats) return null
   /* Calculation works as follows
@@ -61,75 +68,86 @@ const InteractiveDataTable = ({
     return differenceMatrix
   }
 
-  const sortByOptions = titles.map((title, columnIndex) => ({
-    key: [title === '' ? 'Programme code (label)' : title],
-    text: [title === '' ? 'Programme code (label)' : title],
-    value: columnIndex,
-  }))
+  const handleClick = (sorterName, nameIndex) => {
+    if (sorterName === sorter) setSortDir(-1 * sortDir)
+    setSorter(sorterName)
+    setSelectedIndex(nameIndex)
+  }
+
+  const toggleVisibility = yearIndex => {
+    const arrayToModify = [...visible]
+    arrayToModify[yearIndex] = !visible[yearIndex]
+    setVisible(arrayToModify)
+  }
+
+  const sorterNames = shortNames || titles.map(title => (title === '' ? 'Code' : title))
 
   const differenceToPrevYears = calculatDiffToPrevYear(dataProgrammeStats)
-  const yearRef = React.createRef()
+
   return (
-    <div className={`table-container${wideTable ? '-wide' : ''}`}>
+    <div>
+      <Menu compact attached="top">
+        <Menu.Item style={{ cursor: 'default', borderRadius: '1px', padding: '5px' }} active color="black">
+          Sort by:
+        </Menu.Item>
+        {sorterNames.map((sorterName, nameIndex) => (
+          <Menu.Item
+            data-cy={`Menu-${cypress}-${sorterName}`}
+            color={sorter === sorterName ? 'blue' : 'black'}
+            key={sorterName}
+            active={sorter === sorterName}
+            onClick={() => handleClick(sorterName, nameIndex)}
+            style={{ borderRadius: '1px', fontSize: '14px', padding: '5px' }}
+            icon={sortDir === 1 ? 'triangle down' : 'triangle up'}
+            content={sorterName}
+          />
+        ))}
+      </Menu>
       <Table data-cy={`Table-${cypress}`} celled>
         <Table.Header>
-          <Table.Row style={{ max_height: '30px' }}>
-            <Table.HeaderCell colSpan={100}>
-              <Header as="h5">
-                <Header.Content>
-                  Sort programme level breakdown charts by:{' '}
-                  <Dropdown
-                    className="ui primary"
-                    header="Sorting order descending"
-                    options={sortByOptions}
-                    defaultValue={sortByOptions[0].text}
-                    onChange={(_event, data) => sortValuesByColumn(data.value)}
-                    closeOnChange
-                  />
-                </Header.Content>
-              </Header>
-            </Table.HeaderCell>
-          </Table.Row>
-          <Table.Row key={`randow-header-row-${Math.random()}`}>
+          <Table.Row key={`randow-header-row-${Math.random()}`} textAlign="center">
             {titles?.map(title => (
               <Table.HeaderCell key={title}>{title}</Table.HeaderCell>
             ))}
           </Table.Row>
         </Table.Header>
-
         <Table.Body>
           {dataStats?.map((yearArray, yearIndex) => (
-            <ToggleTableView
-              yearArray={yearArray}
-              cypress={cypress}
-              yearIndex={yearIndex}
-              ref={yearRef}
-              key={`togglable-${Math.random()}`}
-            >
-              <Table.Cell data-cy={`Cell-${cypress}-${yearIndex}`} key={`stack-cell${Math.random()}`} colSpan={100}>
-                <CollapsedStackedBar
-                  data={
-                    sortbyColumn === 0
-                      ? sortedKeys?.map(programme => dataProgrammeStats[programme][yearIndex].slice(sliceStart))
-                      : keyOrder[yearIndex]?.map(programme =>
-                          dataProgrammeStats[programme][yearIndex].slice(sliceStart)
-                        )
-                  }
-                  labels={sortbyColumn === 0 ? sortedKeys : keyOrder[yearIndex]}
-                  differenceData={Object.keys(differenceToPrevYears)?.reduce(
-                    (yearlyObject, programme) => ({
-                      ...yearlyObject,
-                      [programme]: differenceToPrevYears[programme][yearIndex],
-                    }),
-                    {}
-                  )}
-                  longLabels={programmeNames}
-                  language={language}
-                  names={titles?.slice(sliceStart)}
-                  extraHeight={extraHeight}
-                />
-              </Table.Cell>
-            </ToggleTableView>
+            <React.Fragment key={`random-fragment-key-${Math.random()}`}>
+              <BasicRow
+                icon={visible[yearIndex] ? 'angle down' : 'angle right'}
+                yearArray={yearArray}
+                cypress={visible[yearIndex] ? `Hide-${cypress}` : `Show-${cypress}`}
+                yearIndex={yearIndex}
+                toggleVisibility={() => toggleVisibility(yearIndex)}
+              />
+              <Table.Row key={`stack-row-key-${Math.random()}`} style={{ display: visible[yearIndex] ? '' : 'none' }}>
+                <Table.Cell data-cy={`Cell-${cypress}-${yearIndex}`} key={`stack-cell${Math.random()}`} colSpan={100}>
+                  <CollapsedStackedBar
+                    data={
+                      sortbyColumn === 0
+                        ? sortedKeys?.map(programme => dataProgrammeStats[programme][yearIndex].slice(sliceStart))
+                        : keyOrder[yearIndex]?.map(programme =>
+                            dataProgrammeStats[programme][yearIndex].slice(sliceStart)
+                          )
+                    }
+                    labels={sortbyColumn === 0 ? sortedKeys : keyOrder[yearIndex]}
+                    differenceData={Object.keys(differenceToPrevYears)?.reduce(
+                      (yearlyObject, programme) => ({
+                        ...yearlyObject,
+                        [programme]: differenceToPrevYears[programme][yearIndex],
+                      }),
+                      {}
+                    )}
+                    longLabels={programmeNames}
+                    language={language}
+                    names={titles?.slice(sliceStart)}
+                    extraHeight={extraHeight}
+                    studentsTable={studentsTable}
+                  />
+                </Table.Cell>
+              </Table.Row>
+            </React.Fragment>
           ))}
         </Table.Body>
       </Table>

services/frontend/src/components/FacultyStatistics/TimesAndPaths/BarChart.jsx

@@ -0,0 +1,208 @@
+/* eslint-disable react/no-this-in-sfc */
+import React from 'react'
+import ReactHighcharts from 'react-highcharts'
+
+const BarChart = ({
+  data,
+  goal,
+  handleClick,
+  facultyGraph = true,
+  year,
+  label,
+  programmeNames,
+  language = null,
+  showMeanTime,
+  classSizes,
+  level,
+  goalExceptions,
+}) => {
+  let modData = null
+  if (!facultyGraph && goalExceptions.needed && ['master', 'bcMsCombo'].includes(level)) {
+    // change colors for longer medicine goal times
+    modData = JSON.parse(JSON.stringify(data))
+    for (let i = 0; i < modData.length; i++) {
+      if (Object.keys(goalExceptions).includes(modData[i].code)) {
+        const realGoal = goal + goalExceptions[modData[i].code]
+        if (modData[i].y <= realGoal) {
+          modData[i].color = '#90A959'
+        } else if (modData[i].y <= realGoal + 12) {
+          modData[i].color = '#FEE191'
+        } else {
+          modData[i].color = '#FB6962'
+        }
+        modData[i].realGoal = realGoal
+      }
+    }
+  }
+
+  const maxValue = data.reduce((max, { y }) => {
+    return y > max ? y : max
+  }, goal * 2)
+
+  const getClassSize = category => {
+    if (facultyGraph) return classSizes[category]
+    if (level === 'master' || level === 'bcMsCombo') {
+      return classSizes[category][level][year]
+    }
+    return classSizes[category][year]
+  }
+
+  const getPercentage = (amount, category) => {
+    const percent = Math.round((amount / getClassSize(category)) * 100 * 10) / 10
+    return Number.isNaN(percent) ? 0 : percent
+  }
+
+  const getDataLabel = (amount, category) => {
+    if (label === 'Start year') {
+      return `${amount} graduated (${getPercentage(amount, category)} % of class)`
+    }
+    return `${amount} graduated`
+  }
+
+  const getHeight = () => {
+    const t = data.length > 8 ? 35 : 55
+    return data.length * t + 100
+  }
+
+  const getTooltipText = (name, code, amount, y, statistics, realGoal) => {
+    const sortingText =
+      label === 'Start year'
+        ? `<b>From class of ${facultyGraph ? name : year}, ${amount}/${getClassSize(code)} students have graduated</b>`
+        : `<b>${amount} students graduated in year ${facultyGraph ? name : year}</b>`
+    const timeText = `<br /><p>${sortingText}, <br /><b>${
+      showMeanTime ? 'mean' : 'median'
+    } study time: ${y} months</p></b>`
+    const statisticsText = `<br /><p>${statistics.onTime} graduated on time</p><br />
+        <p>${statistics.yearOver} graduated max year overtime</p>
+        <br /><p>${statistics.wayOver} graduated over year late</p>`
+
+    if (!facultyGraph) {
+      const goalText = realGoal ? `<br /><p><b>** Exceptional goal time: ${realGoal} months **</b></p>` : ''
+      return `<b>${
+        programmeNames[code]?.[language] ? programmeNames[code]?.[language] : programmeNames[code]?.fi
+      }</b><br />${code}${timeText}${statisticsText}${goalText}`
+    }
+    return `${timeText}${statisticsText}`
+  }
+
+  const config = {
+    chart: {
+      type: 'bar',
+      width: 700,
+      margin: [70, 0],
+      height: getHeight(),
+    },
+    title: { text: ' ' },
+    tooltip: {
+      backgroundColor: 'white',
+      fontSize: '25px',
+      // eslint-disable-next-line
+      formatter: function() {
+        return getTooltipText(
+          this.point.name,
+          this.point.code,
+          this.point.amount,
+          this.y,
+          this.point.statistics,
+          this.point?.realGoal
+        )
+      },
+    },
+    plotOptions: {
+      series: {
+        dataLabels: {
+          enabled: true,
+          inside: true,
+          overflow: 'allow',
+        },
+        pointPadding: 0.0,
+      },
+    },
+    series: [
+      {
+        data: modData || data,
+        dataLabels: [
+          {
+            align: 'left',
+            color: '#424949',
+            style: {
+              textOutline: 'none',
+            },
+            // eslint-disable-next-line
+            formatter: function () {
+              return getDataLabel(this.point.amount, this.point.code ? this.point.code : this.point.name)
+            },
+          },
+        ],
+        showInLegend: false,
+        zones: [
+          {
+            value: goal + 0.1,
+            color: '#90A959',
+          },
+          {
+            value: goal + 12.1,
+            color: '#FEE191',
+          },
+          {
+            color: '#FB6962',
+          },
+        ],
+        point: {
+          events: {
+            click(e) {
+              handleClick(e, facultyGraph)
+            },
+          },
+        },
+      },
+    ],
+    xAxis: {
+      type: 'category',
+      title: {
+        text: facultyGraph ? label : 'Programme',
+        align: 'high',
+        offset: 0,
+        rotation: 0,
+        y: -10,
+      },
+    },
+    yAxis: {
+      min: 0,
+      max: maxValue,
+      title: { text: 'Graduation time (months)' },
+      labels: {
+        overflow: 'justify',
+      },
+      allowDecimals: false,
+      showFirstLabel: false,
+      plotLines: [
+        {
+          color: '#90A959',
+          width: 2,
+          value: goal,
+          dashStyle: 'shortDash',
+        },
+        {
+          color: '#FEE191',
+          width: 2,
+          value: goal + 12,
+          dashStyle: 'shortDash',
+        },
+      ],
+    },
+    credits: {
+      text: 'oodikone | TOSKA',
+    },
+  }
+
+  if (!facultyGraph) config.title.text = `Year ${year} by ${label.toLowerCase()}`
+
+  return (
+    <div className={`${facultyGraph ? 'faculty' : 'programmes'}-graph`}>
+      <ReactHighcharts config={config} />
+    </div>
+  )
+}
+
+export default BarChart

services/frontend/src/components/FacultyStatistics/TimesAndPaths/GraduationTimes.jsx

@@ -0,0 +1,91 @@
+import React, { useState } from 'react'
+import { Divider, Message } from 'semantic-ui-react'
+import BarChart from './BarChart'
+import useLanguage from '../../LanguagePicker/useLanguage'
+import '../faculty.css'
+
+const GraduationTimes = ({
+  title,
+  data,
+  level,
+  goal,
+  label,
+  levelProgrammeData,
+  programmeNames,
+  showMeanTime,
+  classSizes,
+  groupBy,
+  goalExceptions,
+}) => {
+  const [programmeData, setProgrammeData] = useState(false)
+  const [year, setYear] = useState(null)
+  const { language } = useLanguage()
+  if (!data.some(a => a.amount > 0)) return null
+
+  const handleClick = (e, isFacultyGraph) => {
+    if (isFacultyGraph) {
+      setYear(e.point.name)
+      setProgrammeData(true)
+    } else {
+      setProgrammeData(false)
+      setYear(null)
+    }
+  }
+
+  return (
+    <div className={`graduation-times-${level}`} data-cy={`Section-${level}`}>
+      <Divider horizontal>{title}</Divider>
+      <div>
+        {level === 'bcMsCombo' && groupBy === 'byStartYear' && (
+          <div className="graduations-message">
+            <Message compact>
+              Programme class sizes for recent years are not reliable as students might still lack relevant master
+              studies data in Sisu
+            </Message>
+          </div>
+        )}
+        {goalExceptions.needed && ['master', 'bcMsCombo'].includes(level) && (
+          <div className="graduations-message">
+            <Message compact>
+              <b>Different goal times</b> have been taken into account in all numbers and programme level bar coloring,
+              but the faculty level bar color is based on the typical goal time of {goal} months
+            </Message>
+          </div>
+        )}
+        <div className="graduations-chart-container">
+          <BarChart
+            data={data}
+            goal={goal}
+            handleClick={handleClick}
+            label={label}
+            programmeNames={programmeNames}
+            showMeanTime={showMeanTime}
+            classSizes={classSizes?.[level]}
+          />
+          {!programmeData ? (
+            <div className="graduations-message">
+              <Message compact>Click a bar to view that year's programme level breakdown</Message>
+            </div>
+          ) : (
+            <BarChart
+              data={levelProgrammeData[year]?.data}
+              goal={goal}
+              facultyGraph={false}
+              handleClick={handleClick}
+              year={year}
+              label={label}
+              programmeNames={programmeNames}
+              language={language}
+              showMeanTime={showMeanTime}
+              classSizes={classSizes?.programmes}
+              level={level}
+              goalExceptions={goalExceptions}
+            />
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default GraduationTimes

services/frontend/src/components/FacultyStatistics/TimesAndPaths/ProgrammeSelector.jsx

@@ -0,0 +1,34 @@
+import React from 'react'
+import { Dropdown } from 'semantic-ui-react'
+
+import '../../StudyProgramme/studyprogramme.css'
+
+const ProgrammeSelector = ({ programme, setProgramme, programmes }) => {
+  if (!programmes) return null
+
+  const handleStudytrackChange = (event, { value }) => {
+    event.preventDefault()
+    setProgramme(value)
+  }
+
+  return (
+    <div className="programme-selector">
+      <h4>Choose programme</h4>
+      <Dropdown
+        fluid
+        selection
+        name="programme"
+        placeholder="All programmes of the faculty"
+        value={programme}
+        onChange={handleStudytrackChange}
+        options={programmes.map(({ code, name }) => ({
+          key: code,
+          value: code,
+          text: code === 'faculty' ? name?.fi : `${name?.fi}, ${code}`,
+        }))}
+      />
+    </div>
+  )
+}
+
+export default ProgrammeSelector

services/frontend/src/components/FacultyStatistics/TimesAndPaths/index.js

@@ -0,0 +1,134 @@
+import React, { useState } from 'react'
+import { Divider, Loader } from 'semantic-ui-react'
+
+import { useGetFacultyGraduationTimesQuery } from 'redux/facultyStats'
+import Toggle from '../../StudyProgramme/Toggle'
+import InfoBox from '../../Info/InfoBox'
+import GraduationTimes from './GraduationTimes'
+import InfotoolTips from '../../../common/InfoToolTips'
+import '../faculty.css'
+
+const TimesAndPathsView = ({ faculty, studyProgrammes, setStudyProgrammes }) => {
+  const toolTips = InfotoolTips.Faculty
+  const [showMeanTime, setShowMeanTime] = useState(false)
+  const [groupByStartYear, setGroupByStartYear] = useState(false)
+  const studyProgrammeFilter = studyProgrammes ? 'ALL_PROGRAMMES' : 'NEW_STUDY_PROGRAMMES'
+  const graduationStats = useGetFacultyGraduationTimesQuery({ id: faculty?.code, studyProgrammeFilter })
+
+  const groupBy = groupByStartYear ? 'byStartYear' : 'byGradYear'
+  const label = groupByStartYear ? 'Start year' : 'Graduation year'
+  const data = showMeanTime ? graduationStats?.data?.[groupBy].means : graduationStats?.data?.[groupBy].medians
+  const goals = graduationStats?.data?.goals
+  const goalExceptions = { ...goals?.exceptions, needed: faculty?.code === 'H30' }
+  const programmeData = showMeanTime
+    ? graduationStats?.data?.[groupBy].programmes.means
+    : graduationStats?.data?.[groupBy].programmes.medians
+  const programmeNames = graduationStats?.data?.programmeNames
+  const classSizes = graduationStats?.data?.classSizes
+  const commonProps = { label, programmeNames, showMeanTime, classSizes, goalExceptions }
+
+  const isFetchingOrLoading = graduationStats.isLoading || graduationStats.isFetching
+
+  const isError = graduationStats.isError || (graduationStats.isSuccess && !graduationStats.data)
+
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+
+  const getDivider = (title, toolTipText) => (
+    <>
+      <div className="divider">
+        <Divider data-cy={`Section-${toolTipText}`} horizontal>
+          {title}
+        </Divider>
+      </div>
+      <InfoBox content={toolTips[toolTipText]} />
+    </>
+  )
+
+  return (
+    <div className="programmes-overview">
+      <div className="toggle-container">
+        <Toggle
+          cypress="ProgrammeToggle"
+          toolTips={toolTips.ProgrammeToggle}
+          firstLabel="New study programmes"
+          secondLabel="All study programmes"
+          value={studyProgrammes}
+          setValue={setStudyProgrammes}
+        />
+      </div>
+
+      {isFetchingOrLoading ? (
+        <Loader active style={{ marginTop: '15em' }} />
+      ) : (
+        <>
+          {graduationStats.isSuccess && graduationStats.data && (
+            <>
+              {getDivider('Average graduation times', 'AverageGraduationTimes')}
+              <div className="toggle-container">
+                <Toggle
+                  cypress="GroupByToggle"
+                  firstLabel="Group by: Graduation year"
+                  secondLabel="Starting year"
+                  value={groupByStartYear}
+                  setValue={setGroupByStartYear}
+                />
+                <Toggle
+                  cypress="GraduationTimeToggle"
+                  firstLabel="Median time"
+                  secondLabel="Mean time"
+                  value={showMeanTime}
+                  setValue={setShowMeanTime}
+                />
+              </div>
+              <div>
+                <GraduationTimes
+                  level="bachelor"
+                  title="Bachelor"
+                  data={data?.bachelor}
+                  goal={goals?.bachelor}
+                  levelProgrammeData={programmeData?.bachelor}
+                  {...commonProps}
+                />
+                <GraduationTimes
+                  level="bcMsCombo"
+                  title="Bachelor + Master"
+                  data={data?.bcMsCombo}
+                  goal={goals?.bcMsCombo}
+                  levelProgrammeData={programmeData?.bcMsCombo}
+                  groupBy={groupBy}
+                  {...commonProps}
+                />
+                <GraduationTimes
+                  level="master"
+                  title="Master"
+                  data={data?.master}
+                  goal={goals?.master}
+                  levelProgrammeData={programmeData?.master}
+                  {...commonProps}
+                />
+                <GraduationTimes
+                  level="doctor"
+                  title="Doctor"
+                  data={data?.doctor}
+                  goal={goals?.doctor}
+                  levelProgrammeData={programmeData?.doctor}
+                  {...commonProps}
+                />
+                <GraduationTimes
+                  level="licentiate"
+                  title="Licentiate"
+                  data={data?.licentiate}
+                  goal={goals?.licentiate}
+                  levelProgrammeData={programmeData?.licentiate}
+                  {...commonProps}
+                />
+              </div>
+            </>
+          )}
+        </>
+      )}
+    </div>
+  )
+}
+
+export default TimesAndPathsView

services/frontend/src/components/FacultyStatistics/UpdateView/index.jsx

@@ -0,0 +1,57 @@
+import React, { useState } from 'react'
+import { Button, Icon, Loader } from 'semantic-ui-react'
+import { useUpdateFacultyBasicViewQuery, useUpdateFacultyProgressViewQuery } from '../../../redux/facultyStats'
+
+const getStatusIcon = stats => {
+  if (stats.isLoading) return <Loader active />
+  if (stats.isSuccess) return <Icon name="check" color="green" />
+  if (stats.isError) return <Icon name="close" color="red" />
+  return ''
+}
+
+const UpdateView = ({ faculty }) => {
+  const [skipBasic, setSkipBasic] = useState(true)
+  const [skipCredits, setSkipCredits] = useState(true)
+  const [skipThesis, setSkipThesis] = useState(true)
+  const [skipProgressTab, setSkipProgressTab] = useState(true)
+  const basicBasicTabStats = useUpdateFacultyBasicViewQuery({ id: faculty, statsType: 'STUDENT' }, { skip: skipBasic })
+
+  const creditsBasicTabStats = useUpdateFacultyBasicViewQuery(
+    { id: faculty, statsType: 'CREDITS' },
+    { skip: skipCredits }
+  )
+  const thesisBasicTabStats = useUpdateFacultyBasicViewQuery({ id: faculty, statsType: 'THESIS' }, { skip: skipThesis })
+  const progressViewStats = useUpdateFacultyProgressViewQuery({ id: faculty }, { skip: skipProgressTab })
+  return (
+    <div className="update-view">
+      <div className="button-container">
+        <h3>Update data in Basic information -tab</h3>
+        <p>
+          <i>(For large faculties updates can take a couple of minutes)</i>
+        </p>
+        <h4>Update students data on Basic information -view</h4>
+        <Button disabled={basicBasicTabStats.isLoading} color="blue" onClick={() => setSkipBasic(false)}>
+          Update
+        </Button>
+        {getStatusIcon(basicBasicTabStats)}
+        <h4>Update credits data on Basic information -view</h4>
+        <Button disabled={creditsBasicTabStats.isLoading} color="blue" onClick={() => setSkipCredits(false)}>
+          Update
+        </Button>
+        {getStatusIcon(creditsBasicTabStats)}
+        <h4>Update thesiswriters data on Basic information -view</h4>
+        <Button disabled={thesisBasicTabStats.isLoading} color="blue" onClick={() => setSkipThesis(false)}>
+          Update
+        </Button>
+        {getStatusIcon(thesisBasicTabStats)}
+        <h3>Update data in Progress and student populations -tab</h3>
+        <Button disabled={progressViewStats.isLoading} color="blue" onClick={() => setSkipProgressTab(false)}>
+          Update
+        </Button>
+        {getStatusIcon(progressViewStats)}
+      </div>
+    </div>
+  )
+}
+
+export default UpdateView

services/frontend/src/components/FacultyStatistics/faculty.css

@@ -137,3 +137,16 @@
 .update-view .button-container {
   margin: 1em 0;
 }
+
+.graduations-message {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  margin-bottom: 5px;
+}
+
+.graduations-chart-container {
+  display: flex;
+  justify-content: space-between;
+  padding-bottom: 100px;
+}

services/frontend/src/components/FacultyStatistics/facultyHelpers.js

@@ -0,0 +1,42 @@
+/*
+  Order of the programme keys (such as TKT, PSYK) is chosen by "old" code:
+  KH -> MH -> T -> FI -> K- -> Numbers containing letters at end -> Y- -> Number
+  Take
+*/
+const regexValuesAll = [
+  /^KH/,
+  /^MH/,
+  /^T/,
+  /^LI/,
+  /^K-/,
+  /^FI/,
+  /^00901$/,
+  /^00910$/,
+  /^\d.*a$/,
+  /^Y/,
+  /\d$/,
+  /^\d.*e$/,
+]
+
+const testKey = value => {
+  for (let i = 0; i < regexValuesAll.length; i++) {
+    if (regexValuesAll[i].test(value)) {
+      return i
+    }
+  }
+  return 6
+}
+
+const sortProgrammeKeys = programmeKeys => {
+  return programmeKeys.sort((a, b) => {
+    if (testKey(a[1]) - testKey(b[1]) === 0 && testKey(a[0]) - testKey(b[0])) {
+      return a[1].localeCompare(b[1])
+    }
+    if (testKey(a[1]) - testKey(b[1]) === 0) {
+      return a[0].localeCompare(b[0])
+    }
+    return testKey(a[1]) - testKey(b[1])
+  })
+}
+
+export default sortProgrammeKeys

services/frontend/src/components/FacultyStatistics/index.jsx

@@ -2,31 +2,36 @@ import React, { useCallback, useEffect, useState } from 'react'
 import { withRouter, useHistory } from 'react-router-dom'
 import { Segment, Header, Tab, Loader } from 'semantic-ui-react'
 import { useGetFacultiesQuery } from 'redux/facultyStats'
+import { useGetAuthorizedUserQuery } from 'redux/auth'
 import { getTextIn } from 'common'
 import { useTabs, useTitle } from '../../common/hooks'
 import FacultySelector from './facultySelector'
 import BasicOverview from './BasicOverview'
 import ProgrammeOverview from './FacultyProgrammeOverview'
+import TimesAndPathsView from './TimesAndPaths'
+import UpdateView from './UpdateView'
 import useLanguage from '../LanguagePicker/useLanguage'
 import TSA from '../../common/tsa'
 
-const ignore = ['Y', 'H99', 'Y01', 'H92', 'H930']
-
 const FacultyStatistics = props => {
   useTitle('Faculties')
   const history = useHistory()
   const { language } = useLanguage()
   const allFaculties = useGetFacultiesQuery()
-  const faculties = allFaculties?.data && allFaculties.data.filter(f => !ignore.includes(f.code))
+  const faculties = allFaculties?.data
 
   const { match } = props
   const { facultyCode } = match.params
   const faculty = faculties && facultyCode && faculties.find(f => f.code === facultyCode)
   const facultyName = faculty && getTextIn(faculty.name, language)
 
+  const { isAdmin, rights, iamRights } = useGetAuthorizedUserQuery()
   const [tab, setTab] = useTabs('p_tab', 0, history)
   const [academicYear, setAcademicYear] = useState(false)
-
+  const [studyProgrammes, setStudyProgrammes] = useState(false)
+  const [specialGroups, setSpecialGroups] = useState(false)
+  const [graduatedGroup, setGraduatedGroup] = useState(false)
+  const requiredRights = { rights, iamRights, isAdmin }
   useEffect(() => {
     if (!facultyName) {
       return
@@ -56,13 +61,52 @@ const FacultyStatistics = props => {
     const panes = [
       {
         menuItem: 'Basic information',
-        render: () => <BasicOverview faculty={faculty} academicYear={academicYear} setAcademicYear={setAcademicYear} />,
+        render: () => (
+          <BasicOverview
+            faculty={faculty}
+            academicYear={academicYear}
+            setAcademicYear={setAcademicYear}
+            studyProgrammes={studyProgrammes}
+            setStudyProgrammes={setStudyProgrammes}
+            specialGroups={specialGroups}
+            setSpecialGroups={setSpecialGroups}
+            language={language}
+          />
+        ),
       },
       {
         menuItem: 'Programmes and student populations',
-        render: () => <ProgrammeOverview />,
+        render: () => (
+          <ProgrammeOverview
+            faculty={faculty}
+            language={language}
+            graduatedGroup={graduatedGroup}
+            setGraduatedGroup={setGraduatedGroup}
+            specialGroups={specialGroups}
+            setSpecialGroups={setSpecialGroups}
+            requiredRights={requiredRights}
+          />
+        ),
+      },
+      {
+        menuItem: 'Graduation times',
+        render: () => (
+          <TimesAndPathsView
+            faculty={faculty}
+            studyProgrammes={studyProgrammes}
+            setStudyProgrammes={setStudyProgrammes}
+          />
+        ),
       },
     ]
+
+    if (isAdmin) {
+      panes.push({
+        menuItem: 'Update statistics',
+        render: () => <UpdateView faculty={faculty?.code} />,
+      })
+    }
+
     return panes
   }
 
@@ -81,7 +125,7 @@ const FacultyStatistics = props => {
     )
 
   return (
-    <div className="segmentContainer">
+    <div className="segmentContainer" data-cy="FacultySegmentContainer">
       <Segment className="contentSegment">
         <div align="center" style={{ padding: '30px' }}>
           <Header textAlign="center">{facultyName}</Header>

services/frontend/src/components/FigureContainer/index.jsx

@@ -23,7 +23,7 @@ const Header = ({ children, actions, contextItems }) => {
           />
         }
         on="hover"
-        position="bottom"
+        position="bottom left"
       >
         Toggle Fullscreen
       </Popup>
@@ -32,7 +32,9 @@ const Header = ({ children, actions, contextItems }) => {
           <Dropdown position="bottom center" direction="left" icon="ellipsis vertical">
             <Dropdown.Menu>
               {contextItems.map(({ label, onClick }) => (
-                <Dropdown.Item onClick={onClick}>{label}</Dropdown.Item>
+                <Dropdown.Item key={label} onClick={onClick}>
+                  {label}
+                </Dropdown.Item>
               ))}
             </Dropdown.Menu>
           </Dropdown>

services/frontend/src/components/FilterView/filters/courses/CourseCard.jsx

@@ -10,9 +10,19 @@ const filterTexts = {
   [FilterType.PASSED_AFTER_FAILURE]: { label: 'Passed After Failure' },
   [FilterType.FAILED]: { label: 'Failed' },
   [FilterType.FAILED_MANY_TIMES]: { label: 'Failed Multiple Times' },
-  [FilterType.NOT_PARTICIPATED]: { label: 'Not Participated' },
+  // [FilterType.NOT_PARTICIPATED]: { label: 'Not Participated' },
   [FilterType.DID_NOT_PASS]: { label: "Didn't Pass" },
 }
+// a bandaid solution to prevent oodikone crashing
+const translate = {
+  ALL: 'all',
+  PASSED: 'passed',
+  PASSED_AFTER_FAILURE: 'retryPassed',
+  FAILED: 'failed',
+  FAILED_MANY_TIMES: 'failedMany',
+  NOT_PARTICIPATED: 'notParticipated',
+  DID_NOT_PASS: 'didNotPass',
+}
 
 const CourseCard = ({ course, filterType, onChange }) => {
   const { language } = useLanguage()
@@ -27,7 +37,7 @@ const CourseCard = ({ course, filterType, onChange }) => {
   return (
     <>
       <Label style={{ marginTop: '0.5rem' }}>
-        {getTextIn(course.name, language)}
+        {getTextIn(course.course.name, language)}
 
         <Dropdown
           text={filterTexts[filterType].label}
@@ -35,7 +45,7 @@ const CourseCard = ({ course, filterType, onChange }) => {
           fluid
           className="mini"
           button
-          data-cy={`${name}-${course.code}-dropdown`}
+          data-cy={`${name}-${course.course.code}-dropdown`}
           style={{ marginTop: '0.5rem' }}
         >
           <Dropdown.Menu>
@@ -50,6 +60,8 @@ const CourseCard = ({ course, filterType, onChange }) => {
                   />
                 )
               }
+              if (Object.keys(course?.students[translate[type]]).length === 0)
+                return <Dropdown.Item key={label} text={label} value={type} onClick={onClick} disabled />
               return <Dropdown.Item key={label} text={label} value={type} onClick={onClick} />
             })}
           </Dropdown.Menu>
@@ -60,7 +72,7 @@ const CourseCard = ({ course, filterType, onChange }) => {
           size="tiny"
           onClick={clear}
           icon
-          data-cy={`${name}-${course.code}-clear`}
+          data-cy={`${name}-${course.course.code}-clear`}
           style={{ marginTop: '0.5rem' }}
         >
           <Icon name="close" />

services/frontend/src/components/FilterView/filters/courses/index.jsx

@@ -62,7 +62,7 @@ const CourseFilterCard = ({ courseStats, options, onOptionsChange }) => {
       />
       {Object.entries(courseFilters).map(([code]) => (
         <CourseCard
-          course={courseStats[code].course}
+          course={courseStats[code]}
           filterType={courseFilters[code] ?? FilterType.ALL}
           onChange={type => setCourseFilter(code, type)}
           key={`course-filter-selected-course-${code}`}
@@ -83,9 +83,10 @@ const filterFunctions = {
   [FilterType.PASSED_AFTER_FAILURE]: createFilterFunc('retryPassed'),
   [FilterType.FAILED]: createFilterFunc('failed'),
   [FilterType.FAILED_MANY_TIMES]: createFilterFunc('failedMany'),
-  [FilterType.NOT_PARTICIPATED]: createFilterFunc('all', true),
-  [FilterType.DID_NOT_PASS]: ({ studentNumber }, { students }) =>
-    !Object.keys(students.all).includes(studentNumber) || Object.keys(students.failed).includes(studentNumber),
+  [FilterType.DID_NOT_PASS]: createFilterFunc('didNotPass'),
+  // [FilterType.NOT_PARTICIPATED]: createFilterFunc('all', true),
+  // [FilterType.DID_NOT_PASS]: ({ studentNumber }, { students }) =>
+  //   !Object.keys(students.all).includes(studentNumber) || Object.keys(students.failed).includes(studentNumber),
 }
 
 const filter = createFilter({

services/frontend/src/components/FilterView/filters/index.js

@@ -13,7 +13,7 @@ import date from './date'
 import enrollmentStatus from './enrollmentStatus'
 import studyTrack from './studyTrack'
 import studyrightStatus from './studyrightStatus'
-import cherrypick from './cherrypick'
+import studentNumber from './studentNumber'
 import hops from './hops'
 
 export const genderFilter = gender
@@ -31,5 +31,5 @@ export const creditDateFilter = date
 export const enrollmentStatusFilter = enrollmentStatus
 export const studyTrackFilter = studyTrack
 export const studyrightStatusFilter = studyrightStatus
-export const cherrypickFilter = cherrypick
+export const studentNumberFilter = studentNumber
 export const hopsFilter = hops

services/frontend/src/components/FilterView/filters/studentNumber.jsx

@@ -115,7 +115,7 @@ const EditableList = ({ value, onChange, renderLabel }) => {
   )
 }
 
-const CherryPickFilterCard = ({ options, onOptionsChange, withoutSelf }) => {
+const StudentNumberFilterCard = ({ options, onOptionsChange, withoutSelf }) => {
   const students = withoutSelf()
 
   const swapLists = () => {
@@ -177,16 +177,16 @@ const CherryPickFilterCard = ({ options, onOptionsChange, withoutSelf }) => {
 }
 
 export default createFilter({
-  key: 'CherryPick',
+  key: 'studentNumber',
 
-  title: 'Cherry-picker',
+  title: 'Student number *',
 
   defaultOptions: {
     allowlist: [],
     blocklist: [],
   },
 
-  info: filterInfo.cherrypicker,
+  info: filterInfo.studentNumber,
 
   isActive: ({ allowlist, blocklist }) => allowlist.length > 0 || blocklist.length > 0,
 
@@ -198,7 +198,7 @@ export default createFilter({
     )
   },
 
-  render: props => <CherryPickFilterCard {...props} />,
+  render: props => <StudentNumberFilterCard {...props} />,
 
   actions: {
     addToAllowlist: (options, students) => {

services/frontend/src/components/Info/InfoBox/index.jsx

@@ -4,24 +4,32 @@ import { Message, Icon, Button } from 'semantic-ui-react'
 import ReactMarkdown from 'react-markdown'
 import './infoBox.css'
 
-const InfoBox = ({ content }) => {
+const InfoBox = ({ content, cypress }) => {
   const [open, setOpen] = useState(false)
   const toggleOpen = () => setOpen(prev => !prev)
 
   return open ? (
     <Message icon className="ok-infobox" color="green">
-      <div className="content-container">
+      <div className="content-container" data-cy={`${cypress}-info-content`}>
         <Icon name="info circle" size="huge" />
         <Message.Content className="ok-infobox content">
           <ReactMarkdown escapeHtml={false}>{content}</ReactMarkdown>
         </Message.Content>
       </div>
-      <Button onClick={toggleOpen} className="ok-infobox-close">
+      <Button onClick={toggleOpen} className="ok-infobox-close" data-cy={`${cypress}-close-info`}>
         Close Info
       </Button>
     </Message>
   ) : (
-    <Button icon labelPosition="left" className="ok-infobox-collapsed" onClick={toggleOpen} basic color="green">
+    <Button
+      icon
+      labelPosition="left"
+      className="ok-infobox-collapsed"
+      onClick={toggleOpen}
+      basic
+      color="green"
+      data-cy={`${cypress}-open-info`}
+    >
       <Icon name="info circle" />
       Show Info
     </Button>

services/frontend/src/components/NavigationBar/index.jsx

@@ -9,7 +9,8 @@ import LanguagePicker from '../LanguagePicker'
 import { isDev, adminerUrls } from '../../conf'
 
 const allNavigationItems = {
-  faculty: { path: '/faculties', key: 'faculties', label: 'Faculties', reqRights: ['admin'] },
+  trends: { path: '/trends', key: 'trends', label: 'Trends' },
+  faculty: { path: '/faculties', key: 'faculties', label: 'Faculties' },
   populations: {
     key: 'studyProgramme',
     label: 'Study programme',
@@ -18,11 +19,10 @@ const allNavigationItems = {
       { path: '/study-programme', key: 'overview', label: 'Overview' },
     ],
   },
-  students: { path: '/students', key: 'students', label: 'Student statistics' },
   courseStatistics: { path: '/coursestatistics', key: 'courseStatistics', label: 'Course statistics' },
+  students: { path: '/students', key: 'students', label: 'Student statistics' },
   teachers: { path: '/teachers', key: 'teachers', label: 'Teachers', reqRights: ['teachers'] },
   users: { path: '/users', key: 'users', label: 'Users', reqRights: ['admin'] },
-  trends: { path: '/trends', key: 'trends', label: 'Trends' },
   studyGuidanceGroups: {
     path: '/studyguidancegroups',
     key: 'studyGuidanceGroups',
@@ -30,9 +30,12 @@ const allNavigationItems = {
     reqRights: ['studyGuidanceGroups'],
   },
   customPopulations: {
-    path: '/custompopulation',
     key: 'customPopulation',
     label: 'Custom populations',
+    items: [
+      { path: '/custompopulation', key: 'customSearch', label: 'Search by Studentnumbers' },
+      { path: '/openunipopulation', key: 'openUniSearch', label: 'Fetch Open Uni Students by Courses' },
+    ],
   },
   updater: { path: '/updater', key: 'updater', label: 'Updater', reqRights: ['admin'] },
   feedback: { path: '/feedback', key: 'feedback', label: 'Give feedback' },
@@ -50,10 +53,15 @@ const NavigationBar = () => {
       if (key === 'populations') {
         if (!isAdmin && rights.length === 0 && iamRights.length === 0) return
       }
-      if (key === 'students' || key === 'customPopulations') {
+      if (key === 'students') {
         if (!checkUserAccess(['admin', 'studyGuidanceGroups'], roles) && rights.length === 0) return
+      }
+      if (key === 'customPopulations') {
+        if (!checkUserAccess(['admin', 'studyGuidanceGroups', 'openUniSearch'], roles) && rights.length === 0) return
       } else if (key === 'courseStatistics') {
         if (!checkUserAccess(['courseStatistics', 'admin'], roles) && rights.length === 0) return
+      } else if (key === 'faculty') {
+        if (!checkUserAccess(['facultyStatistics', 'admin'], roles)) return
       }
       const { reqRights } = allNavigationItems[key]
       if (!reqRights || reqRights.every(r => roles.includes(r))) {
@@ -73,23 +81,33 @@ const NavigationBar = () => {
       </span>
     </Menu.Item>
   )
+  const showSearch = item => {
+    if (item.key === 'class' || item.key === 'overview') return true
+    if (checkUserAccess(['openUniSearch', 'admin'], roles) && item.key === 'openUniSearch') return true
+    if ((checkUserAccess(['studyGuidanceGroups', 'admin'], roles) || rights.length > 0) && item.key === 'customSearch')
+      return true
+    return false
+  }
 
   const renderNavigationRoutes = () =>
     Object.values(visibleNavigationItems).map(({ items, path, key, label, tag }) =>
       items ? (
         <Menu.Item as={Dropdown} key={`menu-item-drop-${key}`} tabIndex="-1" text={label} data-cy={`navbar-${key}`}>
           <Dropdown.Menu>
-            {items.map(i => (
-              <Dropdown.Item
-                as={NavLink}
-                key={`menu-item-${i.path}`}
-                to={i.path}
-                tabIndex="-1"
-                data-cy={`navbar-${i.key}`}
-              >
-                {i.label}
-              </Dropdown.Item>
-            ))}
+            {items.map(
+              i =>
+                showSearch(i) && (
+                  <Dropdown.Item
+                    as={NavLink}
+                    key={`menu-item-${i.path}`}
+                    to={i.path}
+                    tabIndex="-1"
+                    data-cy={`navbar-${i.key}`}
+                  >
+                    {i.label}
+                  </Dropdown.Item>
+                )
+            )}
           </Dropdown.Menu>
         </Menu.Item>
       ) : (

services/frontend/src/components/PopulationCourseStats/GradeDistribution/index.jsx

@@ -30,7 +30,8 @@ const mapCourseData = course => ({
 })
 
 const GradeDistribution = ({ flat, onlyIamRights }) => {
-  const { modules, courseStatistics, onGoToCourseStatisticsClick } = UsePopulationCourseContext()
+  const { modules, courseStatistics, onGoToCourseStatisticsClick, toggleGroupExpansion, expandedGroups } =
+    UsePopulationCourseContext()
 
   const columns = useMemo(() => {
     const columns = [
@@ -147,7 +148,13 @@ const GradeDistribution = ({ flat, onlyIamRights }) => {
 
   return (
     <>
-      <SortableTable title="Grade distribution of courses" data={data} columns={columns} />
+      <SortableTable
+        title="Grade distribution of courses"
+        data={data}
+        columns={columns}
+        toggleGroupExpansion={toggleGroupExpansion}
+        expandedGroups={expandedGroups}
+      />
     </>
   )
 }

services/frontend/src/components/PopulationCourseStats/PassFailEnrollments/index.jsx

@@ -27,7 +27,8 @@ const createModuleAggregateRow = ({ definition, children }) => ({
 })
 
 const PassFailEnrollments = ({ flat, onlyIamRights }) => {
-  const { modules, courseStatistics, onGoToCourseStatisticsClick } = UsePopulationCourseContext()
+  const { modules, courseStatistics, onGoToCourseStatisticsClick, toggleGroupExpansion, expandedGroups } =
+    UsePopulationCourseContext()
 
   const columns = useMemo(() => {
     const columns = [
@@ -236,6 +237,8 @@ const PassFailEnrollments = ({ flat, onlyIamRights }) => {
         title="Pass and fail statistics of courses with course enrollment details"
         data={data}
         columns={columns}
+        toggleGroupExpansion={toggleGroupExpansion}
+        expandedGroups={expandedGroups}
       />
     </>
   )

services/frontend/src/components/PopulationCourseStats/PassingSemesters/index.jsx

@@ -35,7 +35,7 @@ const yearColumn = (year, cumulative) => ({
 })
 
 const PassingSemesters = ({ onlyIamRights }) => {
-  const { modules, onGoToCourseStatisticsClick } = UsePopulationCourseContext()
+  const { modules, onGoToCourseStatisticsClick, toggleGroupExpansion, expandedGroups } = UsePopulationCourseContext()
   const [cumulativeStats, setCumulativeStats] = useState(false)
 
   const columns = useMemo(() => {
@@ -154,6 +154,8 @@ const PassingSemesters = ({ onlyIamRights }) => {
         }
         columns={columns}
         data={data}
+        toggleGroupExpansion={toggleGroupExpansion}
+        expandedGroups={expandedGroups}
       />
     </div>
   )

services/frontend/src/components/PopulationCourseStats/Students/index.jsx

@@ -9,7 +9,8 @@ import { getTextIn } from '../../../common'
 import { useStudentNameVisibility } from '../../StudentNameVisibilityToggle'
 
 const Students = ({ filteredStudents }) => {
-  const { courseStatistics, onGoToCourseStatisticsClick, modules } = UsePopulationCourseContext()
+  const { courseStatistics, onGoToCourseStatisticsClick, modules, toggleGroupExpansion, expandedGroups } =
+    UsePopulationCourseContext()
   const { visible: namesVisible, toggle: toggleStudentNames } = useStudentNameVisibility()
   const [page, setPage] = useState(0)
 
@@ -181,6 +182,8 @@ const Students = ({ filteredStudents }) => {
         }
         columns={columns}
         data={data}
+        toggleGroupExpansion={toggleGroupExpansion}
+        expandedGroups={expandedGroups}
       />
     </div>
   )

services/frontend/src/components/PopulationCourseStats/index.jsx

@@ -10,7 +10,6 @@ import './populationCourseStats.css'
 import { PopulationCourseContext } from './PopulationCourseContext'
 import sendEvent from '../../common/sendEvent'
 import GradeDistribution from './GradeDistribution'
-import PassFail from './PassFail'
 import PassFailEnrollments from './PassFailEnrollments'
 import Students from './Students'
 import { getTextIn } from '../../common'
@@ -243,16 +242,24 @@ const PopulationCourseStats = props => {
     setFilterFields({ ...filterFields, [field]: '' })
   }
 
-  const toggleGroupExpansion = code => {
-    const newExpandedGroups = new Set(expandedGroups)
-    if (newExpandedGroups.has(code)) {
-      newExpandedGroups.delete(code)
-      sendAnalytics('Courses of Population collapsed group', code)
+  const toggleGroupExpansion = (code, close = false, all = null) => {
+    if (all) {
+      sendAnalytics('Courses of Population expanded all groups', '')
+      setExpandedGroups(new Set(all))
+    } else if (close) {
+      sendAnalytics('Courses of Population collapsed all groups', '')
+      setExpandedGroups(new Set())
     } else {
-      newExpandedGroups.add(code)
-      sendAnalytics('Courses of Population expanded group', code)
+      const newExpandedGroups = new Set(expandedGroups)
+      if (newExpandedGroups.has(code)) {
+        newExpandedGroups.delete(code)
+        sendAnalytics('Courses of Population collapsed group', code)
+      } else {
+        newExpandedGroups.add(code)
+        sendAnalytics('Courses of Population expanded group', code)
+      }
+      setExpandedGroups(newExpandedGroups)
     }
-    setExpandedGroups(newExpandedGroups)
   }
 
   const getFilterValue = field => (field in filterFields ? filterFields[field] || '' : '')
@@ -287,23 +294,13 @@ const PopulationCourseStats = props => {
     sortCriteria,
     reversed,
     courses,
+    toggleGroupExpansion,
+    expandedGroups,
   }
 
   const panes = [
     {
       menuItem: 'pass/fail',
-      render: () => (
-        <Tab.Pane className="menuTab">
-          <PassFail
-            expandedGroups={expandedGroups}
-            toggleGroupExpansion={toggleGroupExpansion}
-            onlyIamRights={props.onlyIamRights}
-          />
-        </Tab.Pane>
-      ),
-    },
-    {
-      menuItem: 'pass/fail w/ enrollments ',
       render: () => (
         <Tab.Pane className="menuTab">
           <PassFailEnrollments

services/frontend/src/components/PopulationCourses/index.jsx

@@ -45,7 +45,7 @@ const PopulationCourses = ({ query = {}, filteredStudents, selectedStudentsByYea
       !mandatoryCourses.pending &&
       !queryHasBeenUpdated() &&
       !populationSelectedStudentCourses.pending &&
-      mandatoryCourses.data
+      mandatoryCourses.data.length > 0
     )
       fetch(mandatoryCourses.data.map(({ code }) => code))
   }, [query, filteredStudents, mandatoryCourses, populationSelectedStudentCourses])

services/frontend/src/components/PopulationDetails/CreditGainStats/CollapsibleCreditRow.jsx

@@ -31,7 +31,7 @@ const CollapsibleCreditRow = ({ min, max, studentCount, filteredLength, months }
         <Table.Cell collapsing>
           <ExternalCreditFilterToggle min={min} max={max} />
         </Table.Cell>
-        <Table.Cell>
+        <Table.Cell key={`${min} ≤ credits`}>
           {max === 0 ? 0 : `${min} ≤ credits`}
           {max > 0 && ` < ${max}`}
           {isCollapsible && <Icon name={isCollapsed ? 'caret down' : 'caret right'} color="grey" />}

services/frontend/src/components/PopulationDetails/CreditGainStats/CreditDistributionDevelopment.jsx

@@ -7,7 +7,7 @@ import chroma from 'chroma-js'
 import { useLocation } from 'react-router-dom'
 import { useGetSemestersQuery } from 'redux/semesters'
 import useLanguage from 'components/LanguagePicker/useLanguage'
-import { cherrypickFilter } from 'components/FilterView/filters'
+import { studentNumberFilter } from 'components/FilterView/filters'
 import { getMonths } from '../../../common/query'
 import useFilters from '../../FilterView/useFilters'
 
@@ -255,7 +255,7 @@ const CreditDistributionDevelopment = ({ students, query }) => {
         point: {
           events: {
             click(e) {
-              filterDispatch(cherrypickFilter.actions.addToAllowlist(e.point.custom.students))
+              filterDispatch(studentNumberFilter.actions.addToAllowlist(e.point.custom.students))
             },
           },
         },

services/frontend/src/components/PopulationDetails/CreditGainStats/CreditsGainedTab.jsx

@@ -13,11 +13,11 @@ const admissionTypes = [
   null,
 ]
 
-const CreditsGainedTab = ({ allStudents, query }) => {
+const CreditsGainedTab = ({ allStudents, query, creditDateFilterOptions }) => {
   const [show, setShow] = useState(false)
   if (!allStudents || !allStudents.length || !query) return null
 
-  const { studyRights } = query
+  const { studyRights, year } = query
 
   const filterFunction = (student, type) =>
     student.studyrights.some(
@@ -26,7 +26,14 @@ const CreditsGainedTab = ({ allStudents, query }) => {
 
   const getCreditsGainedTable = type => {
     const filteredStudents = allStudents.filter(s => filterFunction(s, type))
-    return <CreditsGainedTable type={type || 'Ei valintatapaa'} filteredStudents={filteredStudents} />
+    return (
+      <CreditsGainedTable
+        type={type || 'Ei valintatapaa'}
+        filteredStudents={filteredStudents}
+        year={year}
+        creditDateFilterOptions={creditDateFilterOptions}
+      />
+    )
   }
 
   const admissionTypesAvailable = !allStudents.every(s => filterFunction(s, null))
@@ -35,7 +42,12 @@ const CreditsGainedTab = ({ allStudents, query }) => {
     <Grid>
       <Grid.Row>
         <Grid.Column width={16} data-cy="credits-gained-main-table">
-          <CreditsGainedTable type="All students of the population" filteredStudents={allStudents} />
+          <CreditsGainedTable
+            type="All students of the class"
+            filteredStudents={allStudents}
+            year={year}
+            creditDateFilterOptions={creditDateFilterOptions}
+          />
         </Grid.Column>
       </Grid.Row>
       {admissionTypesAvailable && (

services/frontend/src/components/PopulationDetails/CreditGainStats/CreditsGainedTable.jsx

@@ -1,20 +1,74 @@
-import React, { useMemo } from 'react'
-import { useLocation } from 'react-router-dom'
+import moment from 'moment/moment'
+import React from 'react'
 import { Table } from 'semantic-ui-react'
-import { getMonths } from '../../../common/query'
 import CollapsibleCreditRow from './CollapsibleCreditRow'
 
-const CreditsGainedTable = ({ filteredStudents, type }) => {
-  const months = getMonths(useLocation())
-  const creditList = useMemo(() => filteredStudents.map(student => student.credits), [filteredStudents])
-
+const CreditsGainedTable = ({ filteredStudents, type, year, creditDateFilterOptions }) => {
   if (!filteredStudents || !filteredStudents.length || !type) return null
-
+  let creditList = []
   const studentCount = (min, max = Infinity) =>
     max === 0
       ? creditList.filter(credits => credits === 0).length
       : creditList.filter(credits => credits < max && credits >= min).length
 
+  const getMonths = (start, end) => {
+    const lastDayOfMonth = moment(end).endOf('month')
+    return Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(start))).asMonths())
+  }
+
+  const start = moment(`${year}-08-01`)
+  const end = moment().format('YYYY-MM-DD')
+
+  let title = ''
+  let months = 0
+  const { startDate, endDate } = creditDateFilterOptions || { start, end }
+  if (startDate !== null && endDate !== null) {
+    months = getMonths(startDate, endDate)
+    title = `${moment(startDate).format('DD.MM.YYYY')} and ${moment(endDate).format('DD.MM.YYYY')}`
+    creditList = filteredStudents.map(student =>
+      student.courses.length > 0
+        ? student.courses.reduce(
+            (results, course) =>
+              results +
+              (course.passed &&
+                startDate.isSameOrBefore(course.date, 'day') &&
+                endDate.isSameOrAfter(course.date, 'day'))
+                ? course.credits
+                : 0,
+            0
+          )
+        : 0
+    )
+  } else if (startDate !== null) {
+    months = getMonths(startDate, end)
+    title = `${moment(startDate).format('DD.MM.YYYY')} and ${moment(end).format('DD.MM.YYYY')}`
+    creditList = filteredStudents.map(student =>
+      student.courses.length > 0
+        ? student.courses.reduce(
+            (results, course) =>
+              results + (course.passed && startDate.isSameOrBefore(course.date, 'day')) ? course.credits : 0,
+            0
+          )
+        : 0
+    )
+  } else if (endDate !== null) {
+    months = getMonths(start, endDate)
+    title = `${moment(start).format('DD.MM.YYYY')} and ${moment(endDate).format('DD.MM.YYYY')}`
+    creditList = filteredStudents.map(student =>
+      student.courses.lenght > 0
+        ? student.courses.reduce(
+            (results, course) =>
+              results + (course.passed && endDate.isSameOrAfter(course.date, 'day')) ? course.credits : 0,
+            0
+          )
+        : 0
+    )
+  } else {
+    months = getMonths(start, end)
+    title = `${moment(start).format('DD.MM.YYYY')} and ${moment(end).format('DD.MM.YYYY')}`
+    creditList = filteredStudents.map(student => student.credits)
+  }
+
   const limits = [
     [Math.ceil(months * (60 / 12))],
     [Math.ceil(months * (45 / 12)), Math.ceil(months * (60 / 12))],
@@ -24,27 +78,30 @@ const CreditsGainedTable = ({ filteredStudents, type }) => {
     [null, 0],
   ]
 
+  const monthString = months === 1 ? 'Month' : 'Months'
   return (
     <div className="credits-gained-table" data-cy={`credits-gained-table-${type}`}>
       <h3>{type}</h3>
       <Table celled>
         <Table.Header>
-          <Table.Row>
+          <Table.Row key={`credits-gained-table-${type}`}>
             <Table.HeaderCell collapsing />
-            <Table.HeaderCell>Credits Gained During First {months} Months</Table.HeaderCell>
-            <Table.HeaderCell>
+            <Table.HeaderCell key={`${title}-${type}`}>
+              Gredits Gained Between <br /> {title} <br /> ({months} {monthString})
+            </Table.HeaderCell>
+            <Table.HeaderCell key={`credits-number-of-students-${type}`}>
               Number of Students
               <br />
               <span style={{ fontWeight: 100 }}>(n={filteredStudents.length})</span>
             </Table.HeaderCell>
-            <Table.HeaderCell>Percentage of Population</Table.HeaderCell>
+            <Table.HeaderCell key={`credits-percentage-of-students-${type}`}>Percentage of Population</Table.HeaderCell>
           </Table.Row>
         </Table.Header>
 
         <Table.Body data-cy="credits-gained-table-body">
           {limits.map(([min, max]) => (
             <CollapsibleCreditRow
-              key={`table-row-${min}-${max}`}
+              key={`credits-table-row-${min}-${max}-${type}`}
               min={min}
               max={max}
               studentCount={studentCount}

services/frontend/src/components/PopulationDetails/CreditGainStats/index.jsx

@@ -8,11 +8,15 @@ import CreditDistributionDevelopment from './CreditDistributionDevelopment'
 import infotooltips from '../../../common/InfoToolTips'
 import './creditGainStats.css'
 
-const CreditGainStats = ({ filteredStudents, query }) => {
+const CreditGainStats = ({ filteredStudents, query, creditDateFilterOptions }) => {
   const renderCreditsGainTab = useCallback(() => {
     return (
       <Tab.Pane attached={false}>
-        <CreditsGainedTab allStudents={filteredStudents} query={query} />
+        <CreditsGainedTab
+          allStudents={filteredStudents}
+          query={query}
+          creditDateFilterOptions={creditDateFilterOptions}
+        />
       </Tab.Pane>
     )
   }, [filteredStudents])