Fix tests

It can be a bit misleading

.github/workflows/release-sis-updater-scheduler.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-scheduler"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-scheduler.yaml'
-      - 'updater/sis-updater-scheduler/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-scheduler"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
-        cache-to: type=inline
-        context: updater/sis-updater-scheduler/
-        push: true
-        tags: toska/sis-updater-scheduler:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-scheduler
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-scheduler

.github/workflows/release-sis-updater-worker.yaml

@@ -1,54 +0,0 @@
-name: "Release sis-updater-worker"
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - '.github/workflows/release-sis-updater-worker.yaml'
-      - 'updater/sis-updater-worker/**'
-
-jobs:
-  build_and_release:
-    name: "Build and release sis-updater-worker"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: docker/setup-buildx-action@v1
-    - name: "Login to DockerHub"
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: "Build and push"
-      uses: docker/build-push-action@v2
-      with:
-        cache-from: type=registry,ref=toska/sis-updater-worker:latest
-        cache-to: type=inline
-        context: updater/sis-updater-worker/
-        push: true
-        tags: toska/sis-updater-worker:latest
-        target: production
-    - name: "Send notification to slack"
-      uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-      with:
-        webhook-url: ${{ secrets.WEBHOOK_URL }}
-        message-type: deployment
-        softa-url: https://oodikone.cs.helsinki.fi
-        deployment-target: production
-        softa-to-deploy: sis-updater-worker
-
-  report_failure:
-    runs-on: ubuntu-latest
-    name: "Report deployment failure to oodikone notifications"
-    needs:
-    - build_and_release
-    if: failure()
-    steps:
-      - name: "Send notification to slack"
-        uses:  UniversityOfHelsinkiCS/slackbot-action@v1.4.2
-        with:
-          webhook-url: ${{ secrets.WEBHOOK_URL }}
-          message-type: deployment-failure
-          deployment-target: production
-          softa-to-deploy: sis-updater-worker

.github/workflows/test-and-release-updater.yaml

@@ -0,0 +1,138 @@
+name: 'Test and release updater'
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.github/workflows/test-and-release-updater.yaml'
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
+
+jobs:
+  build_worker:
+    name: 'Build sis-updater-worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          context: updater/sis-updater-worker/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/worker.tar
+          tags: toska/sis-updater-worker:latest
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/worker.tar
+
+  build_scheduler:
+    name: 'Build sis-updater-scheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Build the image'
+        uses: docker/build-push-action@v2
+        with:
+          context: updater/sis-updater-scheduler/
+          target: production
+          push: false
+          outputs: type=docker,dest=/tmp/scheduler.tar
+          tags: toska/sis-updater-scheduler:latest
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+      - name: 'Store built image'
+        uses: actions/upload-artifact@v2
+        with:
+          path: /tmp/scheduler.tar
+
+  test_updater:
+    name: 'Test updater'
+    runs-on: ubuntu-latest
+    needs:
+      - build_worker
+      - build_scheduler
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Login to toska docker registry'
+        uses: docker/login-action@v1
+        with:
+          registry: registry.toska.cs.helsinki.fi
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+      - name: 'Download built images'
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp/
+      - name: 'Load worker image'
+        run: docker load --input /tmp/artifact/worker.tar
+      - name: 'Load scheduler image'
+        run: docker load --input /tmp/artifact/scheduler.tar
+      - name: 'Start worker'
+        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+      - name: 'Run hourly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: hourly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run weekly jobs'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: weekly
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run prepurge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: prepurge
+          EXIT_AFTER_IMMEDIATES: yes
+      - name: 'Run purge'
+        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        env:
+          SCHEDULE_IMMEDIATE: purge
+          EXIT_AFTER_IMMEDIATES: yes
+
+  release_updater:
+    name: 'Release updater'
+    runs-on: ubuntu-latest
+    needs:
+      - test_updater
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - name: 'Login to DockerHub'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 'Build and push scheduler'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-scheduler:latest
+          cache-to: type=inline
+          context: updater/sis-updater-scheduler/
+          push: true
+          tags: toska/sis-updater-scheduler:latest
+          target: production
+      - name: 'Build and push worker'
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry,ref=toska/sis-updater-worker:latest
+          cache-to: type=inline
+          context: updater/sis-updater-worker/
+          push: true
+          tags: toska/sis-updater-worker:latest
+          target: production
+      - name: 'Send notification to slack'
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        with:
+          webhook-url: ${{ secrets.WEBHOOK_URL }}
+          message-type: deployment
+          softa-url: https://oodikone.cs.helsinki.fi
+          deployment-target: production
+          softa-to-deploy: sis-updater

cypress/integration/Course_statistics.js

@@ -20,15 +20,13 @@ describe('Course Statistics tests', () => {
 
       cy.contains('.tabular.menu a', 'Table').click()
       cy.contains('All')
-      cy.contains('.modeSelectorRow a', 'Attempts').click()
-      cy.contains('.modeSelectorRow a', 'Students').click()
+      cy.cs('viewMode-Attempts').click()
+      cy.cs('viewMode-Students').click()
 
       cy.contains('.tabular.menu a', 'Pass rate chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Pass rate chart')
 
       cy.contains('.tabular.menu a', 'Grade distribution chart').click()
-      cy.get('div.modeSelectorContainer').click()
       cy.contains('svg', 'Grades')
 
       cy.contains('a', 'New query').click()
@@ -186,7 +184,7 @@ describe('Course Statistics tests', () => {
       cy.contains('td', '010674989').siblings().contains('1')
     })
 
-    it.skip('Language distribution is correct', () => {
+    it('Language distribution is correct', () => {
       cy.get("input[placeholder='Search by a course code']").type('TKT20003')
       cy.contains(/^TKT20003$/).click()
       cy.contains('Fetch statistics').should('be.enabled').click()
@@ -194,7 +192,7 @@ describe('Course Statistics tests', () => {
       cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click()
       cy.contains('Population of course Käyttöjärjestelmät 2019-2020')
       cy.contains('Language distribution').click()
-      cy.contains('td', 'finnish').siblings().contains('122')
+      cy.contains('td', 'finnish').siblings().contains('107')
       cy.contains('td', 'english').siblings().contains('5')
     })
 
@@ -304,7 +302,7 @@ describe('Course Statistics tests', () => {
     })
   })
 
-  it('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
+  it.only('Some features of Course Statistics are hidden for courseStatistics-users without other rights', () => {
     cy.init('/coursestatistics', 'onlycoursestatistics')
     cy.get('[data-cy=navbar-courseStatistics]').click()
     cy.get('[data-cy=course-code-input]').type('TKT20003')
@@ -314,10 +312,14 @@ describe('Course Statistics tests', () => {
     cy.contains('Faculty statistics').should('not.exist')
     cy.contains('Show population').should('not.exist')
 
+    cy.cs('viewMode-Attempts').click()
+
     const attemptsTableContents = [
       // [time, passed, failed, passrate]
       ['Total*', 288, 213, 75, '73.96 %'],
+      // ['Total', 295, 216, 79, '73.22 %'],
       ['2020-2021', '5 or less students', 'NA', 'NA', 'NA'],
+      //['2020-2021', 5, 2, 3, '40.00 %'],
       ['2019-2020', 164, 121, 43, '73.78 %'],
       ['2018-2019', 85, 60, 25, '70.59 %'],
       ['2017-2018', 39, 32, 7, '82.05 %'],

cypress/integration/Filters.js

@@ -159,8 +159,8 @@ describe('Population Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown(0)
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(14)
-      checkFilteringResult(121)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(7)
+      checkFilteringResult(141)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })
@@ -285,8 +285,8 @@ describe('Course Statistics', () => {
       cy.cs('startYearAtUni-dropdown').selectFromDropdown([0])
       checkFilteringResult(1)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
-      cy.cs('startYearAtUni-dropdown').selectFromDropdown(12)
-      checkFilteringResult(29)
+      cy.cs('startYearAtUni-dropdown').selectFromDropdown(3)
+      checkFilteringResult(84)
       clearSemanticUIMultipleDropDownSelection('startYearAtUni-dropdown')
     })
   })

cypress/integration/Population_statistics.js

@@ -80,7 +80,7 @@ describe('Population Statistics tests', () => {
 
       cy.intercept('/api/v3/courseyearlystats**').as('coursePage')
       // eslint-disable-next-line cypress/no-unnecessary-waiting
-      cy.wait(10000) // a bit hacky way, wait until ui is ready
+      cy.wait(15000) // a bit hacky way, wait until ui is ready
       cy.get('[data-cy=expand-TKT1]').click()
       cy.get('[data-cy=coursestats-link-TKT10002]').click()
       cy.wait('@coursePage')

cypress/integration/Student_statistics.js

@@ -38,7 +38,7 @@ describe('Student Statistics tests', () => {
     cy.url().should('include', '/students')
     cy.get('.prompt').type(student.studentnumber)
     cy.contains(student.studentnumber).click()
-    cy.contains('Started: 01.08.2011')
+    cy.contains('Started: 01.08.2017')
     cy.contains('Tietojenkäsittelytieteen kandiohjelma (01.08.2017 - 01.08.2025')
     cy.contains(student.lastname).should('not.exist')
     cy.contains(student.firstnames).should('not.exist')

cypress/integration/Studyprogramme_overview.js

@@ -7,163 +7,311 @@ const deleteTag = name => {
   cy.contains(name).should('not.exist')
 }
 
-describe('Studyprogramme overview', () => {
-  // check is used by couple of tests
-  const checkProgressAndProductivity = (user = 'basic') => {
-    // Column amount depends on user. Note that with dev user, you'll see even more
-    // stuff than with admin or basic.
-    const CSBachPopulationProgress2018ForAdmin = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ]
-    const CSBachPopulationProgress2018ForBasic = [
-      '162',
-      '44 (27%)',
-      '118 (72%)',
-      '161 (99%)',
-      '161',
-      '1',
-      '11',
-      '11',
-      '152',
-      '125',
-      '78',
-      '42',
-      '16',
-    ] // cancelled and transferred from are missing
-
-    const population = user === 'basic' ? CSBachPopulationProgress2018ForBasic : CSBachPopulationProgress2018ForAdmin
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-
-    cy.contains('2018-2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(population[index])
-      })
-    const populationproductivity2019 = ['7388.0', '17', '7388.00', '0.00', '349.00']
-    cy.get('table')
-      .eq(1)
-      .contains('2019')
-      .siblings()
-      .each((elem, index) => {
-        cy.wrap(elem).contains(populationproductivity2019[index])
-      })
-  }
-
-  describe('when opening programme page with admin user', () => {
-    beforeEach(() => {
-      cy.init('/study-programme/KH50_005', 'admin')
-    })
-
-    it('progress should not be recalculating when opened for the first time', () => {
-      cy.contains('Recalculating').should('not.exist')
-      cy.contains('Last updated').should('exist')
-    })
-
-    // Following vars are used to compare values before and after stats calculation
-    // Taken from https://docs.cypress.io/api/commands/should#Compare-text-values-of-two-elements
-    const normalizeText = s => s.replace(/\s/g, '').toLowerCase()
-    let originalProgressCalculatedText
-    let originalProductivityCalculatedText
-    const doRecalculation = () => {
-      // Wait for page to load before clicking to admin page, TODO: change to cy.intercept
-      cy.wait(5000)
-      cy.get('.attached').contains('Admin').click()
-      cy.contains('recalculate productivity').click()
-      cy.contains('recalculate throughput').click()
-      cy.get('.attached').contains('Overview').click() // TODO: add data-cy tags for tabs
-      // Wait to "recalculating" to disappear, TODO: change this to cy.intercept
-      cy.wait(5000)
-      cy.reload()
+const getEmptyYears = isAcademicYear => {
+  const thisYear = new Date().getFullYear()
+  const years = []
+  for (let year = thisYear; year > 2021; year--) {
+    if (isAcademicYear) {
+      years.push(`${year} - ${year + 1}`)
+    } else {
+      years.push(year)
     }
+  }
+  return years
+}
 
-    it('renders progress and productivity tables with calculated status', () => {
-      doRecalculation()
-
-      // check contains also study programme options for admin
-      cy.contains('Study programme options')
-      cy.get('table').should('have.length', 3)
-
-      checkProgressAndProductivity('admin')
-
-      // Grab update dates to be compared later
-      cy.cs('throughputUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProgressCalculatedText = normalizeText(text)
-          expect(originalProgressCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-
-      cy.cs('productivityUpdateStatus')
-        .invoke('text')
-        .then(text => {
-          originalProductivityCalculatedText = normalizeText(text)
-          expect(originalProductivityCalculatedText).not.to.contain('recalculating')
-          expect(originalProgressCalculatedText).not.to.contain('refresh')
-        })
-    })
-
-    it('renders progress and productivity tables with calculated status after recalculating stats again', () => {
-      doRecalculation()
-      checkProgressAndProductivity('admin')
-
-      // Check new calculation statuses are reported
-      const newProgressCalculatedTextElement = cy.cs('throughputUpdateStatus').invoke('text')
-      const newProductivityCalculatedTextElement = cy.cs('productivityUpdateStatus').invoke('text')
-
-      newProgressCalculatedTextElement.should(text => {
-        const newProgressCalculatedText = normalizeText(text)
-        expect(newProgressCalculatedText).not.to.contain('recalculating')
-        expect(newProgressCalculatedText).not.to.contain('refresh')
-        expect(newProgressCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
-
-      newProductivityCalculatedTextElement.should(text => {
-        const newProductivityCalculatedText = normalizeText(text)
-        expect(newProductivityCalculatedText).not.to.contain('recalculating')
-        expect(newProductivityCalculatedText).not.to.contain('refresh')
-        expect(newProductivityCalculatedText).not.to.contain(originalProgressCalculatedText)
-      })
-    })
-  })
-
+describe('Studyprogramme overview', () => {
   describe('when opening programme page with basic user', () => {
     beforeEach(() => {
-      cy.init('/study-programme/KH50_005')
+      cy.init('/study-programme')
     })
 
-    it('renders progress and productivity tables with previosly calculated values', () => {
-      // check doesn't contain study programme options for admin
-      cy.contains('Study programme options').should('not.exist')
-      cy.get('table').should('have.length', 2)
-      checkProgressAndProductivity()
-    })
+    /* Basic information overview -tests*/
 
-    it('can open Thesis page', () => {
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Basic information -tab loads', () => {
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-      cy.contains('Thesis Courses').click()
-      cy.contains('Add thesis course').click()
-      cy.contains('No results')
+      cy.get('[data-cy=Section-StudentsOfTheStudyprogramme]')
+      cy.get('[data-cy=Section-CreditsProducedByTheStudyprogramme]')
+      cy.get('[data-cy=Section-GraduatedAndThesisWritersOfTheProgramme]')
+      cy.get('[data-cy=Section-ProgrammesAfterGraduation]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
     })
 
-    it('can move to Population statistics page by clickin', () => {
-      cy.get('i.level.up.alternate.icon').eq(0).click()
-      cy.contains('Students (10)')
+    it('Basic information contains correct students', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Started, Graduated, Cancelled, Transferred Away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 1, 0],
+        [2020, 10, 36, 16, 3, 2],
+        [2019, 86, 17, 14, 0, 1],
+        [2018, 162, 1, 3, 0, 0],
+        [2017, 173, 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudentsOfTheStudyprogramme')
     })
 
+    it('Basic information contains correct credits', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Total, Major credits, Non-major credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        [2021, 0, 0, 0, 0],
+        [2020, 4928, 548, 4213, 167],
+        [2019, 7737, 1644, 5744, 349],
+        [2018, 6730, 1820, 4638, 272],
+        [2017, 2900, 788, 1724, 388],
+      ]
+
+      cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
+    })
+
+    it('Basic information contains correct thesis writers and graduates', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      const years = getEmptyYears()
+      const tableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 36, 35],
+        [2019, 17, 25],
+        [2018, 1, 5],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(tableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it('Special studyrights can be excluded and basic data changes accordingly', () => {
+      cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=StudentToggle]').click()
+      const years = getEmptyYears()
+      const studentTableContents = [
+        // [Year, Started, Graduated]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 22, 0],
+        [2019, 12, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0]),
+        [2021, 0, 0, 0],
+        [2020, 117.5, 0, 7.5],
+        [2019, 67, 0, 0],
+        [2018, 0, 0, 0],
+        [2017, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+
+      const graduatedTableContents = [
+        // [Year, Graduated, Wrote thesis]
+        ...years.map(year => [year, 0, 0]),
+        [2021, 0, 0],
+        [2020, 0, 0],
+        [2019, 0, 0],
+        [2018, 0, 0],
+        [2017, 0, 0],
+      ]
+
+      cy.checkTableStats(graduatedTableContents, 'GraduatedAndThesisWritersOfTheProgramme')
+    })
+
+    it('Year can be changed to academic year, and data changes accordingly', () => {
+      cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
+      cy.get('[data-cy=YearToggle]').click()
+      const isAcademicYear = true
+      const years = getEmptyYears(isAcademicYear)
+      const studentTableContents = [
+        // [Year, Started, Graduated, Cancelled, Transferred away, Transferred to]
+        ...years.map(year => [year, 0, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0, 0],
+        ['2020 - 2021', 5, 0, 1, 0, 0],
+        ['2019 - 2020', 27, 0, 1, 0, 0],
+        ['2018 - 2019', 4, 0, 0, 0, 0],
+        ['2017 - 2018', 0, 0, 0, 0, 0],
+      ]
+
+      cy.checkTableStats(studentTableContents, 'StudentsOfTheStudyprogramme')
+
+      const creditTableContents = [
+        // [Year, Total, Major students credits, Non-major students credits, Transferred credits]
+        ...years.map(year => [year, 0, 0, 0, 0]),
+        ['2021 - 2022', 0, 0, 0, 0],
+        ['2020 - 2021', 0, 0, 0, 0],
+        ['2019 - 2020', 338.5, 0, 331, 7.5],
+        ['2018 - 2019', 173, 0, 173, 0],
+        ['2017 - 2018', 42, 0, 42, 0],
+      ]
+
+      cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
+    })
+
+    it('Basic information graphs render', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+
+      cy.get('[data-cy=Graph-StudentsOfTheStudyprogramme')
+        .should('contain', 'Started')
+        .should('contain', 'Graduated')
+        .should('contain', 'Transferred away')
+
+      cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
+        .should('contain', 'Major students credits')
+        .should('contain', 'Non-major students credits')
+        .should('contain', 'Transferred credits')
+        .should('contain', 788)
+        .should('contain', 1820)
+
+      cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
+        .should('contain', 'Graduated students')
+        .should('contain', 'Wrote thesis')
+        .should('contain', 1)
+        .should('contain', 5)
+        .should('contain', 17)
+        .should('contain', 25)
+        .should('contain', 36)
+        .should('contain', 35)
+
+      cy.get('[data-cy=Graph-ProgrammesAfterGraduation')
+        .should('contain', 'Tietojenkäsittelytieteen maisteriohjelma')
+        .should('contain', 'Datatieteen maisteriohjelma')
+        .should('contain', 'Matematiikan ja tilastotieteen maisteriohjelma')
+        .should('contain', 'Kielellisen diversiteetin ja digitaalisten ihmistieteiden maisteriohjelma')
+        .should('contain', 1)
+        .should('contain', 4)
+        .should('contain', 13)
+    })
+
+    /* Studytrack overview -tests*/
+
+    // If the backend breaks for one of the sections, the section header is not rendered and this will fail
+    it('Studytracks and student populations -tab loads', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      cy.get('[data-cy=Section-StudytrackOverview]')
+      cy.get('[data-cy=Section-StudytrackProgress]')
+      cy.get('[data-cy=Section-AverageGraduationTimes]')
+    })
+
+    it('Students of the studyprogramme are shown correctly', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        // [Year, All, Started studying, Currently enrolled, Absent, Cancelled, Graduated, Men, Women, Finnish]
+        [
+          '2020 - 2021',
+          12,
+          '100.0 %',
+          10,
+          '83.3 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          10,
+          '83.3 %',
+          2,
+          '16.7 %',
+          12,
+          '100.0 %',
+        ],
+        [
+          '2019 - 2020',
+          86,
+          '100.0 %',
+          86,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          1,
+          '1.2 %',
+          1,
+          '1.2 %',
+          65,
+          '75.6 %',
+          21,
+          '24.4 %',
+          86,
+          '100.0 %',
+        ],
+        [
+          '2018 - 2019',
+          162,
+          '100.0 %',
+          161,
+          '99.4 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          11,
+          '6.8 %',
+          11,
+          '6.8 %',
+          118,
+          '72.8 %',
+          44,
+          '27.2 %',
+          161,
+          '99.4 %',
+        ],
+        [
+          '2017 - 2018',
+          171,
+          '100.0 %',
+          171,
+          '100.0 %',
+          0,
+          '0 %',
+          0,
+          '0 %',
+          21,
+          '12.3 %',
+          42,
+          '24.6 %',
+          135,
+          '78.9 %',
+          36,
+          '21.1 %',
+          166,
+          '97.1 %',
+        ],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackOverview')
+    })
+
+    it('Student progress data is shown correctly', () => {
+      cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
+      cy.get('.attached').contains('Studytracks and student populations').click()
+      const tableContents = [
+        ['2020 - 2021', 12, 5, 2, 2, 0, 2, 1],
+        ['2019 - 2020', 86, 13, 15, 32, 11, 2, 13],
+        ['2018 - 2019', 162, 7, 16, 35, 33, 27, 44],
+        ['2017 - 2018', 171, 10, 22, 18, 22, 24, 75],
+      ]
+
+      cy.checkTableStats(tableContents, 'StudytrackProgress')
+    })
+
+    /* Tag-tests*/
+
     it('can create and delete tags for population', () => {
       const name = `tag-${new Date().getTime()}`
       cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()

cypress/support/commands.js

@@ -104,3 +104,18 @@ Cypress.Commands.add('selectStudyProgramme', name => {
   cy.cs('select-study-programme').click().children().contains(name).click()
   cy.contains('See population').click()
 })
+
+Cypress.Commands.add('checkTableStats', (correctStats, tableName) => {
+  cy.get(`[data-cy=Table-${tableName}]>tbody`).within(() => {
+    correctStats.forEach((values, trIndex) => {
+      cy.get('tr')
+        .eq(trIndex)
+        .within(() => {
+          values.forEach((value, tdIndex) => {
+            cy.get('td').eq(tdIndex).contains(value)
+          })
+        })
+    })
+    cy.get('tr').should('have.length', correctStats.length)
+  })
+})

docker-compose.ci.yml

@@ -103,9 +103,35 @@ services:
     profiles:
       - updater
 
+  sis-updater-scheduler:
+    <<: *defaults
+    build:
+      target: development
+      context: updater/sis-updater-scheduler
+    container_name: sis-updater-scheduler
+    depends_on:
+      - redis
+      - sis-importer-db
+      - sis-updater-nats
+    environment:
+      REDIS_URI: //redis:6379
+      SECRET_TOKEN: dev
+      SIS_IMPORTER_DATABASE: sis-importer-db
+      SIS_IMPORTER_HOST: sis-importer-db
+      SIS_IMPORTER_PASSWORD: postgres
+      SIS_IMPORTER_USER: postgres
+      SIS_NATS_TOKEN: dev
+      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
+    ports:
+      - 8082:8082
+    profiles:
+      - updater
+
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:

docker-compose.yml

@@ -1,3 +1,5 @@
+version: '2.1'
+
 x-defaults: &defaults
   restart: unless-stopped
 
@@ -97,7 +99,7 @@ services:
 
   sis-updater-nats:
     <<: *defaults
-    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
+    command: '-cid sis-updater-nats -m 8222 -p 4222 --max_channels 1000 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore'
     container_name: sis-updater-nats
     image: nats-streaming:0.16.2
     profiles:
@@ -122,6 +124,8 @@ services:
       SIS_IMPORTER_USER: postgres
       SIS_NATS_TOKEN: dev
       SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
+      SCHEDULE_IMMEDIATE: '${SCHEDULE_IMMEDIATE}'
+      EXIT_AFTER_IMMEDIATES: '${EXIT_AFTER_IMMEDIATES}'
     ports:
       - 8082:8082
     profiles:

services/backend/Dockerfile

@@ -17,7 +17,7 @@ ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT
 ARG SENTRY_RELEASE
 ENV SENTRY_RELEASE=$SENTRY_RELEASE
 COPY --chown=node:node ./package* /usr/src/app/
-RUN npm ci --production
+RUN npm ci --production && apk add --no-cache vim curl nano
 COPY --chown=node:node . /usr/src/app/
 USER node
 CMD ["npm", "start"]

services/backend/src/database/connection.js

@@ -16,7 +16,7 @@ class DbConnection extends EventEmitter {
       pool: {
         max: 25,
         min: 0,
-        acquire: 10000,
+        acquire: 30000,
         idle: 300000000,
       },
       logging: false,

services/backend/src/events.js

@@ -10,9 +10,8 @@ const {
 } = require('./services/trends')
 const { refreshAssociationsInRedis } = require('./services/studyrights')
 const { getAllProgrammes } = require('./services/studyrights')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('./services/studyprogramme')
+const { updateBasicView, updateStudytrackView } = require('./services/studyprogrammeUpdates')
 const { findAndSaveTeachers } = require('./services/topteachers')
-const { setProductivity, setThroughput, patchProductivity, patchThroughput } = require('./services/analyticsService')
 const { isProduction } = require('./conf-backend')
 const { getCurrentSemester } = require('./services/semesters')
 const logger = require('./util/logger')
@@ -24,47 +23,21 @@ const refreshStudyrightAssociations = async () => {
   logger.info('Studyright associations refreshed!')
 }
 
-const refreshOverview = async () => {
-  logger.info('Refreshing throughput and productivity for programmes...')
+const refreshNewOverviews = async () => {
+  logger.info('Refreshing studyprogramme and studytrack overview statistics for all programmes')
   const codes = (await getAllProgrammes()).map(p => p.code).filter(code => code.includes('KH') || code.includes('MH'))
   let ready = 0
   for (const code of codes) {
-    let programmeStatsSince = new Date('2017-07-31')
-    if (code.includes('MH') || code.includes('KH')) {
-      programmeStatsSince = new Date('2017-07-31')
-    } else {
-      programmeStatsSince = new Date('2000-07-31')
-    }
     try {
-      await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-      const data = await throughputStatsForStudytrack(code, programmeStatsSince.getFullYear())
-      await setThroughput(data)
+      await updateBasicView(code)
+      await updateStudytrackView(code)
+      ready += 1
     } catch (e) {
-      logger.error({ message: `failed to recalculate throughput for ${code}`, meta: e })
-      try {
-        await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-      } catch (e) {
-        logger.error({ message: 'failed to update throughtput status to error', meta: e })
-      }
+      logger.error({ message: `Failed to update overview stats for programme ${code}`, meta: e })
     }
-    try {
-      await patchProductivity({ [code]: { status: 'RECALCULATING' } })
-      const data = await productivityStatsForStudytrack(code, programmeStatsSince)
-      await setProductivity(data)
-    } catch (e) {
-      logger.error({ message: `failed to recalculate productivity for ${code}`, meta: e })
-      try {
-        await patchProductivity({
-          [code]: { status: 'RECALCULATION ERRORED' },
-        })
-      } catch (e) {
-        logger.error({ message: 'failed to update productivity status to error', meta: e })
-      }
-    }
-    ready += 1
     logger.info(`${ready}/${codes.length} programmes done`)
   }
-  logger.info('Throughput and productivity for programmes refreshed')
+  logger.info('Studyprogramme and studytrack overview stats refreshed!')
 }
 
 const refreshTeacherLeaderboard = async () => {
@@ -147,7 +120,7 @@ const refreshProtoCProgrammeToRedis = async () => {
 }
 
 const refreshStatistics = async () => {
-  const statfuncs = [refreshStudyrightAssociations, refreshOverview, refreshTeacherLeaderboard]
+  const statfuncs = [refreshStudyrightAssociations, refreshTeacherLeaderboard]
   logger.info('Refreshing statistics')
   for (const func of statfuncs) {
     await func()
@@ -166,9 +139,10 @@ const refreshTrends = async () => {
 
 const startCron = () => {
   if (isProduction) {
+    logger.info('Cronjob for refreshing stats started: runs at 3am every day.')
     // refresh 3am every day
     schedule('0 3 * * *', async () => {
-      for (const func of [refreshStatistics, refreshTrends]) {
+      for (const func of [refreshStatistics, refreshTrends, refreshNewOverviews]) {
         await func()
       }
     })

services/backend/src/rapodiff.js

@@ -1,8 +1,17 @@
 /* eslint no-console: 0 */
 
+// script needs rapo certs, so you need to run this in production pannu
+//
+// possible workflows:
+// - go to oodikone pannu, then "docker exec -it backend sh", then run this script with "npm
+// run rapodiff". Modify with vim / nano until you're happy, then copy changes to local version and commit.
+// - or bind mount a file to backend container in pannu, overriding this file. Modify that file
+// inside or outside container, however you want. Keep file in pannu, or put it to pannu's git repo (oodikone-server-setup).
+
 const axios = require('axios').default
 const fs = require('fs')
 const https = require('https')
+const _ = require('lodash')
 
 const { optimizedStatisticsOf } = require('./services/populations')
 
@@ -38,16 +47,18 @@ const main = async () => {
   const oodikoneStuff = await optimizedStatisticsOf(oodikoneQuery)
   const oodikoneStudents = oodikoneStuff.students.map(s => s.studentNumber)
 
-  const { data: rapoStuff } = await api.get(
+  const rapoUrl = encodeURI(
     `studyrights?faculty=${facultyCode}&presence=${presence}&education=${programme}&studyrightStartDate=${studyrightStartDate}`
   )
 
-  console.log('=== RAPOSTUFF ===')
-  console.log(JSON.stringify(rapoStuff, null, 2))
+  const { data: rapoStuff } = await api.get(rapoUrl)
+  const rapoStudents = rapoStuff.map(s => s.opiskelijanumero)
 
-  console.log('=== OODIKONE ===')
-  console.log(JSON.stringify(oodikoneStuff, null, 2))
-  console.log(JSON.stringify(oodikoneStudents, null, 2))
+  const onlyInOodikone = _.difference(oodikoneStudents, rapoStudents)
+  const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
+
+  console.log('Only in Oodikone:', onlyInOodikone)
+  console.log('Only in Rapo:', onlyInRapo)
 }
 
-main()
+main().then(() => process.exit(0))

services/backend/src/routes/elementdetails.js

@@ -1,23 +1,7 @@
 const router = require('express').Router()
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForStudytrack,
-  throughputStatsForStudytrack,
-  optionData,
-} = require('../services/studyprogramme')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
-const {
-  getProductivity,
-  setProductivity,
-  getThroughput,
-  setThroughput,
-  patchProductivity,
-  patchThroughput,
-} = require('../services/analyticsService')
 const { getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
-const logger = require('../util/logger')
-
-const programmeStatsSince = new Date('2017-07-31')
 
 router.get('/elementdetails/all', async (req, res) => {
   const elementdetails = await getAllElementDetails()
@@ -29,22 +13,6 @@ router.get('/studyprogrammes', async (req, res) => {
   res.json(studyrights)
 })
 
-router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
-  const code = req.params.id
-
-  let level
-  if (code.includes('MH')) {
-    level = 'MSC'
-  } else if (code.includes('KH')) {
-    level = 'BSC'
-  } else {
-    return res.json([])
-  }
-
-  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
-  res.json(data)
-})
-
 router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   if (req.params.id) {
     const codes = await MandatoryCourses.byStudyprogramme(req.params.id)
@@ -54,109 +22,6 @@ router.get('/v2/studyprogrammes/:id/mandatory_courses', async (req, res) => {
   }
 })
 
-router.get('/v2/studyprogrammes/:id/productivity', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    let data = null
-    try {
-      data = await getProductivity(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} productivity`)
-    }
-    if (!data) {
-      try {
-        const stats = await productivityStatsForStudytrack(code, programmeStatsSince)
-        data = await setProductivity(stats)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json(data)
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/productivity/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await productivityStatsForStudytrack(code, new Date('2017-07-31'))
-      await setProductivity(data)
-    } else {
-      const data = await productivityStatsForStudytrack(code, new Date('2000-07-31'))
-      await setProductivity(data)
-    }
-  } catch (e) {
-    try {
-      await patchProductivity({
-        [code]: { status: 'RECALCULATION ERRORED' },
-      })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-
-    logger.error(`Failed to update productivity stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Produdtiviy stats recalculation for studyprogramme ${code} done`)
-})
-
-router.get('/v2/studyprogrammes/:id/throughput', async (req, res) => {
-  const code = req.params.id
-  if (code) {
-    const thesisPromise = findProgrammeTheses(req.params.id)
-
-    let data = null
-    try {
-      data = await getThroughput(code)
-    } catch (e) {
-      logger.error(`Failed to get code ${code} throughput`)
-    }
-    if (!data) {
-      try {
-        const result = await throughputStatsForStudytrack(req.params.id, programmeStatsSince.getFullYear())
-        data = await setThroughput(result)
-      } catch (e) {
-        logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      }
-    }
-    return res.json({ ...data, thesis: await thesisPromise })
-  } else {
-    res.status(422).end()
-  }
-})
-
-router.get('/v2/studyprogrammes/throughput/recalculate', async (req, res) => {
-  const code = req.query.code
-
-  await patchThroughput({ [code]: { status: 'RECALCULATING' } })
-  res.status(200).end()
-
-  try {
-    if (code.includes('MH') || code.includes('KH')) {
-      const data = await throughputStatsForStudytrack(code, 2017)
-      await setThroughput(data)
-    } else {
-      const data = await throughputStatsForStudytrack(code, 2000)
-      await setThroughput(data)
-    }
-  } catch (e) {
-    try {
-      await patchThroughput({ [code]: { status: 'RECALCULATION ERRORED' } })
-    } catch (e) {
-      logger.error(`Failed to update code ${code} patching status to recalculation erroring`)
-      return
-    }
-    logger.error(`Failed to update throughput stats for code: ${code}, reason: ${e.message}`)
-  }
-  logger.info(`Throughput stats recalculation for studyprogramme ${code} done`)
-})
-
 router.get('/v2/studyprogrammes/:id/thesis', async (req, res) => {
   const { id } = req.params
   if (id) {

services/backend/src/routes/studyProgramme.js

@@ -1,11 +1,8 @@
 const router = require('express').Router()
-const {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-} = require('../services/studyprogrammeStats')
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
 const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
-
 const {
   getBasicStats,
   setBasicStats,
@@ -16,16 +13,18 @@ const {
   getStudytrackStats,
   setStudytrackStats,
 } = require('../services/analyticsService')
+const { updateBasicView, updateStudytrackView } = require('../services/studyprogrammeUpdates')
 const logger = require('../util/logger')
 
 router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getBasicStats(code, yearType)
+      data = await getBasicStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} basic stats: ${e}`)
     }
@@ -34,8 +33,9 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
         const result = await getBasicStatsForStudytrack({
           studyprogramme: req.params.id,
           yearType,
+          specialGroups,
         })
-        data = await setBasicStats(result, yearType)
+        data = await setBasicStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} basic stats: ${e}`)
       }
@@ -49,11 +49,12 @@ router.get('/v2/studyprogrammes/:id/basicstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getCreditStats(code, yearType)
+      data = await getCreditStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} credit stats`)
     }
@@ -62,8 +63,9 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
         const result = await getCreditStatsForStudytrack({
           studyprogramme: req.params.id,
           yearType,
+          specialGroups,
         })
-        data = await setCreditStats(result, yearType)
+        data = await setCreditStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} credit stats`)
       }
@@ -77,11 +79,12 @@ router.get('/v2/studyprogrammes/:id/creditstats', async (req, res) => {
 router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getGraduationStats(code, yearType)
+      data = await getGraduationStats(code, yearType, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} graduation stats`)
     }
@@ -90,8 +93,9 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
         const result = await getGraduationStatsForStudytrack({
           studyprogramme: req.params.id,
           yearType,
+          specialGroups,
         })
-        data = await setGraduationStats(result, yearType)
+        data = await setGraduationStats(result, yearType, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} graduation stats`)
       }
@@ -104,18 +108,20 @@ router.get('/v2/studyprogrammes/:id/graduationstats', async (req, res) => {
 
 router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   const code = req.params.id
+  const graduated = req.query?.graduated
+  const specialGroups = req.query?.special_groups
 
   if (code) {
     let data = null
     try {
-      data = await getStudytrackStats(code)
+      data = await getStudytrackStats(code, graduated, specialGroups)
     } catch (e) {
       logger.error(`Failed to get code ${code} studytrack stats: ${e}`)
     }
     if (!data) {
       try {
-        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: req.params.id })
-        data = await setStudytrackStats(result)
+        const result = await getStudytrackStatsForStudyprogramme({ studyprogramme: code, specialGroups, graduated })
+        data = await setStudytrackStats(result, graduated, specialGroups)
       } catch (e) {
         logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
       }
@@ -126,4 +132,34 @@ router.get('/v2/studyprogrammes/:id/studytrackstats', async (req, res) => {
   }
 })
 
+router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateBasicView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} basic stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
+router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
+  const code = req.params.id
+  if (code) {
+    let result = null
+    try {
+      result = await updateStudytrackView(code)
+    } catch (e) {
+      logger.error(`Failed to update code ${code} studytrack stats: ${e}`)
+    }
+    return res.json(result)
+  } else {
+    res.status(422).end()
+  }
+})
+
 module.exports = router

services/backend/src/services/analyticsService.js

@@ -1,106 +1,23 @@
 const { redisClient } = require('../services/redis')
 const moment = require('moment')
 
-const createRedisKeyForProductivity = id => `PRODUCTIVITY_${id}`
-const createRedisKeyForThroughput = id => `THROUGHPUT_${id}`
-const createRedisKeyForBasicStats = (id, yearType) => `BASIC_STATS_${id}_${yearType}`
-const createRedisKeyForCreditStats = (id, yearType) => `CREDIT_STATS_${id}_${yearType}`
-const createRedisKeyForGraduationStats = (id, yearType) => `GRADUATION_STATS_${id}_${yearType}`
-const createRedisKeyForStudytrackStats = id => `STUDYTRACK_STATS_${id}`
+const createRedisKeyForBasicStats = (id, yearType, specialGroups) => `BASIC_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForCreditStats = (id, yearType, specialGroups) => `CREDIT_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForGraduationStats = (id, yearType, specialGroups) =>
+  `GRADUATION_STATS_${id}_${yearType}_${specialGroups}`
+const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
+  `STUDYTRACK_STATS_${id}_${graduated}_${specialGroups}`
 
-const getProductivity = async id => {
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataToRedis = {
-    ...data,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchProductivity = async data => {
-  const { id } = data
-  const redisKey = createRedisKeyForProductivity(id)
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    ...data,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getThroughput = async id => {
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return {
-    [id]: JSON.parse(dataFromRedis),
-  }
-}
-
-const setThroughput = async data => {
-  const { id, data: dataToSet } = data
-  const redisKey = createRedisKeyForThroughput(id)
-  const dataToRedis = {
-    data: dataToSet.years,
-    totals: dataToSet.totals,
-    stTotals: dataToSet.stTotals,
-    status: 'DONE',
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(dataToRedis))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: dataToRedis,
-  }
-}
-
-const patchThroughput = async data => {
-  const [id, dataToPatch] = Object.entries(data)[0]
-  const redisKey = createRedisKeyForThroughput(id)
-  const { status } = dataToPatch
-  const dataFromRedis = JSON.parse(await redisClient.getAsync(redisKey))
-  const patchedData = {
-    ...dataFromRedis,
-    status,
-    lastUpdated: moment().format(),
-  }
-  const setOperationStatus = await redisClient.setAsync(redisKey, JSON.stringify(patchedData))
-  if (setOperationStatus !== 'OK') return null
-  return {
-    [id]: patchedData,
-  }
-}
-
-const getBasicStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+const getBasicStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setBasicStats = async (data, yearType) => {
+const setBasicStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForBasicStats(id, yearType)
+  const redisKey = createRedisKeyForBasicStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -111,16 +28,16 @@ const setBasicStats = async (data, yearType) => {
   return dataToRedis
 }
 
-const getCreditStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+const getCreditStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setCreditStats = async (data, yearType) => {
+const setCreditStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForCreditStats(id, yearType)
+  const redisKey = createRedisKeyForCreditStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -131,16 +48,16 @@ const setCreditStats = async (data, yearType) => {
   return dataToRedis
 }
 
-const getGraduationStats = async (id, yearType) => {
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+const getGraduationStats = async (id, yearType, specialGroups) => {
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setGraduationStats = async (data, yearType) => {
+const setGraduationStats = async (data, yearType, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForGraduationStats(id, yearType)
+  const redisKey = createRedisKeyForGraduationStats(id, yearType, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -151,16 +68,16 @@ const setGraduationStats = async (data, yearType) => {
   return dataToRedis
 }
 
-const getStudytrackStats = async id => {
-  const redisKey = createRedisKeyForStudytrackStats(id)
+const getStudytrackStats = async (id, graduated, specialGroups) => {
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
   return JSON.parse(dataFromRedis)
 }
 
-const setStudytrackStats = async data => {
+const setStudytrackStats = async (data, graduated, specialGroups) => {
   const { id } = data
-  const redisKey = createRedisKeyForStudytrackStats(id)
+  const redisKey = createRedisKeyForStudytrackStats(id, graduated, specialGroups)
   const dataToRedis = {
     ...data,
     status: 'DONE',
@@ -172,12 +89,6 @@ const setStudytrackStats = async data => {
 }
 
 module.exports = {
-  getProductivity,
-  setProductivity,
-  patchProductivity,
-  getThroughput,
-  setThroughput,
-  patchThroughput,
   getBasicStats,
   setBasicStats,
   getCreditStats,

services/backend/src/services/newStudyprogramme.js

@@ -12,12 +12,17 @@ const {
   SemesterEnrollment,
   Semester,
 } = require('../models')
-const { formatStudyright, formatStudent } = require('./studyprogrammeHelpers')
+const { formatStudyright, formatStudent, formatTransfer } = require('./studyprogrammeHelpers')
+const { getCurrentSemester } = require('./semesters')
 
 const whereStudents = studentnumbers => {
   return studentnumbers ? studentnumbers : { [Op.not]: null }
 }
 
+const sinceDate = since => {
+  return since ? { [Op.gte]: since } : { [Op.not]: null }
+}
+
 const studytrackStudents = async studentnumbers =>
   (
     await Student.findAll({
@@ -29,24 +34,8 @@ const studytrackStudents = async studentnumbers =>
     })
   ).map(formatStudent)
 
-const enrolledStudyrights = async studytrack => {
-  const studyrights = await Studyright.findAll({
-    attributes: ['studyrightid', 'studystartdate', 'enddate', 'prioritycode'],
-    include: [
-      {
-        model: StudyrightElement,
-        required: true,
-        where: { code: studytrack },
-      },
-      {
-        model: Student,
-        attributes: ['studentnumber'],
-        required: true,
-      },
-    ],
-  })
-
-  const studentnumbers = studyrights.map(s => s.student.studentnumber)
+const enrolledStudents = async (studytrack, since, studentnumbers) => {
+  const currentSemester = await getCurrentSemester()
 
   const students = await Student.findAll({
     attributes: ['studentnumber'],
@@ -71,6 +60,9 @@ const enrolledStudyrights = async studytrack => {
           {
             model: Semester,
             required: true,
+            where: {
+              semestercode: currentSemester.semestercode,
+            },
           },
         ],
         where: {
@@ -85,32 +77,73 @@ const enrolledStudyrights = async studytrack => {
     },
   })
 
-  let enrollments = new Set()
+  return students.filter(s => s.semester_enrollments?.length)
+}
 
-  students.forEach(student => {
-    student.semester_enrollments.forEach(e => {
-      const enrollmentStartDate = e.semester.startdate
-      const enrollmentEndDate = e.semester.enddate
-      const correctStudyRight = studyrights.find(
-        s =>
-          student.studentnumber === s.student.studentnumber &&
-          s.studystartdate <= enrollmentStartDate && // Studying in the programme should have started on the same day or earlier than the enrollment
-          s.enddate >= enrollmentEndDate && // Studying in the programme should have ended on the same day or later than the enrollment
-          (!s.canceldate || s.canceldate > enrollmentStartDate) && // Studyright should not be canceled OR the possible canceldate should be after the enrollmentStartDate
-          (s.prioritycode === 1 || s.prioritycode === 30) // Studyright should be primary or the student should have graduated
-      )
-      if (correctStudyRight) {
-        enrollments.add({
-          studentnumber: student.studentnumber,
-          startdate: e.semester.startdate,
-        })
-      }
-    })
+const absentStudents = async (studytrack, studentnumbers) => {
+  const currentSemester = await getCurrentSemester()
+  const students = await Student.findAll({
+    attributes: ['studentnumber'],
+    include: [
+      {
+        model: Studyright,
+        include: [
+          {
+            model: StudyrightElement,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        ],
+        attributes: ['studyrightid'],
+      },
+      {
+        model: SemesterEnrollment,
+        attributes: ['semestercode'],
+        where: {
+          enrollmenttype: 2,
+          semestercode: currentSemester.semestercode,
+        },
+      },
+    ],
+    where: {
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
   })
 
-  return enrollments
+  return students
 }
 
+const allStudyrights = async (studytrack, since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
+        },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
 const startedStudyrights = async (studytrack, since, studentnumbers) =>
   (
     await Studyright.findAll({
@@ -142,73 +175,100 @@ const startedStudyrights = async (studytrack, since, studentnumbers) =>
   ).map(formatStudyright)
 
 const graduatedStudyRights = async (studytrack, since, studentnumbers) =>
-  await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
-      include: {
-        model: ElementDetail,
-        required: true,
-        where: {
-          code: studytrack,
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: {
+            model: ElementDetail,
+            required: true,
+            where: {
+              code: studytrack,
+            },
+          },
         },
-      },
-    },
-    where: {
-      graduated: 1,
-      enddate: {
-        [Op.gte]: since,
-      },
-      student_studentnumber: whereStudents(studentnumbers),
-    },
-  })
-
-const cancelledStudyRights = async (studytrack, since) => {
-  return await Studyright.findAll({
-    include: {
-      model: StudyrightElement,
-      required: true,
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
       where: {
-        code: {
-          [Op.eq]: studytrack,
+        graduated: 1,
+        enddate: sinceDate(since),
+        student_studentnumber: whereStudents(studentnumbers),
+      },
+    })
+  ).map(formatStudyright)
+
+const followingStudyrights = async (since, studentnumbers) =>
+  (
+    await Studyright.findAll({
+      include: [
+        {
+          model: StudyrightElement,
+          required: true,
+          include: [
+            {
+              model: ElementDetail,
+              attributes: ['name'],
+            },
+          ],
+          attributes: ['code'],
         },
+        {
+          model: Student,
+          attributes: ['studentnumber'],
+          required: true,
+        },
+      ],
+      where: {
+        studystartdate: {
+          [Op.gte]: since,
+        },
+        extentcode: 2,
+        student_studentnumber: whereStudents(studentnumbers),
       },
-    },
-    where: {
-      canceldate: {
-        [Op.gte]: since,
+    })
+  ).map(formatStudyright)
+
+const transfersAway = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        sourcecode: studytrack,
       },
-    },
-  })
+      distinct: true,
+      col: 'studentnumber',
+    })
+  ).map(formatTransfer)
+
+const transfersTo = async (studytrack, since) =>
+  (
+    await Transfer.findAll({
+      where: {
+        transferdate: {
+          [Op.gte]: since,
+        },
+        targetcode: studytrack,
+      },
+      distinct: true,
+      col: 'studentnumber',
+    })
+  ).map(formatTransfer)
+
+const allTransfers = async (studytrack, since) => {
+  const transferredTo = await transfersTo(studytrack, since)
+  const transferredAway = await transfersAway(studytrack, since)
+  return [...transferredTo, ...transferredAway]
 }
 
-const transfersAway = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      sourcecode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const transfersTo = async (studytrack, since) => {
-  return await Transfer.findAll({
-    where: {
-      transferdate: {
-        [Op.gte]: since,
-      },
-      targetcode: studytrack,
-    },
-    distinct: true,
-    col: 'studentnumber',
-  })
-}
-
-const getProgrammesStudents = async studyprogramme =>
+const getProgrammesStudyrights = async studyprogramme =>
   (
     await Studyright.findAll({
       attributes: ['studyrightid', 'studystartdate', 'enddate', 'graduated', 'prioritycode', 'extentcode'],
@@ -264,14 +324,11 @@ const getCreditsForStudyProgramme = async (provider, since) =>
 
 const getTransferredCredits = async (provider, since) =>
   await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'credittypecode', 'student_studentnumber'],
+    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
     include: {
       model: Course,
       attributes: ['code'],
       required: true,
-      where: {
-        is_study_module: false,
-      },
       include: {
         model: Organization,
         required: true,
@@ -284,13 +341,16 @@ const getTransferredCredits = async (provider, since) =>
       credittypecode: {
         [Op.eq]: [9],
       },
+      isStudyModule: {
+        [Op.not]: true,
+      },
       attainment_date: {
         [Op.gte]: since,
       },
     },
   })
 
-const getThesisCredits = async (provider, since, thesisType) =>
+const getThesisCredits = async (provider, since, thesisType, studentnumbers) =>
   await Credit.findAll({
     attributes: ['id', 'course_code', 'credits', 'attainment_date', 'student_studentnumber'],
     include: {
@@ -320,18 +380,22 @@ const getThesisCredits = async (provider, since, thesisType) =>
       attainment_date: {
         [Op.gte]: since,
       },
+      student_studentnumber: whereStudents(studentnumbers),
     },
   })
 
 module.exports = {
   studytrackStudents,
-  enrolledStudyrights,
+  enrolledStudents,
+  absentStudents,
+  allStudyrights,
   startedStudyrights,
   graduatedStudyRights,
-  cancelledStudyRights,
+  followingStudyrights,
   transfersAway,
   transfersTo,
-  getProgrammesStudents,
+  allTransfers,
+  getProgrammesStudyrights,
   getCreditsForStudyProgramme,
   getTransferredCredits,
   getThesisCredits,

services/backend/src/services/populations.js

@@ -363,8 +363,10 @@ const studentnumbersWithAllStudyrightElements = async (
   exchangeStudents,
   cancelledStudents,
   nondegreeStudents,
-  transferredStudents,
-  tag
+  transferredOutStudents,
+  tag,
+  transferredToStudents,
+  graduatedStudents
 ) => {
   // eslint-disable-line
 
@@ -497,11 +499,11 @@ const studentnumbersWithAllStudyrightElements = async (
     return studyRights.includes(newestStudytrack.code)
   })
 
-  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+  let studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
 
   // fetch students that have transferred out of the programme and filter out these studentnumbers
-  if (!transferredStudents) {
-    const transferredOutStudents = (
+  if (!transferredOutStudents) {
+    const transfersOut = (
       await Transfer.findAll({
         attributes: ['studentnumber'],
         where: {
@@ -519,8 +521,63 @@ const studentnumbersWithAllStudyrightElements = async (
       })
     ).map(s => s.studentnumber)
 
-    const notTransferredStudents = studentnumberlist.filter(sn => !transferredOutStudents.includes(sn))
-    return notTransferredStudents
+    studentnumberlist = studentnumberlist.filter(sn => !transfersOut.includes(sn))
+  }
+
+  // fetch students that have transferred to the programme and filter out these studentnumbers
+  if (transferredToStudents) {
+    const transfersTo = (
+      await Transfer.findAll({
+        attributes: ['studentnumber'],
+        where: {
+          targetcode: {
+            [Op.in]: studyRights,
+          },
+          transferdate: {
+            [Op.gt]: startDate,
+          },
+          studentnumber: {
+            [Op.in]: studentnumberlist,
+          },
+        },
+        raw: true,
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !transfersTo.includes(sn))
+  }
+
+  // fetch students that have graduated from the programme and filter out these studentnumbers
+  if (graduatedStudents) {
+    const graduated = (
+      await Student.findAll({
+        attributes: ['studentnumber'],
+        include: [
+          {
+            model: Studyright,
+            include: [
+              {
+                model: StudyrightElement,
+                required: true,
+                where: {
+                  code: {
+                    [Op.in]: studyRights,
+                  },
+                },
+              },
+            ],
+            where: {
+              graduated: 1,
+            },
+          },
+        ],
+        where: {
+          studentnumber: {
+            [Op.in]: studentnumbers,
+          },
+        },
+      })
+    ).map(s => s.studentnumber)
+    studentnumberlist = studentnumberlist.filter(sn => !graduated.includes(sn))
   }
 
   return studentnumberlist

services/backend/src/services/studyprogrammeBasics.js

@@ -0,0 +1,200 @@
+const { indexOf } = require('lodash')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsArray,
+  getStartDate,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  allStudyrights,
+  graduatedStudyRights,
+  startedStudyrights,
+  transfersAway,
+  transfersTo,
+} = require('./newStudyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+// const getAllStats = async ({ studyprogramme, years, isAcademicYear, includeAllSpecials }) => {
+//   const { graphStats, tableStats } = getStatsBasis(years)
+
+//   await Promise.all(
+//     years.map(async year => {
+//       const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+//       const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+//         codes: [studyprogramme],
+//         startDate,
+//         endDate,
+//         includeAllSpecials,
+//       })
+//       const all = await allStudyrights(studyprogramme, null, studentnumbersOfTheYear)
+
+//       graphStats[indexOf(years, year)] = all.length
+//       tableStats[year] += all.length
+//     })
+//   )
+//   return { graphStats, tableStats }
+// }
+
+const getStartedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+      const studyrights = await startedStudyrights(studyprogramme, since, studentnumbersOfTheYear)
+
+      studyrights.forEach(({ studystartdate }) => {
+        const startYear = defineYear(studystartdate, isAcademicYear)
+        graphStats[indexOf(years, startYear)] += 1
+        tableStats[startYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      studyrights.forEach(({ enddate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        graphStats[indexOf(years, graduationYear)] += 1
+        tableStats[graduationYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getCancelledStats = async ({ studyprogramme, years, isAcademicYear }) => {
+  const studyrights = await allStudyrights(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ canceldate }) => {
+    if (canceldate !== null && canceldate !== undefined) {
+      const cancelYear = defineYear(canceldate, isAcademicYear)
+      graphStats[indexOf(years, cancelYear)] += 1
+      tableStats[cancelYear] += 1
+    }
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredAwayStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersAway(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getTransferredToStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const studyrights = await transfersTo(studyprogramme, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  studyrights.forEach(({ transferdate }) => {
+    const transferYear = defineYear(transferdate, isAcademicYear)
+    graphStats[indexOf(years, transferYear)] += 1
+    tableStats[transferYear] += 1
+  })
+
+  return { graphStats, tableStats }
+}
+
+const getBasicStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const started = await getStartedStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const cancelled = await getCancelledStats(queryParameters)
+  const transferredAway = await getTransferredAwayStats(queryParameters)
+  const transferredTo = await getTransferredToStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['basics'][specialGroups]
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          started.tableStats[year],
+          graduated.tableStats[year],
+          cancelled.tableStats[year],
+          transferredAway.tableStats[year],
+          transferredTo.tableStats[year],
+        ]
+      : [year, started.tableStats[year], graduated.tableStats[year]]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    graphStats: includeAllSpecials
+      ? [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+          {
+            name: 'Cancelled',
+            data: cancelled.graphStats,
+          },
+          {
+            name: 'Transferred away',
+            data: transferredAway.graphStats,
+          },
+          {
+            name: 'Transferred To',
+            data: transferredTo.graphStats,
+          },
+        ]
+      : [
+          {
+            name: 'Started',
+            data: started.graphStats,
+          },
+          {
+            name: 'Graduated',
+            data: graduated.graphStats,
+          },
+        ],
+    tableStats,
+    titles,
+  }
+}
+
+module.exports = {
+  getBasicStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeCredits.js

@@ -0,0 +1,128 @@
+const { indexOf } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getStatsBasis,
+  defineYear,
+  isMajorStudentCredit,
+  getYearsArray,
+  getStartDate,
+  isSpecialGroupCredit,
+  tableTitles,
+} = require('./studyprogrammeHelpers')
+const {
+  getProgrammesStudyrights,
+  getCreditsForStudyProgramme,
+  getTransferredCredits,
+  allTransfers,
+} = require('./newStudyprogramme')
+
+// Fetches all the credits for the studyprogramme and divides them into major-students and non-major students credits
+// Division is done on the basis that whether the student had a primary studyright to the programme on the attainment_date
+// If special groups are excluded, the transfer students are filtered away from the major-students as well
+const getRegularCreditStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  let studyrights = await getProgrammesStudyrights(studyprogramme, since)
+  const transfers = (await allTransfers(studyprogramme, since)).map(t => t.studyrightid)
+
+  if (!includeAllSpecials) {
+    studyrights.filter(s => !transfers.includes(s.studyrightid))
+  }
+
+  const credits = await getCreditsForStudyProgramme(providercode, since)
+
+  let majors = getStatsBasis(years)
+  let nonMajors = getStatsBasis(years)
+
+  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
+    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+
+    if (!includeAllSpecials && isSpecialGroupCredit(studyright, attainment_date, transfers)) {
+      return
+    }
+
+    if (isMajorStudentCredit(studyright, attainment_date)) {
+      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      majors.tableStats[attainmentYear] += credits || 0
+    } else {
+      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
+      nonMajors.tableStats[attainmentYear] += credits || 0
+    }
+  })
+
+  return { majors, nonMajors }
+}
+
+// Fetches all the credits with the type "transferred / hyväksiluettu" and divides them by year
+const getTransferredCreditStats = async ({ studyprogramme, since, years, isAcademicYear }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const credits = await getTransferredCredits(providercode, since)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  credits.forEach(({ attainment_date, credits }) => {
+    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+    graphStats[indexOf(years, attainmentYear)] += credits || 0
+    tableStats[attainmentYear] += credits || 0
+  })
+
+  return { graphStats, tableStats }
+}
+
+// Fetches all credits for the studytrack and combines them into the statistics for the table
+// and graph in the studyprogramme overview
+const getCreditStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const { majors, nonMajors } = await getRegularCreditStats(queryParameters)
+  const transferred = await getTransferredCreditStats(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+  const titles = tableTitles['credits'][specialGroups]
+
+  const tableStats = reversedYears.map(year =>
+    includeAllSpecials
+      ? [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          nonMajors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+      : [
+          year,
+          majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
+          majors.tableStats[year],
+          transferred.tableStats[year],
+        ]
+  )
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    graphStats: [
+      {
+        name: 'Major students credits',
+        data: majors.graphStats,
+      },
+      {
+        name: 'Non-major students credits',
+        data: nonMajors.graphStats,
+      },
+      {
+        name: 'Transferred credits',
+        data: transferred.graphStats,
+      },
+    ],
+    titles,
+  }
+}
+
+module.exports = {
+  getCreditStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeGraduations.js

@@ -0,0 +1,205 @@
+const moment = require('moment')
+const { indexOf, uniqBy } = require('lodash')
+
+const { mapToProviders } = require('../util/utils')
+const {
+  getCorrectStudentnumbers,
+  getStatsBasis,
+  defineYear,
+  getYearsObject,
+  getAcademicYearsObject,
+  getYearsArray,
+  getMedian,
+  getMean,
+  getStartDate,
+  getThesisType,
+} = require('./studyprogrammeHelpers')
+const {
+  graduatedStudyRights,
+  transfersAway,
+  transfersTo,
+  getThesisCredits,
+  followingStudyrights,
+} = require('./newStudyprogramme')
+const { getYearStartAndEndDates } = require('../util/semester')
+
+const getGraduatedStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      studyrights.forEach(({ enddate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        graphStats[indexOf(years, graduationYear)] += 1
+        tableStats[graduationYear] += 1
+      })
+    })
+  )
+  return { graphStats, tableStats }
+}
+
+const getThesisStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const providercode = mapToProviders([studyprogramme])[0]
+  const thesisType = getThesisType(studyprogramme)
+  const { graphStats, tableStats } = getStatsBasis(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const credits = await getThesisCredits(providercode, since, thesisType, studentnumbersOfTheYear)
+
+      credits?.forEach(({ attainment_date }) => {
+        const attainmentYear = defineYear(attainment_date, isAcademicYear)
+        graphStats[indexOf(years, attainmentYear)] += 1
+        tableStats[attainmentYear] += 1
+      })
+    })
+  )
+
+  return { graphStats, tableStats }
+}
+
+const getGraduationTimeStats = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  let graduationAmounts = getYearsObject(years)
+  let graduationTimes = getYearsObject(years, true)
+  let totalAmounts = isAcademicYear ? getAcademicYearsObject(years) : getYearsObject(years)
+
+  await Promise.all(
+    years.map(async year => {
+      const { startDate, endDate } = getYearStartAndEndDates(year, isAcademicYear)
+      const studentnumbersOfTheYear = await getCorrectStudentnumbers({
+        codes: [studyprogramme],
+        startDate,
+        endDate,
+        includeAllSpecials,
+      })
+
+      const studyrights = await graduatedStudyRights(studyprogramme, since, studentnumbersOfTheYear)
+      totalAmounts[year] = studentnumbersOfTheYear.length
+
+      studyrights.forEach(({ enddate, studystartdate }) => {
+        const graduationYear = defineYear(enddate, isAcademicYear)
+        const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+        graduationAmounts[graduationYear] += 1
+        graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
+      })
+    })
+  )
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  const medians = getYearsObject(years, true)
+  const means = getYearsObject(years, true)
+
+  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
+  years.forEach(year => {
+    const median = getMedian(graduationTimes[year])
+    const mean = getMean(graduationTimes[year])
+    medians[year] = [
+      ['', median],
+      ['', comparisonValue - median],
+    ]
+    means[year] = [
+      ['', mean],
+      ['', comparisonValue - mean],
+    ]
+  })
+  return { medians, means, graduationAmounts, totalAmounts }
+}
+
+const getProgrammesAfterGraduation = async ({ studyprogramme, since, years, isAcademicYear, includeAllSpecials }) => {
+  const graduated = await graduatedStudyRights(studyprogramme, since)
+  const graduatedStudents = graduated.map(s => s.studentnumber)
+  let studyrightsAfterThisProgramme = await followingStudyrights(since, graduatedStudents)
+  const studyprogrammeCodes = uniqBy(studyrightsAfterThisProgramme, 'code').map(s => ({ code: s.code, name: s.name }))
+
+  const stats = {}
+  studyprogrammeCodes.forEach(c => (stats[c.code] = { ...c, ...getYearsObject(years) }))
+
+  if (!includeAllSpecials) {
+    const transferredAway = await transfersAway(studyprogramme, since)
+    const transferredTo = await transfersTo(studyprogramme, since)
+    const transfers = [...transferredAway, ...transferredTo].map(s => s.studyrightid)
+    studyrightsAfterThisProgramme.filter(s => !transfers.includes(s.studyrightid) && !s.canceldate)
+  }
+
+  studyrightsAfterThisProgramme.forEach(({ studystartdate, code }) => {
+    const startYear = defineYear(studystartdate, isAcademicYear)
+    if (years.includes(startYear) && code) {
+      stats[code][startYear] += 1
+    }
+  })
+
+  const tableStats = Object.values(stats)
+    .map(p => [p.code, p.name['fi'], ...years.map(year => p[year])])
+    .filter(p => p[0].includes('MH'))
+
+  const graphStats = Object.values(stats)
+    .map(p => ({ name: p.name['fi'], code: p.code, data: years.map(year => p[year]) }))
+    .filter(p => p.code.includes('MH'))
+
+  return { tableStats, graphStats }
+}
+
+const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType, specialGroups }) => {
+  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
+  const includeAllSpecials = specialGroups === 'SPECIAL_INCLUDED'
+  const since = getStartDate(studyprogramme, isAcademicYear)
+  const years = getYearsArray(since.getFullYear(), isAcademicYear)
+
+  const queryParameters = { studyprogramme, since, years, isAcademicYear, includeAllSpecials }
+  const thesis = await getThesisStats(queryParameters)
+  const graduated = await getGraduatedStats(queryParameters)
+  const graduationTimeStats = await getGraduationTimeStats(queryParameters)
+  const programmesAfterGraduation = await getProgrammesAfterGraduation(queryParameters)
+
+  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
+
+  const titles = ['', 'Graduated', 'Wrote thesis']
+  const programmesAfterTitles = ['Code', 'Programme', ...years]
+  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
+
+  return {
+    id: studyprogramme,
+    years,
+    tableStats,
+    titles,
+    graphStats: [
+      {
+        name: 'Graduated students',
+        data: graduated.graphStats,
+      },
+      {
+        name: 'Wrote thesis',
+        data: thesis.graphStats,
+      },
+    ],
+    graduationMedianTime: graduationTimeStats.medians,
+    graduationMeanTime: graduationTimeStats.means,
+    graduationAmounts: graduationTimeStats.graduationAmounts,
+    totalAmounts: graduationTimeStats.totalAmounts,
+    programmesAfterTableStats: programmesAfterGraduation.tableStats,
+    programmesAfterGraphStats: programmesAfterGraduation.graphStats,
+    programmesAfterTitles,
+  }
+}
+
+module.exports = {
+  getGraduationStatsForStudytrack,
+}

services/backend/src/services/studyprogrammeHelpers.js

@@ -1,16 +1,60 @@
 const { mean } = require('lodash')
+const { studentnumbersWithAllStudyrightElements } = require('./populations')
 
 // Helper functions
+
+const getCorrectStudentnumbers = async ({ codes, startDate, endDate, includeAllSpecials, includeGraduated = true }) => {
+  let studentnumbers = []
+  const exchangeStudents = includeAllSpecials
+  const cancelledStudents = includeAllSpecials
+  const nondegreeStudents = includeAllSpecials
+  const transferredOutStudents = includeAllSpecials
+  const transferredToStudents = !includeAllSpecials
+  const graduatedStudents = !includeGraduated
+
+  studentnumbers = await studentnumbersWithAllStudyrightElements(
+    codes,
+    startDate,
+    endDate,
+    exchangeStudents,
+    cancelledStudents,
+    nondegreeStudents,
+    transferredOutStudents,
+    null,
+    transferredToStudents,
+    graduatedStudents
+  )
+
+  return studentnumbers
+}
+
 const formatStudyright = studyright => {
-  const { studyrightid, studystartdate, enddate, graduated, prioritycode, extentcode, student } = studyright
+  const {
+    studyrightid,
+    studystartdate,
+    enddate,
+    canceldate,
+    graduated,
+    prioritycode,
+    extentcode,
+    student,
+    studyright_elements,
+  } = studyright
+
   return {
     studyrightid,
     studystartdate,
     enddate,
+    canceldate,
     graduated,
     prioritycode,
     extentcode,
     studentnumber: student.studentnumber,
+    code: studyright_elements?.length ? studyright_elements[0].code : null,
+    name:
+      studyright_elements?.length && studyright_elements[0].element_detail && studyright_elements[0].element_detail.name
+        ? studyright_elements[0].element_detail.name
+        : null,
   }
 }
 
@@ -24,6 +68,16 @@ const formatStudent = student => {
   }
 }
 
+const formatTransfer = transfer => {
+  const { sourcecode, transfercode, transferdate, studyrightid } = transfer
+  return {
+    sourcecode,
+    transfercode,
+    transferdate,
+    studyrightid,
+  }
+}
+
 const getYearsArray = (since, isAcademicYear) => {
   const years = []
   for (let i = since; i <= new Date().getFullYear(); i++) {
@@ -41,6 +95,14 @@ const getYearsObject = (years, emptyArrays = false) => {
   return yearsObject
 }
 
+const getAcademicYearsObject = (years, emptyArrays = false) => {
+  let yearsObject = {}
+  for (const year of years) {
+    yearsObject = { ...yearsObject, [`${year} - ${year + 1}`]: emptyArrays ? [] : 0 }
+  }
+  return yearsObject
+}
+
 const getStatsBasis = years => {
   return {
     graphStats: new Array(years.length).fill(0),
@@ -51,10 +113,21 @@ const getStatsBasis = years => {
 const isMajorStudentCredit = (studyright, attainment_date) =>
   studyright &&
   (studyright.prioritycode === 1 || studyright.prioritycode === 30) && // Is studyright state = MAIN or state = GRADUATED
+  studyright.studystartdate && // The student has started studying in the programme
   studyright.studystartdate <= attainment_date && // Has the credit been attained after studying in the programme started
   studyright.enddate >= attainment_date && // Has the credit been attained before the studyright ended
   (!studyright.canceldate || studyright.canceldate >= attainment_date) // If the studyright was cancelled, was the credit attained before it was cancelled
 
+const isSpecialGroupCredit = (studyright, attainment_date, transfers) => {
+  if (!studyright) return true // If there is no studyright matching the credit, is not a major student credit
+  if (studyright.canceldate) return true // Cancelled studyrights are in the special groups
+  if (studyright.studystartdate > attainment_date) return true // Credits before the studyright started are not major student credits
+  if (studyright.enddate && attainment_date > studyright.enddate) return true // Credits after studyright are not major student credits
+  if ([7, 9, 16, 34, 33, 99, 14, 13].includes(studyright.extentcode)) return true // Excludes non-degree studyrights and exchange students
+  if (transfers.includes(studyright.studyrightid)) return true // Excludes both transfers in and out of the programme
+  return false
+}
+
 const getMedian = values => {
   if (values.length === 0) return 0
   values.sort((a, b) => a - b)
@@ -79,8 +152,8 @@ const defineYear = (date, isAcademicYear) => {
 const getStartDate = (studyprogramme, isAcademicYear) => {
   if ((studyprogramme.includes('KH') || studyprogramme.includes('MH')) && isAcademicYear) return new Date('2017-08-01')
   if (studyprogramme.includes('KH') || studyprogramme.includes('MH')) return new Date('2017-01-01')
-  if (isAcademicYear) return new Date('2000-08-01')
-  return new Date('2000-01-01')
+  if (isAcademicYear) return new Date('2017-08-01')
+  return new Date('2017-01-01')
 }
 
 // There are 9 course_unit_types
@@ -114,7 +187,7 @@ const getPercentage = (value, total) => {
 
 const getEmptyArray = length => new Array(length).fill(0)
 
-const getCreditGraphStats = years => ({
+const getBachelorCreditGraphStats = years => ({
   lte30: {
     name: 'Less than 30 credits',
     data: getEmptyArray(years.length),
@@ -141,21 +214,117 @@ const getCreditGraphStats = years => ({
   },
 })
 
-const creditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const getMasterCreditGraphStats = years => ({
+  lte200: {
+    name: 'Less than 200 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte220: {
+    name: '200-219 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte240: {
+    name: '220-239 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte260: {
+    name: '240-259 credits',
+    data: getEmptyArray(years.length),
+  },
+  lte280: {
+    name: '260-279 credits',
+    data: getEmptyArray(years.length),
+  },
+  mte280: {
+    name: 'More than 280 credits',
+    data: getEmptyArray(years.length),
+  },
+})
+
+const getCreditGraphStats = (studyprogramme, years) =>
+  studyprogramme.includes('KH') ? getBachelorCreditGraphStats(years) : getMasterCreditGraphStats(years)
+
+const bachelorCreditThresholds = ['lte30', 'lte60', 'lte90', 'lte120', 'lte150', 'mte150']
+const masterCreditThresholds = ['lte200', 'lte220', 'lte240', 'lte260', 'lte280', 'mte280']
+const bachelorCreditAmounts = [30, 60, 90, 120, 150, 150]
+const masterCreditAmounts = [200, 220, 240, 260, 280, 280]
+
+const getCreditThresholds = studyprogramme => {
+  if (studyprogramme.includes('KH')) {
+    return { creditThresholdKeys: bachelorCreditThresholds, creditThresholdAmounts: bachelorCreditAmounts }
+  }
+  return { creditThresholdKeys: masterCreditThresholds, creditThresholdAmounts: masterCreditAmounts }
+}
+
+const tableTitles = {
+  basics: {
+    SPECIAL_EXCLUDED: ['', 'Started studying', 'Graduated'],
+    SPECIAL_INCLUDED: ['', 'Started studying', 'Graduated', 'Cancelled', 'Transferred away', 'Transferred to'],
+  },
+  credits: {
+    SPECIAL_EXCLUDED: ['', 'Total', 'Major students credits', 'Transferred credits'],
+    SPECIAL_INCLUDED: ['', 'Total', 'Major students credits', 'Non major students credits', 'Transferred credits'],
+  },
+  creditProgress: {
+    bachelor: [
+      '',
+      'All',
+      '< 30 credits',
+      '30-59 credits',
+      '60-89 credits',
+      '90-119 credits',
+      '120-149 credits',
+      '> 150 credits',
+    ],
+    master: [
+      '',
+      'All',
+      '< 200 credits',
+      '200-219 credits',
+      '220-239 credits',
+      '240-259 credits',
+      '260-279 credits',
+      '> 280 credits',
+    ],
+  },
+  studytracks: [
+    '',
+    'All',
+    'Started studying',
+    'Currently enrolled',
+    'Absent',
+    'Cancelled',
+    'Graduated',
+    'Men',
+    'Women',
+    'Finnish',
+  ],
+}
+
+const getCreditProgressTableTitles = studyprogramme =>
+  studyprogramme.includes('KH') ? tableTitles.bachelorCreditProgress : tableTitles.masterCreditProgress
 
 module.exports = {
+  getCorrectStudentnumbers,
   formatStudyright,
   formatStudent,
+  formatTransfer,
   getYearsArray,
   getYearsObject,
+  getAcademicYearsObject,
   getStatsBasis,
   isMajorStudentCredit,
+  isSpecialGroupCredit,
   getMedian,
   getMean,
   defineYear,
   getStartDate,
   getThesisType,
   getPercentage,
+  getBachelorCreditGraphStats,
+  getMasterCreditGraphStats,
   getCreditGraphStats,
-  creditThresholds,
+  getCreditThresholds,
+  tableTitles,
+  getCreditProgressTableTitles,
 }

services/backend/src/services/studyprogrammeStats.js

@@ -1,315 +0,0 @@
-const moment = require('moment')
-const { indexOf } = require('lodash')
-
-const { mapToProviders } = require('../util/utils')
-const {
-  getStatsBasis,
-  defineYear,
-  isMajorStudentCredit,
-  getYearsObject,
-  getYearsArray,
-  getMedian,
-  getMean,
-  getStartDate,
-  getThesisType,
-} = require('./studyprogrammeHelpers')
-const {
-  graduatedStudyRights,
-  startedStudyrights,
-  cancelledStudyRights,
-  transfersAway,
-  transfersTo,
-  getProgrammesStudents,
-  getCreditsForStudyProgramme,
-  getTransferredCredits,
-  getThesisCredits,
-} = require('./newStudyprogramme')
-
-// const getEnrolledStats = async (studytrack, since, years, isAcademicYear) => {
-//   const students = await enrolledStudyrights(studytrack)
-//   const { graphStats, tableStats } = getStatsBasis(years)
-
-//   students.forEach(({ startdate }) => {
-//     if (startdate.getMonth() === 7) {
-//       const enrolledYear = defineYear(startdate, isAcademicYear)
-
-//       graphStats[indexOf(years, enrolledYear)] += 1
-//       tableStats[enrolledYear] += 1
-//     }
-//   })
-
-//   return { graphStats, tableStats }
-// }
-
-const getGraduatedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ enddate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    graphStats[indexOf(years, graduationYear)] += 1
-    tableStats[graduationYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getStartedStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await startedStudyrights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ studystartdate }) => {
-    const startYear = defineYear(studystartdate, isAcademicYear)
-    graphStats[indexOf(years, startYear)] += 1
-    tableStats[startYear] += 1
-  })
-  return { graphStats, tableStats }
-}
-
-const getCancelledStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await cancelledStudyRights(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ canceldate }) => {
-    const cancelYear = defineYear(canceldate, isAcademicYear)
-    graphStats[indexOf(years, cancelYear)] += 1
-    tableStats[cancelYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredAwayStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersAway(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getTransferredToStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await transfersTo(studytrack, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  studyrights.forEach(({ transferdate }) => {
-    const transferYear = defineYear(transferdate, isAcademicYear)
-    graphStats[indexOf(years, transferYear)] += 1
-    tableStats[transferYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getRegularCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const studyrights = await getProgrammesStudents(studytrack)
-  const credits = await getCreditsForStudyProgramme(providercode, since)
-
-  let majors = getStatsBasis(years)
-  let nonMajors = getStatsBasis(years)
-
-  credits.forEach(({ student_studentnumber, attainment_date, credits }) => {
-    const studyright = studyrights.find(studyright => studyright.studentnumber == student_studentnumber)
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-
-    if (isMajorStudentCredit(studyright, attainment_date)) {
-      majors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      majors.tableStats[attainmentYear] += credits || 0
-    } else {
-      nonMajors.graphStats[indexOf(years, attainmentYear)] += credits || 0
-      nonMajors.tableStats[attainmentYear] += credits || 0
-    }
-  })
-
-  return { majors, nonMajors }
-}
-
-const getTransferredCreditStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const credits = await getTransferredCredits(providercode, since)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date, credits }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += credits || 0
-    tableStats[attainmentYear] += credits || 0
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getThesisStats = async (studytrack, since, years, isAcademicYear) => {
-  const providercode = mapToProviders([studytrack])[0]
-  const thesisType = getThesisType(studytrack)
-  const credits = await getThesisCredits(providercode, since, thesisType)
-  const { graphStats, tableStats } = getStatsBasis(years)
-
-  credits.forEach(({ attainment_date }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
-    graphStats[indexOf(years, attainmentYear)] += 1
-    tableStats[attainmentYear] += 1
-  })
-
-  return { graphStats, tableStats }
-}
-
-const getGraduationTimeStats = async (studytrack, since, years, isAcademicYear) => {
-  const studyrights = await graduatedStudyRights(studytrack, since)
-  let graduationAmounts = getYearsObject(years)
-  let graduationTimes = getYearsObject(years, true)
-
-  studyrights.forEach(({ enddate, studystartdate }) => {
-    const graduationYear = defineYear(enddate, isAcademicYear)
-    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
-    graduationAmounts[graduationYear] += 1
-    graduationTimes[graduationYear] = [...graduationTimes[graduationYear], timeToGraduation]
-  })
-
-  const medians = getYearsObject(years, true)
-  const means = getYearsObject(years, true)
-
-  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
-  const comparisonValue = studytrack.includes('KH') ? 72 : 48
-
-  // HighCharts graph require the data to have this format (ie. actual value, "empty value")
-  years.forEach(year => {
-    const median = getMedian(graduationTimes[year])
-    const mean = getMean(graduationTimes[year])
-    medians[year] = [
-      ['', median],
-      ['', comparisonValue - median],
-    ]
-    means[year] = [
-      ['', mean],
-      ['', comparisonValue - mean],
-    ]
-  })
-  return { medians, means, graduationAmounts }
-}
-
-const getBasicStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const started = await getStartedStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const cancelled = await getCancelledStats(studyprogramme, since, years, isAcademicYear)
-  const transferredAway = await getTransferredAwayStats(studyprogramme, since, years, isAcademicYear)
-  const transferredTo = await getTransferredToStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    started.tableStats[year],
-    graduated.tableStats[year],
-    cancelled.tableStats[year],
-    transferredAway.tableStats[year],
-    transferredTo.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    graphStats: [
-      {
-        name: 'Started',
-        data: started.graphStats,
-      },
-      {
-        name: 'Graduated',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Cancelled',
-        data: cancelled.graphStats,
-      },
-      {
-        name: 'Transferred away',
-        data: transferredAway.graphStats,
-      },
-      {
-        name: 'Transferred To',
-        data: transferredTo.graphStats,
-      },
-    ],
-    tableStats,
-  }
-}
-
-const getCreditStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const { majors, nonMajors } = await getRegularCreditStats(studyprogramme, since, years, isAcademicYear)
-  const transferred = await getTransferredCreditStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [
-    year,
-    majors.tableStats[year] + nonMajors.tableStats[year] + transferred.tableStats[year],
-    majors.tableStats[year],
-    nonMajors.tableStats[year],
-    transferred.tableStats[year],
-  ])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Major students credits',
-        data: majors.graphStats,
-      },
-      {
-        name: 'Non-major students credits',
-        data: nonMajors.graphStats,
-      },
-      {
-        name: 'Transferred credits',
-        data: transferred.graphStats,
-      },
-    ],
-  }
-}
-
-const getGraduationStatsForStudytrack = async ({ studyprogramme, yearType }) => {
-  const isAcademicYear = yearType === 'ACADEMIC_YEAR'
-  const since = getStartDate(studyprogramme, isAcademicYear)
-  const years = getYearsArray(since.getFullYear(), isAcademicYear)
-  const thesis = await getThesisStats(studyprogramme, since, years, isAcademicYear)
-  const graduated = await getGraduatedStats(studyprogramme, since, years, isAcademicYear)
-  const graduationTimeStats = await getGraduationTimeStats(studyprogramme, since, years, isAcademicYear)
-
-  const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
-  const tableStats = reversedYears.map(year => [year, graduated.tableStats[year], thesis.tableStats[year]])
-
-  return {
-    id: studyprogramme,
-    years,
-    tableStats,
-    graphStats: [
-      {
-        name: 'Graduated students',
-        data: graduated.graphStats,
-      },
-      {
-        name: 'Wrote thesis',
-        data: thesis.graphStats,
-      },
-    ],
-    graduationMedianTime: graduationTimeStats.medians,
-    graduationMeanTime: graduationTimeStats.means,
-    graduationAmounts: graduationTimeStats.graduationAmounts,
-  }
-}
-
-module.exports = {
-  getBasicStatsForStudytrack,
-  getCreditStatsForStudytrack,
-  getGraduationStatsForStudytrack,
-}

services/backend/src/services/studyprogrammeUpdates.js

@@ -0,0 +1,84 @@
+const { getBasicStatsForStudytrack } = require('../services/studyprogrammeBasics')
+const { getCreditStatsForStudytrack } = require('../services/studyprogrammeCredits')
+const { getGraduationStatsForStudytrack } = require('../services/studyprogrammeGraduations')
+const { getStudytrackStatsForStudyprogramme } = require('../services/studytrackStats')
+const {
+  setBasicStats,
+  setCreditStats,
+  setGraduationStats,
+  setStudytrackStats,
+} = require('../services/analyticsService')
+
+const updateBasicView = async code => {
+  const yearTypes = ['ACADEMIC_YEAR', 'CALENDAR_YEAR']
+  const special = ['SPECIAL_INCLUDED', 'SPECIAL_EXCLUDED']
+
+  yearTypes.forEach(async yearType => {
+    special.forEach(async specialGroups => {
+      const basicStats = await getBasicStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setBasicStats(basicStats, yearType, specialGroups)
+
+      const creditStats = await getCreditStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setCreditStats(creditStats, yearType, specialGroups)
+
+      const graduationStats = await getGraduationStatsForStudytrack({
+        studyprogramme: code,
+        yearType,
+        specialGroups,
+      })
+      await setGraduationStats(graduationStats, yearType, specialGroups)
+    })
+  })
+
+  return 'OK'
+}
+
+const updateStudytrackView = async code => {
+  const specialIncluded = 'SPECIAL_INCLUDED'
+  const specialExcluded = 'SPECIAL_EXCLUDED'
+  const graduatedIncluded = 'GRADUATED_INCLUDED'
+  const graduatedExcluded = 'GRADUATED_EXCLUDED'
+
+  const statsSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialIncluded,
+    graduated: graduatedIncluded,
+  })
+  await setStudytrackStats(statsSpecialAndGraduated, graduatedIncluded, specialIncluded)
+
+  const statsSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialIncluded,
+    graduated: graduatedExcluded,
+  })
+  await setStudytrackStats(statsSpecialNotGraduated, graduatedExcluded, specialIncluded)
+
+  const statsNotSpecialAndGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialExcluded,
+    graduated: graduatedIncluded,
+  })
+  await setStudytrackStats(statsNotSpecialAndGraduated, graduatedIncluded, specialExcluded)
+
+  const statsNotSpecialNotGraduated = await getStudytrackStatsForStudyprogramme({
+    studyprogramme: code,
+    specialGroups: specialExcluded,
+    graduated: graduatedExcluded,
+  })
+  await setStudytrackStats(statsNotSpecialNotGraduated, graduatedExcluded, specialExcluded)
+
+  return 'OK'
+}
+
+module.exports = {
+  updateBasicView,
+  updateStudytrackView,
+}

services/backend/src/services/studytrackStats.js

@@ -1,83 +1,143 @@
 const { indexOf } = require('lodash')
+const moment = require('moment')
 
 const { getAssociations } = require('./studyrights')
-const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const {
+  getMedian,
+  getMean,
   getStartDate,
   getYearsArray,
   getPercentage,
-  getCreditGraphStats,
   getYearsObject,
-  creditThresholds,
+  getAcademicYearsObject,
+  getCorrectStudentnumbers,
+  getCreditThresholds,
+  getCreditGraphStats,
+  tableTitles,
 } = require('./studyprogrammeHelpers')
-const { studytrackStudents, startedStudyrights, graduatedStudyRights } = require('./newStudyprogramme')
+const {
+  studytrackStudents,
+  allStudyrights,
+  startedStudyrights,
+  enrolledStudents,
+  absentStudents,
+  graduatedStudyRights,
+} = require('./newStudyprogramme')
 const { getAcademicYearDates } = require('../util/semester')
 
-const getStudentData = students => {
-  const data = { female: 0, male: 0, finnish: 0, lte30: 0, lte60: 0, lte90: 0, lte120: 0, lte150: 0, mte150: 0 }
+const getStudentData = (students, thresholdKeys, thresholdAmounts) => {
+  let data = { female: 0, male: 0, finnish: 0 }
+  thresholdKeys.forEach(t => (data[t] = 0))
+
   students.forEach(({ gender_code, home_country_en, creditcount }) => {
     data.male += gender_code === '1' ? 1 : 0
     data.female += gender_code === '2' ? 1 : 0
     data.finnish += home_country_en === 'Finland' ? 1 : 0
-    data.lte30 += creditcount < 30 ? 1 : 0
-    data.lte60 += creditcount >= 30 && creditcount < 60 ? 1 : 0
-    data.lte90 += creditcount >= 60 && creditcount < 90 ? 1 : 0
-    data.lte120 += creditcount >= 90 && creditcount < 120 ? 1 : 0
-    data.lte150 += creditcount >= 120 && creditcount < 150 ? 1 : 0
-    data.mte150 += creditcount >= 150 ? 1 : 0
+    data[thresholdKeys[0]] += creditcount < thresholdAmounts[0] ? 1 : 0
+    data[thresholdKeys[1]] += creditcount >= thresholdAmounts[0] && creditcount < thresholdAmounts[1] ? 1 : 0
+    data[thresholdKeys[2]] += creditcount >= thresholdAmounts[1] && creditcount < thresholdAmounts[2] ? 1 : 0
+    data[thresholdKeys[3]] += creditcount >= thresholdAmounts[2] && creditcount < thresholdAmounts[3] ? 1 : 0
+    data[thresholdKeys[4]] += creditcount >= thresholdAmounts[3] && creditcount < thresholdAmounts[4] ? 1 : 0
+    data[thresholdKeys[5]] += creditcount >= thresholdAmounts[5] ? 1 : 0
   })
   return data
 }
 
+const getGraduationTimeStats = async ({
+  year,
+  graduated,
+  studyprogramme,
+  track,
+  graduationMeanTime,
+  graduationMedianTime,
+  graduationAmounts,
+}) => {
+  // Count how long each student took to graduate
+  let graduationTimes = []
+  graduated.forEach(({ enddate, studystartdate }) => {
+    const timeToGraduation = moment(enddate).diff(moment(studystartdate), 'months')
+    graduationAmounts[track][year] += 1
+    graduationTimes = [...graduationTimes, timeToGraduation]
+  })
+
+  // The maximum amount of months in the graph depends on the studyprogramme intended graduation time
+  const comparisonValue = studyprogramme.includes('KH') ? 72 : 48
+
+  // HighCharts graph requires the data to have this format (ie. actual value, "empty value")
+  const median = getMedian(graduationTimes)
+  const mean = getMean(graduationTimes)
+  graduationMedianTime[track][year] = [
+    ['', median],
+    ['', comparisonValue - median],
+  ]
+  graduationMeanTime[track][year] = [
+    ['', mean],
+    ['', comparisonValue - mean],
+  ]
+}
+
 // Goes through the programme and all its studytracks for the said year and adds the wanted stats to the data objects
-const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studytrackNames, year, years, data }) => {
-  const { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks } = data
+const getStudytrackDataForTheYear = async ({
+  studyprogramme,
+  special,
+  includeGraduated,
+  studytracks,
+  studytrackNames,
+  year,
+  years,
+  data,
+  creditThresholdKeys,
+  creditThresholdAmounts,
+}) => {
+  const {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMeanTime,
+    graduationMedianTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  } = data
   const { startDate, endDate } = getAcademicYearDates(year)
 
   await Promise.all(
     studytracks.map(async track => {
       const codes = studyprogramme === track ? [studyprogramme] : [studyprogramme, track]
-      const studentnumbers = await studentnumbersWithAllStudyrightElements(
+
+      const studentnumbers = await getCorrectStudentnumbers({
         codes,
         startDate,
         endDate,
-        true,
-        true,
-        true,
-        true
-      )
+        includeAllSpecials: special,
+        includeGraduated,
+      })
 
-      // All the stats are counted for the students who actually started studying
+      // Get all the studyrights and students for the calculations
+      const all = await allStudyrights(track, null, studentnumbers)
+      const students = await studytrackStudents(studentnumbers)
+      const studentData = getStudentData(students, creditThresholdKeys, creditThresholdAmounts)
       const started = await startedStudyrights(track, startDate, studentnumbers)
-      const startedStudentnumbers = [...new Set(started.map(s => s.studentnumber))]
-      const students = await studytrackStudents(startedStudentnumbers)
-      const studentData = getStudentData(students)
-      const graduated = await graduatedStudyRights(track, startDate, studentnumbers)
+      const enrolled = await enrolledStudents(track, startDate, studentnumbers)
+      const absent = await absentStudents(track, studentnumbers)
+      const cancelled = all.filter(s => s.canceldate !== null && s.canceldate !== undefined)
+      const graduated = await graduatedStudyRights(track, null, studentnumbers)
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
-      if (started.length === 0) {
+      if (all.length === 0) {
         emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
         return
       }
 
       // Count stats for creditgraph for the year per track
-      creditThresholds.forEach(
+      creditThresholdKeys.forEach(
         threshold => (creditGraphStats[track][threshold].data[indexOf(years, year)] = studentData[threshold])
       )
-
       // Count stats for the credit progress table for the year per track
       creditTableStats[track] = [
         ...creditTableStats[track],
-        [
-          `${year} - ${year + 1}`,
-          started.length,
-          studentData.lte30,
-          studentData.lte60,
-          studentData.lte90,
-          studentData.lte120,
-          studentData.lte150,
-          studentData.mte150,
-        ],
+        [`${year} - ${year + 1}`, all.length, ...creditThresholdKeys.map(threshold => studentData[threshold])],
       ]
 
       // Count stats for the main studytrack table grouped by tracks
@@ -85,16 +145,24 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
         ...mainStatsByTrack[track],
         [
           `${year} - ${year + 1}`,
+          all.length,
+          getPercentage(all.length, all.length),
           started.length,
-          getPercentage(started.length, started.length),
-          studentData.male,
-          getPercentage(studentData.male, started.length),
-          studentData.female,
-          getPercentage(studentData.female, started.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, started.length),
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          cancelled.length,
+          getPercentage(cancelled.length, all.length),
           graduated.length,
-          getPercentage(graduated.length, started.length),
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
         ],
       ]
 
@@ -102,19 +170,41 @@ const getStudytrackDataForTheYear = async ({ studyprogramme, studytracks, studyt
       mainStatsByYear[year] = [
         ...mainStatsByYear[year],
         [
-          studytrackNames[track]?.name['fi'] || `${year} - ${year + 1}`,
+          studytrackNames[track]?.name['fi']
+            ? `${studytrackNames[track]?.name['fi']}, ${track}`
+            : `${year} - ${year + 1}`,
+          all.length,
+          getPercentage(all.length, all.length),
           started.length,
-          getPercentage(started.length, started.length),
-          studentData.male,
-          getPercentage(studentData.male, started.length),
-          studentData.female,
-          getPercentage(studentData.female, started.length),
-          studentData.finnish,
-          getPercentage(studentData.finnish, started.length),
+          getPercentage(started.length, all.length),
+          enrolled.length,
+          getPercentage(enrolled.length, all.length),
+          absent.length,
+          getPercentage(absent.length, all.length),
+          cancelled.length,
+          getPercentage(cancelled.length, all.length),
           graduated.length,
-          getPercentage(graduated.length, started.length),
+          getPercentage(graduated.length, all.length),
+          studentData.male,
+          getPercentage(studentData.male, all.length),
+          studentData.female,
+          getPercentage(studentData.female, all.length),
+          studentData.finnish,
+          getPercentage(studentData.finnish, all.length),
         ],
       ]
+
+      // Count stats for the graduation time charts grouped by year
+      totalAmounts[track][`${year} - ${year + 1}`] = await all.length
+      await getGraduationTimeStats({
+        year: `${year} - ${year + 1}`,
+        graduated,
+        studyprogramme,
+        track,
+        graduationMedianTime,
+        graduationMeanTime,
+        graduationAmounts,
+      })
     })
   )
 }
@@ -133,25 +223,45 @@ const getStudytrackOptions = (studyprogramme, studytrackNames, studytracks, empt
 }
 
 // Creates empty objects for each statistic type, which are then updated with the studytrack data
-const getEmptyStatsObjects = (years, studytracks) => {
+const getEmptyStatsObjects = (years, studytracks, studyprogramme) => {
   const mainStatsByYear = getYearsObject(years, true)
   const mainStatsByTrack = {}
   const creditGraphStats = {}
   const creditTableStats = {}
+  const graduationMedianTime = {}
+  const graduationMeanTime = {}
+  const graduationAmounts = {}
+  const totalAmounts = {}
   const emptyTracks = new Map()
 
   studytracks.forEach(async track => {
     mainStatsByTrack[track] = []
-    creditGraphStats[track] = await getCreditGraphStats(years)
+    creditGraphStats[track] = getCreditGraphStats(studyprogramme, years)
     creditTableStats[track] = []
+    graduationMedianTime[track] = getAcademicYearsObject(years, true)
+    graduationMeanTime[track] = getAcademicYearsObject(years, true)
+    graduationAmounts[track] = getAcademicYearsObject(years)
+    totalAmounts[track] = getAcademicYearsObject(years)
   })
 
-  return { mainStatsByYear, mainStatsByTrack, creditGraphStats, creditTableStats, emptyTracks }
+  return {
+    mainStatsByYear,
+    mainStatsByTrack,
+    creditGraphStats,
+    creditTableStats,
+    graduationMedianTime,
+    graduationMeanTime,
+    graduationAmounts,
+    totalAmounts,
+    emptyTracks,
+  }
 }
 
 // Combines all the data for the Populations and Studytracks -view
-const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
+const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, graduated, specialGroups }) => {
   const isAcademicYear = true
+  const special = specialGroups === 'SPECIAL_INCLUDED'
+  const includeGraduated = graduated === 'GRADUATED_INCLUDED'
   const since = getStartDate(studyprogramme, isAcademicYear)
   const years = getYearsArray(since.getFullYear())
 
@@ -162,17 +272,22 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
 
   const studytrackNames = associations.studyTracks
 
-  const data = getEmptyStatsObjects(years, studytracks)
+  const data = getEmptyStatsObjects(years, studytracks, studyprogramme)
+  const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(studyprogramme)
 
   await Promise.all(
     years.map(async year => {
       return await getStudytrackDataForTheYear({
         studyprogramme,
+        special,
+        includeGraduated,
         studytracks,
         studytrackNames,
         year,
         years,
         data,
+        creditThresholdKeys,
+        creditThresholdAmounts,
       })
     })
   )
@@ -186,7 +301,14 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme }) => {
     mainStatsByYear: data.mainStatsByYear,
     creditTableStats: data.creditTableStats,
     creditGraphStats: data.creditGraphStats,
+    graduationMedianTime: data.graduationMedianTime,
+    graduationMeanTime: data.graduationMeanTime,
+    graduationAmounts: data.graduationAmounts,
+    totalAmounts: data.totalAmounts,
     studytrackOptions,
+    includeGraduated,
+    populationTitles: tableTitles['studytracks'],
+    creditTableTitles: tableTitles['creditProgress'][studyprogramme.includes('KH') ? 'bachelor' : 'master'],
   }
 }
 

services/backend/src/util/semester.js

@@ -36,9 +36,27 @@ const getAcademicYearDates = year => {
   }
 }
 
+const getYearStartAndEndDates = (year, isAcademicYear) => {
+  if (isAcademicYear) {
+    const firstYear = year.slice(0, 4)
+    return {
+      startDate: new Date(`${firstYear}-${semesterStart['FALL']}`).toISOString(),
+      endDate: new Date(
+        `${moment(firstYear, 'YYYY').add(1, 'years').format('YYYY')}-${semesterEnd['SPRING']}`
+      ).toISOString(),
+    }
+  }
+
+  return {
+    startDate: new Date(`${year}-01-01`).toISOString(),
+    endDate: new Date(`${year}-12-31`).toISOString(),
+  }
+}
+
 module.exports = {
   getPassingSemester,
   getAcademicYearDates,
+  getYearStartAndEndDates,
   semesterEnd,
   semesterStart,
 }

services/frontend/src/common/InfoToolTips/courseStatistics.js

@@ -1,7 +1,7 @@
 export default {
-  0: {
-    Attempts: `
-      Table - Attempts
+  Tables: {
+    ATTEMPTS: `
+      **Table - Attempts**
 
       Taulukko näyttää suorituskerrat lukuvuositasolla.
       Samalla opiskelijalla voi olla useampia suorituskertoja.
@@ -13,8 +13,8 @@ export default {
       Arvosanajakaumaa katsoessasi voit muuttaa luvut prosenteiksi, klikkaamalla painiketta "Relative".
       Mahdollinen sarake "Other passed" sisältää sekä arvosanan "Hyv." saaneet että kurssin hyväksilukeneet.
     `,
-    Students: `
-      Table - Students
+    STUDENT: `
+      **Table - Students**
 
       Taulukko näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
       Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -26,17 +26,17 @@ export default {
       Total-rivillä näkyy niiden opiskelijoiden kokonaismäärä, jotka ovat kurssista joskus arvosanan saaneet.
     `,
   },
-  1: {
-    Attempts: `
-      Pass rate chart - Attempts
+  PassRate: {
+    ATTEMPTS: `
+      **Pass rate chart - Attempts**
 
       Graafi näyttää suorituskerrat lukuvuositasolla jaoteltuina hyväksyttyihin ja hylättyihin suorituksiin.
       Hyväksytyt suoritukset sisältävät arvosanat 1-5 sekä arvosanan Hyv.
       Hylätyt suoritukset sisältävät arvosanat Hyl. ja 0, sekä suoritukset, joille opettaja on antanut merkinnän
       "Luop" (Luopunut suorituksesta) tai "Eisa" (Ei saapunut tenttiin).
     `,
-    Students: `
-      Pass rate chart - Students
+    STUDENT: `
+      **Pass rate chart - Students**
 
       Graafi näyttää yksittäiset opiskelijat, jotka ovat kurssia suorittaneet.
       Jokainen opiskelija on jaoteltu luokkiin: "läpäissyt ensiyrittämällä", "läpäissyt lopulta" tai "ei koskaan läpäissyt kurssia".
@@ -46,18 +46,9 @@ export default {
       myöhemmin keväällä 2019, tulisi merkityksi lukuvuodelle 2017-2018 pylvääseen "passed eventually".        
     `,
   },
-  2: {
-    Attempts: `
-      Grade distribution
-  
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-    Students: `
-      Grade distribution
+  GradeDistribution: `
+    **Grade distribution**
 
-      Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
-      "Other passed" sisältää sekä arvosanan "Hyv." että kurssin hyväksiluvut.
-    `,
-  },
+    Graafi näyttää suorituskertojen arvosanajakauman eri lukuvuosille. Mahdollinen kategoria
+  `,
 }

services/frontend/src/common/InfoToolTips/filters.js

@@ -6,7 +6,8 @@ export default {
   },
   transferred: {
     label: 'Include and exclude students from this program',
-    short: 'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet tutkinto-ohjelmaan',
+    short:
+      'Mahdollisuus suodattaa opiskelijat, jotka ovat siirtyneet nykyiseen 2017 alkaneeseen tutkinto-ohjelmaan vanhasta tutkinto-ohjelmasta',
   },
   courseCredits: {
     label: null,

services/frontend/src/common/InfoToolTips/index.jsx

@@ -1,5 +1,4 @@
 import PopulationStatistics from './populationStatistics'
-import PopulationOverview from './populationOverview'
 import CoolDataScience from './coolDataScience'
 import CourseStatistics from './courseStatistics'
 import CoursePopulation from './coursePopulation'
@@ -7,7 +6,6 @@ import Studyprogramme from './studyprogramme'
 
 export default {
   PopulationStatistics,
-  PopulationOverview,
   CoolDataScience,
   CourseStatistics,
   CoursePopulation,

services/frontend/src/common/InfoToolTips/populationOverview.js

@@ -1,24 +0,0 @@
-export default {
-  PopulationProgress: `Tämä koontisivu sisältää koulutusohjelman kaikkien opiskelijoiden perustiedot lukuvuosittain. Luvut perustuvat opinto-oikeuksiin kyseisessä ohjelmassa.
-
-  Population progress:
-  Vuosittain ohjelmassa aloittaneet, ohjelman keskeyttäneet / peruneet sekä ohjelmaan siirtyneet opiskelijat. 
-  Huomattavaa on, että sama opiskelija voi olla kahdessa luokassa, jos hän on esimerkiksi aloittanut opinnot jo, mutta keskeyttänyt ne lukuvuoden aikana.
-  Tarkemmat tiedot saat esille klikkaamalla nuolta kunkin lukuvuoden kohdalla.`,
-  YearlyProductivity: `
-  Yearly productivity:
-  Koulutusohjelman kunakin kalenterivuonna tuottamat opintopisteet, valmistuneiden määrän, ohjelman pääaineopiskelijoiden
-  opintopisteet sekä muiden kuin pääaineopiskelijoiden opintopisteet. Mukana on myös kuhunkin koulutusohjelmaan hyväksiluetut opintopisteet.
-  
-  Vanhojen koulutusohjelmien lakkauttaminen ja opiskelijoiden siirtäminen uusiin ohjelmiin voi aiheuttaa tavallista suuremmat opintopistemäärät lukuvuodelle 2020-2021.`,
-  Year: `Lukuvuosi`,
-  Students: `Opiskelijat, jotka ovat koulutusohjelmassa aloittaneet, opinto-oikeutensa peruneet sekä koulutusohjelmaan tai siitä pois siirtyneet.`,
-  Started: `Koulutusohjelmassa kunakin lukuvuonna aloittaneet opiskelijat. EI sisällä ohjelmaan siirtyneitä opiskelijoita.`,
-  Cancelled: `Koulutusohjelmaan oikeuttavan opinto-oikeutensa kunakin lukuvuonna peruttaneet`,
-  Graduated: `Tiettynä lukuvuonna aloittaneista opiskelijoista on valmistunut`,
-  Transferred: `Koulutusohjelmaan kyseisenä lukuvuonna siirtyneet opiskelijat. 
-  
-  Vanhojen koulutusohjelmien lakkauttaminen aiheuttaa tavallista suuremmat siirtoluvut lukuvuodelle 2020-2021.`,
-  Credits: `Kunakin vuonna aloittaneista opiskelijoista ne, jotka ovat ylittäneet tietyn opintopisterajapyykin`,
-  Thesis: `Loppututkielman kirjoittaneiden määrä`,
-}

services/frontend/src/common/InfoToolTips/studyprogramme.js

@@ -1,43 +1,74 @@
 export default {
+  YearToggle: `
+  Academic year: Näyttää tilastot akateemisille vuosille jaoteltuna, esimerkiksi 1.8.2019-31.7.2020
+  Calendar year: Näyttää tilastot kalenterivuosille jaoteltuna, esimerkiksi 1.1.2019-31.12.2019 
+  `,
+  StudentToggle: `
+  All studyrights included: Valinnassa ovat mukana vaihto-opiskelijat, erillisoikeudella opiskelevat, opintonsa keskeyttäneet opiskelijat, ohjelmaan siirtyneet opiskelijat sekä ohjelmasta pois siirtyneet opiskelijat.
+  
+  Special studyrights excluded: Valinnasta ja kaikista luvuista on poissuljettu kaikki edellä mainitut erikoisryhmät.
+  `,
+  GradToggle: `
+  Graduated included: Valinnassa ovat mukana ohjelmasta jo valmistuneet opiskelijat.
+  
+  Graduated excluded: Valinnasta on poistettu ohjelmasta jo valmistuneet opiskelijat.
+  `,
   StudentsOfTheStudyprogramme: `
-  **Started**: Sisältää kyseisenä vuonna aloitetut opinto-oikeudet riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
+  **All**: Sisältää kaikki kyseisenä vuonna alkaneet opinto-oikeudet riippumatta siitä, onko opintoja aloitettu tai onko niitä aloitettu ko. vuonna. Mukana myös ohjelmaan siirtyneet opiskelijat.
+
+  **Started studying**: Sisältää kyseisenä vuonna aloittaneet opiskelijat (aloitetut opinto-oikeudet) riippumatta siitä, minä vuonna kyseinen opiskeluoikeus on myönnetty. Opiskelija on myös voinut myöhemmin keskeyttää opintonsa tai siirtyä pois ohjelmasta.\n
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
+  
   **Cancelled**: Sisältää kyseisenä vuonna perutut opiskeluoikeudet riippumatta siitä, onko kyseinen opiskelija ikinä edes alkanut opiskelemaan ohjelmassa tai opiskelun aloitusvuodesta.
   
-  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opinto-oikeudet. Sama opiskelija voi esiintyä tilastoissa useana vuonna. Kyseiset opiskelijat ovat saattaneet aloittaa ohjelmassa tai eivät koskaan ole aloittaneet ohjelmassa. 
+  **Transferred away**: Sisältää kyseisenä vuonna pois ohjelmasta siirtyneet opiskelijat (opinto-oikeudet). Sama opiskelija voi esiintyä tilastoissa useana vuonna, mikäli hän on siirtynyt ohjelmaan ja siitä pois useasti.
   
-  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opinto-oikeudet.
+  **Transferred to**: Sisältää kyseisenä vuonna ohjelmaan siirtyneet opiskelijat (opinto-oikeudet).
 
   HUOM! Ohjelmaan siirtyneiden määrä on erityisen suuri vuonna 2020, jolloin vanhoista koulutusohjelmista luovuttiin ja opiskelijat siirtyivät uusiin ohjelmiin.
   `,
   CreditsProducedByTheStudyprogramme: `
-  Sisältää opintopisteet suoritusvuosittain (Sisussa näkyvä kurssin suorituspäivä)
+  Sisältää opintopisteet suoritusvuosittain. Suoritukset on jaoteltu Sisussa näkyvän kurssin suorituspäivän mukaan.
 
-  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan.
+  **Major students' credits**: Sisältää ne opintopisteet, joiden suorittajalla on suoritushetkellä ollut ensisijainen opinto-oikeus kyseiseen koulutusohjelmaan. Sisältää myös ohjelmaan siirtyneet ensisijaiset opiskelijat.
 
-  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan.
+  **Non-major students' credits**: Sisältää koulutusohjelman tuottamat opintopisteet, joiden suorittajalla EI ole ollut suoritushetkellä ensisijaista opinto-oikeutta kyseiseen koulutusohjelmaan. Sisältää
+  esimerkiksi vaihto-opiskelijat ja erillis-opinto-oikeudella opiskelevat. Sisältää myös opintopisteet, jotka opiskelija on suorittanut koulutusohjelmaan ennen opintojen aloittamista koulutusohjelmassa tai opintojen päätyttyä.
   
-  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan.
+  **Transferred credits**: Sisältää opintopisteet, jotka on suoritettu kyseisenä vuonna, ja hyväksiluettu tähän koulutusohjelmaan. Mukana ovat kaikenlaisilla opinto-oikeuksilla hyväksiluetut opintopisteet.
   `,
   GraduatedAndThesisWritersOfTheProgramme: `
-  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
+  **Wrote thesis**: Sisältää kyseisenä vuonna kandidaatintutkielman tai pro gradu-tutkielman hyväksytysti suorittaneet opiskelijat. Mukana ei ole mahdollisia arvosanan korotuksia.
 
   **Graduated**: Sisältää kyseisenä vuonna valmistuneet opiskelijat riippumatta siitä, minä vuonna opiskeluoikeus ohjelmaan on myönnetty. Jaottelu on tehty opinto-oikeuden päättymispäivän (=valmistumispäivän) perusteella.
   `,
   AverageGraduationTimes: `
-  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **mediaanin** kuukausina.
+  **Median time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **mediaanin** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
 
-  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opintojen keston **keskiarvon** kuukausina.
+  **Mean time**: Näyttää kyseisenä vuonna valmistuneiden opiskelijoiden opintojen keston **keskiarvon** kuukausina. Luku sisältää mahdolliset poissaolokuukaudet.
   
-  Mukana ovat **vain valmistuneet opiskelijat**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
-  StudytrackOverview: `
-  Taulukko näyttää kyseisenä lukuvuonna aloittaneet pääaineopiskelijat. Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
+  n = **valmistuneet opiskelijat / kaikki ohjelmassa kyseisenä vuonna alkaneet opinto-oikeudet**. Kandiohjelmissa tavoiteaika on 36 kk, maisteriohjelmissa 24 kk`,
+  ProgrammesAfterGraduation: `
+  Sisältää maisteriohjelmat, joissa tämän kandiohjelman opiskelijat ovat aloittaneet valmistuttuaan kandeiksi. Vuosijaottelu on tehty maisteriohjelmassa aloittamisen perusteella.
+
+  Mukana ovat **vain valmistuneet opiskelijat ja vain maisteriohjelmat**.
+  `,
+  StudytrackOverview: `  
+  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  
+  HUOM!
+  **All** Sisältää kaikki opiskelijat, joiden opinto-oikeudella on aloituspäivä kyseisenä vuonna, riippumatta siitä, onko opiskelija aloittanut opiskelemaan ko. vuonna.
+  **Started studying** Sisältää kaikki opiskelijat, jotka ovat aloittaneet opiskelemaan koulutusohjelmassa kyseisenä lukuvuonna.
+  
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.
+  Taulukko näyttää myös näistä opiskelijoista miesten, naisten ja suomalaisten osuudet sekä valmistuneiden määrän. 
 
   Yläosan valikosta on mahdollista valita tarkasteluun yhden opintosuunnan opiskelijat. Sekä opiskelijoiden yleistiedot että edistyminen opintopisteittäin
   kuvaavat tällöin kyseisen opintosuunnan opiskelijoita.
   `,
   StudytrackProgress: `
-  Lähtökohtaisesti kuvaa koulutusohjelman kaikkien pääaineoopiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  Kuvaa koulutusohjelman kyseiseen ohjelmaan opinto-oikeudella varustettujen opiskelijoiden etenemistä opintopisteissä mitattuna. Esimerkiksi 2020-2021 aloittaneiden opintopistejakaumaa.
+  Mukana ovat oletusarvoisesti myös vaihto-opiskelijat, tutkinnonsuoritusoikeudesta luopuneet ja erillisopinto-oikeudella opiskelevat.   
 
   Mikäli yläosan valikosta valittuna on jokin tietty opintosuunta, myös nämä graafit ja taulukot kuvaavat kyseisen opintosuunnan opiskelijoita.    
   `,

services/frontend/src/components/CoolDataScience/Status.jsx

@@ -59,6 +59,7 @@ const StatusContainer = ({
   max1,
   showYearlyValues,
   showRelativeValues,
+  showCountingFrom,
   showByYear,
   showStudentCounts,
   clickable,
@@ -84,11 +85,31 @@ const StatusContainer = ({
   let yearlyValues = null
 
   if (showYearlyValues) {
-    yearlyValues = _.orderBy(Object.entries(stats.yearly), ([y]) => y, ['desc']).map(([year, yearStats]) => ({
-      label: showByYear ? year : `${year}-${`${Number(year) + 1}`.slice(-2)}`,
-      accumulated: showStudentCounts && stats.type === 'course' ? yearStats.accStudents : yearStats.acc,
-      total: showStudentCounts && stats.type === 'course' ? yearStats.totalStudents : yearStats.total,
-    }))
+    yearlyValues = _.chain(stats.yearly)
+      .toPairs()
+      .map(([year, stats]) => ({
+        dateRange: {
+          start: showByYear ? moment([year]).startOf('year') : moment([year]).month(7).date(1),
+          end: showByYear
+            ? moment([year]).startOf('year')
+            : moment([parseInt(year, 10) + 1])
+                .month(6)
+                .date(31),
+        },
+        stats,
+      }))
+      .filter(({ dateRange: { start } }) => moment(showCountingFrom).isSameOrAfter(start))
+      .orderBy(({ dateRange: { start } }) => start, ['desc'])
+      .map(({ dateRange, stats }) => ({
+        label: _.chain([dateRange.start, dateRange.end])
+          .map(date => date.year())
+          .uniq()
+          .join('-')
+          .value(),
+        accumulated: showStudentCounts && stats.type === 'course' ? stats.accStudents : stats.acc,
+        total: showStudentCounts && stats.type === 'course' ? stats.totalStudents : stats.total,
+      }))
+      .value()
   }
 
   return (
@@ -108,7 +129,7 @@ const StatusContainer = ({
 }
 
 const StatusContent = ({ data, settings }) => {
-  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts } = settings
+  const { showByYear, showRelativeValues, showYearlyValues, showStudentCounts, showCountingFrom } = settings
 
   return (
     <DrillStack
@@ -124,6 +145,7 @@ const StatusContent = ({ data, settings }) => {
           showRelativeValues={showRelativeValues}
           showYearlyValues={showYearlyValues}
           showStudentCounts={showStudentCounts}
+          showCountingFrom={showCountingFrom}
           min1={-2 * medianDiff}
           max1={2 * medianDiff}
           showByYear={showByYear}

services/frontend/src/components/CourseStatistics/ResultTabs/HelpButton.jsx

@@ -0,0 +1,31 @@
+import React from 'react'
+import ReactMarkdown from 'react-markdown'
+
+import { Popup, Button, Icon } from 'semantic-ui-react'
+
+import infotooltips from '../../../common/InfoToolTips'
+
+export default ({ tab, viewMode }) => {
+  let content = infotooltips.CourseStatistics[tab]
+
+  if (viewMode) {
+    content = content[viewMode]
+  }
+
+  content = content.replace(/\n +/g, '\n')
+
+  return (
+    <Popup
+      on="click"
+      wide="very"
+      trigger={
+        <Button icon labelPosition="left">
+          <Icon name="question circle outline" />
+          Help
+        </Button>
+      }
+    >
+      <ReactMarkdown>{content}</ReactMarkdown>
+    </Popup>
+  )
+}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/attempts.jsx

@@ -1,10 +1,10 @@
 import React from 'react'
 import qs from 'query-string'
+import _, { uniq } from 'lodash'
 import { Link } from 'react-router-dom'
-import { arrayOf, number, oneOfType, shape, string, bool } from 'prop-types'
 import { connect } from 'react-redux'
 import { Header, Icon, Item } from 'semantic-ui-react'
-import { uniq } from 'lodash'
+
 import SortableTable from '../../../../SortableTable'
 import { defineCellColor, getGradeSpread, getThesisGradeSpread, isThesisGrades, THESIS_GRADE_KEYS } from '../util'
 
@@ -40,7 +40,7 @@ const getTableData = (stats, notThesisGrades, isRelative) =>
       passRate: stat.attempts.passRate,
       attempts,
       rowObfuscated,
-      ...gradeSpread,
+      ..._.mapValues(gradeSpread, x => x[0]),
     }
   })
 
@@ -69,14 +69,13 @@ const getGradeColumns = (notThesisGrades, addHTAndTT) => {
 }
 
 const AttemptsTable = ({
-  stats,
-  name,
+  data: { stats, name },
+  settings: { showGrades },
   alternatives,
   separate,
   isRelative,
   userHasAccessToAllStats,
   headerVisible = false,
-  showGrades,
 }) => {
   const {
     attempts: { grades },
@@ -147,7 +146,7 @@ const AttemptsTable = ({
       <SortableTable
         defaultdescending
         getRowKey={s => s.code}
-        tableProps={{ celled: true, structured: true }}
+        tableProps={{ celled: true, style: { width: 'auto' } }}
         columns={columns}
         data={data}
       />
@@ -158,19 +157,4 @@ const AttemptsTable = ({
   )
 }
 
-AttemptsTable.propTypes = {
-  stats: arrayOf(shape({})).isRequired,
-  name: oneOfType([number, string]).isRequired,
-  alternatives: arrayOf(string).isRequired,
-  separate: bool,
-  isRelative: bool.isRequired,
-  userHasAccessToAllStats: bool.isRequired,
-  headerVisible: bool.isRequired,
-  showGrades: bool.isRequired,
-}
-
-AttemptsTable.defaultProps = {
-  separate: false,
-}
-
 export default connect(null)(AttemptsTable)

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/student.jsx

@@ -1,4 +1,4 @@
-import React from 'react'
+import React, { useMemo } from 'react'
 import qs from 'query-string'
 import { Link } from 'react-router-dom'
 import { Header, Icon, Item } from 'semantic-ui-react'
@@ -10,7 +10,7 @@ import { defineCellColor } from '../util'
 
 const formatPercentage = p => `${(p * 100).toFixed(2)} %`
 
-const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllStats, headerVisible = false }) => {
+const getColumns = (showDetails, userHasAccessToAllStats, alternatives, separate) => {
   const showPopulation = (yearcode, years) => {
     const queryObject = {
       from: yearcode,
@@ -23,6 +23,86 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
     return `/coursepopulation?${searchString}`
   }
 
+  const columns = [
+    {
+      key: 'TIME',
+      title: 'Time',
+      getRowVal: s => s.code,
+      getRowContent: s => (
+        <div style={{ whiteSpace: 'nowrap' }}>
+          {s.name}
+          {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
+          {s.name !== 'Total' && userHasAccessToAllStats && (
+            <Item as={Link} to={showPopulation(s.code, s.name, s)}>
+              <Icon name="level up alternate" />
+            </Item>
+          )}
+        </div>
+      ),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'TOTAL',
+      title: 'Total Students',
+      getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.students.total),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'PASS_RATE',
+      title: 'Pass-%',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.passRate),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.passRate || 0)),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'PASS_FIRST',
+      title: 'On First Attempt',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
+      getCellProps: s => defineCellColor(s),
+      headerProps: { style: { borderLeft: '0' } },
+      onlyInDetailedView: true,
+    },
+    {
+      key: 'PASS_EVENTUALLY',
+      title: 'Eventually',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
+      getCellProps: s => defineCellColor(s),
+      headerProps: { style: { borderLeft: '0' } },
+      onlyInDetailedView: true,
+    },
+    {
+      key: 'FAIL_RATE',
+      title: 'Fail-%',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.failRate),
+      getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
+      getCellProps: s => defineCellColor(s),
+    },
+    {
+      key: 'NEVER PASSED',
+      title: 'Yet to Pass',
+      getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.neverPassed || 0),
+      getCellProps: s => defineCellColor(s),
+      headerProps: { style: { borderLeft: '0' } },
+      onlyInDetailedView: true,
+    },
+  ]
+
+  return columns.filter(column => showDetails || !column.onlyInDetailedView)
+}
+
+const StudentTable = ({
+  data: { name, stats },
+  settings: { showDetails },
+  alternatives,
+  separate,
+  userHasAccessToAllStats,
+  headerVisible = false,
+}) => {
+  const columns = useMemo(
+    () => getColumns(showDetails, userHasAccessToAllStats, alternatives, separate),
+    [showDetails, userHasAccessToAllStats, alternatives, separate]
+  )
+
   return (
     <div>
       {headerVisible && (
@@ -33,88 +113,8 @@ const StudentTable = ({ stats, name, alternatives, separate, userHasAccessToAllS
       <SortableTable
         defaultdescending
         getRowKey={s => s.code}
-        tableProps={{ celled: true, structured: true }}
-        columns={[
-          {
-            key: 'TIME',
-            title: 'Time',
-            getRowVal: s => s.code,
-            getRowContent: s => (
-              <div>
-                {s.name}
-                {s.name === 'Total' && !userHasAccessToAllStats && <strong>*</strong>}
-                {s.name !== 'Total' && userHasAccessToAllStats && (
-                  <Item as={Link} to={showPopulation(s.code, s.name, s)}>
-                    <Icon name="level up alternate" />
-                  </Item>
-                )}
-              </div>
-            ),
-            getCellProps: s => defineCellColor(s),
-            headerProps: { rowSpan: 2, width: 3 },
-          },
-          {
-            key: 'TOTAL',
-            title: 'All students',
-            getRowVal: s => (s.rowObfuscated ? '5 or less students' : s.students.total),
-            getCellProps: s => defineCellColor(s),
-            headerProps: { rowSpan: 2, width: 3 },
-          },
-          {
-            key: 'PASSED',
-            title: 'Passed',
-            parent: true,
-            headerProps: { colSpan: 3, width: 5 },
-          },
-          {
-            key: 'PASS_FIRST',
-            title: 'first try',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedFirst || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'PASS_EVENTUALLY',
-            title: 'passed eventually',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.passedEventually || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'PASS_RATE',
-            title: 'percentage',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.passRate),
-            getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.passRate || 0)),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 1 },
-            child: true,
-          },
-          {
-            key: 'FAIL',
-            title: 'Failed',
-            parent: true,
-            headerProps: { colSpan: 3, width: 5 },
-          },
-          {
-            key: 'NEVER PASSED',
-            title: 'never passed',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.categories.neverPassed || 0),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 2 },
-            child: true,
-          },
-          {
-            key: 'FAIL_RATE',
-            title: 'percentage',
-            getRowVal: s => (s.rowObfuscated ? 'NA' : s.students.failRate),
-            getRowContent: s => (s.rowObfuscated ? 'NA' : formatPercentage(s.students.failRate || 0)),
-            getCellProps: s => defineCellColor(s),
-            cellProps: { width: 1 },
-            child: true,
-          },
-        ]}
+        tableProps={{ celled: true, fixed: true, style: { width: 'auto' } }}
+        columns={columns}
         data={stats}
       />
       {!userHasAccessToAllStats && (

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/distribution.jsx

@@ -1,12 +1,11 @@
 /* eslint-disable quotes */
 import React from 'react'
-import { Grid } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { Menu, Radio } from 'semantic-ui-react'
+import HelpButton from '../HelpButton'
 import { gradeGraphOptions } from '../../../../constants'
 import {
   getDataObject,
   getMaxValueOfSeries,
-  dataSeriesType,
   getGradeSpread,
   getThesisGradeSpread,
   isThesisSeries,
@@ -71,61 +70,47 @@ const getGradeSeries = series => {
       }
 }
 
-const Distribution = ({ primary, comparison, isRelative, userHasAccessToAllStats }) => {
-  const primaryStats = primary.stats.filter(stat => stat.name !== 'Total' || isRelative)
-
-  const statYears = primaryStats.map(year => year.name)
-  const comparisonStats = comparison ? comparison.stats.filter(stat => stat.name !== 'Total' || isRelative) : []
-  const primaryGrades = primaryStats.flatMap(s => s.attempts.grades)
-  const comparisonGrades = comparisonStats.flatMap(s => s.attempts.grades)
-
-  const gradeGraphSeries = getGradeSeries(primaryGrades)
-  const comparisonGraphSeries = getGradeSeries(comparisonGrades)
-
-  const maxGradeValue = isRelative ? 1 : getMaxValueOfSeries(gradeGraphSeries.absolute)
-
-  const primaryDistributionOptions = comparison
-    ? gradeGraphOptions(statYears, maxGradeValue, 'Primary Grades')
-    : gradeGraphOptions(statYears, maxGradeValue, 'Grades')
-  const comparisonDistributionOptions = gradeGraphOptions(statYears, maxGradeValue, 'Comparison Grades')
+export const DistributionSettings = ({ value, onChange }) => {
+  const { isRelative } = value
 
   return (
-    <>
-      <Grid.Row>
-        <Grid.Column>
-          <StackedBarChart
-            options={primaryDistributionOptions}
-            series={isRelative ? gradeGraphSeries.relative : gradeGraphSeries.absolute}
-          />
-        </Grid.Column>
-      </Grid.Row>
-      {comparison && (
-        <Grid.Row>
-          <Grid.Column>
-            <StackedBarChart
-              options={comparisonDistributionOptions}
-              series={isRelative ? comparisonGraphSeries.relative : comparisonGraphSeries.absolute}
-            />
-          </Grid.Column>
-        </Grid.Row>
-      )}
-      {!userHasAccessToAllStats && (
-        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
-      )}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show relative"
+          checked={isRelative}
+          onChange={() => onChange({ ...value, isRelative: !isRelative })}
+        />
+        <Menu.Item>
+          <HelpButton tab="GradeDistribution" />
+        </Menu.Item>
+      </Menu.Item>
+    </Menu>
   )
 }
 
-Distribution.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  isRelative: bool,
-  userHasAccessToAllStats: bool.isRequired,
-}
+export const Distribution = ({ data, settings: { isRelative }, userHasAccessToAllStats }) => {
+  const stats = data.stats.filter(stat => stat.name !== 'Total' || isRelative)
 
-Distribution.defaultProps = {
-  comparison: undefined,
-  isRelative: false,
-}
+  const statYears = stats.map(year => year.name)
+  const grades = stats.flatMap(s => s.attempts.grades)
 
-export default Distribution
+  const gradeGraphSeries = getGradeSeries(grades)
+
+  const maxGradeValue = isRelative ? 1 : getMaxValueOfSeries(gradeGraphSeries.absolute)
+
+  const primaryDistributionOptions = gradeGraphOptions(statYears, maxGradeValue, 'Grades')
+
+  return (
+    <div>
+      <StackedBarChart
+        options={primaryDistributionOptions}
+        series={isRelative ? gradeGraphSeries.relative : gradeGraphSeries.absolute}
+      />
+      {!userHasAccessToAllStats && (
+        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
+      )}
+    </div>
+  )
+}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/passRate.jsx

@@ -1,17 +1,10 @@
 /* eslint-disable quotes */
 import React from 'react'
-import { Grid } from 'semantic-ui-react'
-import { bool } from 'prop-types'
+import { Menu, Radio } from 'semantic-ui-react'
 import StackedBarChart from '../../../StackedBarChart'
 import { passRateAttemptGraphOptions, passRateStudGraphOptions } from '../../../../constants'
-import {
-  viewModeNames,
-  getDataObject,
-  getMaxValueOfSeries,
-  dataSeriesType,
-  viewModeType,
-  absoluteToRelative,
-} from './util'
+import HelpButton from '../HelpButton'
+import { viewModeNames, getDataObject, getMaxValueOfSeries, absoluteToRelative } from './util'
 
 const getPassRateAttemptSeriesFromStats = stats => {
   const all = []
@@ -70,62 +63,62 @@ const getPassRateStudSeriesFromStats = stats => {
   }
 }
 
-const PassRate = ({ primary, comparison, viewMode, isRelative = false, userHasAccessToAllStats }) => {
-  const isAttemptsMode = viewMode === viewModeNames.ATTEMPTS
-
-  const primaryStats = primary.stats.filter(stat => stat.name !== 'Total')
-  const statYears = primaryStats.map(year => year.name)
-  const comparisonStats = comparison ? comparison.stats : []
-  const passGraphSerieFn = isAttemptsMode ? getPassRateAttemptSeriesFromStats : getPassRateStudSeriesFromStats
-
-  const passGraphSerie = passGraphSerieFn(primaryStats)
-  const comparisonGraphSerie = passGraphSerieFn(comparisonStats)
-
-  const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
-  const graphOptionsFn = isAttemptsMode ? passRateAttemptGraphOptions : passRateStudGraphOptions
-  const primaryGraphOptions = comparison
-    ? graphOptionsFn(statYears, maxPassRateVal, 'Primary pass rate chart', isRelative)
-    : graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
-  const comparisonGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Comparison pass rate chart', isRelative)
+export const PassRateSettings = ({ value, onChange }) => {
+  const { viewMode, isRelative } = value
 
   return (
-    <>
-      <Grid.Row>
-        <Grid.Column>
-          <StackedBarChart
-            options={primaryGraphOptions}
-            series={isRelative ? passGraphSerie.relative : passGraphSerie.absolute}
-          />
-        </Grid.Column>
-      </Grid.Row>
-      {comparison && (
-        <Grid.Row>
-          <Grid.Column>
-            <StackedBarChart
-              options={comparisonGraphOptions}
-              series={isRelative ? comparisonGraphSerie.relative : comparisonGraphSerie.absolute}
-            />
-          </Grid.Column>
-        </Grid.Row>
-      )}
-      {!userHasAccessToAllStats && (
-        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
-      )}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      {Object.entries(viewModeNames).map(([key, name]) => (
+        <Menu.Item
+          active={viewMode === key}
+          name={name}
+          onClick={() =>
+            onChange({
+              ...value,
+              viewMode: key,
+            })
+          }
+        />
+      ))}
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show relative"
+          checked={isRelative}
+          onChange={() => onChange({ ...value, isRelative: !isRelative })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <HelpButton tab="PassRate" viewMode={viewMode} />
+      </Menu.Item>
+    </Menu>
   )
 }
 
-PassRate.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  viewMode: viewModeType.isRequired,
-  isRelative: bool,
-  userHasAccessToAllStats: bool.isRequired,
-}
+export const PassRate = ({ data, settings: { viewMode, isRelative }, userHasAccessToAllStats }) => {
+  const isAttemptsMode = viewMode === 'ATTEMPTS'
 
-PassRate.defaultProps = {
-  comparison: undefined,
-  isRelative: false,
-}
+  const stats = data.stats.filter(stat => stat.name !== 'Total')
+  const statYears = stats.map(year => year.name)
+  // const comparisonStats = comparison ? comparison.stats : []
+  const passGraphSerieFn = isAttemptsMode ? getPassRateAttemptSeriesFromStats : getPassRateStudSeriesFromStats
 
-export default PassRate
+  const passGraphSerie = passGraphSerieFn(stats)
+  // const comparisonGraphSerie = passGraphSerieFn(comparisonStats)
+
+  const maxPassRateVal = isRelative ? 1 : getMaxValueOfSeries(passGraphSerie.absolute)
+  const graphOptionsFn = isAttemptsMode ? passRateAttemptGraphOptions : passRateStudGraphOptions
+  const primaryGraphOptions = graphOptionsFn(statYears, maxPassRateVal, 'Pass rate chart', isRelative)
+
+  return (
+    <div>
+      <StackedBarChart
+        options={primaryGraphOptions}
+        series={isRelative ? passGraphSerie.relative : passGraphSerie.absolute}
+      />
+      {!userHasAccessToAllStats && (
+        <span className="totalsDisclaimer">* Years with 5 students or less are shown as 0 in the chart</span>
+      )}
+    </div>
+  )
+}

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/tables.jsx

@@ -1,85 +1,61 @@
-import React, { Fragment } from 'react'
-import { connect } from 'react-redux'
-import { Grid } from 'semantic-ui-react'
-import { string, arrayOf, bool } from 'prop-types'
-import { dataSeriesType, viewModeNames, viewModeType } from './util'
+import React from 'react'
+import { useSelector } from 'react-redux'
+import { Menu, Radio } from 'semantic-ui-react'
+import { viewModeNames } from './util'
+import HelpButton from '../HelpButton'
 import StudentTable from './Tables/student'
 import AttemptsTable from './Tables/attempts'
 
-const Tables = ({
-  primary,
-  comparison,
-  viewMode,
-  alternatives,
-  separate,
-  isRelative,
-  showGrades,
-  userHasAccessToAllStats,
-}) => {
-  const getViewMode = statistics => {
-    const { name, stats } = statistics
-    const headerVisible = !!comparison
-
-    switch (viewMode) {
-      case viewModeNames.ATTEMPTS:
-        return (
-          <AttemptsTable
-            separate={separate}
-            name={name}
-            stats={stats}
-            alternatives={alternatives}
-            isRelative={isRelative}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-            headerVisible={headerVisible}
-            showGrades={showGrades}
-          />
-        )
-      case viewModeNames.STUDENT:
-        return (
-          <StudentTable
-            separate={separate}
-            name={name}
-            stats={stats}
-            alternatives={alternatives}
-            userHasAccessToAllStats={userHasAccessToAllStats}
-            headerVisible={headerVisible}
-          />
-        )
-      default:
-        return null
-    }
-  }
+export const TablesSettings = ({ value, onChange }) => {
+  const { viewMode, showDetails, showGrades } = value
 
   return (
-    <>
-      <Grid.Column id="PrimaryDataTable">{getViewMode(primary)}</Grid.Column>
-      {comparison && <Grid.Column id="ComparisonDataTable">{getViewMode(comparison)}</Grid.Column>}
-    </>
+    <Menu secondary style={{ marginBottom: 0 }}>
+      {Object.entries(viewModeNames).map(([key, name]) => (
+        <Menu.Item
+          active={viewMode === key}
+          data-cy={`viewMode-${name}`}
+          name={name}
+          onClick={() =>
+            onChange({
+              ...value,
+              viewMode: key,
+            })
+          }
+        />
+      ))}
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show details"
+          disabled={viewMode !== 'STUDENT'}
+          checked={showDetails}
+          onChange={() => onChange({ ...value, showDetails: !showDetails })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <Radio
+          toggle
+          label="Show grades"
+          data-cy="gradeToggle"
+          disabled={viewMode !== 'ATTEMPTS'}
+          checked={showGrades}
+          onChange={() => onChange({ ...value, showGrades: !showGrades })}
+        />
+      </Menu.Item>
+      <Menu.Item>
+        <HelpButton tab="Tables" viewMode={viewMode} />
+      </Menu.Item>
+    </Menu>
   )
 }
 
-Tables.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  viewMode: viewModeType.isRequired,
-  alternatives: arrayOf(string).isRequired,
-  separate: bool,
-  isRelative: bool.isRequired,
-  showGrades: bool.isRequired,
-  userHasAccessToAllStats: bool.isRequired,
+export const Tables = props => {
+  const alternatives = useSelector(state => state.courseStats.data[state.singleCourseStats.selectedCourse].alternatives)
+
+  const viewModes = { ATTEMPTS: AttemptsTable, STUDENT: StudentTable }
+
+  const Content = viewModes[props.settings.viewMode]
+
+  return <Content {...props} alternatives={alternatives} />
 }
-
-Tables.defaultProps = {
-  comparison: null,
-  separate: false,
-}
-
-const mapStateToProps = state => {
-  const { selectedCourse } = state.singleCourseStats
-
-  return {
-    alternatives: state.courseStats.data[selectedCourse].alternatives,
-  }
-}
-
-export default connect(mapStateToProps)(Tables)

services/frontend/src/components/CourseStatistics/ResultTabs/Panes/util.js

@@ -1,8 +1,8 @@
 import { arrayOf, number, oneOfType, shape, string, oneOf } from 'prop-types'
 
 export const viewModeNames = {
-  ATTEMPTS: 'Attempts',
   STUDENT: 'Students',
+  ATTEMPTS: 'Attempts',
 }
 
 export const getDataObject = (name, data, stack) => ({ name, data, stack })

services/frontend/src/components/CourseStatistics/ResultTabs/index.jsx

@@ -1,62 +1,91 @@
-import React, { useEffect, useState } from 'react'
-import { Tab, Grid, Radio, Menu } from 'semantic-ui-react'
+import React, { useState } from 'react'
+import { Tab, Segment, Menu, Icon } from 'semantic-ui-react'
 import { withRouter } from 'react-router-dom'
-import { shape, bool } from 'prop-types'
-import { dataSeriesType, viewModeNames } from './Panes/util'
-import PassRate from './Panes/passRate'
-import Distribution from './Panes/distribution'
-import Tables from './Panes/tables'
-import InfoBox from '../../Info/InfoBox'
+import { PassRate, PassRateSettings } from './Panes/passRate'
+import { Distribution, DistributionSettings } from './Panes/distribution'
+import { Tables, TablesSettings } from './Panes/tables'
 import { useTabs } from '../../../common/hooks'
 
 import './resultTabs.css'
 import TSA from '../../../common/tsa'
-import infotooltips from '../../../common/InfoToolTips'
 
 const ANALYTICS_CATEGORY = 'Course Statistics'
 const sendAnalytics = (action, name, value) => TSA.Matomo.sendEvent(ANALYTICS_CATEGORY, action, name, value)
 
-const paneViewIndex = {
-  TABLE: 0,
-  PASS_RAGE: 1,
-  GRADE_DISTRIBUTION: 2,
+const PaneContent = ({ component: Component, settings: SettingsComponent, initialSettings, datasets, ...rest }) => {
+  const [settings, setSettings] = useState(initialSettings)
+  const [splitDirection, setSplitDirection] = useState('column')
+
+  return (
+    <Tab.Pane>
+      <Segment basic>
+        <div style={{ display: 'flex', marginBottom: '2em' }}>
+          <SettingsComponent value={settings} onChange={setSettings} />
+          <div style={{ flexGrow: 1 }} />
+          {datasets.filter(i => i).length > 1 && (
+            <div style={{ display: 'flex', alignItems: 'center', gap: '1em' }}>
+              <label>Split direction: </label>
+              <Menu style={{ margin: 0 }}>
+                <Menu.Item active={splitDirection === 'row'} onClick={() => setSplitDirection('row')}>
+                  <Icon name="arrows alternate horizontal" />
+                </Menu.Item>
+                <Menu.Item active={splitDirection === 'column'} onClick={() => setSplitDirection('column')}>
+                  <Icon name="arrows alternate vertical" />
+                </Menu.Item>
+              </Menu>
+            </div>
+          )}
+        </div>
+        <div style={{ display: 'flex', flexDirection: splitDirection, gap: '2em' }}>
+          {datasets
+            .filter(i => i)
+            .map(data => (
+              <div style={{ flexGrow: 1, flexBasis: 1 }}>
+                <h3>{data.name}</h3>
+                <Component data={data} settings={settings} {...rest} />
+              </div>
+            ))}
+        </div>
+      </Segment>
+    </Tab.Pane>
+  )
 }
 
-const ResultTabs = props => {
-  const [tab, setTab] = useTabs('cs_tab', 0, props.history)
-  const [viewMode, setViewMode] = useState(viewModeNames.ATTEMPTS)
-  const [selectedView, setSelectedView] = useState(true)
-  const [isRelative, setIsRelative] = useState(false)
-  const [showGrades, setShowGrades] = useState(false)
+const ResultTabs = ({ primary, comparison, history }) => {
+  const [tab, setTab] = useTabs('cs_tab', 0, history)
+  // const [viewMode, setViewMode] = useState(viewModeNames.STUDENT)
+  // const [selectedView, setSelectedView] = useState(false)
+  // const [isRelative, setIsRelative] = useState(false)
+  // const [showGrades, setShowGrades] = useState(false)
+  const { userHasAccessToAllStats } = primary
 
-  const handleModeChange = newViewMode => {
+  /* const handleModeChange = newViewMode => {
     sendAnalytics(`Current view mode '${newViewMode}'`, 'Course statistics')
     setViewMode(newViewMode)
-  }
+  } */
 
-  useEffect(() => {
+  /* useEffect(() => {
     const newViewMode = selectedView ? viewModeNames.ATTEMPTS : viewModeNames.STUDENT
     handleModeChange(newViewMode)
-  }, [selectedView])
+  }, [selectedView]) */
 
   const handleTabChange = (...params) => {
-    const resetViewMode = params[1].activeIndex === paneViewIndex.TABLE && viewMode === viewModeNames.ATTEMPTS
     const { activeIndex } = params[1]
     const currentTab = params[1].panes[activeIndex]
     sendAnalytics(`Current tab '${currentTab.menuItem.content}'`, 'Course statistics')
     setTab(...params)
-    setViewMode(resetViewMode ? viewMode : viewModeNames.ATTEMPTS)
+    // setViewMode(resetViewMode ? viewMode : viewModeNames.ATTEMPTS)
   }
 
-  const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
+  /* const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
     <div className="toggleContainer">
       <label className="toggleLabel">{firstLabel}</label>
       <Radio toggle data-cy="gradeToggle" checked={value} onChange={() => setValue(!value)} />
       <label className="toggleLabel">{secondLabel}</label>
     </div>
-  )
+  ) */
 
-  const renderViewModeSelector = () => {
+  /* const renderViewModeSelector = () => {
     const isTogglePane = tab !== 0
     const getButtonMenu = () => (
       <Menu secondary>
@@ -72,7 +101,7 @@ const ResultTabs = props => {
       return (
         <div className="chartToggleContainer">
           {tab === 1 && getRadioButton('Student', 'Attempts', selectedView, setSelectedView)}
-          {(tab === 2 || props.comparison) && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
+          {(tab === 2 || comparison) && getRadioButton('Absolute', 'Relative', isRelative, setIsRelative)}
         </div>
       )
     }
@@ -84,16 +113,13 @@ const ResultTabs = props => {
         <InfoBox content={infotooltips.CourseStatistics[tab][viewMode]} />
       </div>
     )
-  }
+  } */
 
-  const getPanes = () => {
-    const { primary, comparison, separate } = props
-    const { userHasAccessToAllStats } = primary
-
-    const paneMenuItems = [
+  /* const getPanes = () => {
+    return [
       {
         menuItem: { key: 'Table', icon: 'table', content: 'Table' },
-        renderFn: () => (
+        render: () => (
           <Tables
             separate={separate}
             comparison={comparison}
@@ -107,7 +133,7 @@ const ResultTabs = props => {
       },
       {
         menuItem: { key: 'pass', icon: 'balance', content: 'Pass rate chart' },
-        renderFn: () => (
+        render: () => (
           <PassRate
             comparison={comparison}
             primary={primary}
@@ -119,7 +145,7 @@ const ResultTabs = props => {
       },
       {
         menuItem: { key: 'grade', icon: 'chart bar', content: 'Grade distribution chart' },
-        renderFn: () => (
+        render: () => (
           <Distribution
             comparison={comparison}
             primary={primary}
@@ -128,7 +154,7 @@ const ResultTabs = props => {
           />
         ),
       },
-    ]
+    ];
 
     return paneMenuItems.map(p => {
       const { menuItem, renderFn } = p
@@ -142,25 +168,52 @@ const ResultTabs = props => {
         ),
       }
     })
-  }
+  } */
+
+  const paneTypes = [
+    {
+      label: 'Tables',
+      icon: 'table',
+      initialSettings: { showDetails: false, viewMode: 'STUDENT' },
+      settings: TablesSettings,
+      component: Tables,
+    },
+    {
+      label: 'Pass rate chart',
+      icon: 'balance',
+      initialSettings: { viewMode: 'STUDENT' },
+      settings: PassRateSettings,
+      component: PassRate,
+    },
+    {
+      label: 'Grade distribution chart',
+      icon: 'chart bar',
+      initialSettings: { isRelative: false, viewMode: 'STUDENT' },
+      settings: DistributionSettings,
+      component: Distribution,
+    },
+  ]
+
+  const panes = paneTypes.map(
+    ({ icon, label, initialSettings, component: Component, settings: SettingsComponent }) => ({
+      menuItem: { icon, content: label },
+      render: () => (
+        <PaneContent
+          component={Component}
+          settings={SettingsComponent}
+          userHasAccessToAllStats={userHasAccessToAllStats}
+          initialSettings={initialSettings}
+          datasets={[primary, comparison]}
+        />
+      ),
+    })
+  )
 
   return (
     <div>
-      <Tab id="CourseStatPanes" panes={getPanes()} onTabChange={handleTabChange} activeIndex={tab} />
+      <Tab id="CourseStatPanes" panes={panes} onTabChange={handleTabChange} activeIndex={tab} />
     </div>
   )
 }
 
-ResultTabs.propTypes = {
-  primary: dataSeriesType.isRequired,
-  comparison: dataSeriesType,
-  history: shape({}).isRequired,
-  separate: bool,
-}
-
-ResultTabs.defaultProps = {
-  comparison: undefined,
-  separate: false,
-}
-
 export default withRouter(ResultTabs)

services/frontend/src/components/CreditAccumulationGraphHighCharts/index.jsx

@@ -226,7 +226,7 @@ const CreditAccumulationGraphHighCharts = ({ students, singleStudent, absences,
   const seriesData = useMemo(() => createStudentCreditLines(students, singleStudent), [students, singleStudent])
 
   if (singleStudent) {
-    const starting = _.chain(students[0].studyRights || students[0].courses)
+    const starting = _.chain(students[0].studyrights || students[0].courses)
       .map(elem => new Date(elem.startdate || elem.date))
       .sortBy()
       .head()

services/frontend/src/components/Frontpage/OoodiToOodikone.jsx

@@ -0,0 +1,46 @@
+import React, { useState, useEffect } from 'react'
+import oodiTXT from '../../static/oodi.txt'
+import './oodiToOodikone.css'
+
+const OodiToOodikone = () => {
+  const [oodis, setOodis] = useState([])
+  useEffect(() => {
+    const getOodis = async () => {
+      try {
+        const data = await fetch(oodiTXT)
+        setOodis(
+          (await data.text())
+            .split('**')
+            .map(o => o.trim())
+            .filter(oodi => !!oodi)
+        )
+      } catch (error) {
+        setOodis([])
+      }
+    }
+    getOodis()
+  }, [])
+
+  if (!oodis || oodis.length === 0) {
+    return null
+  }
+  const oodi = oodis[Math.floor(Math.random() * oodis.length)].split('\n')
+  return (
+    <blockquote cite="http://www.helsinki.fi/discovery">
+      <div style={{ textAlign: 'center' }}>
+        {oodi.map((l, index) => (
+          // eslint-disable-next-line react/no-array-index-key
+          <p key={index} style={{ marginBottom: '0.6rem' }}>
+            {l}
+          </p>
+        ))}
+      </div>
+
+      <p style={{ marginTop: '1.3em', textAlign: 'center' }}>
+        - <a href="http://www.helsinki.fi/discovery">R. U. Nokone</a>
+      </p>
+    </blockquote>
+  )
+}
+
+export default OodiToOodikone

services/frontend/src/components/Frontpage/index.jsx

@@ -1,51 +1,11 @@
-import React, { useEffect, useState } from 'react'
+import React from 'react'
 import { Container, Header, Image, Divider, List } from 'semantic-ui-react'
 import moment from 'moment'
 import { useGetAuthorizedUserQuery } from 'redux/auth'
-import { images, checkUserAccess } from '../common'
-import { useTitle } from '../common/hooks'
-import { builtAt } from '../conf'
-import oodiTXT from '../static/oodi.txt'
-
-const OodiToOodikone = () => {
-  const [oodis, setOodis] = useState([])
-  useEffect(() => {
-    const getOodis = async () => {
-      try {
-        const data = await fetch(oodiTXT)
-        setOodis(
-          (await data.text())
-            .split('**')
-            .map(o => o.trim())
-            .filter(oodi => !!oodi)
-        )
-      } catch (error) {
-        setOodis([])
-      }
-    }
-    getOodis()
-  }, [])
-
-  if (!oodis || oodis.length === 0) {
-    return null
-  }
-  const oodi = oodis[Math.floor(Math.random() * oodis.length)].split('\n')
-  return (
-    <div style={{ margin: 'auto', width: '50%' }}>
-      {oodi.map((l, index) => (
-        // eslint-disable-next-line react/no-array-index-key
-        <p key={index} style={{ textAlign: 'center', fontStyle: 'italic', margin: 0 }}>
-          {index === 0 && '\u201C'}
-          {l}
-          {index === oodi.length - 1 && '\u201D'}
-        </p>
-      ))}
-      <p style={{ marginTop: '1.3em', textAlign: 'center' }}>
-        - <a href="http://www.helsinki.fi/discovery">R. U. Nokone</a>
-      </p>
-    </div>
-  )
-}
+import { images, checkUserAccess } from '../../common'
+import { useTitle } from '../../common/hooks'
+import { builtAt } from '../../conf'
+import OodiToOodikone from './OoodiToOodikone'
 
 const FrontPage = () => {
   const { rights, roles } = useGetAuthorizedUserQuery()

services/frontend/src/components/Frontpage/oodiToOodikone.css

@@ -0,0 +1,31 @@
+blockquote {
+  font-style: italic;
+  line-height: 1.618;
+  font-size: 1.2em;
+  width: 30em;
+  position: relative;
+  padding: 40px 20px;
+}
+blockquote::before {
+  content: open-quote;
+  top: 0;
+  left: 0;
+}
+
+blockquote::after {
+  content: close-quote;
+  bottom: 0;
+  right: 0;
+}
+
+blockquote::before,
+blockquote::after {
+  display: block;
+  width: 60px;
+  height: 60px;
+  line-height: 1.618;
+  font-size: 2em;
+  border-radius: 100%;
+  text-align: center;
+  position: absolute;
+}

services/frontend/src/components/LanguagePicker/useLanguage.jsx

@@ -7,22 +7,25 @@ const LanguageContext = createContext([[], () => {}])
 LanguageContext.displayName = 'Language'
 
 export const LanguageProvider = ({ children }) => {
-  const [state, setState] = useState(LANGUAGE_CODES[0])
+  let language = 'fi'
+  const [state, setState] = useState(language)
   const user = useGetAuthorizedUserQuery()
+  if (user) {
+    language = user.language
+  }
 
   // Load selected language.
   useEffect(() => {
     if (LANGUAGE_CODES.includes(user?.language)) {
       setState(user.language)
     }
-  }, [user])
+  }, [language])
 
   return <LanguageContext.Provider value={[state, setState]}>{children}</LanguageContext.Provider>
 }
 
 export default () => {
   const [state, setState] = useContext(LanguageContext)
-
   const setLanguage = newLanguage => {
     if (!LANGUAGE_CODES.includes(newLanguage)) {
       throw new Error('Illegal language code passed to useLanguage hook!')

services/frontend/src/components/NavigationBar/index.jsx

@@ -66,7 +66,7 @@ const NavigationBar = () => {
 
   // Min-content sets logo always to the left
   const renderHome = () => (
-    <Menu.Item as={Link} to="/" tabIndex="-1" style={{ width: 'min-content' }}>
+    <Menu.Item as={Link} to="/" tabIndex="-1">
       <span className="logo">
         <h2 className="logoText">oodikone</h2>
       </span>

services/frontend/src/components/PopulationCourseStats/CollapsibleModuleTable.jsx

@@ -1,11 +1,11 @@
 import React from 'react'
 import { Table, Icon } from 'semantic-ui-react'
 import { func, arrayOf, object, number, instanceOf } from 'prop-types'
-import { useLanguage } from '../../common/hooks'
+import useLanguage from '../LanguagePicker/useLanguage'
 import { getTextIn } from '../../common'
 
 const CollapsibleModuleTable = ({ modules, emptyColSpan, children, expandedGroups, toggleGroupExpansion }) => {
-  const language = useLanguage()
+  const { language } = useLanguage()
 
   if (!modules) return null
 

services/frontend/src/components/PopulationCourseStats/PassFail/PassFailRow.jsx

@@ -5,12 +5,12 @@ import { shape, string } from 'prop-types'
 import { getTextIn } from '../../../common'
 import FilterToggleIcon from '../../FilterToggleIcon'
 import { UsePopulationCourseContext } from '../PopulationCourseContext'
-import { useLanguage } from '../../../common/hooks'
+import useLanguage from '../../LanguagePicker/useLanguage'
 import { isCourseSelected, toggleCourseSelection } from '../../FilterView/filters/courses'
 import useFilters from '../../FilterView/useFilters'
 
 const PassFailRow = ({ courseStats }) => {
-  const language = useLanguage()
+  const { language } = useLanguage()
   const { useFilterSelector, filterDispatch } = useFilters()
   const { onGoToCourseStatisticsClick } = UsePopulationCourseContext()
   const { course, stats } = courseStats

services/frontend/src/components/Routes/index.jsx

@@ -26,7 +26,7 @@ const retry = async (fn, retriesLeft = 3, interval = 500) =>
       })
   })
 
-const FrontPage = React.lazy(() => retry(() => import('../FrontPage')))
+const FrontPage = React.lazy(() => retry(() => import('../Frontpage')))
 const Populations = React.lazy(() => retry(() => import('../PopulationStatistics')))
 const StudentStatistics = React.lazy(() => retry(() => import('../StudentStatistics')))
 const CourseStatistics = React.lazy(() => retry(() => import('../CourseStatistics')))

services/frontend/src/components/StudyProgramme/BachelorsTable/index.jsx

@@ -1,114 +0,0 @@
-import React, { useState, useEffect } from 'react'
-import { Table, Header, Grid, Segment } from 'semantic-ui-react'
-import { shape, any, arrayOf, bool, oneOfType, array } from 'prop-types'
-
-const DIRECTIONS = {
-  ASC: 'ascending', // true
-  DESC: 'descending', // false
-}
-
-function sortBy(column, data) {
-  const sortByString = column === 'name' || column === 'code'
-  return data.concat().sort((a, b) => {
-    const keyA = column === 'name' ? a[column].fi : a[column]
-    const keyB = column === 'name' ? b[column].fi : b[column]
-    return sortByString ? keyB.localeCompare(keyA) : keyB - keyA
-  })
-}
-
-const BachelorsTable = ({ bachelors, loading }) => {
-  const [bachelorsData, setBachelorsData] = useState({})
-  const [sortColumn, setSortColumn] = useState(null)
-  const [direction, setDirection] = useState(false)
-
-  useEffect(() => setBachelorsData(bachelors), [bachelors])
-
-  useEffect(() => {
-    if (!bachelorsData || !bachelorsData.data) return
-    const data = sortBy(sortColumn, bachelorsData.data)
-    if (direction) data.reverse()
-    setBachelorsData({ ...bachelorsData, data })
-  }, [direction, sortColumn])
-
-  const getSorted = column => {
-    if (column !== sortColumn) return null
-    return direction ? DIRECTIONS.ASC : DIRECTIONS.DESC
-  }
-
-  const setSorted = column => {
-    if (column === sortColumn) setDirection(!direction)
-    else {
-      setSortColumn(column)
-      setDirection(false)
-    }
-  }
-
-  if (!loading && (!bachelorsData.data || !bachelorsData.data.length)) return null
-  return (
-    <>
-      <Header>
-        <Grid columns={2}>
-          <Grid.Row>
-            <Grid.Column>Study programme options</Grid.Column>
-          </Grid.Row>
-        </Grid>
-      </Header>
-      {bachelorsData && bachelorsData.years && `Statistics from ${bachelorsData.years.length} years`}
-      <Segment basic loading={loading} style={{ overflowX: 'auto' }}>
-        <Table celled structured compact striped selectable sortable>
-          {bachelorsData && bachelorsData.data ? (
-            <>
-              <Table.Header>
-                <Table.Row>
-                  <Table.HeaderCell sorted={getSorted('name')} onClick={() => setSorted('name')}>
-                    Name
-                  </Table.HeaderCell>
-                  <Table.HeaderCell sorted={getSorted('code')} onClick={() => setSorted('code')}>
-                    Code
-                  </Table.HeaderCell>
-                  {bachelorsData.years.map(year => (
-                    <Table.HeaderCell sorted={getSorted(year)} onClick={() => setSorted(year)} key={year}>
-                      {year}
-                    </Table.HeaderCell>
-                  ))}
-                  <Table.HeaderCell sorted={getSorted('total')} onClick={() => setSorted('total')}>
-                    Total
-                  </Table.HeaderCell>
-                </Table.Row>
-              </Table.Header>
-              <Table.Body>
-                {bachelorsData.data.map(item => (
-                  <Table.Row key={item.code}>
-                    <Table.Cell>{item.name.fi}</Table.Cell>
-                    <Table.Cell>{item.code}</Table.Cell>
-                    {bachelorsData.years.map(year => (
-                      <Table.Cell key={year}>{item[year]}</Table.Cell>
-                    ))}
-                    <Table.Cell>{item.total}</Table.Cell>
-                  </Table.Row>
-                ))}
-              </Table.Body>
-            </>
-          ) : null}
-        </Table>
-      </Segment>
-    </>
-  )
-}
-
-BachelorsTable.propTypes = {
-  bachelors: oneOfType([
-    shape({
-      data: arrayOf(any),
-      years: arrayOf(any),
-    }),
-    array,
-  ]),
-  loading: bool.isRequired,
-}
-
-BachelorsTable.defaultProps = {
-  bachelors: null,
-}
-
-export default BachelorsTable

services/frontend/src/components/StudyProgramme/BasicOverview/BarChart.jsx

@@ -4,7 +4,7 @@ const ReactHighcharts = require('react-highcharts')
 
 const colors = ['#003E65', '#1392c2', '#036415']
 
-const BarChart = ({ data }) => {
+const BarChart = ({ cypress, data }) => {
   const dataWithColors = data?.graphStats?.map((series, index) => ({ ...series, color: colors[index] }))
 
   const defaultConfig = {
@@ -51,7 +51,7 @@ const BarChart = ({ data }) => {
 
   if (!data) return <></>
   return (
-    <div className="graph-container">
+    <div className="graph-container" data-cy={`Graph-${cypress}`}>
       <ReactHighcharts config={defaultConfig} />
     </div>
   )

services/frontend/src/components/StudyProgramme/BasicOverview/DataTable.jsx

@@ -1,12 +1,12 @@
 import React from 'react'
 import { Table } from 'semantic-ui-react'
 
-const DataTable = ({ data, titles }) => {
+const DataTable = ({ data, cypress, titles, wideTable }) => {
   if (!data || !titles) return null
 
   return (
-    <div className="table-container">
-      <Table celled>
+    <div className={`table-container${wideTable ? '-wide' : ''}`}>
+      <Table data-cy={`Table-${cypress}`} celled>
         <Table.Header>
           <Table.Row>
             {titles?.map(title => (

services/frontend/src/components/StudyProgramme/BasicOverview/GaugeChart.jsx

@@ -0,0 +1,82 @@
+import React from 'react'
+
+const ReactHighcharts = require('react-highcharts')
+
+const GaugeChart = ({ cypress, data, year, graduationAmount, studyprogramme }) => {
+  if (!data || !graduationAmount || !data.length) return null
+
+  const thresholdValues = studyprogramme.includes('KH') ? [36, 41] : [24, 27]
+
+  const getColor = data => {
+    if (data <= thresholdValues[0]) return ['#90A959', '#EBECF0']
+    if (data > thresholdValues[0] && data <= thresholdValues[1]) return ['#FEE191', '#EBECF0']
+    return ['#FB6962', '#EBECF0']
+  }
+
+  if (!data) return null
+  const defaultConfig = {
+    chart: {
+      plotBackgroundColor: null,
+      plotBorderWidth: 0,
+      plotShadow: false,
+      width: 260,
+      height: 300,
+    },
+    title: {
+      text: `${data[0][1]} kk`,
+      align: 'center',
+      verticalAlign: 'middle',
+      y: 60,
+    },
+    subtitle: {
+      text: `${year}<br/> n = ${graduationAmount}`,
+      style: {
+        fontWeight: 'bold',
+        fontSize: '15px',
+      },
+    },
+    credits: {
+      text: '',
+    },
+    exporting: {
+      enabled: false,
+    },
+    accessibility: {
+      point: {
+        valueSuffix: '%',
+      },
+    },
+    plotOptions: {
+      pie: {
+        dataLabels: {
+          title: '{point.percentage:.1f}%',
+          distance: -50,
+          style: {
+            fontWeight: 'bold',
+            color: 'white',
+          },
+        },
+        startAngle: -90,
+        endAngle: 90,
+        center: ['50%', '75%'],
+        size: '100%',
+        colors: getColor(data[0][1]),
+      },
+    },
+    tooltip: {
+      enabled: false,
+    },
+    series: [
+      {
+        type: 'pie',
+        name: '',
+        innerSize: '40%',
+        data,
+      },
+    ],
+  }
+
+  return <ReactHighcharts data-cy={`Graph-${cypress}`} config={defaultConfig} />
+}
+
+export default GaugeChart

services/frontend/src/components/StudyProgramme/BasicOverview/LineGraph.jsx

@@ -7,8 +7,11 @@ require('highcharts-exporting')(ReactHighcharts.Highcharts)
 
 const colors = ['#003E65', '#1392c2', '#E68825', '#333737', '#036415']
 
-const LineGraph = ({ data }) => {
-  const dataWithColors = data?.graphStats.map((series, index) => ({ ...series, color: colors[index] }))
+const LineGraph = ({ cypress, data }) => {
+  const dataWithColors = data?.graphStats.map((series, index) => ({
+    ...series,
+    color: colors[index],
+  }))
 
   const defaultConfig = {
     title: {
@@ -46,7 +49,7 @@ const LineGraph = ({ data }) => {
 
   if (!data) return <></>
   return (
-    <div className="graph-container">
+    <div className="graph-container" data-cy={`Graph-${cypress}`}>
       <ReactHighcharts config={defaultConfig} />
     </div>
   )

services/frontend/src/components/StudyProgramme/BasicOverview/StackedBarChart.jsx

@@ -2,10 +2,10 @@ import React from 'react'
 
 const ReactHighcharts = require('react-highcharts')
 
-const colors = ['#003E65', '#1392c2', '#036415']
+const colors = ['#003E65', '#1392c2', '#036415', '#bdc2c7', '#E68825', '#333737', '#54997b', '#195f8a']
 
-const StackedBarChart = ({ data }) => {
-  const dataWithColors = data?.graphStats?.map((series, index) => ({ ...series, color: colors[index] }))
+const StackedBarChart = ({ cypress, data, labels, wideTable }) => {
+  const dataWithColors = data?.map((series, index) => ({ ...series, color: colors[index] }))
 
   const defaultConfig = {
     title: {
@@ -16,7 +16,7 @@ const StackedBarChart = ({ data }) => {
       text: 'oodikone | TOSKA',
     },
     xAxis: {
-      categories: data?.years,
+      categories: labels,
     },
     chart: {
       type: 'column',
@@ -55,7 +55,7 @@ const StackedBarChart = ({ data }) => {
 
   if (!data) return <></>
   return (
-    <div className="graph-container">
+    <div className={`graph-container${wideTable ? '-narrow' : ''}`} data-cy={`Graph-${cypress}`}>
       <ReactHighcharts config={defaultConfig} />
     </div>
   )

services/frontend/src/components/StudyProgramme/BasicOverview/index.jsx

@@ -0,0 +1,184 @@
+import React, { useState } from 'react'
+import { Divider, Loader, Message, Radio } from 'semantic-ui-react'
+
+import { useGetBasicStatsQuery, useGetCreditStatsQuery, useGetGraduationStatsQuery } from 'redux/studyProgramme'
+import WithHelpTooltip from '../../Info/InfoWithHelpTooltip'
+import LineGraph from './LineGraph'
+import StackedBarChart from './StackedBarChart'
+import BarChart from './BarChart'
+import GaugeChart from './GaugeChart'
+import DataTable from './DataTable'
+import InfoBox from '../../Info/InfoBox'
+
+import InfotoolTips from '../../../common/InfoToolTips'
+import '../studyprogramme.css'
+
+const getRadioButton = (cypress, toolTip, firstLabel, secondLabel, value, setValue) => (
+  <div className="radio-toggle">
+    <label className="toggle-label">{firstLabel}</label>
+    <Radio data-cy={cypress} toggle checked={value} onChange={() => setValue(!value)} />
+    {toolTip ? (
+      <WithHelpTooltip tooltip={{ short: toolTip }}>
+        <label className="toggle-label">{secondLabel}</label>
+      </WithHelpTooltip>
+    ) : (
+      <label className="toggle-label">{secondLabel}</label>
+    )}
+  </div>
+)
+
+const Overview = ({ studyprogramme, specialGroups, setSpecialGroups, academicYear, setAcademicYear }) => {
+  const [showMeanTime, setShowMeanTime] = useState(true)
+  const toolTips = InfotoolTips.Studyprogramme
+  const yearType = academicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR'
+  const special = specialGroups ? 'SPECIAL_EXCLUDED' : 'SPECIAL_INCLUDED'
+  const basics = useGetBasicStatsQuery({ id: studyprogramme, yearType, specialGroups: special })
+  const credits = useGetCreditStatsQuery({ id: studyprogramme, yearType, specialGroups: special })
+  const graduations = useGetGraduationStatsQuery({ id: studyprogramme, yearType, specialGroups: special })
+
+  const getDivider = (title, toolTipText) => (
+    <>
+      <div className="divider">
+        <Divider data-cy={`Section-${toolTipText}`} horizontal>
+          {title}
+        </Divider>
+      </div>
+      <InfoBox content={toolTips[toolTipText]} />
+    </>
+  )
+
+  const isFetchingOrLoading =
+    basics.isLoading ||
+    credits.isLoading ||
+    graduations.isLoading ||
+    basics.isFetching ||
+    credits.isFetching ||
+    graduations.isFetching
+
+  const isError =
+    (basics.isError && credits.isError && graduations.isError) ||
+    (basics.isSuccess &&
+      !basics.data &&
+      credits.isSuccess &&
+      !credits.data &&
+      graduations.isSuccess &&
+      !graduations.data)
+
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+
+  return (
+    <div className="studyprogramme-overview">
+      <div className="toggle-container">
+        {getRadioButton(
+          'YearToggle',
+          toolTips.YearToggle,
+          'Calendar year ',
+          'Academic year',
+          academicYear,
+          setAcademicYear
+        )}
+        {getRadioButton(
+          'StudentToggle',
+          toolTips.StudentToggle,
+          'All studyrights included',
+          'Special studyrights excluded',
+          specialGroups,
+          setSpecialGroups
+        )}
+      </div>
+
+      {isFetchingOrLoading ? (
+        <Loader active style={{ marginTop: '10em' }} />
+      ) : (
+        <>
+          {!studyprogramme.includes('KH') && !studyprogramme.includes('MH') && (
+            <Message>
+              Please note, that the data is not complete only for current Master's and Bachelor's programmes
+            </Message>
+          )}
+          {basics.isSuccess && basics.data && (
+            <>
+              {getDivider('Students of the studyprogramme', 'StudentsOfTheStudyprogramme')}
+              <div className="section-container">
+                <LineGraph cypress="StudentsOfTheStudyprogramme" data={basics?.data} />
+                <DataTable
+                  cypress="StudentsOfTheStudyprogramme"
+                  data={basics?.data?.tableStats}
+                  titles={basics?.data?.titles}
+                />
+              </div>
+            </>
+          )}
+          {credits.isSuccess && credits.data && (
+            <>
+              {getDivider('Credits produced by the studyprogramme', 'CreditsProducedByTheStudyprogramme')}
+              <div className="section-container">
+                <StackedBarChart
+                  cypress="CreditsProducedByTheStudyprogramme"
+                  data={credits?.data?.graphStats}
+                  labels={credits?.data?.years}
+                />
+                <DataTable
+                  cypress="CreditsProducedByTheStudyprogramme"
+                  data={credits?.data?.tableStats}
+                  titles={credits?.data?.titles}
+                />
+              </div>
+            </>
+          )}
+          {graduations.isSuccess && graduations.data && (
+            <>
+              {getDivider('Graduated and thesis writers of the programme', 'GraduatedAndThesisWritersOfTheProgramme')}
+              <div className="section-container">
+                <BarChart cypress="GraduatedAndThesisWritersOfTheProgramme" data={graduations?.data} />
+                <DataTable
+                  cypress="GraduatedAndThesisWritersOfTheProgramme"
+                  data={graduations?.data?.tableStats}
+                  titles={graduations?.data?.titles}
+                />
+              </div>
+              {getDivider('Average graduation times', 'AverageGraduationTimes')}
+              {getRadioButton('GraduationTimeToggle', null, 'Mean time', 'Median time', showMeanTime, setShowMeanTime)}
+              <div className="section-container-centered">
+                {graduations?.data?.years.map(year => (
+                  <GaugeChart
+                    key={year}
+                    year={year}
+                    data={
+                      showMeanTime
+                        ? graduations?.data?.graduationMeanTime[year]
+                        : graduations?.data?.graduationMedianTime[year]
+                    }
+                    graduationAmount={graduations?.data?.graduationAmounts[year]}
+                    totalAmount={graduations?.data?.totalAmounts[year]}
+                    studyprogramme={studyprogramme}
+                  />
+                ))}
+              </div>
+              {studyprogramme.includes('KH') && (
+                <>
+                  {getDivider('Primary master programme studies after this programme', 'ProgrammesAfterGraduation')}
+                  <div className="section-container">
+                    <StackedBarChart
+                      cypress="ProgrammesAfterGraduation"
+                      wideTable
+                      data={graduations?.data?.programmesAfterGraphStats}
+                      labels={graduations?.data?.years}
+                    />
+                    <DataTable
+                      wideTable
+                      data={graduations?.data?.programmesAfterTableStats}
+                      titles={graduations?.data?.programmesAfterTitles}
+                    />
+                  </div>
+                </>
+              )}
+            </>
+          )}
+        </>
+      )}
+    </div>
+  )
+}
+
+export default Overview

services/frontend/src/components/StudyProgramme/NewOverview/index.jsx

@@ -1,105 +0,0 @@
-import React, { useState } from 'react'
-import { Divider, Loader, Radio } from 'semantic-ui-react'
-
-import { useGetBasicStatsQuery, useGetCreditStatsQuery, useGetGraduationStatsQuery } from 'redux/studyProgramme'
-import LineGraph from './LineGraph'
-import StackedBarChart from './StackedBarChart'
-import BarChart from './BarChart'
-import GaugeChart from './GaugeChart'
-import DataTable from './DataTable'
-import InfoBox from '../../Info/InfoBox'
-
-import InfotoolTips from '../../../common/InfoToolTips'
-import '../studyprogramme.css'
-
-const basicsTitles = ['', 'Started', 'Graduated', 'Cancelled', 'Transferred away', 'Transferred to']
-const creditsTitles = ['', 'Total', 'Major students credits', 'Non major students credits', 'Transferred credits']
-const graduationsTitles = ['', 'Graduated', 'Wrote thesis']
-
-const getRadioButton = (firstLabel, secondLabel, value, setValue) => (
-  <div className="radio-toggle">
-    <label className="toggle-label">{firstLabel}</label>
-    <Radio toggle checked={value} onChange={() => setValue(!value)} />
-    <label className="toggle-label">{secondLabel}</label>
-  </div>
-)
-
-const Overview = ({ studyprogramme }) => {
-  const [academicYear, setAcademicYear] = useState(false)
-  const [showMeanTime, setShowMeanTime] = useState(true)
-  const toolTips = InfotoolTips.Studyprogramme
-  const yearType = academicYear ? 'ACADEMIC_YEAR' : 'CALENDAR_YEAR'
-  const basics = useGetBasicStatsQuery({ id: studyprogramme, yearType })
-  const credits = useGetCreditStatsQuery({ id: studyprogramme, yearType })
-  const graduations = useGetGraduationStatsQuery({ id: studyprogramme, yearType })
-
-  const getDivider = (title, toolTipText) => (
-    <>
-      <div className="divider">
-        <Divider horizontal>{title}</Divider>
-      </div>
-      <InfoBox content={toolTips[toolTipText]} />
-    </>
-  )
-
-  return (
-    <div className="studyprogramme-overview">
-      <div style={{ display: 'flex', justifyContent: 'center', margin: '10px' }}>
-        <p style={{ color: 'red' }}>
-          Please note that this view is still very much a work in progress. This view is only visible to some admins.
-        </p>
-      </div>
-      {getRadioButton('kalenterivuosi', 'lukuvuosi', academicYear, setAcademicYear)}
-      {basics.isLoading || credits.isLoading || graduations.isLoading ? (
-        <Loader active={basics.isLoading || credits.isLoading || graduations.isLoading} />
-      ) : (
-        <>
-          {getDivider('Students of the studyprogramme', 'StudentsOfTheStudyprogramme')}
-          <div className="section-container">
-            <LineGraph data={basics?.data} />
-            <DataTable titles={basicsTitles} data={basics?.data?.tableStats} />
-          </div>
-          {getDivider('Credits produced by the studyprogramme', 'CreditsProducedByTheStudyprogramme')}
-          <div className="section-container">
-            <StackedBarChart data={credits?.data} />
-            <DataTable titles={creditsTitles} data={credits?.data?.tableStats} />
-          </div>
-          {getDivider('Graduated and thesis writers of the programme', 'GraduatedAndThesisWritersOfTheProgramme')}
-          <div className="section-container">
-            <BarChart data={graduations?.data} />
-            <DataTable titles={graduationsTitles} data={graduations?.data?.tableStats} />
-          </div>
-          {getDivider('Average graduation times', 'AverageGraduationTimes')}
-          {getRadioButton('Mean time', 'Median time', showMeanTime, setShowMeanTime)}
-          {showMeanTime ? (
-            <div className="section-container">
-              {graduations?.data?.years.map(year => (
-                <GaugeChart
-                  key={year}
-                  year={year}
-                  data={graduations?.data?.graduationMedianTime[year]}
-                  amount={graduations?.data?.graduationAmounts[year]}
-                  studyprogramme={studyprogramme}
-                />
-              ))}
-            </div>
-          ) : (
-            <div className="section-container">
-              {graduations?.data?.years.map(year => (
-                <GaugeChart
-                  key={year}
-                  year={year}
-                  data={graduations?.data?.graduationMeanTime[year]}
-                  amount={graduations?.data?.graduationAmounts[year]}
-                  studyprogramme={studyprogramme}
-                />
-              ))}
-            </div>
-          )}
-        </>
-      )}
-    </div>
-  )
-}
-
-export default Overview

services/frontend/src/components/StudyProgramme/NewStudytrackOverview/StudytrackDataTable.jsx

@@ -1,93 +0,0 @@
-import React, { useState } from 'react'
-import { Icon, Table } from 'semantic-ui-react'
-
-const getKey = year => `${year}-${Math.random()}`
-
-const getFirstCell = (yearlyData, year, show, singleTrack) => {
-  if (yearlyData.length === 1 || singleTrack) {
-    return <Table.Cell key={getKey(year)}>{year}</Table.Cell>
-  }
-  return (
-    <Table.Cell key={getKey(year)}>
-      <Icon name={`${show ? 'angle down' : 'angle right'}`} />
-      {year}
-    </Table.Cell>
-  )
-}
-
-const getRow = ({ yearlyData, array, show, setShow, singleTrack }) => {
-  if (array[0].includes('20') && !singleTrack) {
-    return (
-      <Table.Row key={getKey(array[0])} className="header-row" onClick={() => setShow(!show)}>
-        {array.map((value, index) =>
-          index === 0 ? (
-            getFirstCell(yearlyData, array[0], show, singleTrack)
-          ) : (
-            <Table.Cell textAlign="left">{value}</Table.Cell>
-          )
-        )}
-      </Table.Row>
-    )
-  }
-
-  if (show || singleTrack) {
-    return (
-      <Table.Row key={getKey(array[0])} className="regular-row">
-        {array.map((value, index) =>
-          index === 0 && !singleTrack ? (
-            <Table.Cell textAlign="left" style={{ paddingLeft: '50px' }} key={getKey(array[0])}>
-              {value}
-            </Table.Cell>
-          ) : (
-            <Table.Cell textAlign="left" key={getKey(array[0])}>
-              {value}
-            </Table.Cell>
-          )
-        )}
-      </Table.Row>
-    )
-  }
-
-  return null
-}
-
-const StudytrackDataTable = ({ allData, dataOfStudytrack, titles }) => {
-  const [show, setShow] = useState(false)
-
-  if (!allData && !dataOfStudytrack) return null
-
-  const sortedMainStats = Object.values(allData)
-    .reverse()
-    .sort((a, b) => a[0] - b[0])
-
-  return (
-    <div className="datatable">
-      <Table celled>
-        <Table.Header>
-          <Table.Row>
-            {titles.map((title, index) => (
-              <Table.HeaderCell
-                key={title}
-                colSpan={index === 0 ? 1 : 2}
-                textAlign="left"
-                style={{ fontWeight: 'bold' }}
-              >
-                {title}
-              </Table.HeaderCell>
-            ))}
-          </Table.Row>
-        </Table.Header>
-
-        <Table.Body>
-          {dataOfStudytrack
-            ? dataOfStudytrack.map(array =>
-                getRow({ yearlyData: dataOfStudytrack, array, show, setShow, singleTrack: true })
-              )
-            : sortedMainStats?.map(yearlyData => yearlyData.map(array => getRow({ yearlyData, array, show, setShow })))}
-        </Table.Body>
-      </Table>
-    </div>
-  )
-}
-
-export default StudytrackDataTable

services/frontend/src/components/StudyProgramme/NewStudytrackOverview/index.jsx

@@ -1,93 +0,0 @@
-import React, { useEffect, useState } from 'react'
-import { Divider, Loader } from 'semantic-ui-react'
-import { useGetStudytrackStatsQuery } from 'redux/studyProgramme'
-import InfoBox from '../../Info/InfoBox'
-import BarChart from './BarChart'
-import BasicDataTable from './BasicDataTable'
-import StudytrackDataTable from './StudytrackDataTable'
-import StudytrackSelector from './StudytrackSelector'
-
-import InfotoolTips from '../../../common/InfoToolTips'
-import '../studyprogramme.css'
-
-const populationTitles = ['', 'All started', 'Men', 'Women', 'Finnish', 'Graduated']
-const creditTableTitles = [
-  '',
-  'All started',
-  '< 30 credits',
-  '30-59 credits',
-  '60-89 credits',
-  '90-119 credits',
-  '120-149 credits',
-  '> 150 credits',
-]
-
-const StudytrackOverview = ({ studyprogramme }) => {
-  const toolTips = InfotoolTips.Studyprogramme
-  const stats = useGetStudytrackStatsQuery({ id: studyprogramme })
-  const [track, setTrack] = useState(studyprogramme)
-
-  useEffect(() => {
-    if (!track && stats?.data?.mainStatsByTrack[studyprogramme]) {
-      setTrack(studyprogramme)
-    }
-  }, [studyprogramme, track, stats])
-
-  const getDivider = (title, toolTipText) => (
-    <>
-      <div className="divider">
-        <Divider horizontal>{title}</Divider>
-      </div>
-      <InfoBox content={toolTips[toolTipText]} />
-    </>
-  )
-
-  return (
-    <div className="studytrack-overview">
-      <div style={{ display: 'flex', justifyContent: 'center', margin: '10px' }}>
-        <p style={{ color: 'red' }}>
-          Please note that this view is still very much a work in progress. This view is only visible to some admins.
-        </p>
-      </div>
-      {stats.isLoading ? (
-        <Loader active={stats.isLoading} />
-      ) : (
-        <>
-          <StudytrackSelector track={track} setTrack={setTrack} studytracks={stats?.data?.studytrackOptions} />
-          {getDivider(
-            `Students of ${
-              track === '' || track === 'studyprogramme'
-                ? 'the studyprogramme by starting year'
-                : `the studytrack ${track} by starting year`
-            }`,
-            'StudytrackOverview'
-          )}
-          <StudytrackDataTable
-            track={track || studyprogramme}
-            titles={populationTitles}
-            allData={stats?.data?.mainStatsByYear}
-            dataOfStudytrack={track && track !== studyprogramme ? stats?.data?.mainStatsByTrack[track] : null}
-          />
-          {getDivider(
-            `Progress of students of ${
-              track === '' || track === 'studyprogramme'
-                ? 'the studyprogramme by starting year'
-                : `the studytrack ${track} by starting year`
-            }`,
-            'StudytrackProgress'
-          )}
-          <div className="section-container">
-            <BarChart data={stats?.data} track={track || studyprogramme} />
-            <BasicDataTable
-              data={stats?.data?.creditTableStats}
-              track={track || studyprogramme}
-              titles={creditTableTitles}
-            />
-          </div>
-        </>
-      )}
-    </div>
-  )
-}
-
-export default StudytrackOverview

services/frontend/src/components/StudyProgramme/Overview/index.jsx

@@ -1,62 +0,0 @@
-import React, { useEffect } from 'react'
-import { connect } from 'react-redux'
-import ProductivityTable from '../ProductivityTable'
-import ThroughputTable from '../ThroughputTable'
-import BachelorsTable from '../BachelorsTable'
-import { getProductivity } from '../../../redux/productivity'
-import { getThroughput } from '../../../redux/throughput'
-import { getBachelors } from '../../../redux/studyProgrammeBachelors'
-import { isNewHYStudyProgramme } from '../../../common'
-
-const Overview = props => {
-  const {
-    productivity,
-    throughput,
-    bachelors,
-    studyprogramme,
-    dispatchGetProductivity,
-    dispatchGetThroughput,
-    dispatchGetBachelors,
-    history,
-  } = props
-
-  useEffect(() => {
-    dispatchGetProductivity(studyprogramme)
-    dispatchGetThroughput(studyprogramme)
-    dispatchGetBachelors(studyprogramme)
-  }, [])
-  return (
-    <>
-      <ThroughputTable
-        throughput={throughput.data[studyprogramme]}
-        thesis={throughput.data.thesis}
-        loading={throughput.pending}
-        error={throughput.error}
-        studyprogramme={studyprogramme}
-        history={history}
-        newProgramme={isNewHYStudyProgramme(studyprogramme)}
-      />
-      <ProductivityTable
-        productivity={productivity.data[studyprogramme]}
-        thesis={throughput.data.thesis}
-        loading={productivity.pending}
-        error={productivity.error}
-        showCredits={isNewHYStudyProgramme(studyprogramme)}
-        newProgramme={isNewHYStudyProgramme(studyprogramme)}
-      />
-      <BachelorsTable bachelors={bachelors.data} loading={throughput.pending} />
-    </>
-  )
-}
-
-const mapStateToProps = ({ studyProgrammeProductivity, studyProgrammeThroughput, studyProgrammeBachelors }) => ({
-  productivity: studyProgrammeProductivity,
-  throughput: studyProgrammeThroughput,
-  bachelors: studyProgrammeBachelors,
-})
-
-export default connect(mapStateToProps, {
-  dispatchGetProductivity: getProductivity,
-  dispatchGetThroughput: getThroughput,
-  dispatchGetBachelors: getBachelors,
-})(Overview)

services/frontend/src/components/StudyProgramme/ProductivityTable/index.jsx

@@ -1,162 +0,0 @@
-import React, { useState, useEffect } from 'react'
-import moment from 'moment'
-import { Table, Header, Grid, Label, Segment, Dropdown } from 'semantic-ui-react'
-import { connect } from 'react-redux'
-import { shape, number, arrayOf, bool, string, oneOfType } from 'prop-types'
-import { getProductivity } from '../../../redux/productivity'
-import InfoBox from '../../Info/InfoBox'
-import infotooltips from '../../../common/InfoToolTips'
-import './productivityTable.css'
-
-const ProductivityTable = ({ productivity, thesis, loading, error, showCredits, newProgramme }) => {
-  const [selectedYear, setYear] = useState(newProgramme ? 2017 : null)
-
-  const years = productivity
-    ? productivity.data
-        .map(stats => ({ key: stats.year, text: stats.year, value: stats.year }))
-        .sort((year1, year2) => Number(year2.value) - Number(year1.value))
-    : []
-
-  useEffect(() => {
-    if (!selectedYear && years.length > 5) {
-      setYear(years[5].value)
-    } else if (!selectedYear && years.length > 1) {
-      setYear(years[years.length - 1].value)
-    }
-  }, [years])
-
-  if (error) return <h1>Oh no so error {error}</h1>
-  let thesisTypes = []
-  if (thesis) {
-    thesisTypes = thesis.map(t => t.thesisType)
-  }
-
-  const headerList = [
-    'Year',
-    'Credits',
-    thesisTypes.includes('MASTER') && 'Masters Thesis',
-    thesisTypes.includes('BACHELOR') && 'Bachelors Thesis',
-    'Graduated',
-    'Credits for studyprogramme majors',
-    'Credits for non major students',
-    'Hyväksiluettu (not included in Credits column)',
-  ].filter(_ => _)
-
-  const handleChange = (event, { value }) => {
-    event.preventDefault()
-    setYear(value)
-  }
-
-  const creditCell = content =>
-    showCredits ? (
-      <Table.Cell>{content}</Table.Cell>
-    ) : (
-      <Table.Cell
-        className="productivity-table__not-available"
-        title="Credits productivity statistics not available for old programmes"
-      >
-        Not available
-      </Table.Cell>
-    )
-
-  return (
-    <>
-      <div style={{ marginTop: '2em' }}>
-        <InfoBox content={infotooltips.PopulationOverview.YearlyProductivity} />
-      </div>
-      <Header>
-        <Grid columns={2}>
-          <Grid.Row>
-            <Grid.Column>
-              Yearly productivity
-              {productivity && (
-                <Header.Subheader data-cy="productivityUpdateStatus">
-                  {`Last updated ${
-                    productivity.lastUpdated
-                      ? moment(productivity.lastUpdated).format('HH:mm:ss MM-DD-YYYY')
-                      : 'unknown'
-                  }`}
-                  {productivity.status === 'RECALCULATING' && (
-                    <Label content="Recalculating! Refresh page in a few minutes" color="red" />
-                  )}
-                </Header.Subheader>
-              )}
-            </Grid.Column>
-          </Grid.Row>
-        </Grid>
-      </Header>
-      <div>Statistics from selected year onwards</div>
-      {years.length > 4 ? <Dropdown options={years} onChange={handleChange} selection value={selectedYear} /> : null}
-      <Segment basic loading={loading} style={{ overflowX: 'auto' }}>
-        <Table structured celled compact striped selectable className="fixed-header">
-          <Table.Header>
-            <Table.Row>
-              {headerList.map(header => (
-                <Table.HeaderCell key={header}>{header}</Table.HeaderCell>
-              ))}
-            </Table.Row>
-          </Table.Header>
-          <Table.Body>
-            {productivity && productivity.data
-              ? productivity.data
-                  .filter(stats => Number(stats.year) >= Number(selectedYear))
-                  .sort((year1, year2) => year2.year - year1.year)
-                  .map(year => (
-                    <Table.Row key={year.year}>
-                      <Table.Cell>{year.year}</Table.Cell>
-                      {creditCell(year.credits.toFixed(2))}
-                      {thesisTypes.includes('BACHELOR') && <Table.Cell>{year.bThesis}</Table.Cell>}
-                      {thesisTypes.includes('MASTER') && <Table.Cell>{year.mThesis}</Table.Cell>}
-                      <Table.Cell>{year.graduated}</Table.Cell>
-                      {creditCell(year.creditsForMajors.toFixed(2))}
-                      {creditCell((year.credits - year.creditsForMajors).toFixed(2))}
-                      {creditCell(year.transferredCredits.toFixed(2))}
-                    </Table.Row>
-                  ))
-              : null}
-          </Table.Body>
-        </Table>
-      </Segment>
-    </>
-  )
-}
-
-ProductivityTable.propTypes = {
-  productivity: shape({
-    lastUpdated: string,
-    status: string,
-    data: arrayOf(
-      shape({
-        year: oneOfType([number, string]),
-        credits: number,
-        mThesis: number,
-        bThesis: number,
-        graduated: number,
-        creditsForMajors: number,
-        transferredCredits: number,
-      })
-    ),
-  }),
-  thesis: arrayOf(
-    shape({
-      programmeCode: string,
-      courseCode: string,
-      thesisType: string,
-      createdAt: string,
-      updatedAt: string,
-    })
-  ),
-  loading: bool.isRequired,
-  error: bool.isRequired,
-  showCredits: bool.isRequired,
-  newProgramme: bool.isRequired,
-}
-
-ProductivityTable.defaultProps = {
-  productivity: null,
-  thesis: undefined,
-}
-
-export default connect(null, {
-  dispatchGetProductivity: getProductivity,
-})(ProductivityTable)

services/frontend/src/components/StudyProgramme/ProductivityTable/productivityTable.css

@@ -1,3 +0,0 @@
-.productivity-table__not-available {
-  color: rgba(40, 40, 40, 0.3);
-}

services/frontend/src/components/StudyProgramme/StudyTrackOverview/index.jsx

@@ -1,265 +0,0 @@
-import React, { useEffect, useState } from 'react'
-import { string, func, shape, bool } from 'prop-types'
-import { connect } from 'react-redux'
-import { uniqBy } from 'lodash'
-import { Dropdown, Tab, Message, Button } from 'semantic-ui-react'
-import ThroughputTable from '../ThroughputTable'
-import { getProductivity } from '../../../redux/productivity'
-import { getThroughput } from '../../../redux/throughput'
-import { isNewHYStudyProgramme, textAndDescriptionSearch, getTextIn } from '../../../common'
-import ProtoC from '../../CoolDataScience/ProtoC'
-
-const Overview = ({
-  language,
-  throughput,
-  studyprogramme,
-  dispatchGetProductivity,
-  dispatchGetThroughput,
-  history,
-  studyprogrammes,
-}) => {
-  const [selectedTrack, setTrack] = useState('')
-  const [selectedYear, setYear] = useState('')
-  const [throughputData, setData] = useState(null)
-  const [studytrackOptions, setStudytrackOptions] = useState([])
-  const [yearOptions, setYearOptions] = useState([])
-  const [activeIndex, setIndex] = useState(0)
-
-  useEffect(() => {
-    dispatchGetProductivity(studyprogramme)
-    dispatchGetThroughput(studyprogramme)
-  }, [])
-
-  // set dropdown options
-  useEffect(() => {
-    if (throughput.data[studyprogramme]) {
-      const years = throughput.data[studyprogramme].data.map(data => data.year)
-      const filteredStudytracks = years.reduce((acc, curr) => {
-        if (studyprogrammes[studyprogramme].enrollmentStartYears[curr.slice(0, 4)])
-          acc.push(...Object.values(studyprogrammes[studyprogramme].enrollmentStartYears[curr.slice(0, 4)].studyTracks))
-        return uniqBy(acc, 'code')
-      }, [])
-
-      const yearDropdownOptions = years.map(year => ({
-        key: year,
-        text: year,
-        value: year,
-      }))
-
-      const dropdownOptions = filteredStudytracks.map(st => ({
-        key: st.code,
-        text: getTextIn(st.name, language),
-        description: st.code,
-        value: st.code,
-      }))
-      if (yearDropdownOptions.length > 0) {
-        setYear(yearDropdownOptions[0].value)
-      }
-      setYearOptions(yearDropdownOptions)
-      setStudytrackOptions(dropdownOptions)
-    }
-  }, [throughput, studyprogrammes])
-
-  // mankel throughput data after selecting studytrack
-  useEffect(() => {
-    if (throughput.data[studyprogramme] && selectedTrack) {
-      if (throughput.data[studyprogramme].data[0].studytrackdata) {
-        const selectedData = throughput.data[studyprogramme].data.reduce((acc, curr) => {
-          if (curr.studytrackdata[selectedTrack]) acc.push(curr.studytrackdata[selectedTrack])
-          return acc
-        }, [])
-        const newData = {
-          data: selectedData,
-          lastUpdated: throughput.data[studyprogramme].lastUpdated,
-          status: throughput.data[studyprogramme].status,
-          totals: throughput.data[studyprogramme].stTotals[selectedTrack],
-        }
-        setData(newData)
-      }
-    }
-  }, [selectedTrack, activeIndex])
-
-  // mankel throughput data after selecting a year
-  useEffect(() => {
-    if (throughput.data[studyprogramme] && selectedYear) {
-      if (throughput.data[studyprogramme].data[0].studytrackdata) {
-        const selectedYearData = throughput.data[studyprogramme].data.find(data => data.year === selectedYear)
-        const filteredStudytracks = Object.values(studyprogrammes[studyprogramme].enrollmentStartYears).reduce(
-          (acc, curr) => {
-            acc.push(...Object.values(curr.studyTracks))
-            return uniqBy(acc, 'code')
-          },
-          []
-        )
-        const studytrackYearData = Object.keys(selectedYearData.studytrackdata).reduce((acc, curr) => {
-          const studytrack = filteredStudytracks.find(st => st.code === curr)
-          if (studytrack) {
-            const newStudyTrackObject = {
-              ...selectedYearData.studytrackdata[curr],
-              // oh pls no, pls fix asap
-              // need to fix logic in throughputtable component so that we can name this better
-              year: `${getTextIn(studytrack.name, language)}, ${curr}`,
-            }
-            acc.push(newStudyTrackObject)
-          }
-          return acc
-        }, [])
-        const newData = {
-          data: studytrackYearData,
-          lastUpdated: throughput.data[studyprogramme].lastUpdated,
-          status: throughput.data[studyprogramme].status,
-          totals: {
-            ...selectedYearData,
-            credits: selectedYearData.creditValues,
-            genders: {
-              Male: Number(selectedYearData.genders.Male) || 0,
-              Female: Number(selectedYearData.genders.Female) || 0,
-              Other: Number(selectedYearData.genders.Other) || 0,
-            },
-            students:
-              (Number(selectedYearData.genders.Male) || 0) +
-              (Number(selectedYearData.genders.Female) || 0) +
-              (Number(selectedYearData.genders.Other) || 0),
-          },
-        }
-        setData(newData)
-      }
-    }
-  }, [selectedYear, activeIndex])
-
-  const handleStudytrackChange = (event, { value }) => {
-    event.preventDefault()
-    setTrack(value)
-  }
-
-  const handleYearChange = (event, { value }) => {
-    event.preventDefault()
-    setYear(value)
-  }
-
-  const renderSelection = year => {
-    if (year && yearOptions.length < 10)
-      return (
-        <>
-          <h3>Year selection</h3>
-          <Button.Group>
-            {yearOptions.map(option => (
-              <Button
-                key={option.key}
-                content={option.text}
-                style={{ borderRadius: '1px' }}
-                onClick={e => handleYearChange(e, option)}
-                active={option.value === selectedYear}
-                color={option.value === selectedYear ? 'blue' : 'black'}
-                basic={option.value !== selectedYear}
-              />
-            ))}
-          </Button.Group>
-        </>
-      )
-    return (
-      <>
-        <h3>{year ? 'Year' : 'Studytrack'} selection</h3>
-        <Dropdown
-          options={year ? yearOptions : studytrackOptions}
-          onChange={year ? handleYearChange : handleStudytrackChange}
-          defaultValue={year ? yearOptions[0].value : studytrackOptions[0].value}
-          selection
-          placeholder={`Select ${year ? 'year' : ' studytrack'}`}
-          search={textAndDescriptionSearch}
-          fluid
-        />
-      </>
-    )
-  }
-
-  const renderTable = year => {
-    if (!throughputData || throughputData.data.length < 1) {
-      return (
-        <>
-          {renderSelection(year)}
-          <h2>No {year ? 'year' : 'studytrack'} selected or no data available for selection</h2>
-        </>
-      )
-    }
-    return (
-      <>
-        {renderSelection(year)}
-        <ThroughputTable
-          throughput={throughputData}
-          thesis={throughput.data.thesis}
-          loading={throughput.pending}
-          error={throughput.error}
-          studyprogramme={studyprogramme}
-          studytrack={selectedTrack}
-          history={history}
-          newProgramme={isNewHYStudyProgramme(studyprogramme)}
-          isStudytrackView={year}
-        />
-      </>
-    )
-  }
-  const handleTabChange = (event, { activeIndex }) => {
-    setData(null)
-    if (activeIndex === 1) {
-      setTrack(studytrackOptions[0].value)
-      setYear('')
-    } else if (activeIndex === 0) {
-      setYear(yearOptions[0].value)
-      setTrack('')
-    }
-    setIndex(activeIndex)
-  }
-
-  const panes = [
-    {
-      menuItem: 'Year specific',
-      render: () => renderTable(true),
-    },
-    {
-      menuItem: 'Studytrack specific',
-      render: () => renderTable(false),
-    },
-  ]
-
-  if (studyprogramme.includes('KH'))
-    panes.push({
-      menuItem: 'Studytrack Graph',
-      render: () => <ProtoC programme={studyprogramme} />,
-    })
-
-  return (
-    <>
-      <Message
-        content="Here you can see statistics on studytrack level. In year specific tab you can compare statistics for all studytracks within selected year. 
-                  In studytrack specific tab you can compare different years from one selected studytrack."
-      />
-      <Tab panes={panes} activeIndex={activeIndex} onTabChange={handleTabChange} />
-    </>
-  )
-}
-
-Overview.propTypes = {
-  studyprogramme: string.isRequired,
-  dispatchGetProductivity: func.isRequired,
-  dispatchGetThroughput: func.isRequired,
-  history: shape({}).isRequired,
-  studyprogrammes: shape({}).isRequired,
-  language: string.isRequired,
-  throughput: shape({
-    error: bool,
-    pending: bool,
-    data: shape({}),
-  }).isRequired // eslint-disable-line
-}
-
-const mapStateToProps = ({ studyProgrammeThroughput, populationProgrammes, settings }) => ({
-  throughput: studyProgrammeThroughput,
-  studyprogrammes: populationProgrammes.data.programmes || {},
-  language: settings.language,
-})
-
-export default connect(mapStateToProps, {
-  dispatchGetProductivity: getProductivity,
-  dispatchGetThroughput: getThroughput,
-})(Overview)

services/frontend/src/components/StudyProgramme/StudytrackOverview/BarChart.jsx

@@ -6,7 +6,7 @@ const ReactHighcharts = require('react-highcharts')
 const colors = ['#333737', '#bdc2c7', '#1392c2', '#195f8a', '#54997b', '#036415']
 
 const BarChart = ({ data, track }) => {
-  if (!data || !data.creditGraphStats[track])
+  if (!data || !data.creditGraphStats || !data.creditGraphStats[track])
     return <NoDataMessage message="No progress data for the studytrack found. Try with another studytrack" />
 
   const correctData = data?.creditGraphStats[track]

services/frontend/src/components/StudyProgramme/StudytrackOverview/BasicDataTable.jsx

@@ -7,9 +7,18 @@ const DataTable = ({ data, titles, track }) => {
   if (!data || !data[track]?.length || !titles)
     return <NoDataMessage message="No progress data found for the studytrack. Try with another studytrack" />
 
+  const copy = [...data[track]]
+  const sortedData = copy.sort((a, b) => {
+    if (a[0] < b[0]) return -1
+    if (a[0] > b[0]) return 1
+    return 0
+  })
+
+  sortedData.reverse()
+
   return (
     <div className="table-container">
-      <Table compact celled>
+      <Table data-cy="Table-StudytrackProgress" compact celled>
         <Table.Header>
           <Table.Row>
             {titles?.map(title => (
@@ -19,7 +28,7 @@ const DataTable = ({ data, titles, track }) => {
         </Table.Header>
 
         <Table.Body>
-          {data[track].map(array => (
+          {sortedData.map(array => (
             <Table.Row key={`random-year-key-${Math.random()}`}>
               {array.map(value => (
                 <Table.Cell key={`random-key-${Math.random()}`}>{value}</Table.Cell>

services/frontend/src/components/StudyProgramme/StudytrackOverview/GaugeChart.jsx

@@ -2,8 +2,8 @@ import React from 'react'
 
 const ReactHighcharts = require('react-highcharts')
 
-const GaugeChart = ({ data, year, amount, studyprogramme }) => {
-  if (!data) return null
+const GaugeChart = ({ data, year, graduationAmount, totalAmount, studyprogramme }) => {
+  if (!data || !graduationAmount || !data.length) return null
 
   const thresholdValues = studyprogramme.includes('KH') ? [36, 41] : [24, 27]
 
@@ -29,7 +29,7 @@ const GaugeChart = ({ data, year, amount, studyprogramme }) => {
       y: 60,
     },
     subtitle: {
-      text: `${year}<br/> n = ${amount}`,
+      text: `${year}<br/> n = ${graduationAmount} / ${totalAmount}`,
       style: {
         fontWeight: 'bold',
         fontSize: '15px',
@@ -38,9 +38,6 @@ const GaugeChart = ({ data, year, amount, studyprogramme }) => {
     credits: {
       text: '',
     },
-    tooltip: {
-      pointFormat: '{series.name}: <b>{point.percentage:.1f}%</b>',
-    },
     exporting: {
       enabled: false,
     },
@@ -66,6 +63,9 @@ const GaugeChart = ({ data, year, amount, studyprogramme }) => {
         colors: getColor(data[0][1]),
       },
     },
+    tooltip: {
+      enabled: false,
+    },
     series: [
       {
         type: 'pie',

services/frontend/src/components/StudyProgramme/StudytrackOverview/PopulationLink.jsx

@@ -0,0 +1,21 @@
+import React from 'react'
+import { Icon } from 'semantic-ui-react'
+import { Link } from 'react-router-dom'
+import moment from 'moment'
+
+const PopulationLink = ({ studytrack, studyprogramme, year: yearLabel }) => {
+  const year = Number(yearLabel.slice(0, 4))
+  const months = Math.ceil(moment.duration(moment().diff(`${year}-08-01`)).asMonths())
+  const href = studytrack
+    ? `/populations?months=${months}&semesters=FALL&semesters=` +
+      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%2C"studyTrack"%3A"${studytrack}"%7D&year=${year}`
+    : `/populations?months=${months}&semesters=FALL&semesters=` +
+      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&year=${year}`
+  return (
+    <Link title={`Population statistics of class ${yearLabel}`} to={href}>
+      <Icon name="level up alternate" />
+    </Link>
+  )
+}
+
+export default PopulationLink

services/frontend/src/components/StudyProgramme/StudytrackOverview/StudytrackDataTable.jsx

@@ -0,0 +1,187 @@
+import React, { useState } from 'react'
+import { Icon, Radio, Table } from 'semantic-ui-react'
+import * as _ from 'lodash'
+
+import PopulationLink from './PopulationLink'
+
+const getKey = year => `${year}-${Math.random()}`
+
+const shouldBeHidden = (hidePercentages, value) => hidePercentages && typeof value === 'string' && value.includes('%')
+
+const getFirstCell = (yearlyData, year, show, studyprogramme) => {
+  return (
+    <Table.Cell key={getKey(year)}>
+      {yearlyData.length > 1 && <Icon name={`${show ? 'angle down' : 'angle right'}`} />}
+      {year}
+      <PopulationLink studyprogramme={studyprogramme} year={year} />
+    </Table.Cell>
+  )
+}
+
+const getSingleTrackRow = ({ row, studyprogramme, code, hidePercentages }) => {
+  return (
+    <Table.Row key={getKey(row[0])} className="regular-row">
+      {row.map((value, index) => (
+        <>
+          {shouldBeHidden(hidePercentages, value) ? null : (
+            <Table.Cell textAlign="left" key={getKey(row[0])}>
+              {value}
+              {index === 0 && <PopulationLink studyprogramme={studyprogramme} year={row[0]} studytrack={code} />}
+            </Table.Cell>
+          )}
+        </>
+      ))}
+    </Table.Row>
+  )
+}
+
+const getRow = ({ yearlyData, row, show, setShow, studyprogramme, studytracks, hidePercentages, years }) => {
+  const year = yearlyData && yearlyData[0] && yearlyData[0][0]
+
+  // Get row for the studyprogramme
+  if (years.includes(row[0])) {
+    return (
+      <Table.Row key={getKey(row[0])} className="header-row" onClick={() => setShow(!show)}>
+        {row.map((value, index) =>
+          index === 0 ? (
+            getFirstCell(yearlyData, row[0], show, studyprogramme)
+          ) : (
+            <>
+              {shouldBeHidden(hidePercentages, value) ? null : (
+                <Table.Cell key={getKey(value)} textAlign="left">
+                  {value}
+                </Table.Cell>
+              )}
+            </>
+          )
+        )}
+      </Table.Row>
+    )
+  }
+
+  // Get row for any possible studytrack under the header studyprogramme row, if they are folded open
+  if (show) {
+    return (
+      <Table.Row key={getKey(row[0])} className="regular-row">
+        {row.map((value, index) =>
+          index === 0 ? (
+            <Table.Cell textAlign="left" style={{ paddingLeft: '50px' }} key={getKey(row[0])}>
+              {value} {_.findKey(studytracks, value)}
+              <PopulationLink
+                studyprogramme={studyprogramme}
+                year={year}
+                studytrack={_.findKey(studytracks, s => s === value)}
+              />
+            </Table.Cell>
+          ) : (
+            <>
+              {shouldBeHidden(hidePercentages, value) ? null : (
+                <Table.Cell textAlign="left" key={getKey(row[0])}>
+                  {value}
+                </Table.Cell>
+              )}
+            </>
+          )
+        )}
+      </Table.Row>
+    )
+  }
+
+  return null
+}
+
+const sortTrackDataByYear = data => {
+  if (!data || !data.length) return []
+
+  const copy = [...data]
+  const sortedData = copy.sort((a, b) => {
+    if (a[0] < b[0]) return -1
+    if (a[0] > b[0]) return 1
+    return 0
+  })
+
+  sortedData.reverse()
+  return sortedData
+}
+
+const sortMainDataByYear = data => {
+  if (!data || !data.length) return []
+
+  const sortedData = []
+  data.forEach(arrays => {
+    if (arrays.length) {
+      const copy = [...arrays]
+      const sortedYear = copy.sort((a, b) => {
+        if (a[0] < b[0]) return 1
+        if (a[0] > b[0]) return -1
+        return 0
+      })
+      sortedData.push(sortedYear.reverse())
+    }
+  })
+
+  sortedData.reverse()
+  return sortedData
+}
+
+const getRadioButton = (toolTip, firstLabel, secondLabel, value, setValue) => (
+  <div className="radio-toggle">
+    <label className="toggle-label">{firstLabel}</label>
+    <Radio toggle checked={value} onChange={() => setValue(!value)} />
+    <label className="toggle-label">{secondLabel}</label>
+  </div>
+)
+
+const StudytrackDataTable = ({
+  studyprogramme,
+  dataOfAllTracks,
+  studytracks,
+  singleTrack,
+  dataOfSingleTrack,
+  titles,
+  years,
+}) => {
+  const [show, setShow] = useState(false)
+  const [hidePercentages, setHidePercentages] = useState(false)
+
+  if (!dataOfAllTracks && !dataOfSingleTrack) return null
+
+  const sortedMainStats = sortMainDataByYear(Object.values(dataOfAllTracks))
+  const sortedTrackStats = sortTrackDataByYear(dataOfSingleTrack)
+
+  return (
+    <div className="datatable">
+      {getRadioButton(null, 'Show percentages', 'Hide percentages', hidePercentages, setHidePercentages)}
+      <Table data-cy="Table-StudytrackOverview" celled>
+        <Table.Header>
+          <Table.Row>
+            {titles.map((title, index) => (
+              <Table.HeaderCell
+                key={title}
+                colSpan={index === 0 || hidePercentages ? 1 : 2}
+                textAlign="left"
+                style={{ fontWeight: 'bold' }}
+              >
+                {title}
+              </Table.HeaderCell>
+            ))}
+          </Table.Row>
+        </Table.Header>
+
+        <Table.Body>
+          {singleTrack
+            ? sortedTrackStats.map(row =>
+                getSingleTrackRow({ row, studyprogramme, code: singleTrack, hidePercentages })
+              )
+            : sortedMainStats?.map(yearlyData =>
+                yearlyData.map(row =>
+                  getRow({ yearlyData, row, studyprogramme, show, setShow, studytracks, hidePercentages, years })
+                )
+              )}
+        </Table.Body>
+      </Table>
+    </div>
+  )
+}
+
+export default StudytrackDataTable

services/frontend/src/components/StudyProgramme/StudytrackOverview/index.jsx

@@ -0,0 +1,130 @@
+import React, { useEffect, useState } from 'react'
+import { Divider, Loader, Radio } from 'semantic-ui-react'
+
+import { useGetStudytrackStatsQuery } from 'redux/studyProgramme'
+import WithHelpTooltip from '../../Info/InfoWithHelpTooltip'
+import InfoBox from '../../Info/InfoBox'
+import BarChart from './BarChart'
+import BasicDataTable from './BasicDataTable'
+import GaugeChart from './GaugeChart'
+import StudytrackDataTable from './StudytrackDataTable'
+import StudytrackSelector from './StudytrackSelector'
+
+import InfotoolTips from '../../../common/InfoToolTips'
+import '../studyprogramme.css'
+
+const getRadioButton = (toolTip, firstLabel, secondLabel, value, setValue) => (
+  <div className="radio-toggle">
+    <label className="toggle-label">{firstLabel}</label>
+    <Radio toggle checked={value} onChange={() => setValue(!value)} />
+    <WithHelpTooltip tooltip={{ short: toolTip }}>
+      <label className="toggle-label">{secondLabel}</label>
+    </WithHelpTooltip>
+  </div>
+)
+
+const StudytrackOverview = ({ studyprogramme, specialGroups, setSpecialGroups, graduated, setGraduated }) => {
+  const toolTips = InfotoolTips.Studyprogramme
+  const [showMeanTime, setShowMeanTime] = useState(true)
+  const [track, setTrack] = useState(studyprogramme)
+  const special = specialGroups ? 'SPECIAL_EXCLUDED' : 'SPECIAL_INCLUDED'
+  const grad = graduated ? 'GRADUATED_EXCLUDED' : 'GRADUATED_INCLUDED'
+  const stats = useGetStudytrackStatsQuery({ id: studyprogramme, specialGroups: special, graduated: grad })
+
+  useEffect(() => {
+    if (!track && stats?.data?.mainStatsByTrack[studyprogramme]) {
+      setTrack(studyprogramme)
+    }
+  }, [studyprogramme, track, stats, specialGroups])
+
+  const getDivider = (title, toolTipText) => (
+    <>
+      <div className="divider">
+        <Divider data-cy={`Section-${toolTipText}`} horizontal>
+          {title}
+        </Divider>
+      </div>
+      <InfoBox content={toolTips[toolTipText]} />
+    </>
+  )
+
+  const isError = (stats.isSuccess && !stats.data) || stats.isError
+
+  if (isError) return <h3>Something went wrong, please try refreshing the page.</h3>
+
+  return (
+    <div className="studytrack-overview">
+      {stats.isLoading || stats.isFetching ? (
+        <Loader active style={{ marginTop: '10em' }} />
+      ) : (
+        <>
+          <StudytrackSelector track={track} setTrack={setTrack} studytracks={stats?.data?.studytrackOptions} />
+          <div className="toggle-container">
+            {getRadioButton(
+              toolTips.StudentToggle,
+              'All studyrights included',
+              'Special studyrights excluded',
+              specialGroups,
+              setSpecialGroups
+            )}
+            {getRadioButton(toolTips.GradToggle, 'Graduated included', 'Graduated excluded', graduated, setGraduated)}
+          </div>
+          {getDivider(
+            `Students of ${
+              track === '' || track === studyprogramme
+                ? 'the studyprogramme by starting year'
+                : `the studytrack ${track} by starting year`
+            }`,
+            'StudytrackOverview'
+          )}
+          <StudytrackDataTable
+            studyprogramme={studyprogramme}
+            singleTrack={track !== studyprogramme && track}
+            studytracks={stats?.data?.studytrackOptions}
+            titles={stats?.data?.populationTitles}
+            dataOfAllTracks={stats?.data?.mainStatsByYear}
+            dataOfSingleTrack={track && track !== studyprogramme ? stats?.data?.mainStatsByTrack[track] : null}
+            years={stats?.data?.years}
+          />
+          {getDivider(
+            `Progress of students of ${
+              track === '' || track === studyprogramme
+                ? 'the studyprogramme by starting year'
+                : `the studytrack ${track} by starting year`
+            }`,
+            'StudytrackProgress'
+          )}
+          <div className="section-container">
+            <BarChart data={stats?.data} track={track || studyprogramme} />
+            <BasicDataTable
+              cypress="StudytrackProgress"
+              data={stats?.data?.creditTableStats}
+              track={track || studyprogramme}
+              titles={stats?.data?.creditTableTitles}
+            />
+          </div>
+          {stats?.isSuccess && stats?.data?.includeGraduated && (
+            <>
+              {getDivider('Average graduation times', 'AverageGraduationTimes')}
+              {getRadioButton(null, 'Mean time', 'Median time', showMeanTime, setShowMeanTime)}
+              <div className="section-container-centered">
+                {stats?.data?.years.map(year => (
+                  <GaugeChart
+                    key={year}
+                    year={year}
+                    data={stats?.data?.graduationMedianTime[track][year]}
+                    graduationAmount={stats?.data?.graduationAmounts[track][year]}
+                    totalAmount={stats?.data?.totalAmounts[track][year]}
+                    studyprogramme={studyprogramme}
+                  />
+                ))}
+              </div>
+            </>
+          )}
+        </>
+      )}
+    </div>
+  )
+}
+
+export default StudytrackOverview

services/frontend/src/components/StudyProgramme/ThesisCourses/SearchResult.jsx

@@ -1,56 +0,0 @@
-import React from 'react'
-import { connect } from 'react-redux'
-import { Segment } from 'semantic-ui-react'
-import { getCourseSearchResults } from '../../../selectors/courses'
-import useLanguage from '../../LanguagePicker/useLanguage'
-import SortableTable from '../../SortableTable'
-
-const SearchResult = ({ courses, getCourseActions }) => {
-  const { language } = useLanguage()
-
-  if (!courses || courses.length === 0) return <Segment content="No results" />
-
-  return (
-    <SortableTable
-      getRowKey={c => c.code}
-      tableProps={{ celled: true }}
-      columns={[
-        {
-          key: 'code',
-          title: 'Code',
-          getRowVal: c => c.code,
-          headerProps: { width: 2, textAlign: 'center' },
-        },
-        {
-          key: 'name',
-          title: 'Name',
-          getRowVal: c => c.name[language],
-          headerProps: { width: 12, textAlign: 'left' },
-        },
-        {
-          key: 'actions',
-          title: 'Action',
-          getRowContent: getCourseActions,
-          headerProps: { width: 2, textAlign: 'center' },
-        },
-      ]}
-      data={courses}
-    />
-  )
-}
-
-SearchResult.defaultProps = {
-  courses: [],
-}
-
-const mapStateToProps = state => {
-  const { thesisCourses } = state
-  const { data } = thesisCourses || null
-  const thesisCourseCodes = data.map(c => c.courseCode)
-  const searchResults = getCourseSearchResults(state).courses
-  const filteredResults = searchResults.filter(c => !thesisCourseCodes.includes(c.code))
-  return {
-    courses: filteredResults,
-  }
-}
-export default connect(mapStateToProps)(SearchResult)

services/frontend/src/components/StudyProgramme/ThesisCourses/ThesisCourseList.jsx

@@ -1,67 +0,0 @@
-import React, { useEffect } from 'react'
-import { connect } from 'react-redux'
-import { Dimmer, Loader, Segment, Button } from 'semantic-ui-react'
-import { getThesisCourses, deleteThesisCourse } from '../../../redux/thesisCourses'
-import SortableTable from '../../SortableTable'
-
-const ThesisCourseList = ({ studyprogramme, getThesisCourses, pending, data, deleteThesisCourse }) => {
-  useEffect(() => {
-    getThesisCourses(studyprogramme)
-  }, [studyprogramme])
-
-  const handleDelete = course => () => {
-    deleteThesisCourse(studyprogramme, course)
-  }
-
-  return (
-    <>
-      <Dimmer inverted active={pending}>
-        <Loader />
-      </Dimmer>
-      {data.length === 0 ? (
-        <Segment content="No thesis courses" />
-      ) : (
-        <SortableTable
-          tableProps={{ celled: true }}
-          data={data}
-          getRowKey={c => c.courseCode}
-          columns={[
-            {
-              key: 'code',
-              title: 'Code',
-              headerProps: { width: 2 },
-              getRowVal: c => c.courseCode,
-            },
-            {
-              key: 'type',
-              title: 'Type',
-              headerProps: { width: 13 },
-              getRowVal: c => c.thesisType,
-            },
-            {
-              key: 'action',
-              title: 'Remove',
-              headerProps: { width: 1, textAlign: 'center', onClick: null, sorted: null },
-              cellProps: { textAlign: 'center' },
-              getRowContent: c => <Button basic icon="trash" onClick={handleDelete(c.courseCode)} />,
-            },
-          ]}
-        />
-      )}
-    </>
-  )
-}
-
-ThesisCourseList.defaultProps = {
-  data: [],
-}
-
-const mapStateToProps = ({ thesisCourses }) => {
-  const { pending, data } = thesisCourses
-  return {
-    pending,
-    data,
-  }
-}
-
-export default connect(mapStateToProps, { getThesisCourses, deleteThesisCourse })(ThesisCourseList)

services/frontend/src/components/StudyProgramme/ThesisCourses/index.jsx

@@ -1,39 +0,0 @@
-import React from 'react'
-import { Button, Message } from 'semantic-ui-react'
-import { connect } from 'react-redux'
-import CourseSearchForm from '../../CourseSearchForm'
-import ThesisCourseList from './ThesisCourseList'
-import SearchResult from './SearchResult'
-import Expandable from '../../Expandable'
-import { createNewThesisCourse, getThesisCourses } from '../../../redux/thesisCourses'
-
-const ThesisCourses = ({ studyprogramme, createThesis, getTheses }) => {
-  const handleClick = (course, thesis) => () => {
-    createThesis(studyprogramme, course, thesis).then(() => {
-      getTheses(studyprogramme)
-    })
-  }
-  return (
-    <>
-      <Message content="The set of courses used for calculating thesis productivity statistics" />
-      <Expandable fluid title="Add thesis course">
-        <CourseSearchForm />
-        <SearchResult
-          getCourseActions={({ code }) => (
-            <Button.Group basic fluid width="3">
-              <Button content="Bachelors" onClick={handleClick(code, 'BACHELOR')} />
-              <Button.Or />
-              <Button content="Masters" onClick={handleClick(code, 'MASTER')} />
-            </Button.Group>
-          )}
-        />
-      </Expandable>
-      <ThesisCourseList studyprogramme={studyprogramme} />
-    </>
-  )
-}
-
-export default connect(null, {
-  createThesis: createNewThesisCourse,
-  getTheses: getThesisCourses,
-})(ThesisCourses)

services/frontend/src/components/StudyProgramme/ThroughputTable/index.jsx

@@ -1,390 +0,0 @@
-import React, { useState } from 'react'
-import moment from 'moment'
-import { Header, Table, Icon, Label, Segment, Dropdown, Button, Modal, Popup } from 'semantic-ui-react'
-import { Link } from 'react-router-dom'
-import { flatten, uniq, range } from 'lodash'
-import { useGetAuthorizedUserQuery } from 'redux/auth'
-import { isDev } from '../../../conf'
-import InfoBox from '../../Info/InfoBox'
-import infotooltips from '../../../common/InfoToolTips'
-
-const getMonths = year => {
-  const end = moment()
-  const lastDayOfMonth = moment(end).endOf('month')
-  const start = `${year}-08-01`
-  return Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(start))).asMonths())
-}
-
-const PopulationStatisticsLink = ({ studytrack, studyprogramme, year: yearLabel, children }) => {
-  const year = Number(yearLabel.slice(0, 4))
-  const months = Math.ceil(moment.duration(moment().diff(`${year}-08-01`)).asMonths())
-  const href = studytrack
-    ? `/populations?months=${months}&semesters=FALL&semesters=` +
-      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%2C"studyTrack"%3A"${studytrack}"%7D&year=${year}`
-    : `/populations?months=${months}&semesters=FALL&semesters=` +
-      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&year=${year}`
-  return (
-    <Link title={`Population statistics of class ${yearLabel}`} to={href}>
-      {children}
-    </Link>
-  )
-}
-
-const TotalPopulationLink = ({ confirm, years, studyprogramme, studytrack, children }) => {
-  const confirmWrapper = e => {
-    if (confirm) {
-      // eslint-disable-next-line no-alert
-      const c = window.confirm(
-        `Are you sure you want to see a combined population of ${years.length} different populations?`
-      )
-      if (!c) e.preventDefault()
-    }
-  }
-  const yearsString = years.map(year => year.value).join('&years=')
-  const months = getMonths(Math.min(...years.map(year => Number(year.value))))
-  const href = studytrack
-    ? `/populations?months=${months}&semesters=FALL&semesters=` +
-      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%2C"studyTrack"%3A"${studytrack}"%7D&year=${years[0].value}&years=${yearsString}`
-    : `/populations?months=${months}&semesters=FALL&semesters=` +
-      `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&year=${years[0].value}&years=${yearsString}`
-  return (
-    <Link onClick={confirmWrapper} title="Population statistics of all years" to={href}>
-      {children}
-    </Link>
-  )
-}
-
-const ThroughputTable = ({
-  throughput,
-  thesis,
-  loading,
-  error,
-  studyprogramme,
-  studytrack,
-  history,
-  newProgramme,
-  isStudytrackView,
-}) => {
-  const [lowerYear, setLower] = useState(null)
-  const [upperYear, setUpper] = useState(null)
-  const [yearDifference, setDifference] = useState(null)
-  const { isAdmin } = useGetAuthorizedUserQuery()
-
-  const data = throughput && throughput.data ? throughput.data.filter(year => year.credits.length > 0) : []
-
-  const years =
-    data && !isStudytrackView
-      ? data
-          .map(stats => ({
-            key: stats.year.substring(0, 4),
-            text: stats.year,
-            value: stats.year.substring(0, 4),
-          }))
-          .sort((year1, year2) => Number(year2.value) - Number(year1.value))
-      : []
-
-  if (error) return <h1>Oh no so error {error}</h1>
-
-  // toggle only on dev env and with admin user
-  const GRADUATED_FEATURE_TOGGLED_ON = isAdmin && isDev
-
-  const genders = data.length > 0 ? uniq(flatten(data.map(year => Object.keys(year.genders)))) : []
-  genders.sort()
-
-  const renderGenders = genders.length > 0
-
-  const calculateTotalNationalities = () =>
-    data.length > 0 ? Object.values(throughput.totals.nationalities).reduce((res, curr) => res + curr, 0) : 0
-
-  const renderRatioOfFinns = calculateTotalNationalities() > 0
-  let thesisTypes = []
-  if (thesis) {
-    thesisTypes = thesis.map(t => t.thesisType)
-  }
-
-  const handleLowerBoundChange = (event, { value }) => {
-    event.preventDefault()
-    if (Number(value) > Number(upperYear)) {
-      setUpper(value)
-      setLower(value)
-      setDifference(1)
-    } else if (Number(upperYear) - Number(value) > 5) {
-      setUpper(String(Number(value) + 5))
-      setLower(value)
-      setDifference(6)
-    } else {
-      setLower(value)
-      setDifference(Number(upperYear) - Number(value) + 1)
-    }
-  }
-
-  const handleUpperBoundChange = (event, { value }) => {
-    event.preventDefault()
-    if (Number(value) < Number(lowerYear)) {
-      setUpper(value)
-      setLower(value)
-      setDifference(1)
-    } else if (Number(value) - Number(lowerYear) > 5) {
-      setLower(String(Number(value) - 5))
-      setUpper(value)
-      setDifference(6)
-    } else {
-      setUpper(value)
-      setDifference(Number(value) - Number(lowerYear) + 1)
-    }
-  }
-
-  const pushQueryToUrl = event => {
-    event.preventDefault()
-    const yearRange = range(Number(lowerYear), Number(upperYear) + 1)
-    const months = getMonths(Number(lowerYear))
-    const yearsString = yearRange.join('&years=')
-
-    if (yearRange.length > 1) {
-      history.push(
-        `/populations?months=${months}&semesters=FALL&semesters=` +
-          `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&year=${lowerYear}&years=${yearsString}`
-      )
-    } else {
-      history.push(
-        `/populations?months=${months}&semesters=FALL&semesters=` +
-          `SPRING&studyRights=%7B"programme"%3A"${studyprogramme}"%7D&year=${lowerYear}&years[]=${yearsString}`
-      )
-    }
-  }
-
-  const headerCellPopUp = (title, cell) => {
-    return (
-      <Popup trigger={cell} wide="very" position="left center">
-        <Popup.Content>{infotooltips.PopulationOverview[title]}</Popup.Content>
-      </Popup>
-    )
-  }
-
-  const renderStudentsHeader = () => {
-    let colSpan = 4
-    let rowSpan = 1
-
-    if (renderGenders) colSpan += genders.length
-    if (renderRatioOfFinns) colSpan += 1
-    if (!renderGenders && !renderRatioOfFinns) rowSpan += 1
-
-    return headerCellPopUp(
-      'Students',
-      <Table.HeaderCell colSpan={colSpan} rowSpan={rowSpan}>
-        Students
-      </Table.HeaderCell>
-    )
-  }
-
-  const ratioOfFinnsIn = year => {
-    const total = Object.values(year.nationalities).reduce((res, curr) => res + curr, 0)
-    return (
-      <Table.Cell>
-        {`${year.nationalities.Finland || 0} (${Math.floor((year.nationalities.Finland / total) * 100) || 0}%)`}
-      </Table.Cell>
-    )
-  }
-  return (
-    <>
-      <div style={{ marginTop: '2em' }}>
-        <InfoBox content={infotooltips.PopulationOverview.PopulationProgress} />
-      </div>
-      <Header>
-        {studytrack ? `Population progress for selected studytrack` : 'Population progress'}
-        {throughput && (
-          <Header.Subheader data-cy="throughputUpdateStatus">
-            {`Last updated ${
-              throughput.lastUpdated ? moment(throughput.lastUpdated).format('HH:mm:ss MM-DD-YYYY') : 'unknown'
-            }`}
-            {throughput.status === 'RECALCULATING' && (
-              <Label content="Recalculating! Refresh page in a few minutes" color="red" />
-            )}
-          </Header.Subheader>
-        )}
-      </Header>
-      <Segment basic loading={loading} style={{ overflowX: 'auto' }}>
-        <Table celled structured compact striped selectable className="fixed-header">
-          <Table.Header>
-            <Table.Row>
-              <Table.HeaderCell rowSpan="2">{isStudytrackView ? 'Studytrack' : 'Year'}</Table.HeaderCell>
-              {renderStudentsHeader()}
-
-              {headerCellPopUp(
-                'Graduated',
-                <Table.HeaderCell colSpan={GRADUATED_FEATURE_TOGGLED_ON ? '3' : '1'}>Graduated</Table.HeaderCell>
-              )}
-              {headerCellPopUp('Credits', <Table.HeaderCell colSpan="5">Credits</Table.HeaderCell>)}
-              {(thesisTypes.includes('BACHELOR') || thesisTypes.includes('MASTER')) &&
-                headerCellPopUp('Thesis', <Table.HeaderCell colSpan={thesisTypes.length}>Thesis</Table.HeaderCell>)}
-            </Table.Row>
-
-            <Table.Row>
-              {renderGenders || renderRatioOfFinns ? <Table.HeaderCell content="All" /> : null}
-              {genders.map(gender => (
-                <Table.HeaderCell key={gender} content={gender} />
-              ))}
-              {renderRatioOfFinns ? <Table.HeaderCell content="Finnish" /> : null}
-              {headerCellPopUp(
-                'Started',
-                <Table.HeaderCell rowSpan="2" colSpan="1">
-                  Started
-                </Table.HeaderCell>
-              )}
-              {headerCellPopUp(
-                'Transferred',
-                <Table.HeaderCell rowSpan="2" colSpan="1">
-                  Transferred to
-                </Table.HeaderCell>
-              )}
-              {headerCellPopUp(
-                'Cancelled',
-                <Table.HeaderCell rowSpan="2" colSpan="1">
-                  Cancelled
-                </Table.HeaderCell>
-              )}
-              <Table.HeaderCell>Graduated overall</Table.HeaderCell>
-              {GRADUATED_FEATURE_TOGGLED_ON && (
-                <>
-                  <Table.HeaderCell>Graduated in time</Table.HeaderCell>
-                  <Table.HeaderCell>Graduation median time</Table.HeaderCell>
-                </>
-              )}
-
-              <Table.HeaderCell content="≥ 30" />
-              <Table.HeaderCell content="≥ 60" />
-              <Table.HeaderCell content="≥ 90" />
-              <Table.HeaderCell content="≥ 120" />
-              <Table.HeaderCell content="≥ 150" />
-              {thesisTypes.includes('MASTER') && <Table.HeaderCell content="Master" />}
-              {thesisTypes.includes('BACHELOR') && <Table.HeaderCell content="Bachelor" />}
-            </Table.Row>
-          </Table.Header>
-          <Table.Body>
-            {data
-              .sort((year1, year2) => Number(year2.year.slice(0, 4)) - Number(year1.year.slice(0, 4)))
-              .map(year => (
-                <Table.Row key={year.year}>
-                  <Table.Cell>
-                    {year.year}
-                    {!isStudytrackView && (
-                      <PopulationStatisticsLink
-                        studytrack={studytrack}
-                        studyprogramme={studyprogramme}
-                        year={year.year}
-                      >
-                        <Icon name="level up alternate" />
-                      </PopulationStatisticsLink>
-                    )}
-                  </Table.Cell>
-                  <Table.Cell>{year.credits.length}</Table.Cell>
-                  {genders.map(gender => (
-                    <Table.Cell key={`${year.year} gender:${gender}`}>
-                      {`${year.genders[gender] || 0} (${
-                        Math.floor((year.genders[gender] / year.credits.length) * 100) || 0
-                      }%)`}
-                    </Table.Cell>
-                  ))}
-                  {renderRatioOfFinns && ratioOfFinnsIn(year)}
-                  <Table.Cell>{year.started}</Table.Cell>
-                  <Table.Cell>{year.transferred}</Table.Cell>
-                  <Table.Cell>{year.cancelled}</Table.Cell>
-                  <Table.Cell>{year.graduated}</Table.Cell>
-
-                  {GRADUATED_FEATURE_TOGGLED_ON && (
-                    <>
-                      <Table.Cell>{year.inTargetTime}</Table.Cell>
-                      <Table.Cell>{year.medianGraduationTime ? `${year.medianGraduationTime} months` : '∞'}</Table.Cell>
-                    </>
-                  )}
-
-                  {Object.keys(year.creditValues).map(creditKey => (
-                    <Table.Cell key={`${year.year} credit:${creditKey}`}>{year.creditValues[creditKey]}</Table.Cell>
-                  ))}
-                  {thesisTypes.includes('MASTER') ? <Table.Cell>{year.thesisM}</Table.Cell> : null}
-                  {thesisTypes.includes('BACHELOR') ? <Table.Cell>{year.thesisB}</Table.Cell> : null}
-                </Table.Row>
-              ))}
-          </Table.Body>
-          {throughput && throughput.totals ? (
-            <Table.Footer>
-              <Table.Row>
-                <Table.HeaderCell style={{ fontWeight: 'bold' }}>
-                  All{' '}
-                  {newProgramme && !isStudytrackView && years.length > 0 && years.length < 5 ? (
-                    <TotalPopulationLink confirm studyprogramme={studyprogramme} studytrack={studytrack} years={years}>
-                      <Icon name="level up alternate" />
-                    </TotalPopulationLink>
-                  ) : null}
-                </Table.HeaderCell>
-                <Table.HeaderCell>{throughput.totals.students}</Table.HeaderCell>
-                {Object.keys(throughput.totals.genders)
-                  .sort()
-                  .map(genderKey => (
-                    <Table.HeaderCell key={`${genderKey}total`}>
-                      {`${throughput.totals.genders[genderKey]} (${Math.floor(
-                        (throughput.totals.genders[genderKey] / throughput.totals.students) * 100
-                      )}%)`}
-                    </Table.HeaderCell>
-                  ))}
-                {renderRatioOfFinns ? (
-                  <Table.HeaderCell>
-                    {`${throughput.totals.nationalities.Finland || 0} (${
-                      Math.floor((throughput.totals.nationalities.Finland / calculateTotalNationalities()) * 100) || 0
-                    }%)`}
-                  </Table.HeaderCell>
-                ) : null}
-                <Table.HeaderCell>{throughput.totals.started}</Table.HeaderCell>
-                <Table.HeaderCell>{throughput.totals.transferred}</Table.HeaderCell>
-                <Table.HeaderCell>{throughput.totals.cancelled}</Table.HeaderCell>
-                <Table.HeaderCell>{throughput.totals.graduated}</Table.HeaderCell>
-                {GRADUATED_FEATURE_TOGGLED_ON && (
-                  <>
-                    <Table.HeaderCell>{throughput.totals.inTargetTime}</Table.HeaderCell>
-                    <Table.HeaderCell>
-                      {throughput.totals.medianGraduationTime
-                        ? `${throughput.totals.medianGraduationTime} months`
-                        : '∞'}
-                    </Table.HeaderCell>
-                  </>
-                )}
-                {Object.keys(throughput.totals.credits).map(creditKey => (
-                  <Table.HeaderCell key={`${creditKey}total`}>{throughput.totals.credits[creditKey]}</Table.HeaderCell>
-                ))}
-                {thesisTypes.includes('MASTER') ? (
-                  <Table.HeaderCell>{throughput.totals.thesisM}</Table.HeaderCell>
-                ) : null}
-                {thesisTypes.includes('BACHELOR') ? (
-                  <Table.HeaderCell>{throughput.totals.thesisB}</Table.HeaderCell>
-                ) : null}
-              </Table.Row>
-            </Table.Footer>
-          ) : null}
-        </Table>
-      </Segment>
-      {years.length > 4 ? (
-        <>
-          Statistics from:
-          <Dropdown selection options={years} onChange={handleLowerBoundChange} value={lowerYear} />
-          to:
-          <Dropdown selection options={years} onChange={handleUpperBoundChange} value={upperYear} />
-          <Modal
-            trigger={<Button disabled={!lowerYear || !upperYear}>Show combined population</Button>}
-            header={`Are you sure you want to see a combined population of ${yearDifference} different populations?`}
-            actions={[
-              'Cancel',
-              {
-                key: 'fetch',
-                content: 'Show population',
-                positive: true,
-                onClick: event => pushQueryToUrl(event),
-              },
-            ]}
-          />
-        </>
-      ) : null}
-    </>
-  )
-}
-
-export default ThroughputTable

services/frontend/src/components/StudyProgramme/UpdateView/index.js

@@ -0,0 +1,38 @@
+import React, { useState } from 'react'
+import { Button, Icon, Loader } from 'semantic-ui-react'
+import { useUpdateBasicViewQuery, useUpdateStudytrackViewQuery } from '../../../redux/studyProgramme'
+
+const getStatusIcon = stats => {
+  if (stats.isLoading) return <Loader active />
+  if (stats.isSuccess) return <Icon name="check" color="green" />
+  if (stats.isError) return <Icon name="close" color="red" />
+  return ''
+}
+
+const NewUpdateView = ({ studyprogramme }) => {
+  const [skipBasic, setSkipBasic] = useState(true)
+  const [skipStudytrack, setSkipStudytrack] = useState(true)
+  const basicstats = useUpdateBasicViewQuery({ id: studyprogramme }, { skip: skipBasic })
+  const studytrackstats = useUpdateStudytrackViewQuery({ id: studyprogramme }, { skip: skipStudytrack })
+
+  return (
+    <div className="update-view">
+      <div className="button-container">
+        <h4>Update data on Basic Information -view</h4>
+        <Button disabled={basicstats.isLoading} color="blue" onClick={() => setSkipBasic(false)}>
+          Update
+        </Button>
+        {getStatusIcon(basicstats)}
+      </div>
+      <div className="button-container">
+        <h4>Update data on Populations and Studytracks -view</h4>
+        <Button disabled={studytrackstats.isLoading} color="blue" onClick={() => setSkipStudytrack()}>
+          Update
+        </Button>
+        {getStatusIcon(studytrackstats)}
+      </div>
+    </div>
+  )
+}
+
+export default NewUpdateView

services/frontend/src/components/StudyProgramme/index.jsx

@@ -1,135 +1,95 @@
-import React, { useCallback, useEffect } from 'react'
-import { withRouter } from 'react-router-dom'
-import { connect } from 'react-redux'
-import { Header, Segment, Tab, Button } from 'semantic-ui-react'
-import { isEqual, uniqBy } from 'lodash'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useHistory, withRouter } from 'react-router-dom'
+import { connect, useDispatch, useSelector } from 'react-redux'
+import { Header, Segment, Tab } from 'semantic-ui-react'
 import { useGetAuthorizedUserQuery } from 'redux/auth'
 import DegreeCoursesTable from './DegreeCourses'
 import StudyProgrammeSelector from './StudyProgrammeSelector'
-import Overview from './Overview'
-import NewOverview from './NewOverview'
-import NewStudytrackOverview from './NewStudytrackOverview'
-import StudyTrackOverview from './StudyTrackOverview'
-import ThesisCourses from './ThesisCourses'
+import BasicOverview from './BasicOverview'
+import StudytrackOverview from './StudytrackOverview'
+import UpdateView from './UpdateView'
 import '../PopulationQueryCard/populationQueryCard.css'
 import { getTextIn } from '../../common'
 import { useTabs, useTitle } from '../../common/hooks'
 import TSA from '../../common/tsa'
 import Tags from './Tags'
 
-import { getThroughput } from '../../redux/throughput'
-import { getProductivity } from '../../redux/productivity'
 import { getProgrammes } from '../../redux/populationProgrammes'
 
-import { callApi } from '../../apiConnection'
 import useLanguage from '../LanguagePicker/useLanguage'
 
 const StudyProgramme = props => {
+  const dispatch = useDispatch()
+  const history = useHistory()
+  const programmes = useSelector(state => state.populationProgrammes?.data?.programmes)
   const { language } = useLanguage()
-  const { roles, isAdmin } = useGetAuthorizedUserQuery()
-  const [tab, setTab] = useTabs('p_tab', 0, props.history)
+  const { isAdmin } = useGetAuthorizedUserQuery()
+  const [tab, setTab] = useTabs('p_tab', 0, history)
+  const [academicYear, setAcademicYear] = useState(false)
+  const [specialGroups, setSpecialGroups] = useState(false)
+  const [graduated, setGraduated] = useState(false)
 
   useTitle('Study programmes')
 
   useEffect(() => {
-    props.getProgrammesDispatch()
+    dispatch(getProgrammes())
   }, [])
 
-  const refreshProductivity = () => {
-    callApi('/v2/studyprogrammes/productivity/recalculate', 'get', null, {
-      code: props.match.params.studyProgrammeId,
-    })
-      .then(() => {
-        props.getProductivityDispatch(props.match.params.studyProgrammeId)
-      })
-      .catch(e => {
-        if (e.message.toLowerCase() === 'network error') {
-          window.location.reload(true)
-        }
-      })
-  }
-  const refreshThroughput = () => {
-    callApi('/v2/studyprogrammes/throughput/recalculate', 'get', null, {
-      code: props.match.params.studyProgrammeId,
-    })
-      .then(() => {
-        props.getThroughputDispatch(props.match.params.studyProgrammeId)
-      })
-      .catch(e => {
-        if (e.message.toLowerCase() === 'network error') {
-          window.location.reload(true)
-        }
-      })
-  }
-
-  const SHOW_NEW_OVERVIEW = isAdmin && roles.includes('teachers')
-
   const getPanes = () => {
-    const { match, programmes } = props
+    const { match } = props
     const { studyProgrammeId } = match.params
-    const filteredStudytracks = programmes?.[studyProgrammeId]
-      ? Object.values(programmes?.[studyProgrammeId].enrollmentStartYears).reduce((acc, curr) => {
-          acc.push(...Object.values(curr.studyTracks))
-          return uniqBy(acc, 'code')
-        }, [])
-      : []
     const panes = []
     panes.push({
-      menuItem: 'Overview',
-      render: () => <Overview studyprogramme={studyProgrammeId} history={props.history} />,
+      menuItem: 'Basic information (NEW)',
+      render: () => (
+        <BasicOverview
+          studyprogramme={studyProgrammeId}
+          history={history}
+          specialGroups={specialGroups}
+          setSpecialGroups={setSpecialGroups}
+          academicYear={academicYear}
+          setAcademicYear={setAcademicYear}
+        />
+      ),
     })
-    if (filteredStudytracks.length > 0) {
+    panes.push({
+      menuItem: 'Studytracks and student populations (NEW)',
+      render: () => (
+        <StudytrackOverview
+          studyprogramme={studyProgrammeId}
+          history={history}
+          specialGroups={specialGroups}
+          setSpecialGroups={setSpecialGroups}
+          graduated={graduated}
+          setGraduated={setGraduated}
+        />
+      ),
+    })
+    if (isAdmin) {
       panes.push({
-        menuItem: 'Studytrack overview',
-        render: () => <StudyTrackOverview studyprogramme={studyProgrammeId} history={props.history} admin={isAdmin} />,
+        menuItem: 'Update statistics',
+        render: () => <UpdateView studyprogramme={studyProgrammeId} />,
       })
     }
     panes.push({
       menuItem: 'Degree Courses',
       render: () => <DegreeCoursesTable studyProgramme={studyProgrammeId} />,
     })
-    panes.push({
-      menuItem: 'Thesis Courses',
-      render: () => <ThesisCourses studyprogramme={studyProgrammeId} />,
-    })
     panes.push({
       menuItem: 'Tags',
       render: () => <Tags studyprogramme={studyProgrammeId} />,
     })
-    if (SHOW_NEW_OVERVIEW) {
-      panes.push({
-        menuItem: 'Basic information (NEW)',
-        render: () => <NewOverview studyprogramme={studyProgrammeId} history={props.history} />,
-      })
-    }
-    if (SHOW_NEW_OVERVIEW) {
-      panes.push({
-        menuItem: 'Populations and Studytracks (NEW)',
-        render: () => <NewStudytrackOverview studyprogramme={studyProgrammeId} history={props.history} />,
-      })
-    }
-    if (isAdmin) {
-      panes.push({
-        menuItem: 'Admin',
-        render: () => (
-          <>
-            <Button onClick={() => refreshThroughput()}>recalculate throughput</Button>
-            <Button onClick={() => refreshProductivity()}>recalculate productivity</Button>
-          </>
-        ),
-      })
-    }
     return panes
   }
 
   const handleSelect = useCallback(
     programme => {
-      props.history.push(`/study-programme/${programme}`, { selected: programme })
+      history.push(`/study-programme/${programme}`, { selected: programme })
     },
-    [props.history]
+    [history]
   )
 
-  const { match, programmes } = props
+  const { match } = props
   const { studyProgrammeId } = match.params
   const programmeName = programmes?.[studyProgrammeId] && getTextIn(programmes?.[studyProgrammeId].name, language)
   const panes = getPanes()
@@ -173,22 +133,4 @@ const StudyProgramme = props => {
   )
 }
 
-const mapStateToProps = ({ populationProgrammes }) => {
-  const programmes = populationProgrammes.data ? populationProgrammes.data.programmes : {}
-  return {
-    programmes,
-  }
-}
-
-export default connect(
-  mapStateToProps,
-  {
-    getThroughputDispatch: getThroughput,
-    getProductivityDispatch: getProductivity,
-    getProgrammesDispatch: getProgrammes,
-  },
-  null,
-  {
-    areStatePropsEqual: isEqual,
-  }
-)(withRouter(StudyProgramme))
+export default connect()(withRouter(StudyProgramme))

services/frontend/src/components/StudyProgramme/studyprogramme.css

@@ -1,9 +1,37 @@
+@media only screen and (min-width: 3240px) {
+  .studytrack-overview,
+  .studyprogramme-overview {
+    max-width: 80%;
+  }
+}
+
+@media only screen and (min-width: 3240px) {
+  .studytrack-overview .section-container,
+  .studyprogramme-overview .section-container {
+    justify-content: center !important;
+  }
+}
+
+.studytrack-overview .toggle-container,
+.studyprogramme-overview .toggle-container {
+  display: flex;
+  justify-content: center;
+  margin-top: 1em;
+}
+
 .studyprogramme-overview .radio-toggle {
   display: flex;
   justify-content: center;
-  margin: 20px 35px 35px 35px;
+  margin: 20px 35px 20px 35px;
 }
 
+.studytrack-overview .radio-toggle {
+  display: flex;
+  justify-content: center;
+  margin: 0 35px 70px 35px;
+}
+
+.studytrack-overview .toggle-label,
 .studyprogramme-overview .toggle-label {
   cursor: pointer;
   padding-left: 0.5em;
@@ -22,21 +50,41 @@
   flex-wrap: wrap;
 }
 
+.studytrack-overview .section-container-centered,
+.studyprogramme-overview .section-container-centered {
+  display: flex;
+  justify-content: center;
+  flex-wrap: wrap;
+}
+
 .studytrack-overview .graph-container,
 .studyprogramme-overview .graph-container {
-  min-width: 55%;
+  min-width: 50%;
+  height: 500px;
+}
+
+.studytrack-overview .graph-container-narrow,
+.studyprogramme-overview .graph-container-narrow {
+  min-width: 40%;
   height: 500px;
 }
 
 .studytrack-overview .table-container,
 .studyprogramme-overview .table-container {
-  width: 40%;
-  height: 500px;
-  margin-left: 2em;
+  max-width: 45%;
+  margin-left: 1em;
+  flex-grow: 2;
+}
+
+.studytrack-overview .table-container-wide,
+.studyprogramme-overview .table-container-wide {
+  max-width: 55%;
+  margin-left: 1em;
+  flex-grow: 2;
 }
 
 .studytrack-overview .studytrack-selector {
-  margin: 60px 200px 60px 60px;
+  margin: 60px 60px 30px 60px;
 }
 
 .studytrack-overview .header-row {
@@ -50,3 +98,11 @@
 .studytrack-overview .no-data-message {
   margin: 20px 0;
 }
+
+.update-view {
+  margin: 2em 0;
+}
+
+.update-view .button-container {
+  margin: 1em 0;
+}

services/frontend/src/redux/index.js

@@ -9,9 +9,6 @@ import populationSelectedStudentCourses from './populationSelectedStudentCourses
 import populationProgrammes from './populationProgrammes'
 import populationProgrammesUnfiltered from './populationProgrammesUnfiltered'
 import populationMandatoryCourses from './populationMandatoryCourses'
-import studyProgrammeProductivity from './productivity'
-import studyProgrammeThroughput from './throughput'
-import studyProgrammeBachelors from './studyProgrammeBachelors'
 import faculties from './faculties'
 import units from './units'
 import elementdetails from './elementdetails'
@@ -53,9 +50,6 @@ const store = configureStore({
     populationProgrammes,
     populationProgrammesUnfiltered,
     populationMandatoryCourses,
-    studyProgrammeProductivity,
-    studyProgrammeThroughput,
-    studyProgrammeBachelors,
     faculties,
     units,
     students,

services/frontend/src/redux/productivity.js

@@ -1,13 +0,0 @@
-import { callController } from '../apiConnection/index'
-import listreducer from './common/listreducer'
-
-const prefix = 'STUDYPROGRAMME_PRODUCTIVITY_'
-
-export const getProductivity = studyprogrammeId => {
-  const route = `v2/studyprogrammes/${studyprogrammeId}/productivity`
-  return callController(route, prefix, [], 'get')
-}
-
-const reducer = listreducer(prefix, null, false)
-
-export default reducer

services/frontend/src/redux/studyProgramme.js

@@ -3,20 +3,36 @@ import { RTKApi } from 'apiConnection'
 const studyprogrammeApi = RTKApi.injectEndpoints({
   endpoints: builder => ({
     getBasicStats: builder.query({
-      query: ({ id, yearType }) => `/v2/studyprogrammes/${id}/basicstats?year_type=${yearType}`,
+      query: ({ id, yearType, specialGroups }) =>
+        `/v2/studyprogrammes/${id}/basicstats?year_type=${yearType}&special_groups=${specialGroups}`,
     }),
     getCreditStats: builder.query({
-      query: ({ id, yearType }) => `/v2/studyprogrammes/${id}/creditstats?year_type=${yearType}`,
+      query: ({ id, yearType, specialGroups }) =>
+        `/v2/studyprogrammes/${id}/creditstats?year_type=${yearType}&special_groups=${specialGroups}`,
     }),
     getGraduationStats: builder.query({
-      query: ({ id, yearType }) => `/v2/studyprogrammes/${id}/graduationstats?year_type=${yearType}`,
+      query: ({ id, yearType, specialGroups }) =>
+        `/v2/studyprogrammes/${id}/graduationstats?year_type=${yearType}&special_groups=${specialGroups}`,
     }),
     getStudytrackStats: builder.query({
-      query: ({ id }) => `/v2/studyprogrammes/${id}/studytrackstats`,
+      query: ({ id, graduated, specialGroups }) =>
+        `/v2/studyprogrammes/${id}/studytrackstats?graduated=${graduated}&special_groups=${specialGroups}`,
+    }),
+    updateBasicView: builder.query({
+      query: ({ id }) => `/v2/studyprogrammes/${id}/update_basicview`,
+    }),
+    updateStudytrackView: builder.query({
+      query: ({ id }) => `/v2/studyprogrammes/${id}/update_studytrackview`,
     }),
   }),
   overrideExisting: false,
 })
 
-export const { useGetBasicStatsQuery, useGetCreditStatsQuery, useGetGraduationStatsQuery, useGetStudytrackStatsQuery } =
-  studyprogrammeApi
+export const {
+  useGetBasicStatsQuery,
+  useGetCreditStatsQuery,
+  useGetGraduationStatsQuery,
+  useGetStudytrackStatsQuery,
+  useUpdateBasicViewQuery,
+  useUpdateStudytrackViewQuery,
+} = studyprogrammeApi

services/frontend/src/redux/studyProgrammeBachelors.js

@@ -1,13 +0,0 @@
-import { callController } from '../apiConnection/index'
-import listreducer from './common/listreducer'
-
-const prefix = 'GET_STUDYPROGRAMME_BACHELORS_'
-
-export const getBachelors = studyprogrammeId => {
-  const route = `v2/studyprogrammes/${studyprogrammeId}/optiondata`
-  return callController(route, prefix, [], 'get')
-}
-
-const reducer = listreducer(prefix, null, true)
-
-export default reducer

services/frontend/src/redux/throughput.js

@@ -1,13 +0,0 @@
-import { callController } from '../apiConnection/index'
-import listreducer from './common/listreducer'
-
-const prefix = 'GET_STUDYPROGRAMME_THROUGHPUT_'
-
-export const getThroughput = studyprogrammeId => {
-  const route = `v2/studyprogrammes/${studyprogrammeId}/throughput`
-  return callController(route, prefix, [], 'get')
-}
-
-const reducer = listreducer(prefix, null, false)
-
-export default reducer

updater/sis-updater-scheduler/src/config.js

@@ -13,3 +13,8 @@ module.exports.REDIS_LATEST_MESSAGE_RECEIVED = 'LATEST_MESSAGE_RECEIVED'
 module.exports.LATEST_MESSAGE_RECEIVED_THRESHOLD = 1000 * 60 * 5
 module.exports.REDIS_LAST_WEEKLY_SCHEDULE = 'LAST_WEEKLY_SCHEDULE'
 module.exports.REDIS_LAST_PREPURGE_INFO = 'LAST_PREPURGE_INFO'
+
+module.exports.EXIT_AFTER_IMMEDIATES = process.env.EXIT_AFTER_IMMEDIATES === 'yes'
+module.exports.SCHEDULE_IMMEDIATE = (process.env.SCHEDULE_IMMEDIATE || 'none').split(',')
+module.exports.ENABLE_WORKER_REPORTING =
+  module.exports.EXIT_AFTER_IMMEDIATES || process.env.ENABLE_WORKER_REPORTING === 'yes'

updater/sis-updater-scheduler/src/index.js

@@ -3,7 +3,13 @@ const { logger } = require('./utils/logger')
 const { set: redisSet } = require('./utils/redis')
 const { schedule: scheduleCron } = require('./utils/cron')
 const { stan } = require('./utils/stan')
-const { isDev, REDIS_LAST_WEEKLY_SCHEDULE, REDIS_LAST_HOURLY_SCHEDULE } = require('./config')
+const {
+  isDev,
+  REDIS_LAST_WEEKLY_SCHEDULE,
+  REDIS_LAST_HOURLY_SCHEDULE,
+  EXIT_AFTER_IMMEDIATES,
+  SCHEDULE_IMMEDIATE,
+} = require('./config')
 const { startServer } = require('./server')
 const { scheduleHourly, scheduleWeekly, schedulePrePurge, schedulePurge, isUpdaterActive } = require('./scheduler')
 
@@ -22,17 +28,70 @@ knexConnection.on('error', e => {
   if (!process.env.CI) process.exit(1)
 })
 
+const JOB_TYPES = {
+  hourly: scheduleHourly,
+  weekly: scheduleWeekly,
+  prepurge: schedulePrePurge,
+  purge: schedulePurge,
+}
+
+const scheduleJob = async type => {
+  if (!JOB_TYPES[type]) {
+    logger.error({
+      message: `Cannot schedule unknown job type '${type}'.`,
+      job: type,
+    })
+
+    return
+  }
+
+  logger.info(`Scheduling job type '${type}'.`)
+
+  await JOB_TYPES[type]()
+
+  logger.info(`Job '${type}' finished.`)
+}
+
+const handleImmediates = async () => {
+  try {
+    await Promise.all(SCHEDULE_IMMEDIATE.map(scheduleJob))
+
+    if (EXIT_AFTER_IMMEDIATES) {
+      process.exit(0)
+    }
+  } catch (e) {
+    logger.error({
+      message: `Running immediate job failed: ${e.message}`,
+      meta: e.stack,
+    })
+
+    if (EXIT_AFTER_IMMEDIATES) {
+      process.exit(1)
+    }
+  }
+}
+
 knexConnection.on('connect', async () => {
   logger.info('Knex database connection established successfully')
   startServer()
 
+  await handleImmediates()
+
   // Monday-Friday at every minute 30
   scheduleCron('30 * * * 1-5', async () => {
     // If updater is currently running, then return
     if ((await isUpdaterActive()) || isDev) return
     logger.info('Starting hourly')
 
-    await scheduleHourly()
+    try {
+      await scheduleHourly()
+    } catch (e) {
+      logger.error({
+        message: 'Hourly run failed: ' + e.message,
+        meta: e.stack,
+      })
+    }
+
     await redisSet(REDIS_LAST_HOURLY_SCHEDULE, new Date())
   })
 
@@ -41,7 +100,15 @@ knexConnection.on('connect', async () => {
     if (isDev) return
     logger.info('Starting weekly')
 
-    await scheduleWeekly()
+    try {
+      await scheduleWeekly()
+    } catch (e) {
+      logger.error({
+        message: 'Weekly run failed: ' + e.message,
+        meta: e.stack,
+      })
+    }
+
     await redisSet(REDIS_LAST_WEEKLY_SCHEDULE, new Date())
   })
 
@@ -49,7 +116,15 @@ knexConnection.on('connect', async () => {
   scheduleCron('0 12 * * MON', async () => {
     if (isDev) return
     logger.info('Starting prepurge')
-    await schedulePrePurge()
+
+    try {
+      await schedulePrePurge()
+    } catch (e) {
+      logger.error({
+        message: 'Prepurge failed: ' + e.message,
+        meta: e.stack,
+      })
+    }
   })
 
   // Sunday at 4 AM
@@ -57,6 +132,14 @@ knexConnection.on('connect', async () => {
     // If updater is currently running, then return
     if ((await isUpdaterActive()) || isDev) return
     logger.info('Starting purge')
-    await schedulePurge()
+
+    try {
+      await schedulePurge()
+    } catch (e) {
+      logger.error({
+        message: 'Purge failed: ' + e.message,
+        meta: e.stack,
+      })
+    }
   })
 })

updater/sis-updater-scheduler/src/scheduler.js

@@ -1,9 +1,11 @@
 const { chunk } = require('lodash')
 const { eachLimit } = require('async')
+const uuid = require('uuid')
 const { knexConnection } = require('./db/connection')
-const { stan } = require('./utils/stan')
+const { stan, opts } = require('./utils/stan')
 const { incrby: redisIncrementBy, get: redisGet } = require('./utils/redis')
 const {
+  NATS_GROUP,
   SIS_UPDATER_SCHEDULE_CHANNEL,
   SIS_MISC_SCHEDULE_CHANNEL,
   CHUNK_SIZE,
@@ -14,6 +16,7 @@ const {
   REDIS_LAST_HOURLY_SCHEDULE,
   REDIS_LATEST_MESSAGE_RECEIVED,
   LATEST_MESSAGE_RECEIVED_THRESHOLD,
+  ENABLE_WORKER_REPORTING,
 } = require('./config')
 const { startPrePurge, startPurge } = require('./purge')
 const { logger } = require('./utils/logger')
@@ -36,11 +39,48 @@ const createJobs = async (entityIds, type, channel = SIS_UPDATER_SCHEDULE_CHANNE
   const redisKey = type === 'students' ? REDIS_TOTAL_STUDENTS_KEY : REDIS_TOTAL_META_KEY
   await redisIncrementBy(redisKey, entityIds.length)
 
-  return new Promise((res, rej) => {
-    stan.publish(channel, JSON.stringify({ entityIds, type }), err => {
-      if (err) return rej(err)
-      res()
-    })
+  if (!ENABLE_WORKER_REPORTING) {
+    stan.publish(channel, JSON.stringify({ entityIds, type }))
+    return
+  }
+
+  const id = uuid.v4()
+
+  const completionChannel = stan.subscribe('SIS_COMPLETED_CHANNEL-' + id, NATS_GROUP, opts)
+
+  return new Promise((resolve, reject) => {
+    const messageHandler = msg => {
+      let data = null
+
+      try {
+        data = JSON.parse(msg.getData())
+      } catch (err) {
+        logger.error({
+          message: 'Unable to parse completion message.',
+          meta: err.stack,
+        })
+
+        return
+      }
+
+      if (data.id === id) {
+        if (data.success) {
+          resolve()
+        } else {
+          logger.error('Job failed: ' + data.message)
+          reject(data.message)
+        }
+
+        completionChannel.off('message', messageHandler)
+        completionChannel.unsubscribe()
+      }
+
+      msg.ack()
+    }
+
+    completionChannel.on('message', messageHandler)
+
+    stan.publish(channel, JSON.stringify({ id, entityIds, type }))
   })
 }
 
@@ -195,6 +235,7 @@ const scheduleHourly = async () => {
     await eachLimit(chunk(personsToUpdate, CHUNK_SIZE), 10, async s => await createJobs(s, 'students'))
   } catch (e) {
     logger.error({ message: 'Hourly scheduling failed', meta: e.stack })
+    throw e
   }
 }
 
@@ -209,6 +250,7 @@ const scheduleProgrammes = async () => {
     await createJobs(entityIds, 'programme_modules')
   } catch (e) {
     logger.error({ message: 'Programme module scheduling failed', meta: e.stack })
+    throw e
   }
 }
 
@@ -218,6 +260,7 @@ const scheduleWeekly = async () => {
     await scheduleStudents()
   } catch (e) {
     logger.error({ message: 'Weekly scheduling failed', meta: e.stack })
+    throw e
   }
 }
 
@@ -226,6 +269,7 @@ const schedulePrePurge = async () => {
     await startPrePurge()
   } catch (e) {
     logger.error({ message: 'Purge failed', meta: e.stack })
+    throw e
   }
 }
 
@@ -234,6 +278,7 @@ const schedulePurge = async () => {
     await startPurge()
   } catch (e) {
     logger.error({ message: 'Purge failed', meta: e.stack })
+    throw e
   }
 }
 

updater/sis-updater-worker/src/index.js

@@ -19,9 +19,33 @@ const {
 } = require('./config')
 
 const handleMessage = messageHandler => async msg => {
+  let data = null
+
   try {
-    await messageHandler(JSON.parse(msg.getData()))
+    data = JSON.parse(msg.getData())
+  } catch (err) {
+    logger.error({
+      message: 'Failed to parse message',
+      meta: err.stack,
+    })
+  }
+
+  try {
+    await messageHandler(data)
+
+    if (data.id) {
+      logger.info('Completion Ack ' + data.id + ' (Success)')
+      stan.publish('SIS_COMPLETED_CHANNEL-' + data.id, JSON.stringify({ id: data.id, success: true }))
+    }
   } catch (e) {
+    if (data.id) {
+      logger.info('Completion Ack ' + data.id + ' (Failure)')
+      stan.publish(
+        'SIS_COMPLETED_CHANNEL-' + data.id,
+        JSON.stringify({ id: data.id, success: false, message: e.message })
+      )
+    }
+
     logger.error({ message: 'Failed handling message', meta: e.stack })
   } finally {
     try {