[General tab] Fix page crashing if there are no valid first and last semesters

When studyprogramme and year are selected, the General tab now matches what is shown in the Class statistics page

.eslintrc.json

@@ -8,12 +8,40 @@
     "ecmaVersion": "latest",
     "sourceType": "commonjs"
   },
-  "extends": ["eslint:recommended", "plugin:prettier/recommended", "plugin:cypress/recommended"],
+  "extends": [
+    "eslint:recommended",
+    "airbnb",
+    "plugin:cypress/recommended",
+    "plugin:import/recommended",
+    "plugin:prettier/recommended"
+  ],
   "rules": {
-    "no-console": "error",
-    "no-async-promise-executor": "off",
-    "prettier/prettier": "error",
+    "camelcase": "off",
+    "consistent-return": "off",
     "cypress/no-assigning-return-values": "off",
-    "cypress/unsafe-to-chain-command": "off"
-  }
+    "cypress/unsafe-to-chain-command": "off",
+    "default-param-last": "off",
+    "import/order": "off",
+    "no-async-promise-executor": "off",
+    "no-await-in-loop": "off",
+    "no-console": "error",
+    "no-continue": "off",
+    "no-param-reassign": "off",
+    "no-plusplus": "off",
+    "no-promise-executor-return": "off",
+    "no-restricted-syntax": [
+      "error",
+      {
+        "selector": "ForInStatement",
+        "message": "for..in loops iterate over the entire prototype chain, which is virtually never what you want. Use Object.{keys,values,entries}, and iterate over the resulting array."
+      },
+      "LabeledStatement",
+      "WithStatement"
+    ],
+    "no-return-await": "off",
+    "no-shadow": "off",
+    "prettier/prettier": "error",
+    "quotes": ["error", "single", { "avoidEscape": true }]
+  },
+  "ignorePatterns": ["node_modules/", "build/", "**/*migrations*/**"]
 }

.github/actions/run-tests/action.yaml

@@ -24,7 +24,7 @@ runs:
 
     - name: 'Run services, pull rest of the needed images'
       shell: bash
-      run: docker-compose --file docker-compose.ci.yml up --detach
+      run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
 
     - name: 'Run cypress'
       uses: cypress-io/github-action@v6
@@ -44,4 +44,4 @@ runs:
     - name: 'Print logs on failure'
       shell: bash
       if: failure()
-      run: docker-compose --file docker-compose.ci.yml logs
+      run: docker compose --file docker-compose.ci.yml logs

.github/workflows/production.yaml

@@ -122,7 +122,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment
@@ -142,7 +142,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: 'Send notification to slack'
-        uses: UniversityOfHelsinkiCS/slackbot-action@v1.4.2
+        uses: UniversityOfHelsinkiCS/slackbot-action@v1.5
         with:
           webhook-url: ${{ secrets.WEBHOOK_URL }}
           message-type: deployment-failure

.github/workflows/pull_request.yaml

@@ -0,0 +1,73 @@
+name: 'Run tests'
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'cypress/**'
+      - 'cypress.config.json'
+      - 'services/**'
+      - '!services/backend/src/rapodiff/**'
+      - 'docker-compose.ci.yml'
+      - 'package*.json'
+
+jobs:
+  lint_backend_and_frontend:
+    name: 'Lint backend and frontend'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: 'Install dependencies'
+        run: npm ci && cd services/frontend && npm ci && cd ../backend && npm ci
+      - name: 'Run Stylelint for CSS files'
+        run: npm run stylelint "services/frontend/src/**/*.css"
+      - name: 'Run ESLint for JS files'
+        run: npm run eslint -- --ext .jsx,.js services --report-unused-disable-directives
+
+  test_end_to_end:
+    runs-on: ubuntu-latest
+    needs:
+      - lint_backend_and_frontend
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: 'Test course statistics'
+            spec: cypress/e2e/Course_statistics.js
+          - name: 'Test custom population'
+            spec: cypress/e2e/Custom_population.js
+          - name: 'Test population statistics'
+            spec: cypress/e2e/Population_statistics.js
+          - name: 'Test filters'
+            spec: cypress/e2e/Filters.js
+          - name: 'Test students'
+            spec: cypress/e2e/Students.js
+          - name: 'Test completed courses'
+            spec: cypress/e2e/Completed_courses.js
+          - name: 'Test studyprogramme overview'
+            spec: cypress/e2e/Studyprogramme_overview.js
+          - name: 'Test teachers and users'
+            spec: |
+              cypress/e2e/Teachers.js
+              cypress/e2e/Users.js
+          - name: 'Test faculty statistics'
+            spec: cypress/e2e/Faculty_statistics.js
+          - name: 'Test open uni search'
+            spec: cypress/e2e/Openuni.js
+          - name: 'Test language center view'
+            spec: cypress/e2e/Language_center.js
+          - name: 'Test study guidance groups'
+            spec: cypress/e2e/Study_guidance_groups.js
+    name: ${{ matrix.name }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: 'Run tests'
+        uses: ./.github/actions/run-tests
+        with:
+          username: ${{ secrets.TOSKAREGISTRY_USERNAME }}
+          password: ${{ secrets.TOSKAREGISTRY_PASSWORD }}
+          spec: ${{ matrix.spec }}

.github/workflows/test.yaml

@@ -12,8 +12,8 @@ on:
 
 jobs:
   # === Static tests / linting ===
-  lint_backend:
-    name: 'Lint backend'
+  lint_backend_and_frontend:
+    name: 'Lint backend and frontend'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -21,24 +21,11 @@ jobs:
         with:
           node-version: 18
       - name: 'Install dependencies'
-        run: npm ci && cd services/backend && npm ci
-      - name: 'Run backend linter'
-        run: npm run eslint services/backend
-
-  lint_frontend:
-    name: 'Lint frontend'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 18
-      - name: 'Install dependencies'
-        run: npm ci && cd services/frontend && npm ci
-      - name: 'Run frontend linter'
-        run: npm run concurrently \
-          'eslint --ext .jsx,.js services/frontend/src --report-unused-disable-directives' \
-          'stylelint services/frontend/src/**/*.css'
+        run: npm ci && cd services/frontend && npm ci && cd ../backend && npm ci
+      - name: 'Run Stylelint for CSS files'
+        run: npm run stylelint "services/frontend/src/**/*.css"
+      - name: 'Run ESLint for JS files'
+        run: npm run eslint -- --ext .jsx,.js services --report-unused-disable-directives
 
   # === Build database images ===
   build_backend:
@@ -83,7 +70,7 @@ jobs:
       - name: 'Let sis-updater-worker run migrations, then push new version of image to toska hub'
         if: steps.check_migrations.outputs.migrations == 'true'
         run: |
-          docker-compose --file docker-compose.ci.yml up --detach sis-updater-worker
+          docker compose --file docker-compose.ci.yml up --detach sis-updater-worker
           sleep 30 # wait for sis-updater-worker to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh sis-db
 
@@ -116,7 +103,7 @@ jobs:
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         if: steps.check_migrations.outputs.migrations == 'true'
         run: |
-          docker-compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach backend
           sleep 30 # wait for sis-updater-worker to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh kone-db
 
@@ -149,7 +136,7 @@ jobs:
       - name: 'Let backend run migrations, then push new version of image to toska hub'
         if: steps.check_migrations.outputs.migrations == 'true'
         run: |
-          docker-compose --file docker-compose.ci.yml up --detach backend
+          docker compose --file docker-compose.ci.yml up --detach backend
           sleep 30 # wait for sis-updater-worker to start and run migrations
           ./anonyymioodi/commit_and_push_to_toska_hub.sh user-db
 
@@ -218,7 +205,7 @@ jobs:
           password: ${{ secrets.password }}
 
       - name: 'Run services, pull rest of the needed images'
-        run: docker-compose --file docker-compose.ci.yml up --detach
+        run: docker compose --file docker-compose.ci.yml up --detach --quiet-pull
 
       - name: 'Check that exit statuses are not errors'
         run: |
@@ -230,4 +217,4 @@ jobs:
 
       - name: 'Print logs on failure'
         if: failure()
-        run: docker-compose --file docker-compose.ci.yml logs
+        run: docker compose --file docker-compose.ci.yml logs

.github/workflows/updater.yaml

@@ -1,4 +1,4 @@
-name: 'Test and release Updater'
+name: 'Test (and release) Updater'
 
 on:
   push:
@@ -8,8 +8,27 @@ on:
       - '.github/workflows/updater.yaml'
       - 'updater/sis-updater-worker/**'
       - 'updater/sis-updater-scheduler/**'
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'updater/sis-updater-worker/**'
+      - 'updater/sis-updater-scheduler/**'
 
 jobs:
+  lint_scheduler_and_worker:
+    name: 'Lint scheduler and worker'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: 'Install dependencies'
+        run: npm ci && cd updater/sis-updater-scheduler && npm ci && cd ../sis-updater-worker && npm ci
+      - name: 'Run linter'
+        run: npm run eslint updater -- --report-unused-disable-directives
+
   build_worker:
     name: 'Build Updater worker'
     runs-on: ubuntu-latest
@@ -50,6 +69,7 @@ jobs:
     name: 'Test Updater'
     runs-on: ubuntu-latest
     needs:
+      - lint_scheduler_and_worker
       - build_worker
       - build_scheduler
     steps:
@@ -69,24 +89,24 @@ jobs:
       - name: 'Load scheduler image'
         run: docker load --input /tmp/artifact/scheduler.tar
       - name: 'Start worker'
-        run: docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
+        run: docker compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker
       - name: 'Run hourly jobs'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
         env:
           SCHEDULE_IMMEDIATE: hourly
           EXIT_AFTER_IMMEDIATES: yes
       - name: 'Run weekly jobs'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
         env:
           SCHEDULE_IMMEDIATE: weekly
           EXIT_AFTER_IMMEDIATES: yes
       - name: 'Run prepurge'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
         env:
           SCHEDULE_IMMEDIATE: prepurge
           EXIT_AFTER_IMMEDIATES: yes
       - name: 'Run purge'
-        run: docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
+        run: docker compose --file docker-compose.ci.yml run sis-updater-scheduler npm start
         env:
           SCHEDULE_IMMEDIATE: purge
           EXIT_AFTER_IMMEDIATES: yes
@@ -95,6 +115,7 @@ jobs:
     name: 'Build and release Updater worker'
     runs-on: ubuntu-latest
     needs: test_updater
+    if: github.event_name == 'push'
     steps:
       - uses: actions/checkout@v4
 
@@ -121,7 +142,7 @@ jobs:
     name: 'Build and release Updater scheduler'
     runs-on: ubuntu-latest
     needs: test_updater
-
+    if: github.event_name == 'push'
     steps:
       - uses: actions/checkout@v4
 

README.md

@@ -98,6 +98,11 @@ In development, backend and updater use the same redis, but in production they a
 
 See also [older graph](documentation/Oodikone.png), which presents more information.
 
+To read a bit more details about what updater-scheduler and updater-worker do, see their own READMEs:
+
+- [updater-scheduler's README](./updater/sis-updater-scheduler/README.md)
+- [updater-worker's README](./updater/sis-updater-worker/README.md)
+
 ### Documentation
 
 Folder `documentation` currently contains some notes on sis-db schema and is meant as a place for storing useful information on Oodikone to make life easier for future developers. Feel free to add anything relevant!
@@ -118,8 +123,11 @@ Some useful commands are defined in `package.json` and can be run with `npm run
 - `npm run both:real`: starts oodikone and updater with real data
 - `npm run docker:down`: stops the whole environment
 - `npm run testupdater`: runs tests for updater
-- `npm run flushredis`: Clears redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views.
+- `npm run flushredis`: clears redis, which forces big calculations to be redone in e.g. studyprogramme overview and faculty views.
 - `npm run rapodiff`: runs [Rapodiff](./services/backend/src/rapodiff/)
+- `npm run install:local`: installs `node_modules` for each package (backend, frontend, sis-updater-scheduler and sis-updater-worker) locally
+- `npm run install:docker`: installs `node_modules` for each package inside containers
+- `npm run install:both`: runs both `npm run install:local` and `npm run install:docker`
 
 Once you have ran setup for oodikone , you can just execute the first one (`npm run oodikone`). After starting and waiting for a while for containers to compile, oodikone can be accessed at [http://localhost:3000/](http://localhost:3000/) and Adminer (database investigation tool) at [http://localhost:5050/](http://localhost:5050/). Adminer requires you to login with username `postgres` and with any password you choose (for example `p`).
 
@@ -180,7 +188,7 @@ Continuous integration (CI) works with Github actions and is defined in workflow
 
 ### Modules are missing after updating package.json
 
-You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container.
+You should always install the dependencies **inside** the container to have the application **inside** the container access them. Module might be missing for example when someone else installs a new library and you only pull the changes in package.json. Use `docker exec <service> npm ci` to install modules inside the container or `npm run install:docker` to install modules inside all packages (backend, frontend, sis-updater-scheduler and sis-updater-worker).
 
 ### Studyguidance groups don't work on my machine
 

cli.sh

@@ -22,7 +22,7 @@ trap cleanup SIGINT SIGTERM ERR EXIT
 cleanup() {
   trap - SIGINT SIGTERM ERR EXIT
   warningmsg "Trying to run docker-compose down and remove orphans"
-  docker-compose down --remove-orphans
+  docker-compose --profile oodikone --profile updater down --remove-orphans
 }
 
 # === CLI ===

cypress/e2e/Completed_courses.js

@@ -67,7 +67,7 @@ describe('When search modal is opened', () => {
       )
       cy.get('[data-cy="rights-notification"]').should('contain.text', '1')
       cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Completed courses search')
-      cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Student Number')
+      cy.get('[data-cy="completed-courses-table-div"]').should('contain.text', 'Student number')
       cy.get('[data-cy="completed-courses-table-div"]').should('not.contain', '1')
     })
   })

cypress/e2e/Custom_population.js

@@ -132,7 +132,7 @@ describe('Custom population tests', () => {
   })
 
   describe('Custom population search saving', () => {
-    it('Saves a custom population search', () => {
+    it('Saves a custom population search', { retries: 2 }, () => {
       cy.fixture('customPopulations').then(({ studentNumbersForCSStudentsSet1 }) => {
         const students = studentNumbersForCSStudentsSet1
         openCustomPopupForm()

cypress/e2e/Filters.js

@@ -83,9 +83,9 @@ describe('Population Statistics with CS master', () => {
     })
   })
 
-  describe('Studyright Type filter', () => {
+  describe('Studyright type filter', () => {
     it('is visible', () => {
-      cy.contains('Studyright Type')
+      cy.contains('Studyright type')
     })
 
     it('is set to all by default', () => {
@@ -252,8 +252,8 @@ describe('Population Statistics', () => {
     checkFilteringResult(defaultAmountOfStudents)
   })
 
-  it('Studyright Type filter is not visible', () => {
-    cy.contains('Studyright Type').should('not.exist')
+  it('Studyright type filter is not visible', () => {
+    cy.contains('Studyright type').should('not.exist')
   })
 })
 

cypress/e2e/Population_statistics.js

@@ -91,7 +91,7 @@ describe('Population Statistics tests', () => {
       cy.contains('Courses of class').click()
       cy.get('Opiskelijan digitaidot - Kumpula').should('not.exist')
       cy.get('[data-cy=curriculum-picker]').click()
-      cy.contains('2020 - 2022').click()
+      cy.contains('2020 - 2023').click()
       cy.contains('Opiskelijan digitaidot - Kumpula')
     })
 
@@ -106,7 +106,7 @@ describe('Population Statistics tests', () => {
       cy.contains('No tags defined. You can define them here.').find('a').click()
       cy.contains('Tags').click()
       cy.contains('Tietojenkäsittelytieteen kandiohjelma')
-      cy.contains('Create new tag')
+      cy.contains('Create a new tag')
     })
 
     // This test sometimes fails on headless mode. It seems that the click on the
@@ -132,14 +132,14 @@ describe('Population Statistics tests', () => {
       cy.contains('Credit accumulation (for 170 students)')
     })
 
-    it('Credit Statistics, Credits Gained tab works', () => {
+    it('Credit Statistics, Credits gained tab works', () => {
       cy.selectStudyProgramme('Tietojenkäsittelytieteen kandiohjelma')
       cy.contains('Credit statistics')
         .parent()
         .then($parentDiv => {
           if (!$parentDiv.hasClass('active')) cy.contains('Credit statistics').click()
         })
-      cy.contains('Credits Gained').click()
+      cy.contains('Credits gained').click()
       cy.get("[data-cy='credits-gained-main-table']").should('contain', 'All students of the class')
       const today = moment().endOf('month')
       const months = Math.round(moment.duration(moment(today).diff(moment('2017-08-01'))).asMonths())
@@ -172,7 +172,7 @@ describe('Population Statistics tests', () => {
       cy.get("[data-cy='credits-gained-table-Ei valintatapaa']").should('not.exist')
     })
 
-    it('Credit Statistics, Credits Gained tab shows stats by admissions', () => {
+    it('Credit Statistics, Credits gained tab shows stats by admissions', () => {
       cy.visit(pathToCSMaster2019)
       cy.cs('filtered-students')
       cy.contains('Credit statistics')
@@ -181,7 +181,7 @@ describe('Population Statistics tests', () => {
         .then($parentDiv => {
           if (!$parentDiv.hasClass('active')) cy.get("[data-cy='Credit statistics']").click()
         })
-      cy.contains('Credits Gained').click()
+      cy.contains('Credits gained').click()
       cy.get('.credits-gained-divider').click()
       cy.get("[data-cy='credits-gained-table-Avoin väylä']").should('exist')
       cy.get("[data-cy='credits-gained-table-Yhteispisteet']").should('exist')
@@ -221,16 +221,16 @@ describe('Population Statistics tests', () => {
         })
       cy.contains(existing)
       cy.contains(nonExisting).should('not.exist')
-      cy.contains('button', 'Check studentnumbers').click()
-      cy.contains('Check for studentnumbers')
+      cy.contains('button', 'Check student numbers').click()
+      cy.contains('Check for student numbers')
       cy.get('textarea').type(existing).type('{enter}').type(nonExisting)
-      cy.contains('button', 'check students').click()
+      cy.contains('button', 'Check students').click()
       cy.contains('#checkstudentsresults', 'Results').within(() => {
-        cy.contains('Student numbers in list and in oodi').click()
+        cy.contains('Student numbers in list and in Sisu').click()
         cy.contains('#found', existing)
-        cy.contains('Student numbers in list but not in oodi').click()
+        cy.contains('Student numbers in list but not in Sisu').click()
         cy.contains('#notfound', nonExisting)
-        cy.contains('Student numbers in oodi but not in list').click()
+        cy.contains('Student numbers in Sisu but not in list').click()
         cy.contains('#notsearched', '010614509')
       })
     })

cypress/e2e/Study_guidance_groups.js

@@ -143,10 +143,16 @@ describe('Study guidance group tests', () => {
           'Start of studyright',
           'Started in programme',
           'Semesters present',
+          'Enrollment status',
           'Graduation date',
           'Start year at uni',
           'Other programmes',
           'Semesters present amount',
+          'Transferred from',
+          'Gender',
+          'Citizenship',
+          'Curriculum period',
+          'Latest attainment date',
           'Tags',
         ]
         const secondRowHeadings = ['All', 'HOPS', 'Since 1.8.2018']
@@ -154,7 +160,9 @@ describe('Study guidance group tests', () => {
         cy.get('[data-cy="student-table-tabs"] table thead tr').then($tr => {
           const firstRowTexts = Array.from($tr[0].children).map(elem => elem.innerText.replace('\n', ' '))
           expect(firstRowTexts).to.deep.equal(firstRowHeadings)
-          const secondRowTexts = Array.from($tr[1].children).map(elem => elem.innerText)
+          const secondRowTexts = Array.from($tr[1].children)
+            .filter(elem => elem.style.display !== 'none')
+            .map(elem => elem.innerText)
           expect(secondRowTexts).to.deep.equal(secondRowHeadings)
         })
       })

cypress/e2e/Studyprogramme_overview.js

@@ -59,10 +59,10 @@ describe('Studyprogramme overview', () => {
       const tableContents = [
         ...years.map(year => [year, 0, 0]),
         [2021, 0, 0, 0, 0, 0],
-        [2020, 4511, 4511, 0, 0, 167],
-        [2019, 7097, 7097, 0, 0, 349],
-        [2018, 5742, 5742, 0, 0, 272],
-        [2017, 1910, 1910, 0, 0, 388],
+        [2020, 4511, 4511, 0, 0, 406],
+        [2019, 7117, 7097, 0, 0, 434],
+        [2018, 5742, 5737, 0, 0, 150],
+        [2017, 1882, 1882, 0, 0, 186],
       ]
 
       cy.checkTableStats(tableContents, 'CreditsProducedByTheStudyprogramme')
@@ -101,10 +101,10 @@ describe('Studyprogramme overview', () => {
       const creditTableContents = [
         ...years.map(year => [year, 0, 0]),
         [2021, 0, 0, 0, 0, 0],
-        [2020, 4511, 4511, 0, 0, 167],
-        [2019, 7097, 7097, 0, 0, 349],
-        [2018, 5742, 5742, 0, 0, 272],
-        [2017, 1910, 1910, 0, 0, 388],
+        [2020, 4511, 4511, 0, 0, 406],
+        [2019, 7117, 7097, 0, 0, 434],
+        [2018, 5742, 5737, 0, 0, 150],
+        [2017, 1882, 1882, 0, 0, 186],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -141,10 +141,10 @@ describe('Studyprogramme overview', () => {
       const creditTableContents = [
         ...years.map(year => [year, 0, 0]),
         ['2021 - 2022', 0, 0, 0, 0, 0],
-        ['2020 - 2021', 352, 352, 0, 0, 60],
-        ['2019 - 2020', 6966, 6966, 0, 0, 193],
-        ['2018 - 2019', 7777, 7777, 0, 0, 328],
-        ['2017 - 2018', 4165, 4165, 0, 0, 344],
+        ['2020 - 2021', 352, 352, 0, 0, 296],
+        ['2019 - 2020', 6966, 6966, 0, 0, 430],
+        ['2018 - 2019', 7797, 7777, 0, 0, 259],
+        ['2017 - 2018', 4137, 4132, 0, 0, 191],
       ]
 
       cy.checkTableStats(creditTableContents, 'CreditsProducedByTheStudyprogramme')
@@ -160,8 +160,8 @@ describe('Studyprogramme overview', () => {
       cy.get('[data-cy=Graph-CreditsProducedByTheStudyprogramme')
         .should('contain', 'Degree students')
         .should('contain', 'Transferred')
-        .should('contain', 388)
         .should('contain', 7097)
+        .should('contain', 434)
 
       cy.get('[data-cy=Graph-GraduatedAndThesisWritersOfTheProgramme')
         .should('contain', 'Graduated students')
@@ -353,7 +353,7 @@ describe('Studyprogramme overview', () => {
     })
   })
 
-  /* Tag-tests*/
+  /* Tag-tests */
   describe('Basic information -view works for basic user', () => {
     beforeEach(() => {
       cy.init('/study-programme')
@@ -365,7 +365,7 @@ describe('Studyprogramme overview', () => {
       const name = `tag-${new Date().getTime()}`
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
-      cy.contains('Create new tag').click()
+      cy.contains('Create a new tag').click()
       cy.contains(name)
       cy.contains('2018')
       deleteTag(name)
@@ -377,7 +377,7 @@ describe('Studyprogramme overview', () => {
       cy.get('.yearSelectInput').type('2018')
 
       cy.get('.ui > label').click()
-      cy.contains('Create new tag').click()
+      cy.contains('Create a new tag').click()
       cy.contains(name)
       deleteTag(name)
     })
@@ -388,7 +388,7 @@ describe('Studyprogramme overview', () => {
       const student = '010113437'
       cy.get('.tagNameSelectInput > .ui > input').type(name)
       cy.get('.yearSelectInput').type('2018')
-      cy.contains('Create new tag').click()
+      cy.contains('Create a new tag').click()
       cy.contains(name)
 
       cy.contains('Add tags to students').click()

docker-compose.yml

@@ -175,6 +175,8 @@ services:
   jami:
     <<: *defaults
     image: quay.io/toska/jami:production
+    env_file:
+      - .env
     environment:
       - npm_config_cache=/tmp
       - PORT=3003

package-lock.json

@@ -8,10 +8,9 @@
       "license": "ISC",
       "dependencies": {
         "lodash": "^4.17.21",
-        "moment": "^2.29.4"
+        "moment": "^2.30.1"
       },
       "devDependencies": {
-        "concurrently": "^7.4.0",
         "cypress": "^13.2.0",
         "eslint": "^8.56.0",
         "eslint-config-airbnb": "^19.0.4",
@@ -1127,17 +1126,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/cliui": {
-      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
-      "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
-      "dev": true,
-      "dependencies": {
-        "string-width": "^4.2.0",
-        "strip-ansi": "^6.0.0",
-        "wrap-ansi": "^7.0.0"
-      }
-    },
     "node_modules/color-convert": {
       "version": "1.9.3",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -1201,118 +1189,6 @@
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
       "dev": true
     },
-    "node_modules/concurrently": {
-      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-7.4.0.tgz",
-      "integrity": "sha512-M6AfrueDt/GEna/Vg9BqQ+93yuvzkSKmoTixnwEJkH0LlcGrRC2eCmjeG1tLLHIYfpYJABokqSGyMcXjm96AFA==",
-      "dev": true,
-      "dependencies": {
-        "chalk": "^4.1.0",
-        "date-fns": "^2.29.1",
-        "lodash": "^4.17.21",
-        "rxjs": "^7.0.0",
-        "shell-quote": "^1.7.3",
-        "spawn-command": "^0.0.2-1",
-        "supports-color": "^8.1.0",
-        "tree-kill": "^1.2.2",
-        "yargs": "^17.3.1"
-      },
-      "bin": {
-        "conc": "dist/bin/concurrently.js",
-        "concurrently": "dist/bin/concurrently.js"
-      },
-      "engines": {
-        "node": "^12.20.0 || ^14.13.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/open-cli-tools/concurrently?sponsor=1"
-      }
-    },
-    "node_modules/concurrently/node_modules/ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-      "dev": true,
-      "dependencies": {
-        "color-convert": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-      }
-    },
-    "node_modules/concurrently/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
-      "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
-      }
-    },
-    "node_modules/concurrently/node_modules/chalk/node_modules/supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/concurrently/node_modules/color-convert": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-      "dev": true,
-      "dependencies": {
-        "color-name": "~1.1.4"
-      },
-      "engines": {
-        "node": ">=7.0.0"
-      }
-    },
-    "node_modules/concurrently/node_modules/color-name": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-      "dev": true
-    },
-    "node_modules/concurrently/node_modules/has-flag": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/concurrently/node_modules/supports-color": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
-      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/supports-color?sponsor=1"
-      }
-    },
     "node_modules/confusing-browser-globals": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz",
@@ -1575,19 +1451,6 @@
         "node": ">=0.10"
       }
     },
-    "node_modules/date-fns": {
-      "version": "2.29.1",
-      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.1.tgz",
-      "integrity": "sha512-dlLD5rKaKxpFdnjrs+5azHDFOPEu4ANy/LTh04A1DTzMM7qoajmKCBc8pkKRFT41CNzw+4gQh79X5C+Jq27HAw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.11"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/date-fns"
-      }
-    },
     "node_modules/dayjs": {
       "version": "1.11.4",
       "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.4.tgz",
@@ -1902,15 +1765,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/escalade": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
-      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/escape-string-regexp": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
@@ -2862,15 +2716,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/get-caller-file": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
-      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
-      "dev": true,
-      "engines": {
-        "node": "6.* || 8.* || >= 10.*"
-      }
-    },
     "node_modules/get-intrinsic": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
@@ -4756,9 +4601,9 @@
       }
     },
     "node_modules/moment": {
-      "version": "2.29.4",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
-      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
+      "version": "2.30.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
+      "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
       "engines": {
         "node": "*"
       }
@@ -5497,15 +5342,6 @@
         "throttleit": "^1.0.0"
       }
     },
-    "node_modules/require-directory": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
-      "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/require-from-string": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
@@ -5745,12 +5581,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/shell-quote": {
-      "version": "1.7.3",
-      "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.3.tgz",
-      "integrity": "sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==",
-      "dev": true
-    },
     "node_modules/side-channel": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
@@ -5836,12 +5666,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/spawn-command": {
-      "version": "0.0.2-1",
-      "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2-1.tgz",
-      "integrity": "sha512-n98l9E2RMSJ9ON1AKisHzz7V42VDiBQGY6PB1BwRglz99wpVsSuGzQ+jOi6lFXBGVTCrRpltvjm+/XA+tpeJrg==",
-      "dev": true
-    },
     "node_modules/sshpk": {
       "version": "1.18.0",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@@ -6492,15 +6316,6 @@
         "node": ">= 4.0.0"
       }
     },
-    "node_modules/tree-kill": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
-      "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
-      "dev": true,
-      "bin": {
-        "tree-kill": "cli.js"
-      }
-    },
     "node_modules/tsconfig-paths": {
       "version": "3.15.0",
       "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz",
@@ -6936,15 +6751,6 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/y18n": {
-      "version": "5.0.8",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
-      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
-      "dev": true,
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -6960,33 +6766,6 @@
         "node": ">= 14"
       }
     },
-    "node_modules/yargs": {
-      "version": "17.5.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz",
-      "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==",
-      "dev": true,
-      "dependencies": {
-        "cliui": "^7.0.2",
-        "escalade": "^3.1.1",
-        "get-caller-file": "^2.0.5",
-        "require-directory": "^2.1.1",
-        "string-width": "^4.2.3",
-        "y18n": "^5.0.5",
-        "yargs-parser": "^21.0.0"
-      },
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/yargs/node_modules/yargs-parser": {
-      "version": "21.1.1",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
-      "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
-      "dev": true,
-      "engines": {
-        "node": ">=12"
-      }
-    },
     "node_modules/yauzl": {
       "version": "2.10.0",
       "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",

package.json

@@ -3,8 +3,8 @@
   "description": "An application for analyzing university data.",
   "scripts": {
     "cypress": "cypress",
-    "concurrently": "concurrently",
     "eslint": "eslint",
+    "lint:all": "eslint --ext .js,.jsx services/ updater/ --report-unused-disable-directives",
     "stylelint": "stylelint",
     "actionlint": "node-actionlint",
     "prettier": "prettier",
@@ -19,7 +19,10 @@
     "testupdater": "docker-compose --file docker-compose.ci.yml up --detach --no-recreate sis-updater-worker && SCHEDULE_IMMEDIATE=weekly EXIT_AFTER_IMMEDIATES=yes docker-compose --file docker-compose.ci.yml run sis-updater-scheduler npm start",
     "flushredis": "docker exec -it redis redis-cli flushall",
     "prepare": "husky install",
-    "rapodiff": "docker exec oodikone_backend_1 node /opt/app-root/src/src/rapodiff/rapodiff.js"
+    "rapodiff": "docker exec oodikone_backend_1 node /opt/app-root/src/src/rapodiff/rapodiff.js",
+    "install:local": "npm i && cd services/frontend && npm i && cd ../backend && npm i && cd ../../updater/sis-updater-scheduler && npm i && cd ../sis-updater-worker && npm i && cd ../..",
+    "install:docker": "docker exec frontend npm i && docker exec oodikone_backend_1 npm i && docker exec sis-updater-scheduler npm i && docker exec oodikone_sis-updater-worker_1 npm i",
+    "install:both": "npm run install:local && npm run install:docker"
   },
   "repository": {
     "type": "git",
@@ -31,7 +34,6 @@
   },
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
-    "concurrently": "^7.4.0",
     "cypress": "^13.2.0",
     "eslint": "^8.56.0",
     "eslint-config-airbnb": "^19.0.4",
@@ -52,6 +54,6 @@
   },
   "dependencies": {
     "lodash": "^4.17.21",
-    "moment": "^2.29.4"
+    "moment": "^2.30.1"
   }
 }

scripts/functions_for_setup.sh

@@ -204,7 +204,6 @@ set_up_oodikone_from_scratch() {
   infomsg "Cleaning up docker setup and node_modules from possible previous installations"
   local compose_down_args="--remove-orphans --volumes --rmi all"
   "$PROJECT_ROOT"/run.sh both down "$compose_down_args" || infomsg "Cleaning errored, but will continue"
-  docker-compose --file "$PROJECT_ROOT"/docker-compose.test.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
   docker-compose --file "$PROJECT_ROOT"/docker-compose.ci.yml "$compose_down_args" || infomsg "Cleaning errored, but will continue"
   for folder in "${folders_to_set_up[@]}"; do
     cd "$folder" || die "Couldn't change directory to folder $folder"

services/backend/package-lock.json

@@ -13,24 +13,24 @@
         "bullmq": "^4.14.2",
         "cors": "^2.8.5",
         "cron": "^1.8.2",
-        "express": "^4.18.2",
+        "express": "^4.19.2",
         "express-async-errors": "^3.1.1",
         "lodash": "^4.17.21",
         "lru-cache": "^5.1.1",
-        "moment": "^2.29.1",
+        "moment": "^2.30.1",
         "moment-timezone": "^0.5.37",
         "morgan": "^1.10.0",
         "papaparse": "^5.4.1",
-        "pg": "^8.8.0",
+        "pg": "^8.11.5",
         "redis": "^3.1.2",
-        "sequelize": "^6.24.0",
+        "sequelize": "^6.37.3",
         "umzug": "^2.3.0",
         "unfuck-utf8-headers-middleware": "^1.0.1",
         "winston": "^3.8.2",
         "winston-gelf-transporter": "^1.0.2"
       },
       "devDependencies": {
-        "nodemon": "^3.0.3"
+        "nodemon": "^3.1.0"
       }
     },
     "node_modules/@colors/colors": {
@@ -424,12 +424,12 @@
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "node_modules/body-parser": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
-      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
+      "version": "1.20.2",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz",
+      "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==",
       "dependencies": {
         "bytes": "3.1.2",
-        "content-type": "~1.0.4",
+        "content-type": "~1.0.5",
         "debug": "2.6.9",
         "depd": "2.0.0",
         "destroy": "1.2.0",
@@ -437,7 +437,7 @@
         "iconv-lite": "0.4.24",
         "on-finished": "2.4.1",
         "qs": "6.11.0",
-        "raw-body": "2.5.1",
+        "raw-body": "2.5.2",
         "type-is": "~1.6.18",
         "unpipe": "1.0.0"
       },
@@ -468,14 +468,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/buffer-writer": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
-      "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==",
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/bullmq": {
       "version": "4.14.2",
       "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-4.14.2.tgz",
@@ -506,12 +498,18 @@
       }
     },
     "node_modules/call-bind": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
-      "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
+      "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
       "dependencies": {
-        "function-bind": "^1.1.1",
-        "get-intrinsic": "^1.0.2"
+        "es-define-property": "^1.0.0",
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2",
+        "get-intrinsic": "^1.2.4",
+        "set-function-length": "^1.2.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -615,9 +613,9 @@
       }
     },
     "node_modules/content-type": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
-      "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==",
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+      "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
       "engines": {
         "node": ">= 0.6"
       }
@@ -679,6 +677,22 @@
         "ms": "2.0.0"
       }
     },
+    "node_modules/define-data-property": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+      "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+      "dependencies": {
+        "es-define-property": "^1.0.0",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/denque": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
@@ -727,6 +741,25 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/es-define-property": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
+      "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
+      "dependencies": {
+        "get-intrinsic": "^1.2.4"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/escape-html": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -741,16 +774,16 @@
       }
     },
     "node_modules/express": {
-      "version": "4.18.2",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
-      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
+      "version": "4.19.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
+      "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.1",
+        "body-parser": "1.20.2",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
-        "cookie": "0.5.0",
+        "cookie": "0.6.0",
         "cookie-signature": "1.0.6",
         "debug": "2.6.9",
         "depd": "2.0.0",
@@ -790,9 +823,9 @@
       }
     },
     "node_modules/express/node_modules/cookie": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
-      "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+      "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
       "engines": {
         "node": ">= 0.6"
       }
@@ -842,9 +875,9 @@
       "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
     },
     "node_modules/follow-redirects": {
-      "version": "1.15.5",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz",
-      "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==",
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
       "funding": [
         {
           "type": "individual",
@@ -896,9 +929,12 @@
       }
     },
     "node_modules/function-bind": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
     },
     "node_modules/gelf-pro": {
       "version": "1.3.11",
@@ -909,13 +945,18 @@
       }
     },
     "node_modules/get-intrinsic": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz",
-      "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==",
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
+      "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
       "dependencies": {
-        "function-bind": "^1.1.1",
-        "has": "^1.0.3",
-        "has-symbols": "^1.0.3"
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2",
+        "has-proto": "^1.0.1",
+        "has-symbols": "^1.0.3",
+        "hasown": "^2.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -970,15 +1011,15 @@
         "node": ">=10"
       }
     },
-    "node_modules/has": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
-      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+    "node_modules/gopd": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
       "dependencies": {
-        "function-bind": "^1.1.1"
+        "get-intrinsic": "^1.1.3"
       },
-      "engines": {
-        "node": ">= 0.4.0"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/has-flag": {
@@ -990,6 +1031,28 @@
         "node": ">=4"
       }
     },
+    "node_modules/has-property-descriptors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+      "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+      "dependencies": {
+        "es-define-property": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-proto": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz",
+      "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/has-symbols": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
@@ -1001,6 +1064,17 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/hasown": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+      "dependencies": {
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -1322,9 +1396,9 @@
       }
     },
     "node_modules/moment": {
-      "version": "2.29.4",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
-      "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
+      "version": "2.30.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
+      "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
       "engines": {
         "node": "*"
       }
@@ -1425,9 +1499,9 @@
       }
     },
     "node_modules/nodemon": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.3.tgz",
-      "integrity": "sha512-7jH/NXbFPxVaMwmBCC2B9F/V6X1VkEdNgx3iu9jji8WxWcvhMWkmhNWhI5077zknOnZnBzba9hZP6bCPJLSReQ==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.0.tgz",
+      "integrity": "sha512-xqlktYlDMCepBJd43ZQhjWwMw2obW/JRvkrLxq5RCNcuDDX1DbcPT+qT1IlIIdf+DhnWs90JpTMe+Y5KxOchvA==",
       "dev": true,
       "dependencies": {
         "chokidar": "^3.5.2",
@@ -1508,9 +1582,9 @@
       }
     },
     "node_modules/object-inspect": {
-      "version": "1.12.2",
-      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
-      "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==",
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
+      "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -1550,11 +1624,6 @@
         "fn.name": "1.x.x"
       }
     },
-    "node_modules/packet-reader": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
-      "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
-    },
     "node_modules/papaparse": {
       "version": "5.4.1",
       "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz",
@@ -1574,21 +1643,22 @@
       "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
     },
     "node_modules/pg": {
-      "version": "8.8.0",
-      "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz",
-      "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==",
+      "version": "8.11.5",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz",
+      "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==",
       "dependencies": {
-        "buffer-writer": "2.0.0",
-        "packet-reader": "1.0.0",
-        "pg-connection-string": "^2.5.0",
-        "pg-pool": "^3.5.2",
-        "pg-protocol": "^1.5.0",
+        "pg-connection-string": "^2.6.4",
+        "pg-pool": "^3.6.2",
+        "pg-protocol": "^1.6.1",
         "pg-types": "^2.1.0",
         "pgpass": "1.x"
       },
       "engines": {
         "node": ">= 8.0.0"
       },
+      "optionalDependencies": {
+        "pg-cloudflare": "^1.1.1"
+      },
       "peerDependencies": {
         "pg-native": ">=3.0.1"
       },
@@ -1598,10 +1668,16 @@
         }
       }
     },
+    "node_modules/pg-cloudflare": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz",
+      "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
+      "optional": true
+    },
     "node_modules/pg-connection-string": {
-      "version": "2.6.2",
-      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz",
-      "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA=="
+      "version": "2.6.4",
+      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz",
+      "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA=="
     },
     "node_modules/pg-int8": {
       "version": "1.0.1",
@@ -1612,17 +1688,17 @@
       }
     },
     "node_modules/pg-pool": {
-      "version": "3.5.2",
-      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz",
-      "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==",
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz",
+      "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==",
       "peerDependencies": {
         "pg": ">=8.0"
       }
     },
     "node_modules/pg-protocol": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz",
-      "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ=="
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz",
+      "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg=="
     },
     "node_modules/pg-types": {
       "version": "2.2.0",
@@ -1740,9 +1816,9 @@
       }
     },
     "node_modules/raw-body": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
-      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+      "version": "2.5.2",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
+      "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
       "dependencies": {
         "bytes": "3.1.2",
         "http-errors": "2.0.0",
@@ -1922,9 +1998,9 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/sequelize": {
-      "version": "6.37.0",
-      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.0.tgz",
-      "integrity": "sha512-MS6j6aXqWzB3fe9FhmfpQMgVC16bBdYroJCqIqR0l9M2ko8pZdKoi/0PiNWgMyFQDXUHxXyAOG3K07CbnOhteQ==",
+      "version": "6.37.3",
+      "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.3.tgz",
+      "integrity": "sha512-V2FTqYpdZjPy3VQrZvjTPnOoLm0KudCRXfGWp48QwhyPPp2yW8z0p0sCYZd/em847Tl2dVxJJ1DR+hF+O77T7A==",
       "funding": [
         {
           "type": "opencollective",
@@ -2033,19 +2109,39 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/set-function-length": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
+      "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==",
+      "dependencies": {
+        "define-data-property": "^1.1.4",
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2",
+        "get-intrinsic": "^1.2.4",
+        "gopd": "^1.0.1",
+        "has-property-descriptors": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/setprototypeof": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
     },
     "node_modules/side-channel": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
-      "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
+      "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
       "dependencies": {
-        "call-bind": "^1.0.0",
-        "get-intrinsic": "^1.0.2",
-        "object-inspect": "^1.9.0"
+        "call-bind": "^1.0.7",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.4",
+        "object-inspect": "^1.13.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"

services/backend/package.json

@@ -14,24 +14,24 @@
     "bullmq": "^4.14.2",
     "cors": "^2.8.5",
     "cron": "^1.8.2",
-    "express": "^4.18.2",
+    "express": "^4.19.2",
     "express-async-errors": "^3.1.1",
     "lodash": "^4.17.21",
     "lru-cache": "^5.1.1",
-    "moment": "^2.29.1",
+    "moment": "^2.30.1",
     "moment-timezone": "^0.5.37",
     "morgan": "^1.10.0",
     "papaparse": "^5.4.1",
-    "pg": "^8.8.0",
+    "pg": "^8.11.5",
     "redis": "^3.1.2",
-    "sequelize": "^6.24.0",
+    "sequelize": "^6.37.3",
     "umzug": "^2.3.0",
     "unfuck-utf8-headers-middleware": "^1.0.1",
     "winston": "^3.8.2",
     "winston-gelf-transporter": "^1.0.2"
   },
   "devDependencies": {
-    "nodemon": "^3.0.3"
+    "nodemon": "^3.1.0"
   },
   "overrides": {
     "winston-gelf-transporter": {

services/backend/src/conf-backend.js

@@ -29,11 +29,10 @@ const frontUrl = process.env.FRONT_URL
 const backendPort = 8080
 
 // Other stuff
-const DB_URL_KONE = process.env.DB_URL_KONE
-const DB_URL_USER = process.env.DB_URL_USER
+const { DB_URL_KONE, DB_URL_USER, SECRET_TOKEN, SIS_UPDATER_URL, CRYPT_KEY } = process.env
 
 let DB_MAX_CONNECTIONS = parseInt(process.env.DB_MAX_CONNECTIONS, 10)
-if (isNaN(DB_MAX_CONNECTIONS)) {
+if (Number.isNaN(DB_MAX_CONNECTIONS)) {
   DB_MAX_CONNECTIONS = 5 // sequelize's default
 }
 let DB_MAX_CRON_CONNECTIONS = DB_MAX_CONNECTIONS - 5
@@ -42,16 +41,12 @@ if (DB_MAX_CRON_CONNECTIONS < 1) {
 }
 
 const redis = process.env.REDIS
-const SECRET_TOKEN = process.env.SECRET_TOKEN
 const DB_SCHEMA_KONE = process.env.DB_SCHEMA_KONE || 'public'
-const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
 
 const KONE_PASSWORD = process.env.KONE_PASSWORD || 'postgres'
 const USER_PASSWORD = process.env.USER_PASSWORD || 'postgres'
 const SIS_PASSWORD = process.env.SIS_PASSWORD || 'postgres'
 
-const CRYPT_KEY = process.env.CRYPT_KEY
-
 module.exports = {
   DB_URL_USER,
   CRYPT_KEY,

services/backend/src/database/connection.js

@@ -2,9 +2,10 @@ const Sequelize = require('sequelize')
 const EventEmitter = require('events')
 const Umzug = require('umzug')
 const conf = require('../conf-backend')
-const { SIS_DB_URL } = process.env
 const logger = require('../util/logger')
 
+const { SIS_DB_URL } = process.env
+
 class DbConnection extends EventEmitter {
   constructor() {
     super()

services/backend/src/events.js

@@ -13,6 +13,11 @@ const { getAssociations } = require('./services/studyrights')
 const { redisClient } = require('./services/redis')
 const { computeLanguageCenterData, LANGUAGE_CENTER_REDIS_KEY } = require('./services/languageCenterData')
 const { jobMaker } = require('./worker/queue')
+const { deleteOutdatedUsers } = require('./services/userService')
+const {
+  findStudentsCloseToGraduation,
+  CLOSE_TO_GRADUATION_REDIS_KEY,
+} = require('./services/populations/closeToGraduation')
 
 const schedule = (cronTime, func) => new CronJob({ cronTime, onTick: func, start: true, timeZone: 'Europe/Helsinki' })
 
@@ -68,7 +73,7 @@ const refreshProgramme = async code => {
   await updateBasicView(code, '')
   await updateStudytrackView(code, '', associations)
 
-  let combinedProgramme = combinedStudyprogrammes[code] || ''
+  const combinedProgramme = combinedStudyprogrammes[code] || ''
   await updateBasicView(code, combinedProgramme)
   await updateStudytrackView(code, combinedProgramme, associations)
 }
@@ -95,11 +100,19 @@ const refreshLanguageCenterData = async () => {
   logger.info('Language center data refreshed!')
 }
 
+const refreshCloseToGraduating = async () => {
+  logger.info('Updating students close to graduating...')
+  const updatedData = await findStudentsCloseToGraduation()
+  await redisClient.setAsync(CLOSE_TO_GRADUATION_REDIS_KEY, JSON.stringify(updatedData))
+  logger.info('Students close to graduating updated!')
+}
+
 const dailyJobs = () => {
   refreshFaculties()
   refreshProgrammes()
   jobMaker.languagecenter()
   jobMaker.statistics()
+  refreshCloseToGraduating()
 }
 
 const startCron = () => {
@@ -109,6 +122,11 @@ const startCron = () => {
       logger.info('Running daily jobs from cron')
       dailyJobs()
     })
+    schedule('0 4 * * 3', async () => {
+      logger.info("Deleting users who haven't logged in for 18 months")
+      const numberOfDeletedUsers = await deleteOutdatedUsers()
+      logger.info(`Deleted ${numberOfDeletedUsers} users.`)
+    })
     schedule('0 19 * * 1', async () => {
       logger.info('Updating students whose studyplans have not been updated recently')
       jobMaker.studyplansUpdate(4)

services/backend/src/middleware/accessLogger.js

@@ -1,6 +1,7 @@
 const morgan = require('morgan')
 const logger = require('../util/logger')
 const _ = require('lodash')
+const { getFullStudyProgrammeRights } = require('../util/utils')
 
 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
 // without actually using morgan what it's used for (LOGGING REQUESTS!).
@@ -10,8 +11,10 @@ const accessLogger = morgan((tokens, req, res) => {
   const fields = ['method', 'url', 'status', 'response-time', 'remote-addr', 'remote-user', 'user-agent', 'referrer']
   const { user } = req
   const meta = { ...user }
-  fields.forEach(field => (meta[field] = tokens[field](req, res)))
-  meta['time'] = tokens['date'](req, res, 'iso')
+  fields.forEach(field => {
+    meta[field] = tokens[field](req, res)
+  })
+  meta.time = tokens.date(req, res, 'iso')
   if (req.route) {
     meta['req-route'] = req.route.path
   }
@@ -19,13 +22,15 @@ const accessLogger = morgan((tokens, req, res) => {
   const message = [
     `${user.mockedBy ? '(mocking) ' : ''}${user.name}`,
     `${(tokens['response-time'](req, res) || '0').split('.')[0]} ms`.padEnd(8, ' '),
-    tokens['status'](req, res),
-    tokens['method'](req, res),
-    tokens['url'](req, res),
+    tokens.status(req, res),
+    tokens.method(req, res),
+    tokens.url(req, res),
   ].join(' ')
 
-  const usingIamRights = user.iamRights.some(programmeCode => tokens['url'](req, res).includes(programmeCode))
-  const onlyIamRights = !user.isAdmin && user.rights.length === 0
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(user.programmeRights)
+
+  const usingIamRights = user.iamRights.some(programmeCode => tokens.url(req, res).includes(programmeCode))
+  const onlyIamRights = !user.isAdmin && fullStudyProgrammeRights.length === 0
 
   logger.info(message, {
     // don't log student list which might be huge

services/backend/src/middleware/currentUserMiddleware.js

@@ -41,7 +41,7 @@ const currentUserMiddleware = async (req, _res, next) => {
 
   if (!hasRequiredIamGroup(iamGroups, iamRights)) {
     logger.error({ message: 'User does not have required iam group', meta: { iamGroups, iamRights } })
-    throw new ApplicationError('User does not have required iam group', 403, { logoutUrl })
+    throw new ApplicationError(`User '${username}' does not have required iam group`, 403, { logoutUrl })
   }
 
   let user = await getUser({
@@ -52,6 +52,7 @@ const currentUserMiddleware = async (req, _res, next) => {
     iamRights,
     specialGroup,
     sisId,
+    access,
   })
 
   if (showAsUser && user.isAdmin) {

services/backend/src/models/models_user.js

@@ -39,27 +39,6 @@ const AccessGroup = sequelizeUser.define('access_group', {
   },
 })
 
-const UserFaculties = sequelizeUser.define('user_faculties', {
-  userId: {
-    primaryKey: true,
-    type: Sequelize.BIGINT,
-    references: {
-      model: 'users',
-      key: 'id',
-    },
-  },
-  faculty_code: {
-    primaryKey: true,
-    type: Sequelize.STRING,
-  },
-  createdAt: {
-    type: Sequelize.DATE,
-  },
-  updatedAt: {
-    type: Sequelize.DATE,
-  },
-})
-
 const User = sequelizeUser.define(
   'users',
   {
@@ -98,12 +77,9 @@ User.belongsToMany(AccessGroup, {
 })
 AccessGroup.belongsToMany(User, { through: 'user_accessgroup' })
 
-User.hasMany(UserFaculties, { as: 'faculty', foreignKey: 'userId' })
-
 module.exports = {
   User,
   UserElementDetails,
   AccessGroup,
-  UserFaculties,
   sequelizeUser,
 }

services/backend/src/models/studyplan.js

@@ -37,10 +37,13 @@ Studyplan.init(
     included_courses: {
       type: ARRAY(STRING),
     },
+    sisu_id: {
+      type: STRING,
+    },
     completed_credits: {
       type: INTEGER,
     },
-    sisu_id: {
+    curriculum_period_id: {
       type: STRING,
     },
     createdAt: {

services/backend/src/rapodiff/conf.js

@@ -2,8 +2,8 @@
 const rapoToken = process.env.RAPO_NODEPROXY
 const importerDbApiToken = process.env.IMPORTER_DB_API_TOKEN
 
-const nodeproxyUrl = `https://toska.cs.helsinki.fi`
-const importerDbApiUrl = `https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer`
+const nodeproxyUrl = 'https://toska.cs.helsinki.fi'
+const importerDbApiUrl = 'https://api-toska.apps.ocp-prod-0.k8s.it.helsinki.fi/importer'
 
 const bscProgrammesOfFaculties = {
   H10: ['KH10_001'],

services/backend/src/rapodiff/creditsDiff.js

@@ -1,100 +1,10 @@
 /* eslint-disable no-console */
-const { getCreditsProduced } = require('../services/providerCredits')
+const { computeCreditsProduced } = require('../services/providerCredits')
 const { parseCsv } = require('./helpers')
 const _ = require('lodash')
 
 const diffs = []
-let noDiffCounter = 0
-
-const diff = (rapoData, okData, code) => {
-  for (const year of Object.keys(okData)) {
-    if (year !== '2022' && year !== '2023') continue
-    const rapo = rapoData[year]
-    const ok = okData[year]
-    if (!rapo) {
-      if (!ok) console.log(`Year ${year} not in rapo for code ${code}, skipping`)
-      continue
-    }
-    for (const field of ['basic', 'incoming-exchange', 'open-uni', 'agreement', 'separate', 'other']) {
-      const okValue = Math.round(ok[field] || 0)
-      const rapoValue = Math.round(rapo[field] || 0)
-      const diff = Math.abs(okValue - rapoValue)
-      const percentage =
-        rapoValue !== 0 || okValue !== 0 ? Number(((diff / Math.max(okValue, rapoValue)) * 100).toFixed(2)) : 0
-      const diffStr = `${field.padEnd(10, ' ')} ${year} ${code.padEnd(
-        10,
-        ' '
-      )} Difference: ${diff} Rapo: ${rapoValue} Oodikone: ${okValue} Percentage: ${percentage} %`
-      diffs.push({ field, diff, okValue, rapoValue, code, year, diffStr, percentage })
-    }
-  }
-}
-let counter = 0
-// Acual logic in this function
-const process = async data => {
-  // Define here the fields to diff against each other. Must be an array, multiple fields will be summed
-  const rapoProgrammeData = formatData(data.slice(1))
-  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
-  for (const programmeCode of allProgrammeCodes) {
-    const okProgrammeData = await getCreditsProduced(programmeCode, false, true)
-    counter++
-    if (counter % 5 === 0) {
-      console.log(`Done ${Math.round((counter / allProgrammeCodes.length) * 100)} %`)
-    }
-    diff(rapoProgrammeData[programmeCode], okProgrammeData.stats, programmeCode)
-  }
-  const fieldToSortBy = 'percentage'
-  const orderedDiffs = _.orderBy(diffs, fieldToSortBy, 'asc')
-  orderedDiffs.forEach(d => console.log(d.diffStr))
-  console.log(`${diffs.length} diffs found with current settings. Numbers with no diff: ${noDiffCounter}`)
-  const totalDiff = orderedDiffs.reduce((sum, cur) => cur.diff + sum, 0)
-  console.log('Total difference: ', totalDiff)
-}
-
-const programmeCreditsDiff = async fileName => {
-  console.log('Running studyprogramme credits diff')
-  await parseCsv(fileName, process)
-  console.log('Diff completed.')
-}
-
-const processIds = async (rawData, code) => {
-  const data = rawData.slice(1)
-  console.log('amount of rapo credits: ', data.length)
-  const rapoData = data
-    .map(row => ({
-      id: row[4],
-      credits: row[5],
-      type: row[6],
-      incl: row[7],
-    }))
-    .filter(row => row.type === '2' && row.incl === '0')
-  const rapoIds = rapoData.map(row => row.id)
-  const stats = await getCreditsProduced(code)
-  if (!stats.ids) {
-    console.log(stats)
-    console.log('Edit getCreditStats code so that it saves a list of the relevant ids you want to compare.')
-    return
-  }
-  const okIds = stats.ids
-  const notInOk = rapoIds.filter(id => id && !okIds[id.slice(11)]).map(str => str.slice(11))
-  const rapoIdMap = rapoIds.reduce((obj, cur) => {
-    obj[cur] = true
-    return obj
-  }, {})
-  const notInRapo = Object.keys(okIds).filter(id => !rapoIdMap[`ATTAINMENT-${id}`])
-  console.log('\n In rapo, but not in OK: ', notInOk.length)
-  console.dir(notInOk)
-  console.log('\n In OK, but not in Rapo: ', notInRapo.length)
-  console.dir(notInRapo)
-  delete stats.ids
-  console.log(stats['2022'])
-  const rapoCredits = rapoData.reduce((sum, cur) => parseInt(sum, 10) + parseInt(cur.credits, 10), 0)
-  console.log('rapoCredits: ', rapoCredits)
-}
-
-const testNewCalc = async code => {
-  await parseCsv('credits.csv', async data => processIds(data, code))
-}
+const noDiffCounter = 0
 
 // Change weird rapo programmecodes to oodikone format, example: 300-M003 => MH30_003
 const transformProgrammeCode = oldCode => `${oldCode[4]}H${oldCode.slice(0, 2)}_${oldCode.slice(5, 8)}`
@@ -115,10 +25,153 @@ const formatData = data =>
       'open-uni': parseNum(openUni),
       separate: parseNum(special),
       total: parseNum(total),
-      abroad: parseNum(abroad),
-      other: parseNum(other),
+      transferred: parseNum(abroad) + parseNum(other),
     }
     return obj
   }, {})
 
+const diff = (rapoData, okData, code) => {
+  for (const year of Object.keys(okData)) {
+    if (year !== '2022' && year !== '2023') continue
+    const rapo = rapoData[year]
+    const ok = okData[year]
+    if (!rapo) {
+      if (!ok) console.log(`Year ${year} not in rapo for code ${code}, skipping`)
+      continue
+    }
+    for (const field of ['basic', 'incoming-exchange', 'open-uni', 'agreement', 'separate', 'transferred', 'other']) {
+      const okValue = Math.round(ok[field] || 0)
+      const rapoValue = Math.round(rapo[field] || 0)
+      const diff = Math.abs(okValue - rapoValue)
+      const percentage =
+        rapoValue !== 0 || okValue !== 0 ? Number(((diff / Math.max(okValue, rapoValue)) * 100).toFixed(2)) : 0
+      const diffStr = `${field.padEnd(10, ' ')} ${year} ${code.padEnd(
+        10,
+        ' '
+      )} Difference: ${diff} Rapo: ${rapoValue} Oodikone: ${okValue} Percentage: ${percentage} %`
+      diffs.push({ field, diff, okValue, rapoValue, code, year, diffStr, percentage })
+    }
+  }
+}
+let counter = 0
+const process = async data => {
+  const rapoProgrammeData = formatData(data.slice(1))
+  const allProgrammeCodes = [...new Set(Object.keys(rapoProgrammeData))]
+  for (const programmeCode of allProgrammeCodes) {
+    const okProgrammeData = await computeCreditsProduced(programmeCode, false, true)
+    counter++
+    if (counter % 5 === 0) {
+      console.log(`Done ${Math.round((counter / allProgrammeCodes.length) * 100)} %`)
+    }
+    diff(rapoProgrammeData[programmeCode], okProgrammeData.stats, programmeCode)
+  }
+  const fieldToSortBy = 'diff' // 'percentage'
+  const orderedDiffs = _.orderBy(diffs, fieldToSortBy, 'asc')
+  orderedDiffs.forEach(d => console.log(d.diffStr))
+  console.log(`${diffs.length} diffs found with current settings. Numbers with no diff: ${noDiffCounter}`)
+  const totalDiff = orderedDiffs.reduce((sum, cur) => cur.diff + sum, 0)
+  console.log('Total difference: ', totalDiff)
+}
+
+const programmeCreditsDiff = async fileName => {
+  console.log('Running studyprogramme credits diff')
+  await parseCsv(fileName, process)
+  console.log('Diff completed.')
+}
+
+/* 
+separate
+Suoritus on erillisellä opiskeluoikeudella opettajankoulutuksen opintoja suorittavan opiskelijan suorittama
+Suoritus on erillisellä opiskeluoikeudella opiskelevan suorittama
+
+agreement
+Suoritus on korkeakoulujen välisillä yhteistyösopimuksilla opiskelevan suorittama
+
+
+open-uni
+Suoritus on avoimessa korkeakouluopetuksessa suoritettu
+
+incoming-exchange
+Suoritus on tulevan kansainvälisen vaihto-opiskelijan suorittama
+
+basic
+perustutkinto-opiskelija = 1
+ja
+Tuntematon
+tai (ehkä):
+Suoritus on kotimainen harjoittelujakso
+?? Opintosuoritus on täydennyskoulutuksessa suoritettu ??
+
+other
+perustutkinto-opiskelija = 0 && tuntematon
+
+*/
+
+const getCategory = (typeName, isBasic, hyvaksiluettu) => {
+  if (hyvaksiluettu) return 'transferred' // ?? unsure
+  if (isBasic || ['Suoritus on kotimainen harjoittelujakso', 'Tuntematon'].includes(typeName)) return 'basic'
+  if (typeName === 'Suoritus on avoimessa korkeakouluopetuksessa suoritettu') return 'open-uni'
+  if (typeName === 'Suoritus on tulevan kansainvälisen vaihto-opiskelijan suorittama') return 'incoming-exchange'
+  if (typeName.startsWith('Suoritus on erillisellä opiskeluoikeudella')) return 'separate'
+  if (typeName === 'Suoritus on korkeakoulujen välisillä yhteistyösopimuksilla opiskelevan suorittama')
+    return 'agreement'
+  return 'other'
+}
+
+const processIds = async (rawData, code, field) => {
+  const data = rawData.slice(0, 21628) // 11476 = KTM viimeinen
+  const rapoData = data
+    .map(row => ({
+      id: row[0].slice(11),
+      type: getCategory(row[1], row[3] === '1', row[6] === '1'),
+      typeCode: row[8],
+      isModule: row[9] !== '0',
+      credits: parseFloat(row[10]),
+      included: row[4] === '1',
+    }))
+    .filter(c => !c.isModule && (field === 'transferred' || c.included))
+  const rapoIds = rapoData.filter(c => c.credits > 0 && c.type === field).map(row => row.id)
+  const rapoCredits = rapoData.reduce(
+    (stats, cur) => {
+      stats[cur.type] += cur.credits
+      return stats
+    },
+    {
+      transferred: 0,
+      total: '_',
+      basic: 0,
+      'open-uni': 0,
+      separate: 0,
+      'incoming-exchange': 0,
+      agreement: 0,
+      other: 0,
+    }
+  )
+  const stats = await computeCreditsProduced(code, false, true, field)
+  if (!stats.stats.ids) {
+    // console.log(stats)
+    console.log({ rapoCredits })
+    console.log('Edit getCreditsProduced code so that it saves a list of the relevant ids you want to compare.')
+    return
+  }
+  const okIdMap = stats.stats.ids
+  const rapoIdMap = rapoIds.reduce((obj, cur) => {
+    obj[cur] = true
+    return obj
+  }, {})
+  const onlyInRapo = Object.keys(rapoIdMap).filter(id => !okIdMap[id])
+  const onlyInOk = Object.keys(okIdMap).filter(id => !rapoIdMap[id])
+  console.log('\n onlyInRapo: ', onlyInRapo.length)
+  console.dir(onlyInRapo.slice(0, 30))
+  console.log('\n onlyInOk: ', onlyInOk.length)
+  console.dir(onlyInOk.slice(0, 30))
+  delete stats.ids
+  console.log(stats.stats['2022'])
+  console.log('rapoCredits: ', rapoCredits)
+}
+
+const testNewCalc = async (fileName, code, field) => {
+  await parseCsv(fileName, async data => processIds(data, code, field))
+}
+
 module.exports = { programmeCreditsDiff, testNewCalc }

services/backend/src/rapodiff/rapodiff.js

@@ -10,7 +10,7 @@ const { programmeCreditsDiff, testNewCalc } = require('./creditsDiff')
 const modes = {
   credits: () => programmeCreditsDiff('data.csv'),
   // Check credits with current local oodikone data against provided data.csv in rapodiff folder. data.csv can be downloaded from rapo by selecting all faculties, then from the bottom of the page clicking Export > data > csv.
-  check: code => testNewCalc(code),
+  check: (fileName, code, field) => testNewCalc(fileName, code, field),
   // Provide programme code as argument. Parses credits from credits.csv in rapodiff folder. This must be obtained from Rapo people. This tells which credit ids are found in oodikone but not rapo, and vice versa.
   // notice: check mode requires writing ids in to the stats object in credits calculating function.
   // Add something like this there:
@@ -22,6 +22,8 @@ const main = async () => {
   const mode = process.argv[2]
   const modeFunction = modes[mode]
   const arg = process.argv[3]
+  const arg2 = process.argv[4]
+  const arg3 = process.argv[5]
   if (!modeFunction) {
     console.log(
       'Unknown or missing mode. Supply the mode as an argument. Supported modes:\n\n',
@@ -29,7 +31,7 @@ const main = async () => {
     )
     return
   }
-  await modeFunction(arg)
+  await modeFunction(arg, arg2, arg3)
 }
 
 main().then(() => process.exit(0))

services/backend/src/rapodiff/studyrightDiff.js

@@ -6,7 +6,7 @@ const _ = require('lodash')
 
 const { optimizedStatisticsOf } = require('../services/populations')
 const { Student, Transfer, StudyrightElement, Studyright } = require('../models')
-const { rapoToken, tokenImporterApi, importerDbApiUrl, nodeproxyUrl } = require('./conf.js')
+const { rapoToken, tokenImporterApi, importerDbApiUrl, nodeproxyUrl } = require('./conf')
 
 const getFromRapo = async (urlStart, urlEnd) => {
   console.log({ urlStart, urlEnd })
@@ -164,7 +164,7 @@ const programme_diff_year = async (programme, facultyCode, year, brief = false)
   const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
 
   const wrongMissingInRapo = []
-  for (let student of onlyInOodikone) {
+  for (const student of onlyInOodikone) {
     const status = await isAcuallyActive(student)
     if (status) {
       wrongMissingInRapo.push(student)
@@ -173,7 +173,7 @@ const programme_diff_year = async (programme, facultyCode, year, brief = false)
 
   // these are in wrong RAPO-population
   const transferred = []
-  for (let student of onlyInRapo) {
+  for (const student of onlyInRapo) {
     const isTransfer = await hasTransferred(student, programme)
     if (isTransfer) {
       transferred.push(student)
@@ -243,7 +243,7 @@ const facultyDiff = async (faculty, programmes, brief) => {
   console.log('\n')
   console.log(faculty)
   console.log('==')
-  for (let programme of programmes) {
+  for (const programme of programmes) {
     await programme_diff(programme, faculty, brief)
   }
 }
@@ -277,16 +277,16 @@ const msc_programme_diff_year = async (programme, facultyCode, year, brief = fal
   const onlyInRapo = _.difference(rapoStudents, oodikoneStudents)
 
   const wrongInRapo = []
-  for (let student of onlyInOodikone) {
+  for (const student of onlyInOodikone) {
     const status = await isAcuallyActive(student)
     if (status) {
-      //wrongInRapo.push(student)
+      // wrongInRapo.push(student)
     }
   }
 
   // in wrong RAPO population
   const transferred = []
-  for (let student of onlyInRapo) {
+  for (const student of onlyInRapo) {
     const status = await hasTransferred(student, programme)
     if (status) {
       transferred.push(student)

services/backend/src/routes.js

@@ -1,11 +1,13 @@
 const Sentry = require('@sentry/node')
 const express = require('express')
 const cors = require('cors')
+
 const { frontUrl } = require('./conf-backend')
 const shibbolethCharsetMiddleware = require('./middleware/shibbolethCharsetMiddleware')
 const currentUserMiddleware = require('./middleware/currentUserMiddleware')
 const accessLogger = require('./middleware/accessLogger')
 const auth = require('./middleware/auth')
+const errorMiddleware = require('./middleware/errorMiddleware')
 const changelog = require('./routes/changelog')
 const courses = require('./routes/courses')
 const students = require('./routes/students')
@@ -30,9 +32,10 @@ const studyGuidanceGroups = require('./routes/studyGuidanceGroups')
 const studyProgramme = require('./routes/studyProgramme')
 const customOpenUniSearch = require('./routes/customOpenUniSearch')
 const studyProgrammeCriteria = require('./routes/studyProgrammeCriteria')
-const initializeSentry = require('./util/sentry')
 const completedCoursesSearch = require('./routes/completedCoursesSearch')
-const errorMiddleware = require('./middleware/errorMiddleware')
+const closeToGraduation = require('./routes/closeToGraduation')
+const initializeSentry = require('./util/sentry')
+
 module.exports = (app, url) => {
   initializeSentry(app)
   app.use(Sentry.Handlers.requestHandler())
@@ -61,7 +64,7 @@ module.exports = (app, url) => {
   app.use(`${url}/changelog`, changelog)
   app.use(`${url}/completedcoursessearch`, completedCoursesSearch)
   app.use(`${url}/languagecenterdata`, languageCenterData)
-  app.use(`${url}/faculties`, auth.roles(['facultyStatistics', 'katselmusViewer']), faculties)
+  app.use(`${url}/faculties`, faculties)
   app.use(`${url}/university`, university)
   app.use(`${url}/updater`, auth.roles(['admin']), updater)
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
@@ -69,6 +72,7 @@ module.exports = (app, url) => {
   app.use(`${url}/feedback`, feedback)
   app.use(`${url}/custom-population-search`, customPopulationSearch)
   app.use(`${url}/studyguidancegroups`, auth.roles(['studyGuidanceGroups']), studyGuidanceGroups)
+  app.use(`${url}/close-to-graduation`, auth.roles(['fullSisuAccess']), closeToGraduation)
   app.get('*', async (_, res) => {
     const results = { error: 'unknown endpoint' }
     res.status(404).json(results)

services/backend/src/routes/changelog.js

@@ -1,14 +1,16 @@
 const router = require('express').Router()
 const axios = require('axios')
+
 const { isDev } = require('../conf-backend')
+
 const changelog = {}
 
-router.get('/', async (req, res) => {
+router.get('/', async (_req, res) => {
   if (changelog?.data) return res.status(200).send(changelog.data)
   if (isDev)
     return res.status(200).json([
       {
-        description: "### Fake release\nLet's not spam github api in development!",
+        description: "### Fake release\nLet's not spam the GitHub API in development!",
         title: 'Fake title for fake release',
         time: new Date(),
       },

services/backend/src/routes/closeToGraduation.js

@@ -0,0 +1,10 @@
+const { getCloseToGraduationData } = require('../services/populations/closeToGraduation')
+
+const router = require('express').Router()
+
+router.get('/', async (req, res) => {
+  const result = await getCloseToGraduationData()
+  res.json(result)
+})
+
+module.exports = router

services/backend/src/routes/completedCoursesSearch.js

@@ -1,6 +1,7 @@
 const router = require('express').Router()
 const { getCompletedCourses } = require('../services/completedCoursesSearch')
 const { getImporterClient } = require('../util/importerClient')
+
 const importerClient = getImporterClient()
 const _ = require('lodash')
 const {
@@ -10,12 +11,13 @@ const {
   updateSearch,
 } = require('../services/openUni/openUniManageSearches')
 const logger = require('../util/logger')
+const { hasFullAccessToStudentData } = require('../util/utils')
 
 router.get('/', async (req, res) => {
   const studentNumbers = JSON.parse(req.query?.studentlist) || []
   const courseCodes = JSON.parse(req.query?.courselist) || []
   const {
-    user: { isAdmin, studentsUserCanAccess, sisPersonId: teacherId },
+    user: { roles, studentsUserCanAccess, sisPersonId: teacherId },
   } = req
   if (!Array.isArray(studentNumbers)) return res.status(400).json({ error: 'Student numbers must be of type array' })
   if (!Array.isArray(courseCodes)) return res.status(400).json({ error: 'Courses must be of type array' })
@@ -27,7 +29,7 @@ router.get('/', async (req, res) => {
     // have enrolled to their courses in last 8 months
     // (acual logic a bit different, see importer)
     const teacherRightsToStudents = await Promise.race([
-      importerClient.post(`/teacher-rights/`, { teacherId, studentNumbers }),
+      importerClient.post('/teacher-rights/', { teacherId, studentNumbers }),
       answerTimeout,
     ])
     if (teacherRightsToStudents && Array.isArray(teacherRightsToStudents.data))
@@ -36,7 +38,9 @@ router.get('/', async (req, res) => {
     logger.error(`Importer teacher-rights request failed with message: ${e?.message}`)
   }
 
-  const filteredStudentNumbers = isAdmin ? studentNumbers : _.intersection(studentNumbers, studentsUserCanAccess)
+  const filteredStudentNumbers = hasFullAccessToStudentData(roles)
+    ? studentNumbers
+    : _.intersection(studentNumbers, studentsUserCanAccess)
   const completedCourses = await getCompletedCourses(filteredStudentNumbers, courseCodes)
   const discardedStudentNumbers = _.difference(
     studentNumbers,

services/backend/src/routes/courseExclusions.js

@@ -2,7 +2,8 @@ const router = require('express').Router()
 const { addExcludedCourses, removeExcludedCourses } = require('../services/excludedCourses')
 
 router.delete('/v3/course_exclusions/:code', async (req, res) => {
-  const { programmeCode, curriculumVersion, courseCodes } = req.body
+  const { code: programmeCode } = req.params
+  const { curriculumVersion, courseCodes } = req.body
   const result = await removeExcludedCourses({ programmeCode, curriculumVersion, courseCodes })
   if (!result) {
     res.status(400).end()

services/backend/src/routes/courses.js

@@ -3,6 +3,7 @@ const crypto = require('crypto')
 const courseService = require('../services/courses')
 const { faculties } = require('../services/organisations')
 const { validateParamLength } = require('../util')
+const { getFullStudyProgrammeRights } = require('../util/utils')
 
 router.get('/v2/coursesmulti', async (req, res) => {
   let results = { courses: [] }
@@ -34,14 +35,15 @@ router.get('/v2/coursesmulti', async (req, res) => {
 
 router.get('/v3/courseyearlystats', async (req, res) => {
   const {
-    user: { rights, roles },
+    user: { roles, programmeRights },
   } = req
 
   const allowedRoles = roles && ['admin', 'courseStatistics'].find(role => roles.includes(role))
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
 
   // If user has rights to see at least one programme, then they are allowed
   // to see all of them
-  if (!allowedRoles && rights.length < 1) {
+  if (!allowedRoles && fullStudyProgrammeRights.length === 0) {
     return res.status(403).json({ error: 'No programmes so no access to course stats' })
   }
 
@@ -54,7 +56,7 @@ router.get('/v3/courseyearlystats', async (req, res) => {
   } else {
     // Studentnumbers should be obfuscated to all other users except admins
     // and users with rights to any specific study programmes
-    const anonymize = allowedRoles !== 'admin' && rights.length < 1
+    const anonymize = allowedRoles !== 'admin' && fullStudyProgrammeRights.length === 0
     const anonymizationSalt = anonymize ? crypto.randomBytes(12).toString('hex') : null
     const results = await courseService.courseYearlyStats(codes, separate, anonymizationSalt, combineSubstitutions)
     res.json(results)

services/backend/src/routes/customOpenUniSearch.js

@@ -51,7 +51,7 @@ router.put('/searches/:id', async (req, res) => {
   if (!id || !userId) return res.status(422).end()
   const updatedSearch = await updateSearch(userId, id, courseCodes)
 
-  if (!updatedSearch) return res.status(404).json({ error: `Open uni search could not be found` })
+  if (!updatedSearch) return res.status(404).json({ error: 'Open uni search could not be found' })
   return res.json({
     id: updatedSearch.id,
     userId: updatedSearch.userId,
@@ -67,7 +67,7 @@ router.delete('/searches/:id', async (req, res) => {
   if (!id || !userId) return res.status(422).end()
 
   const deletedSearch = await deleteSearch(userId, id)
-  if (!deletedSearch) return res.status(404).json({ error: `Open uni search could not be found` })
+  if (!deletedSearch) return res.status(404).json({ error: 'Open uni search could not be found' })
   return res.json(id)
 })
 

services/backend/src/routes/faculties.js

@@ -21,6 +21,7 @@ const {
 } = require('../services/faculty/facultyService')
 const logger = require('../util/logger')
 const { getFacultyList } = require('../services/faculty/facultyHelpers')
+const auth = require('../middleware/auth')
 
 // Faculty uses a lot of tools designed for Study programme.
 // Some of them have been copied here and slightly edited for faculty purpose.
@@ -30,7 +31,7 @@ router.get('/', async (req, res) => {
   res.json(facultyList)
 })
 
-router.get('/:id/basicstats', async (req, res) => {
+router.get('/:id/basicstats', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
   const programmeFilter = req.query?.programme_filter
@@ -42,8 +43,8 @@ router.get('/:id/basicstats', async (req, res) => {
   const wantedProgrammes = await getProgrammes(code, programmeFilter)
   if (!wantedProgrammes) return res.status(422).end()
 
-  //all programmes are required for correct sorting of transfers
-  let allProgrammeCodes = []
+  // all programmes are required for correct sorting of transfers
+  const allProgrammeCodes = []
   if (programmeFilter === 'NEW_STUDY_PROGRAMMES') {
     const allProgs = await getProgrammes(code, 'ALL_PROGRAMMES')
     allProgs?.data.forEach(prog => allProgrammeCodes.push(prog.code))
@@ -65,14 +66,14 @@ router.get('/:id/basicstats', async (req, res) => {
   return res.json(updatedStats)
 })
 
-router.get('/:id/creditstats', async (req, res) => {
+router.get('/:id/creditstats', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const { year_type: yearType } = req.query
   const stats = await getFacultyCredits(code, yearType === 'ACADEMIC_YEAR')
   return res.json(stats)
 })
 
-router.get('/:id/thesisstats', async (req, res) => {
+router.get('/:id/thesisstats', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const yearType = req.query?.year_type
   const specialGroups = req.query?.special_groups
@@ -90,7 +91,7 @@ router.get('/:id/thesisstats', async (req, res) => {
   return res.json(updateStats)
 })
 
-router.get('/:id/graduationtimes', async (req, res) => {
+router.get('/:id/graduationtimes', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const programmeFilter = req.query?.programme_filter
 
@@ -106,7 +107,7 @@ router.get('/:id/graduationtimes', async (req, res) => {
   return res.json(updatedStats)
 })
 
-router.get('/:id/progressstats', async (req, res) => {
+router.get('/:id/progressstats', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const specialGroups = req.query?.special_groups
   const graduated = req.query?.graduated
@@ -124,7 +125,7 @@ router.get('/:id/progressstats', async (req, res) => {
   return res.json(updateStats)
 })
 
-router.get('/:id/studentstats', async (req, res) => {
+router.get('/:id/studentstats', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const specialGroups = req.query?.special_groups
   const graduated = req.query?.graduated
@@ -142,7 +143,7 @@ router.get('/:id/studentstats', async (req, res) => {
   return res.json(updateStats)
 })
 
-router.get('/:id/update_basicview', async (req, res) => {
+router.get('/:id/update_basicview', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   const statsType = req.query?.stats_type
   if (code) {
@@ -152,7 +153,7 @@ router.get('/:id/update_basicview', async (req, res) => {
   return res.status(422).end()
 })
 
-router.get('/:id/update_progressview', async (req, res) => {
+router.get('/:id/update_progressview', auth.roles(['facultyStatistics', 'katselmusViewer']), async (req, res) => {
   const code = req.params.id
   if (code) {
     let result = null

services/backend/src/routes/population.js

@@ -7,7 +7,7 @@ const studentService = require('../services/students')
 const StudyrightService = require('../services/studyrights')
 const CourseService = require('../services/courses')
 const StatMergeService = require('../services/statMerger')
-const { mapToProviders } = require('../util/utils')
+const { mapToProviders, getFullStudyProgrammeRights, hasFullAccessToStudentData } = require('../util/utils')
 const { encrypt, decrypt } = require('../services/encrypt')
 const { mapCodesToIds } = require('../services/studyprogramme/studyprogrammeHelpers')
 
@@ -92,10 +92,12 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
     return
   }
   const {
-    user: { isAdmin, studentsUserCanAccess },
+    user: { roles, studentsUserCanAccess },
   } = req
   const studentnumbers = await studentService.findByTag(tag)
-  const studentnumberlist = isAdmin ? studentnumbers : _.intersection(studentnumbers, studentsUserCanAccess)
+  const studentnumberlist = hasFullAccessToStudentData(roles)
+    ? studentnumbers
+    : _.intersection(studentnumbers, studentsUserCanAccess)
 
   const result = await populationService.bottlenecksOf(
     {
@@ -119,7 +121,7 @@ router.post('/v2/populationstatistics/coursesbytag', async (req, res) => {
 
 router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, res) => {
   const {
-    user: { isAdmin, studentsUserCanAccess },
+    user: { roles, studentsUserCanAccess },
     body: { studentnumberlist: studentnumbersInReq, ...query },
   } = req
 
@@ -127,7 +129,9 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
     throw new ApplicationError('The body should have a studentnumberlist defined', 400)
   }
 
-  const studentnumberlist = isAdmin ? studentnumbersInReq : _.intersection(studentnumbersInReq, studentsUserCanAccess)
+  const studentnumberlist = hasFullAccessToStudentData(roles)
+    ? studentnumbersInReq
+    : _.intersection(studentnumbersInReq, studentsUserCanAccess)
   const result = await populationService.bottlenecksOf(
     {
       year: query?.year ?? 1900,
@@ -148,7 +152,9 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
 
 router.get('/v3/populationstatistics', async (req, res) => {
   const { year, semesters, studyRights: studyRightsJSON } = req.query
-  const { user } = req
+  const {
+    user: { roles, programmeRights, id: userId },
+  } = req
 
   if (!year || !semesters || !studyRightsJSON) {
     res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })
@@ -156,17 +162,15 @@ router.get('/v3/populationstatistics', async (req, res) => {
   }
   let studyRights = null
   studyRights = JSON.parse(studyRightsJSON)
-  const {
-    user: { rights, iamRights, isAdmin },
-  } = req
+
+  const fullProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+  const programmeRightsCodes = programmeRights.map(({ code }) => code)
 
   try {
     if (
-      !isAdmin &&
-      !rights.includes(studyRights.programme) &&
-      !rights.includes(studyRights.combinedProgramme) &&
-      !iamRights.includes(studyRights.programme) &&
-      !iamRights.includes(studyRights.combinedProgramme)
+      !hasFullAccessToStudentData(roles) &&
+      !programmeRightsCodes.includes(studyRights.programme) &&
+      !programmeRightsCodes.includes(studyRights.combinedProgramme)
     ) {
       res.status(403).json([])
       return
@@ -207,7 +211,7 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
-    res.json(filterPersonalTags(result, user.id))
+    res.json(filterPersonalTags(result, userId))
   } else {
     const result = await populationService.optimizedStatisticsOf({
       ...req.query,
@@ -220,8 +224,12 @@ router.get('/v3/populationstatistics', async (req, res) => {
       return
     }
 
-    // Obfuscate if user has only iam rights
-    if (!isAdmin && !rights.includes(studyRights.programme) && !rights.includes(studyRights.combinedProgramme)) {
+    // Obfuscate if user has only limited study programme rights
+    if (
+      !hasFullAccessToStudentData(roles) &&
+      !fullProgrammeRights.includes(studyRights.programme) &&
+      !fullProgrammeRights.includes(studyRights.combinedProgramme)
+    ) {
       result.students = result.students.map(student => {
         const { iv, encryptedData } = encrypt(student.studentNumber)
         const obfuscatedBirthDate = new Date(new Date(student.birthdate).setMonth(0, 0)) // only birth year for age distribution
@@ -242,14 +250,14 @@ router.get('/v3/populationstatistics', async (req, res) => {
       })
     }
 
-    res.json(filterPersonalTags(result, user.id))
+    res.json(filterPersonalTags(result, userId))
   }
 })
 
 router.get('/v3/populationstatisticsbycourse', async (req, res) => {
   const { coursecodes, from, to, separate: sep, unifyCourses } = req.query
   const {
-    user: { id, isAdmin, studentsUserCanAccess: allStudentsUserCanAccess, rights },
+    user: { id, roles, studentsUserCanAccess: allStudentsUserCanAccess, programmeRights },
   } = req
   const separate = sep ? JSON.parse(sep) : false
 
@@ -285,14 +293,17 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
     studentnumbers
   )
   let courseproviders = []
-  if (!isAdmin) {
+  if (!hasFullAccessToStudentData(roles)) {
     courseproviders = await CourseService.getCourseProvidersForCourses(JSON.parse(coursecodes))
   }
-  const rightsMappedToProviders = mapToProviders(rights)
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+  const rightsMappedToProviders = mapToProviders(fullStudyProgrammeRights)
   const found = courseproviders.some(r => rightsMappedToProviders.includes(r))
 
   const studentsUserCanAccess =
-    isAdmin || found ? new Set(studentnumbers) : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
+    hasFullAccessToStudentData(roles) || found
+      ? new Set(studentnumbers)
+      : new Set(_.intersection(studentnumbers, allStudentsUserCanAccess))
 
   const randomHash = crypto.randomBytes(12).toString('hex')
   const obfuscateStudent = ({ studyrights, studentNumber, courses, gender_code }) => ({
@@ -325,9 +336,11 @@ router.get('/v3/populationstatisticsbycourse', async (req, res) => {
 router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
   const { studentnumberlist, tags } = req.body
   const {
-    user: { isAdmin, id, studentsUserCanAccess },
+    user: { roles, id, studentsUserCanAccess },
   } = req
-  const filteredStudentNumbers = isAdmin ? studentnumberlist : _.intersection(studentnumberlist, studentsUserCanAccess)
+  const filteredStudentNumbers = hasFullAccessToStudentData(roles)
+    ? studentnumberlist
+    : _.intersection(studentnumberlist, studentsUserCanAccess)
 
   const studyProgrammeCode =
     tags?.studyProgramme && tags?.studyProgramme.includes('+')
@@ -356,14 +369,14 @@ router.post('/v3/populationstatisticsbystudentnumbers', async (req, res) => {
 
 router.get('/v3/populationstatistics/studyprogrammes', async (req, res) => {
   const {
-    user: { rights, iamRights, roles },
+    user: { roles, programmeRights },
   } = req
-  if (roles?.includes('admin')) {
+  if (hasFullAccessToStudentData(roles)) {
     const studyrights = await StudyrightService.getAssociations()
     mapCodesToIds(studyrights.programmes)
     res.json(studyrights)
   } else {
-    const allRights = rights.concat(iamRights)
+    const allRights = _.uniq(programmeRights.map(p => p.code))
     // For combined programme
     // If more programmes are combined, then a function might be a better idea to add moar rights
     if (allRights.includes('KH90_001') || allRights.includes('MH90_001')) allRights.push('KH90_001', 'MH90_001')

services/backend/src/routes/programmeModules.js

@@ -1,15 +1,15 @@
 const router = require('express').Router()
 const { getCoursesAndModules, getCurriculumVersions } = require('../services/programmeModules')
 
-router.get('/v3/get_curriculum_options/:code', async (req, res) => {
+router.get('/v3/curriculum-options/:code', async (req, res) => {
   const { code } = req.params
   const result = await getCurriculumVersions(code)
   res.json(result)
 })
 
-router.get('/v3/get_curriculum/:code/:period_ids', async (req, res) => {
-  const { code, period_ids } = req.params
-  const result = await getCoursesAndModules(code, period_ids.replace(' ', '').split(','))
+router.get('/v3/curriculum/:code/:periodIds', async (req, res) => {
+  const { code, periodIds } = req.params
+  const result = await getCoursesAndModules(code, periodIds.replace(' ', '').split(','))
   res.json({
     defaultProgrammeCourses: result.defaultProgrammeCourses.courses,
     defaultProgrammeModules: result.defaultProgrammeCourses.modules,

services/backend/src/routes/students.js

@@ -1,6 +1,8 @@
 const router = require('express').Router()
+
 const Student = require('../services/students')
 const { ApplicationError } = require('../util/customErrors')
+const { hasFullAccessToStudentData } = require('../util/utils')
 
 const filterStudentTags = (student, userId) => {
   return {
@@ -11,7 +13,7 @@ const filterStudentTags = (student, userId) => {
 
 router.get('/students', async (req, res) => {
   const {
-    user: { isAdmin, studentsUserCanAccess },
+    user: { roles, studentsUserCanAccess },
     query: { searchTerm },
   } = req
 
@@ -28,7 +30,7 @@ router.get('/students', async (req, res) => {
 
   let results = []
   if (trimmedSearchTerm) {
-    results = isAdmin
+    results = hasFullAccessToStudentData(roles)
       ? await Student.bySearchTerm(trimmedSearchTerm)
       : await Student.bySearchTermAndStudentNumbers(trimmedSearchTerm, studentsUserCanAccess)
   }
@@ -38,10 +40,10 @@ router.get('/students', async (req, res) => {
 router.get('/students/:id', async (req, res) => {
   const { id: studentId } = req.params
   const {
-    user: { id, isAdmin, studentsUserCanAccess },
+    user: { id, roles, studentsUserCanAccess },
   } = req
 
-  if (!isAdmin && !studentsUserCanAccess.includes(studentId)) {
+  if (!hasFullAccessToStudentData(roles) && !studentsUserCanAccess.includes(studentId)) {
     throw new ApplicationError('Error finding student', 400)
   }
   const student = await Student.withId(studentId)

services/backend/src/routes/studyProgramme.js

@@ -23,7 +23,7 @@ const { ElementDetail } = require('../models')
 
 // For grafana statistics (idea stolen from Norppa)
 const logInfoForGrafana = async (code, combinedProgramme) => {
-  const programme = await ElementDetail.findOne({ where: { code: code } })
+  const programme = await ElementDetail.findOne({ where: { code } })
   const programmeCode = combinedProgramme ? `${programme.code}-${combinedProgramme}` : programme.code
   logger.info('Study Programme', {
     studyprogrammeName: combinedProgramme ? `${programme.name.fi} + maisteri` : programme.name.fi,
@@ -59,7 +59,7 @@ router.get('/v2/studyprogrammes/creditstats', async (req, res) => {
   const codes = JSON.parse(codesListString)
   const stats = {}
   for (const code of codes) {
-    stats[code] = await getCreditsProduced(code, isAcademicYear !== 'false', includeSpecials)
+    stats[code] = await getCreditsProduced(code, isAcademicYear !== 'false', includeSpecials !== 'false')
   }
   return res.json({ stats })
 })
@@ -150,9 +150,8 @@ router.get('/v2/studyprogrammes/:id/update_basicview', async (req, res) => {
       logger.error({ message: `Failed to update code ${code} ${combinedProgramme} basic stats`, meta: `${e}` })
     }
     return res.json(result)
-  } else {
-    res.status(422).end()
   }
+  res.status(422).end()
 })
 
 router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) => {
@@ -167,9 +166,8 @@ router.get('/v2/studyprogrammes/:id/update_studytrackview', async (req, res) =>
       logger.error({ message: `Failed to update code ${code} ${combinedProgramme} studytrack stats`, meta: `${e}` })
     }
     return res.json(result)
-  } else {
-    res.status(422).end()
   }
+  res.status(422).end()
 })
 
 router.get('/v2/studyprogrammes/:id/evaluationstats', async (req, res) => {

services/backend/src/routes/tags.js

@@ -1,9 +1,10 @@
 const router = require('express').Router()
-const Tags = require('../services/tags')
-const TagStudent = require('../services/tagstudent')
-const Students = require('../services/students')
 const { difference } = require('lodash')
 
+const Tags = require('../services/tags')
+const Students = require('../services/students')
+const { getFullStudyProgrammeRights } = require('../util/utils')
+
 const filterRelevantTags = (tags, userId) => {
   return tags.filter(tag => !tag.personal_user_id || tag.personal_user_id === userId)
 }
@@ -12,21 +13,22 @@ const filterRelevantStudentTags = (studentTags, userId) => {
   return studentTags.filter(({ tag }) => !tag.personal_user_id || tag.personal_user_id === userId)
 }
 
-router.get('/tags', async (_req, res) => {
-  const tags = await Tags.findTags()
-  res.status(200).json(tags)
-})
+const userIsUnauthorized = (programmeRights, programmeCodes, roles) =>
+  !programmeRights.includes(programmeCodes[0]) &&
+  !programmeRights.includes(programmeCodes[1]) &&
+  !roles?.includes('admin')
 
 router.get('/tags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
   const {
-    user: { rights, roles, id },
+    user: { roles, id, programmeRights },
   } = req
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
   const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
 
   // Respond with null and 200 instead of 403 if the user isn't authorized to view the tags. This is to avoid unnecessary noise in Sentry
-  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
-    return res.json(null)
+  if (userIsUnauthorized(fullStudyProgrammeRights, programmeCodes, roles)) return res.json(null)
 
   const tags = await Tags.findTagsByStudytrack(studytrack)
   res.status(200).json(filterRelevantTags(tags, id))
@@ -37,12 +39,12 @@ router.post('/tags', async (req, res) => {
     tag: { studytrack, tagname, year, personal_user_id },
   } = req.body
   const {
-    user: { rights, roles, id },
+    user: { roles, id, programmeRights },
   } = req
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
 
   const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
-  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
-    return res.status(403).end()
+  if (userIsUnauthorized(fullStudyProgrammeRights, programmeCodes, roles)) return res.status(403).end()
 
   await Tags.createNewTag({ studytrack, tagname, year, personal_user_id })
   const tags = await Tags.findTagsByStudytrack(studytrack)
@@ -52,18 +54,15 @@ router.post('/tags', async (req, res) => {
 router.delete('/tags', async (req, res) => {
   const { tag } = req.body
   const {
-    user: { rights, roles, id },
+    user: { roles, id, programmeRights },
   } = req
 
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
   const programmeCodes =
     tag.studytrack.includes('KH') && tag.studytrack.includes('MH') ? tag.studytrack.split('-') : [tag.studytrack]
 
-  if (
-    !rights.includes(programmeCodes[0]) &&
-    !rights.includes(programmeCodes[1]) &&
-    !roles?.includes('admin') &&
-    !(tag.personal_user_id === id)
-  )
+  if (userIsUnauthorized(fullStudyProgrammeRights, programmeCodes, roles) && !(tag.personal_user_id === id))
     return res.status(403).end()
 
   await Tags.deleteTag(tag)
@@ -71,57 +70,37 @@ router.delete('/tags', async (req, res) => {
   res.status(200).json(filterRelevantTags(t, id))
 })
 
-router.get('/studenttags', async (req, res) => {
-  const result = await TagStudent.getStudentTags()
-  res.status(200).json(result)
-})
-
 router.get('/studenttags/:studytrack', async (req, res) => {
   const { studytrack } = req.params
   const {
-    user: { rights, roles, id },
+    user: { roles, id, programmeRights },
   } = req
 
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
   const programmeCodes = studytrack.includes('KH') && studytrack.includes('MH') ? studytrack.split('-') : [studytrack]
 
   // Respond with null and 200 instead of 403 if the user isn't authorized to view the tags. This is to avoid unnecessary noise in Sentry
-  if (!rights.includes(programmeCodes[0]) && !rights.includes(programmeCodes[1]) && !roles?.includes('admin'))
+  if (
+    !fullStudyProgrammeRights.includes(programmeCodes[0]) &&
+    !fullStudyProgrammeRights.includes(programmeCodes[1]) &&
+    !roles?.includes('admin')
+  )
     return res.json(null)
 
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrack)
-  res.status(200).json(filterRelevantStudentTags(result, id))
-})
-
-router.get('/studenttags/:studentnumber', async (req, res) => {
-  const { studentnumber } = req.params
-  const { user } = req
-  const result = await TagStudent.getStudentTagsByStudentnumber(studentnumber)
-  res.status(200).json(filterRelevantStudentTags(result, user.id))
-})
-
-router.post('/studenttags/:studentnumber', async (req, res) => {
-  const { tag, studytrack, combinedProgramme } = req.body
-  const {
-    user: { roles, rights, id },
-  } = req
-
-  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
-    return res.status(403).end()
-
-  await TagStudent.createStudentTag(tag)
-  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
+  const result = await Tags.getStudentTagsByStudytrack(studytrack)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
 router.post('/studenttags', async (req, res) => {
   const { tags, studytrack, combinedProgramme } = req.body
   const {
-    user: { rights, roles, id },
+    user: { roles, id, programmeRights },
   } = req
 
-  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
-    return res.status(403).end()
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
+  if (userIsUnauthorized(fullStudyProgrammeRights, [studytrack, combinedProgramme], roles)) return res.status(403).end()
 
   const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
   const existingTags = await Tags.findTagsByStudytrack(studytrackCode)
@@ -138,42 +117,27 @@ router.post('/studenttags', async (req, res) => {
       .status(400)
       .json({ error: `Could not find the following students from the programme: ${missingStudents.join(', ')}` })
 
-  await TagStudent.createMultipleStudentTags(tags)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
+  await Tags.createMultipleStudentTags(tags)
+  const result = await Tags.getStudentTagsByStudytrack(studytrackCode)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 
-router.delete('/studenttags/delete_one', async (req, res) => {
-  const { tag_id, studentnumber, studytrack, combinedProgramme } = req.body
-  const {
-    user: { rights, roles, id },
-  } = req
-
-  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
-    return res.status(403).end()
-  const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
-  const tags = await Tags.findTagsFromStudytrackById(studytrackCode, [tag_id])
-  if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
-
-  await TagStudent.deleteStudentTag(studentnumber, tag_id)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
-  res.status(200).json(filterRelevantStudentTags(result, id))
-})
-
-router.delete('/studenttags/delete_many', async (req, res) => {
+router.delete('/studenttags', async (req, res) => {
   const { tagId, studentnumbers, studytrack, combinedProgramme } = req.body
   const {
-    user: { rights, roles, id },
+    user: { roles, id, programmeRights },
   } = req
 
-  if (!rights.includes(studytrack) && !rights.includes(combinedProgramme) && !roles?.includes('admin'))
-    return res.status(403).end()
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
+  if (userIsUnauthorized(fullStudyProgrammeRights, [studytrack, combinedProgramme], roles)) return res.status(403).end()
+
   const studytrackCode = combinedProgramme ? `${studytrack}-${combinedProgramme}` : studytrack
   const tags = await Tags.findTagsFromStudytrackById(studytrackCode, [tagId])
   if (tags.length === 0) return res.status(403).json({ error: 'The tag does not exist' })
 
-  await TagStudent.deleteMultipleStudentTags(tagId, studentnumbers)
-  const result = await TagStudent.getStudentTagsByStudytrack(studytrackCode)
+  await Tags.deleteMultipleStudentTags(tagId, studentnumbers)
+  const result = await Tags.getStudentTagsByStudytrack(studytrackCode)
   res.status(200).json(filterRelevantStudentTags(result, id))
 })
 

services/backend/src/routes/teachers.js

@@ -2,7 +2,7 @@ const teachers = require('../services/teachers')
 const topteachers = require('../services/topteachers')
 const { isFaculty, providersOfFaculty } = require('../services/organisations')
 const router = require('express').Router()
-const { mapToProviders } = require('../util/utils')
+const { mapToProviders, getFullStudyProgrammeRights } = require('../util/utils')
 
 router.get('/', async (req, res) => {
   const { searchTerm } = req.query
@@ -43,14 +43,16 @@ router.get('/top/categories', async (req, res) => {
 
 router.get('/stats', async (req, res) => {
   const {
-    user: { rights, roles },
+    user: { roles, programmeRights },
   } = req
 
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
   const { providers, semesterStart, semesterEnd } = req.query
   if (!providers || !semesterStart) {
     return res.status(422).send('Missing required query parameters.')
   }
-  const providerRights = mapToProviders(rights)
+  const providerRights = mapToProviders(fullStudyProgrammeRights)
 
   if (!(providers.every(p => providerRights.includes(p)) || roles.includes('admin'))) {
     return res.status(403).send('You do not have rights to see this data')

services/backend/src/routes/university.js

@@ -3,10 +3,11 @@ const { getFacultyList } = require('../services/faculty/facultyHelpers')
 const { getGraduationStats, getFacultyProgressStats } = require('../services/faculty/facultyService')
 
 const { getMedian } = require('../services/studyprogramme/studyprogrammeHelpers')
+
 const degreeNames = ['bachelor', 'bachelorMaster', 'master', 'licentiate', 'doctor']
 
 router.get('/allprogressstats', async (req, res) => {
-  const specialGroups = 'SPECIAL_EXCLUDED'
+  const specialGroups = req.query?.specialsIncluded === 'true' ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
   const graduated = req.query?.graduated
   const allFaculties = await getFacultyList()
   const facultyCodes = allFaculties.map(f => f.code)
@@ -15,7 +16,7 @@ router.get('/allprogressstats', async (req, res) => {
   for (const facultyCode of facultyCodes) {
     const data = await getFacultyProgressStats(facultyCode, specialGroups, graduated)
     if (!data) {
-      return res.status(500).end()
+      return res.status(404).send('Data missing from server: Refreshing faculty data required')
     }
     codeToData[facultyCode] = data
   }
@@ -23,6 +24,9 @@ router.get('/allprogressstats', async (req, res) => {
   const universityData = {
     years: codeToData.H50.years,
     yearlyBachelorTitles: codeToData.H50.yearlyBachelorTitles,
+    yearlyBcMsTitles: codeToData.H50.yearlyBcMsTitles,
+    yearlyMasterTitles: codeToData.H50.yearlyMasterTitles,
+    yearlyLicentiateTitles: codeToData.H50.yearlyLicentiateTitles,
     programmeNames: allFaculties.reduce((obj, fac) => {
       const { name, ...rest } = fac.dataValues
       obj[fac.code] = { ...rest, ...name }
@@ -66,8 +70,8 @@ router.get('/allprogressstats', async (req, res) => {
         'mastersProgStats',
         'doctoralProgStats',
       ]) {
-        if (Object.keys(facultyData[fieldName]).length === 0) continue
         if (!universityData[fieldName]) universityData[fieldName] = {}
+        if (Object.keys(facultyData[fieldName]).length === 0) continue
         universityData[fieldName][facultyCode] = unifyProgressStats(Object.values(facultyData[fieldName]))
       }
     }
@@ -75,7 +79,7 @@ router.get('/allprogressstats', async (req, res) => {
 
   // Remove ELL bachelor+master progresstats for now, because it is problematic due to different credit limits
   // when expanding the table rows to show 'programme'(faculty) specific progress bars
-  delete universityData.bcMsProgStats['H90']
+  delete universityData.bcMsProgStats.H90
 
   return res.status(200).json(universityData)
 })
@@ -176,7 +180,7 @@ router.get('/allgraduationstats', async (req, res) => {
         })
       }
     }
-    // eslint-disable-next-line no-unused-vars
+
     const { programmes, ...rest } = data.classSizes
     universityData.classSizes.programmes[facultyCode] = rest
   }

services/backend/src/routes/updater.js

@@ -8,6 +8,7 @@ const {
   updateStudentsByStudentNumber,
   abort,
   updateCoursesByCourseCode,
+  updateStudentsIndividually,
 } = require('../services/sisUpdaterService')
 
 const logger = require('../util/logger')
@@ -37,8 +38,13 @@ router.get('/update/v2/students', async (req, res) => {
   }
 })
 
+router.get('/update/v2/students_individually', async (req, res) => {
+  await updateStudentsIndividually()
+  res.status(200).send()
+})
+
 router.post('/update/v2/customlist/:type', async (req, res) => {
-  const type = req.params.type
+  const { type } = req.params
   const list = req.body
   const typeToJob = {
     students: updateStudentsByStudentNumber,

services/backend/src/routes/users.js

@@ -65,11 +65,4 @@ router.delete('/:uid/elements', async (req, res) => {
   res.json(user)
 })
 
-router.post('/:uid/faculties', async (req, res) => {
-  const { uid } = req.params
-  const { faculties } = req.body
-  const user = await userService.setFaculties(uid, faculties)
-  res.json(user)
-})
-
 module.exports = router

services/backend/src/services/analyticsService.js

@@ -1,4 +1,4 @@
-const { redisClient } = require('../services/redis')
+const { redisClient } = require('./redis')
 const moment = require('moment')
 
 // Only new bachelor, masters and doctoral programmes get their data updated in redis every night, use redis for them
@@ -14,7 +14,7 @@ const createRedisKeyForStudytrackStats = (id, graduated, specialGroups) =>
 
 const getBasicStats = async (id, combinedProgramme, yearType, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
   const redisKey = createRedisKeyForBasicStats(searchkey, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -66,7 +66,7 @@ const setCreditStats = async (data, isAcademicYear, specialGroups) => {
 
 const getGraduationStats = async (id, combinedProgramme, yearType, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
   const redisKey = createRedisKeyForGraduationStats(searchkey, yearType, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null
@@ -89,7 +89,7 @@ const setGraduationStats = async (data, yearType, specialGroups) => {
 
 const getStudytrackStats = async (id, combinedProgramme, graduated, specialGroups) => {
   if (!isUpdatedNewProgramme(id)) return null
-  let searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
+  const searchkey = combinedProgramme ? `${id}-${combinedProgramme}` : id
   const redisKey = createRedisKeyForStudytrackStats(searchkey, graduated, specialGroups)
   const dataFromRedis = await redisClient.getAsync(redisKey)
   if (!dataFromRedis) return null

services/backend/src/services/completedCoursesSearch.js

@@ -1,17 +1,15 @@
 const { Credit, Enrollment, Student, Course } = require('../models')
-const { Op } = require('sequelize')
+const { Op, where, fn, col } = require('sequelize')
 
 const getCompletedCourses = async (studentNumbers, courseCodes) => {
   let courses = await Course.findAll({
     attributes: ['code', 'name', 'substitutions'],
-    where: {
-      code: {
-        [Op.in]: courseCodes,
-      },
-    },
+    where: where(fn('LOWER', col('code')), {
+      [Op.in]: courseCodes.map(code => code.toLowerCase()),
+    }),
   })
 
-  let fullCourseCodes = [
+  const fullCourseCodes = [
     ...courseCodes,
     ...new Set([...courses.reduce((acc, course) => [...acc, course.code, ...course.substitutions], [])]),
   ]
@@ -63,7 +61,7 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
       ? null
       : courses.find(course => course.substitutions.includes(course_code))?.code
     return {
-      courseCode: originalCode ? originalCode : course_code,
+      courseCode: originalCode || course_code,
       substitution: originalCode ? course_code : null,
       studentNumber: student_studentnumber,
       creditType: credittypecode,
@@ -74,7 +72,7 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
   enrollments = enrollments.map(enrollment => {
     const originalCode = courses.find(course => course.substitutions.includes(enrollment.course_code))?.code
     return {
-      courseCode: originalCode ? originalCode : enrollment.course_code,
+      courseCode: originalCode || enrollment.course_code,
       substitution: originalCode ? enrollment.course_code : null,
       studentNumber: enrollment.studentnumber,
       date: enrollment.enrollment_date_time,
@@ -100,7 +98,8 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
     const previous = studentCredits[credit.studentNumber].credits?.find(c => credit.courseCode === c.courseCode)
     if (previous && previous.date > credit.date) {
       return
-    } else if (previous) {
+    }
+    if (previous) {
       studentCredits[credit.studentNumber].credits = studentCredits[credit.studentNumber].credits.filter(
         c => credit.courseCode !== c.courseCode
       )
@@ -148,20 +147,17 @@ const getCompletedCourses = async (studentNumbers, courseCodes) => {
 
   students.forEach(student => {
     courseCodes.forEach(courseCode => {
-      student.enrollments[courseCode] = student.allEnrollments
+      const [latestEnrollment] = student.allEnrollments
         .filter(e => e.courseCode === courseCode || e.substitution === courseCode)
-        .sort(e => new Date(e.date).getDate())[0]
+        .sort((a, b) => new Date(b.date) - new Date(a.date))
+      student.enrollments[courseCode] = latestEnrollment
     })
   })
 
   // Omit allEnrollments, we're only supposed to show the recent, relevant enrollment,
   // the user does not have rights to see all enrollments.
   return {
-    students: students.map(student => {
-      // eslint-disable-next-line no-unused-vars
-      const { allEnrollments, ...rest } = student
-      return { ...rest }
-    }),
+    students: students.map(({ allEnrollments, ...rest }) => rest),
     courses,
   }
 }

services/backend/src/services/courses/course_stats_counter.js

@@ -49,6 +49,7 @@ class CourseStatsCounter {
     this.grades = {}
   }
 
+  // eslint-disable-next-line class-methods-use-this
   initializePassingSemesters(initialValue = 0) {
     const passingSemesters = {
       BEFORE: initialValue,
@@ -63,7 +64,7 @@ class CourseStatsCounter {
   }
 
   markAttempt() {
-    this.stats.attempts = this.stats.attempts + 1
+    this.stats.attempts += 1
   }
 
   markToAll(studentnumber) {
@@ -164,12 +165,12 @@ class CourseStatsCounter {
   }
 
   getPassingSemestersCumulative() {
-    const passingSemesters = this.stats.passingSemesters
+    const { passingSemesters } = this.stats
     const attemptStats = {
-      BEFORE: passingSemesters['BEFORE'],
+      BEFORE: passingSemesters.BEFORE,
     }
 
-    attemptStats['0-FALL'] = passingSemesters['BEFORE'] + passingSemesters['0-FALL']
+    attemptStats['0-FALL'] = passingSemesters.BEFORE + passingSemesters['0-FALL']
     attemptStats['0-SPRING'] = attemptStats['0-FALL'] + passingSemesters['0-SPRING']
 
     for (let i = 1; i < 7; i++) {
@@ -177,22 +178,21 @@ class CourseStatsCounter {
       attemptStats[spring[i]] = attemptStats[fall[i]] + passingSemesters[spring[i]]
     }
 
-    attemptStats['LATER'] = attemptStats['6-SPRING'] + passingSemesters['LATER']
+    attemptStats.LATER = attemptStats['6-SPRING'] + passingSemesters.LATER
 
     return attemptStats
   }
 
   getFinalStats() {
-    const stats = this.stats
-    const students = this.students
+    const { stats, students } = this
     const studs = this.enrollments
       ? Object.keys(this.enrollments)
           .filter(key => key !== 'ENROLLED' && key !== 'semesters')
           .reduce((acc, key) => [...acc, ...[...this.enrollments[key]].map(studentnumber => studentnumber)], [])
       : []
     const allStudents = Object.keys(students.all).map(student => student)
-    const fileredEnrolledNoGrade = this.enrollments['ENROLLED']
-      ? [...this.enrollments['ENROLLED']]
+    const fileredEnrolledNoGrade = this.enrollments.ENROLLED
+      ? [...this.enrollments.ENROLLED]
           .filter(student => !studs.includes(student) && !allStudents.includes(student))
           .reduce((acc, student) => ({ ...acc, [student]: true }), {})
       : {}

services/backend/src/services/courses/course_yearly_stats_counter.js

@@ -199,20 +199,16 @@ class CourseYearlyStatsCounter {
             category: 'passedFirst',
             code: groupcode,
           })
-        } else {
-          if (student.category == 'passedFirst') {
-            this.students.set(studentnumber, {
-              earliestAttainment: attainment_date,
-              category: 'passedEventually',
-              code: groupcode,
-            })
-          }
+        } else if (student.category === 'passedFirst') {
+          this.students.set(studentnumber, {
+            earliestAttainment: attainment_date,
+            category: 'passedEventually',
+            code: groupcode,
+          })
         }
-      } else {
-        if (student.category === 'neverPassed') {
-          if (passed) {
-            this.students.set(studentnumber, { ...student, category: 'passedEventually' })
-          }
+      } else if (student.category === 'neverPassed') {
+        if (passed) {
+          this.students.set(studentnumber, { ...student, category: 'passedEventually' })
         }
       }
     }
@@ -258,8 +254,8 @@ class CourseYearlyStatsCounter {
         // totals cannot be calculated
         this.obfuscated = true
 
-        let gradeSpread = {}
-        for (const grade in normalStats.attempts.grades) {
+        const gradeSpread = {}
+        for (const grade of Object.keys(normalStats.attempts.grades)) {
           gradeSpread[grade] = []
         }
 

services/backend/src/services/courses/index.js

@@ -1,9 +1,8 @@
-const Sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const { Credit, Course, Organization, Enrollment } = require('../../models')
 
 const { parseCredit } = require('./parseCredits')
 const { parseEnrollment } = require('./parseEnrollments')
-const Op = Sequelize.Op
 const { CourseYearlyStatsCounter } = require('./course_yearly_stats_counter')
 const { sortMainCode } = require('../../util/utils')
 const {
@@ -23,7 +22,7 @@ const unifyOpenUniversity = code => {
 
 const allCodeAlternatives = async code => {
   const course = await Course.findOne({
-    where: { code: code },
+    where: { code },
   })
 
   return sortMainCode([...course.substitutions, code])
@@ -62,7 +61,7 @@ const yearlyStatsOfNew = async (
 
   const counter = new CourseYearlyStatsCounter()
 
-  for (let credit of credits) {
+  for (const credit of credits) {
     const {
       studentnumber,
       grade,

services/backend/src/services/excludedCourses.js

@@ -1,6 +1,5 @@
-const Sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const { ExcludedCourse } = require('../models/models_kone')
-const { Op } = Sequelize
 
 const addExcludedCourses = async (programmecode, coursecodes, curriculum) => {
   return ExcludedCourse.bulkCreate(

services/backend/src/services/faculty/faculty.js

@@ -1,5 +1,4 @@
-const Sequelize = require('sequelize')
-const { Op } = Sequelize
+const { Op } = require('sequelize')
 const {
   ElementDetail,
   Organization,
@@ -22,26 +21,11 @@ const {
   mapCodesToIds,
   facultyProgrammeStudents,
 } = require('./facultyHelpers')
+const { uniqBy } = require('lodash')
 const {
   dbConnections: { sequelize },
 } = require('../../database/connection')
-const getTransferredToAndAway = async (programmeCodes, allProgrammeCodes, since) => {
-  const awayTransfers = await transferredAway(programmeCodes, allProgrammeCodes, since)
-  const toTransfers = await transferredTo(programmeCodes, allProgrammeCodes, since)
-  return [...toTransfers, ...awayTransfers]
-}
 
-const getTransferredInside = async (programmeCodes, allProgrammeCodes, since) => {
-  return await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
-}
-
-const getTransfersOut = async (programmeCode, start, end) => {
-  return await transferredFaculty([], [programmeCode], start, end)
-}
-
-const getTransfersIn = async (programmeCode, start, end) => {
-  return await transferredFaculty([programmeCode], [], start, end)
-}
 const transferredFaculty = async (programmeCodeIn, programmeCodeOut, start, end) =>
   (
     await Transfer.findAll({
@@ -69,7 +53,7 @@ const startedStudyrights = async (faculty, code, since, studyRightWhere) =>
           model: StudyrightElement,
           required: true,
           where: {
-            code: code,
+            code,
           },
           include: {
             model: ElementDetail,
@@ -95,7 +79,7 @@ const graduatedStudyrights = async (faculty, code, since, studyrightWhere) =>
           model: StudyrightElement,
           required: true,
           where: {
-            code: code,
+            code,
           },
           include: {
             model: ElementDetail,
@@ -122,7 +106,7 @@ const studyrightsByRightStartYear = async (faculty, code, since, graduated = 1)
           model: StudyrightElement,
           attributes: ['code', 'startdate'],
           where: {
-            code: code,
+            code,
           },
           required: true,
           include: {
@@ -136,7 +120,7 @@ const studyrightsByRightStartYear = async (faculty, code, since, graduated = 1)
         startdate: {
           [Op.gte]: since,
         },
-        graduated: graduated,
+        graduated,
       },
     })
   ).map(facultyFormatStudyright)
@@ -149,7 +133,7 @@ const getStudyRightsByExtent = async (faculty, startDate, endDate, code, extents
         attributes: [],
         required: true,
         where: {
-          code: code,
+          code,
         },
         include: {
           model: ElementDetail,
@@ -191,7 +175,7 @@ const getStudyRightsByBachelorStart = async (faculty, startDate, endDate, code,
         model: StudyrightElement,
         required: true,
         where: {
-          code: code,
+          code,
         },
       },
       where: {
@@ -278,20 +262,25 @@ const transferredTo = async (programmes, allProgrammeCodes, since) =>
   ).map(formatFacultyTransfer)
 
 const degreeProgrammesOfFaculty = async facultyCode =>
-  (
-    await ProgrammeModule.findAll({
-      attributes: ['code', 'name'],
-      include: {
-        model: Organization,
-        where: {
-          code: {
-            [Op.startsWith]: facultyCode,
+  // Some programmenames are different, causing this to return multiples of same codes.
+  // Hence the uniqBy
+  uniqBy(
+    (
+      await ProgrammeModule.findAll({
+        attributes: ['code', 'name'],
+        include: {
+          model: Organization,
+          where: {
+            code: {
+              [Op.startsWith]: facultyCode,
+            },
           },
         },
-      },
-      group: ['programme_module.code', 'programme_module.name', 'organization.id'],
-    })
-  ).map(facultyFormatProgramme)
+        group: ['programme_module.code', 'programme_module.name', 'organization.id'],
+      })
+    ).map(facultyFormatProgramme),
+    'code'
+  )
 
 const facultyOrganizationId = async faculty => {
   return await Organization.findOne({
@@ -366,6 +355,7 @@ const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
   for (const org of facultyChildOrganizations) {
     const childAssociationProgrammes = await degreeProgrammesOfFaculty(org.code)
     if (childAssociationProgrammes.length > 0) {
+      // eslint-disable-next-line no-loop-func
       childAssociationProgrammes.forEach(prog => {
         if (!(prog.code in allProgrammeCodes)) {
           allProgrammes = allProgrammes.concat([prog])
@@ -379,6 +369,7 @@ const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
         for (const gcOrg of grandChildren) {
           const associatedProgrammes = await degreeProgrammesOfFaculty(gcOrg.code)
           if (associatedProgrammes.length > 0) {
+            // eslint-disable-next-line no-loop-func
             associatedProgrammes.forEach(prog => {
               if (!(prog.code in allProgrammeCodes)) {
                 allProgrammes = allProgrammes.concat([prog])
@@ -400,6 +391,24 @@ const findFacultyProgrammeCodes = async (faculty, programmeFilter) => {
   return allProgrammes
 }
 
+const getTransferredToAndAway = async (programmeCodes, allProgrammeCodes, since) => {
+  const awayTransfers = await transferredAway(programmeCodes, allProgrammeCodes, since)
+  const toTransfers = await transferredTo(programmeCodes, allProgrammeCodes, since)
+  return [...toTransfers, ...awayTransfers]
+}
+
+const getTransferredInside = async (programmeCodes, allProgrammeCodes, since) => {
+  return await transferredInsideFaculty(programmeCodes, allProgrammeCodes, since)
+}
+
+const getTransfersOut = async (programmeCode, start, end) => {
+  return await transferredFaculty([], [programmeCode], start, end)
+}
+
+const getTransfersIn = async (programmeCode, start, end) => {
+  return await transferredFaculty([programmeCode], [], start, end)
+}
+
 module.exports = {
   startedStudyrights,
   graduatedStudyrights,

services/backend/src/services/faculty/facultyBasics.js

@@ -18,7 +18,7 @@ const { codes } = require('../../../config/programmeCodes')
 const filterDuplicateStudyrights = studyrights => {
   // bachelor+master students have two studyrights (separated by two last digits in studyrightid)
   // choose only bachelor one, so we don't count start of masters as starting in faculty
-  let rightsToCount = {}
+  const rightsToCount = {}
 
   studyrights.forEach(right => {
     const id = right.studyrightid.slice(0, -2)
@@ -39,18 +39,22 @@ const addProgramme = async (
   facultyProgrammes,
   includeAllSpecials
 ) => {
-  let { progId, name, code } = facultyProgrammes.filter(prog => prog.code === programme)[0]
+  const { progId, name, code } = facultyProgrammes.filter(prog => prog.code === programme)[0]
   if (!(progId in stats)) {
     stats[progId] = {}
     if (includeAllSpecials) {
-      Object.keys(tableStats).forEach(year => (stats[progId][year] = [0, 0, 0]))
+      Object.keys(tableStats).forEach(year => {
+        stats[progId][year] = [0, 0, 0]
+      })
     } else {
-      Object.keys(tableStats).forEach(year => (stats[progId][year] = [0]))
+      Object.keys(tableStats).forEach(year => {
+        stats[progId][year] = [0]
+      })
     }
   }
   stats[progId][transferYear][index] += 1
   if (!(progId in allBasics.programmeNames)) {
-    allBasics.programmeNames[progId] = { ...name, code: code }
+    allBasics.programmeNames[progId] = { ...name, code }
   }
 }
 
@@ -116,7 +120,7 @@ const getFacultyStarters = async (
     })
   }
   allBasics.studentInfo.graphStats.push({ name: 'Started studying (new in faculty)', data: startedGraphStats })
-  programmeData['started'] = programmeTableStats
+  programmeData.started = programmeTableStats
 
   Object.keys(startedTableStats).forEach(year => {
     counts[year] = [startedTableStats[year]]
@@ -143,9 +147,11 @@ const getFacultyGraduates = async (
   const graduatedGraphStats = [[...graphStats], [...graphStats], [...graphStats], [...graphStats], [...graphStats]]
   const graduatedTableStats = {}
   const programmeTableStats = {}
-  Object.keys(tableStats).forEach(year => (graduatedTableStats[year] = [0, 0, 0, 0, 0]))
+  Object.keys(tableStats).forEach(year => {
+    graduatedTableStats[year] = [0, 0, 0, 0, 0]
+  })
   const keys = Object.keys(codes)
-  let studyrightWhere = getExtentFilter(includeAllSpecials)
+  const studyrightWhere = getExtentFilter(includeAllSpecials)
   for (const code of programmes) {
     let graduatedRights = await graduatedStudyrights(faculty, code, since, studyrightWhere)
     if (!includeAllSpecials) {
@@ -169,7 +175,9 @@ const getFacultyGraduates = async (
 
         if (!(programmeId in programmeTableStats)) {
           programmeTableStats[programmeId] = {}
-          Object.keys(tableStats).forEach(year => (programmeTableStats[programmeId][year] = [0, 0, 0, 0, 0]))
+          Object.keys(tableStats).forEach(year => {
+            programmeTableStats[programmeId][year] = [0, 0, 0, 0, 0]
+          })
         }
         programmeTableStats[programmeId][endYear][0] += 1
 
@@ -204,7 +212,7 @@ const getFacultyGraduates = async (
   allBasics.graduationInfo.graphStats.push({ name: 'Doctors', data: graduatedGraphStats[3] })
   allBasics.graduationInfo.graphStats.push({ name: 'Others', data: graduatedGraphStats[4] })
 
-  programmeData['graduated'] = programmeTableStats
+  programmeData.graduated = programmeTableStats
 
   Object.keys(graduatedTableStats).forEach(year => {
     counts[year] = counts[year].concat(graduatedTableStats[year][0])
@@ -246,13 +254,17 @@ const getFacultyTransfers = async (
     faculty
   )
   let transferGraphStats = []
-  let transferTableStats = {}
+  const transferTableStats = {}
   if (includeAllSpecials) {
     transferGraphStats = [[...graphStats], [...graphStats], [...graphStats]]
-    Object.keys(tableStats).forEach(year => (transferTableStats[year] = [0, 0, 0]))
+    Object.keys(tableStats).forEach(year => {
+      transferTableStats[year] = [0, 0, 0]
+    })
   } else {
     transferGraphStats = [[...graphStats]]
-    Object.keys(tableStats).forEach(year => (transferTableStats[year] = [0]))
+    Object.keys(tableStats).forEach(year => {
+      transferTableStats[year] = [0]
+    })
   }
 
   const programmeTableStats = {}
@@ -320,7 +332,7 @@ const getFacultyTransfers = async (
     allBasics.studentInfo.graphStats.push({ name: 'Transferred away', data: transferGraphStats[1] })
     allBasics.studentInfo.graphStats.push({ name: 'Transferred to', data: transferGraphStats[2] })
   }
-  programmeData['transferred'] = programmeTableStats
+  programmeData.transferred = programmeTableStats
 
   Object.keys(transferTableStats).forEach(year => {
     counts[year] = counts[year].concat(transferTableStats[year])
@@ -328,10 +340,10 @@ const getFacultyTransfers = async (
 }
 
 const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeCodes, programmeFilter, special) => {
-  let counts = {}
-  let countsGraduations = {}
-  let years = []
-  let programmeData = {}
+  const counts = {}
+  const countsGraduations = {}
+  const years = []
+  const programmeData = {}
   const isAcademicYear = yearType === 'ACADEMIC_YEAR'
   const includeAllSpecials = special === 'SPECIAL_INCLUDED'
   const since = isAcademicYear ? new Date('2017-08-01') : new Date('2017-01-01')
@@ -340,7 +352,7 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
   Object.keys(tableStats).forEach(year => years.push(year))
   const wantedProgrammeCodes = programmes.map(prog => prog.code)
 
-  let allBasics = {
+  const allBasics = {
     id: faculty,
     years: [],
     programmeNames: {},
@@ -424,7 +436,7 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
   allBasics.studentInfo.tableStats.reverse()
   allBasics.graduationInfo.tableStats.reverse()
 
-  //combine programme level tablestats
+  // combine programme level tablestats
   // all programmes are not present in all data types, check all found programmes and patch missing details with 0
 
   let allCodes = Object.keys(programmeData.started).concat(
@@ -433,8 +445,8 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
   )
   allCodes = [...new Set(allCodes)]
 
-  let studentInfo = {}
-  let graduationInfo = {}
+  const studentInfo = {}
+  const graduationInfo = {}
 
   allCodes.forEach(code => {
     if (!(code in studentInfo)) studentInfo[code] = {}
@@ -457,12 +469,10 @@ const combineFacultyBasics = async (faculty, programmes, yearType, allProgrammeC
       // trasferred
       if (code in programmeData.transferred) {
         studentInfo[code][year] = studentInfo[code][year].concat(programmeData.transferred[code][year])
+      } else if (includeAllSpecials) {
+        studentInfo[code][year] = studentInfo[code][year].concat([0, 0, 0])
       } else {
-        if (includeAllSpecials) {
-          studentInfo[code][year] = studentInfo[code][year].concat([0, 0, 0])
-        } else {
-          studentInfo[code][year] = studentInfo[code][year].concat([0])
-        }
+        studentInfo[code][year] = studentInfo[code][year].concat([0])
       }
     }
   })

services/backend/src/services/faculty/facultyGraduationTimes.js

@@ -8,7 +8,7 @@ const { getProgrammes } = require('./facultyService')
 
 const sortProgrammes = data => {
   const check = name => {
-    if (isNaN(name[0])) return -1
+    if (Number.isNaN(name[0])) return -1
     return 1
   }
   data.sort((a, b) => {
@@ -22,8 +22,8 @@ const getSortedProgIds = progs => {
 
   for (const prog of progs) {
     if (Object.keys(codes).includes(prog)) {
-      const name = prog === 'T923103' ? codes[prog].toUpperCase() + '-DP' : codes[prog].toUpperCase()
-      programmes = [...programmes, { name: name, code: prog }]
+      const name = prog === 'T923103' ? `${codes[prog].toUpperCase()}-DP` : codes[prog].toUpperCase()
+      programmes = [...programmes, { name, code: prog }]
     } else {
       programmes = [...programmes, { name: prog, code: prog }]
     }
@@ -137,12 +137,10 @@ const getClassSizes = async (faculty, programmeCodes, since, classSizes, program
         if (studyrightid.slice(-2) === '-2') {
           level = 'bcMsCombo'
           actualStartdate = await findBachelorStartdate(studyrightid.replace(/-2$/, '-1'))
+        } else if (code.includes('KH')) {
+          level = 'bachelor' // Some rare faculties have one/two outliers with extent 2 for bachelor
         } else {
-          if (code.includes('KH')) {
-            level = 'bachelor' // Some rare faculties have one/two outliers with extent 2 for bachelor
-          } else {
-            level = 'master'
-          }
+          level = 'master'
         }
       } else if (extentcode === 4) {
         level = 'doctor'
@@ -158,15 +156,15 @@ const getClassSizes = async (faculty, programmeCodes, since, classSizes, program
       if (level !== 'bcMsCombo') {
         classSizes[level][startYear] += 1
         if (level === 'bachelor' && (await hasMS(code, studyrightElements, studyrightid))) {
-          classSizes['bcMsCombo'][startYear] += 1
+          classSizes.bcMsCombo[startYear] += 1
         }
       }
 
       if (!(code in classSizes.programmes)) {
         if (extentcode === 2) {
           classSizes.programmes[code] = {}
-          classSizes.programmes[code]['bcMsCombo'] = getYearsObject({ years, emptyArrays: false })
-          classSizes.programmes[code]['master'] = getYearsObject({ years, emptyArrays: false })
+          classSizes.programmes[code].bcMsCombo = getYearsObject({ years, emptyArrays: false })
+          classSizes.programmes[code].master = getYearsObject({ years, emptyArrays: false })
         } else {
           classSizes.programmes[code] = getYearsObject({ years, emptyArrays: false })
         }
@@ -193,10 +191,10 @@ const count = async (
   goals,
   mode
 ) => {
-  let graduationAmounts = {}
-  let graduationTimes = {}
-  let programmes = getProgrammeObjectBasis(yearsList, levels)
-  let data = {
+  const graduationAmounts = {}
+  const graduationTimes = {}
+  const programmes = getProgrammeObjectBasis(yearsList, levels)
+  const data = {
     medians: {},
     programmes: {
       medians: getProgrammeObjectBasis(yearsList, levels),
@@ -265,7 +263,9 @@ const count = async (
           }
 
           const progStatistics = countTimeCategories(programmes[level][year][prog.code].graduationTimes, goal)
-          Object.keys(progStatistics).forEach(key => (statistics[key] += progStatistics[key]))
+          Object.keys(progStatistics).forEach(key => {
+            statistics[key] += progStatistics[key]
+          })
 
           data.programmes.medians[level][year].programmes = [
             ...data.programmes.medians[level][year].programmes,
@@ -318,7 +318,7 @@ const countGraduationTimes = async (faculty, programmeFilter) => {
     doctor: 48,
     licentiate: 78,
     exceptions: {
-      MH30_004: 6, //months more
+      MH30_004: 6, // months more
       '420420-ma': 6,
       MH30_001: 12,
       '320011-ma': 12,
@@ -363,7 +363,7 @@ const countGraduationTimes = async (faculty, programmeFilter) => {
   )
 
   const classSizes = getProgrammeObjectBasis(yearsList, levels, false)
-  classSizes['programmes'] = {}
+  classSizes.programmes = {}
   await getClassSizes(faculty, programmeCodes, since, classSizes, programmeFilter, years)
 
   return { id: faculty, goals, byGradYear, byStartYear, programmeNames, classSizes }

services/backend/src/services/faculty/facultyHelpers.js

@@ -1,5 +1,4 @@
-const Sequelize = require('sequelize')
-const { Op } = Sequelize
+const { Op } = require('sequelize')
 const { codes } = require('../../../config/programmeCodes')
 const { faculties } = require('../organisations')
 
@@ -124,7 +123,7 @@ const getExtentFilter = includeAllSpecials => {
   if (!includeAllSpecials) {
     filteredExtents.push(6, 7, 9, 13, 14, 18, 22, 23, 34, 99)
   }
-  let studyrightWhere = {
+  const studyrightWhere = {
     extentcode: {
       [Op.notIn]: filteredExtents,
     },
@@ -139,7 +138,7 @@ const mapCodesToIds = data => {
     if (keys.includes(prog.code)) {
       prog.progId = codes[prog.code].toUpperCase()
       if (prog.code === 'T923103' && !prog.progId.includes('-DP')) {
-        prog.progId = prog.progId + '-DP'
+        prog.progId += '-DP'
       }
     } else {
       prog.progId = prog.code

services/backend/src/services/faculty/facultyService.js

@@ -26,13 +26,12 @@ const removeGraduationTimes = data => {
       yearStat.times = null
     })
   )
-  data.byGradYear.medians
 }
 
 const setFacultyProgrammes = async (id, data, programmeFilter) => {
   const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
   const dataToRedis = {
-    data: data,
+    data,
     status: 'DONE',
     lastUpdated: moment().format(),
   }
@@ -41,6 +40,13 @@ const setFacultyProgrammes = async (id, data, programmeFilter) => {
   return dataToRedis
 }
 
+const getFacultyProgrammesFromRedis = async (id, programmeFilter) => {
+  const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
+  const dataFromRedis = await redisClient.getAsync(redisKey)
+  if (!dataFromRedis) return null
+  return JSON.parse(dataFromRedis)
+}
+
 const getProgrammes = async (code, programmeFilter = 'NEW_STUDY_PROGRAMMES') => {
   const programmes = await getFacultyProgrammesFromRedis(code, programmeFilter)
   if (programmes) return programmes
@@ -50,13 +56,6 @@ const getProgrammes = async (code, programmeFilter = 'NEW_STUDY_PROGRAMMES') =>
   return updatedProgrammes
 }
 
-const getFacultyProgrammesFromRedis = async (id, programmeFilter) => {
-  const redisKey = createRedisKeyForFacultyProgrammes(id, programmeFilter)
-  const dataFromRedis = await redisClient.getAsync(redisKey)
-  if (!dataFromRedis) return null
-  return JSON.parse(dataFromRedis)
-}
-
 const setBasicStats = async (data, yearType, programmeFilter, specialGroups) => {
   const { id } = data
   const redisKey = createRedisKeyForBasicStats(id, yearType, programmeFilter, specialGroups)

services/backend/src/services/faculty/facultyStudentProgress.js

@@ -15,15 +15,19 @@ const getStudentData = (
   thresholdKeys = [],
   thresholdAmounts = []
 ) => {
-  let data = {}
-  let programmeData = {}
+  const data = {}
+  const programmeData = {}
   const creditCounts = students.map(({ credits }) =>
     credits
       .filter(credit => moment(credit.attainment_date).isSameOrAfter(startDate))
       .reduce((total, credit) => total + credit.credits, 0)
   )
-  thresholdKeys.forEach(t => (data[t] = 0))
-  limitKeys.forEach(t => (programmeData[t] = 0))
+  thresholdKeys.forEach(t => {
+    data[t] = 0
+  })
+  limitKeys.forEach(t => {
+    programmeData[t] = 0
+  })
 
   // Data is only used for doctoral programmes, otherwise only programmeData is needed
   if (prog === 'T') {
@@ -129,7 +133,7 @@ const combineFacultyStudentProgress = async (faculty, programmes, specialGroups,
     if (year === 'Total') continue
     const { startDate, endDate } = getAcademicYearDates(year, since)
     const lastDayOfMonth = moment().endOf('month')
-    let months = Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(startDate))).asMonths())
+    const months = Math.round(moment.duration(moment(lastDayOfMonth).diff(moment(startDate))).asMonths())
     if (months >= 36) {
       bachelorlimits = createLimits(36, 0)
     } else {
@@ -181,7 +185,7 @@ const combineFacultyStudentProgress = async (faculty, programmes, specialGroups,
       // Get credit threshold values: 'combined study programme' is false and 'only master studyright' is true
 
       if (!(progId in progressStats.programmeNames)) {
-        progressStats.programmeNames[progId] = { code: code, ...name }
+        progressStats.programmeNames[progId] = { code, ...name }
       }
       if (!includeAllSpecials) {
         studyrights = await filterOutTransfers(studyrights, code, startDate, endDate)
@@ -246,7 +250,7 @@ const combineFacultyStudentProgress = async (faculty, programmes, specialGroups,
       } else {
         const all = studyrights.filter(sr => sr.extentcode === 4 || sr.extentcode === 3).map(sr => sr.studentnumber)
         const doctoralStudents = await studytrackStudents(all)
-        let { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(code)
+        const { creditThresholdKeys, creditThresholdAmounts } = getCreditThresholds(code)
 
         const { data, creditCounts } = getStudentData(
           startDate,

services/backend/src/services/faculty/facultyStudents.js

@@ -10,6 +10,7 @@ const { getStudyRightsByExtent, getStudentsByStudentnumbers, getTransfersIn } =
 const { checkTransfers } = require('./facultyHelpers')
 const { inactiveStudyrights, graduatedStudyRights } = require('../studyprogramme/studyrightFinders')
 const { enrolledStudents, absentStudents } = require('../studyprogramme/studentGetters')
+
 const emptyTotals = () => {
   return {
     total: 0,
@@ -58,7 +59,7 @@ const addTotalsProgramme = (programmeTableStats, progId, year, totals) => {
 }
 
 const getStudentData = (students, facultyExtra, year, code) => {
-  let data = { female: 0, male: 0, finnish: 0, otherCountries: 0, otherUnknown: 0 }
+  const data = { female: 0, male: 0, finnish: 0, otherCountries: 0, otherUnknown: 0 }
   students.forEach(({ genderCode, homeCountryEn }) => {
     data.male += genderCode === '1' ? 1 : 0
     data.female += genderCode === '2' ? 1 : 0
@@ -67,11 +68,11 @@ const getStudentData = (students, facultyExtra, year, code) => {
     data.otherCountries += homeCountryEn !== 'Finland' ? 1 : 0
     if (!Object.keys(facultyExtra[year][code]).includes(homeCountryEn) && homeCountryEn !== 'Finland')
       facultyExtra[year][code][homeCountryEn] = 0
-    if (!Object.keys(facultyExtra['Total'][code]).includes(homeCountryEn) && homeCountryEn !== 'Finland')
-      facultyExtra['Total'][code][homeCountryEn] = 0
+    if (!Object.keys(facultyExtra.Total[code]).includes(homeCountryEn) && homeCountryEn !== 'Finland')
+      facultyExtra.Total[code][homeCountryEn] = 0
     if (homeCountryEn !== 'Finland') {
       facultyExtra[year][code][homeCountryEn] += 1
-      facultyExtra['Total'][code][homeCountryEn] += 1
+      facultyExtra.Total[code][homeCountryEn] += 1
     }
   })
   return data
@@ -105,7 +106,7 @@ const getFacultyDataForYear = async ({
   const totals = emptyTotals()
   for (const { progId, code, name } of programmes) {
     if (!Object.keys(facultyExtra[year]).includes(code)) facultyExtra[year][code] = {}
-    if (!Object.keys(facultyExtra['Total']).includes(code)) facultyExtra['Total'][code] = {}
+    if (!Object.keys(facultyExtra.Total).includes(code)) facultyExtra.Total[code] = {}
 
     const toTransfers = await getTransfersIn(code, startDate, endDate)
     const allStudyrights = await getStudyRightsByExtent(faculty, startDate, endDate, code, extents, graduated)
@@ -113,7 +114,7 @@ const getFacultyDataForYear = async ({
     if (includeAllSpecials) {
       allStudents = [...new Set([...allStudyrights.map(sr => sr.studentnumber)])]
     }
-    let studyrights = allStudyrights.filter(s => !checkTransfers(s, toTransfers, toTransfers))
+    const studyrights = allStudyrights.filter(s => !checkTransfers(s, toTransfers, toTransfers))
 
     // Get all the studyrights and students for the calculations
     const studentNbrs = studyrights.map(sr => sr.studentnumber)
@@ -134,7 +135,7 @@ const getFacultyDataForYear = async ({
       progTotals[progId] = emptyTotals()
     }
     if (!(progId in programmeNames)) {
-      programmeNames[progId] = { code: code, ...name }
+      programmeNames[progId] = { code, ...name }
       progTotals[progId] = emptyTotals()
     }
 
@@ -232,12 +233,12 @@ const combineFacultyStudents = async (code, programmes, specialGroups, graduated
 
   const studentsData = {
     id: code,
-    years: years,
-    facultyTableStats: facultyTableStats,
+    years,
+    facultyTableStats,
     facultyTableStatsExtra: facultyExtra,
     programmeStats: programmeTableStats,
     titles: [...tableTitles.studytracksStart, ...tableTitles.studytracksBasic, ...tableTitles.studytracksEnd],
-    programmeNames: programmeNames,
+    programmeNames,
   }
   return studentsData
 }

services/backend/src/services/faculty/facultyThesisWriters.js

@@ -13,10 +13,10 @@ const {
 const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyProgrammes, includeAllSpecials }) => {
   const thesisTypes = ['urn:code:course-unit-type:bachelors-thesis', 'urn:code:course-unit-type:masters-thesis']
 
-  let bachelors = getStatsBasis(years)
-  let masters = getStatsBasis(years)
-  let programmeCounts = {}
-  let programmeNames = {}
+  const bachelors = getStatsBasis(years)
+  const masters = getStatsBasis(years)
+  const programmeCounts = {}
+  const programmeNames = {}
 
   for (const { progId, code, name } of facultyProgrammes) {
     if (code === 'MH70_008_2' || code.startsWith('LIS') || code.startsWith('T')) continue
@@ -36,8 +36,10 @@ const getFacultyThesisWriters = async ({ since, years, isAcademicYear, facultyPr
       if (!(progId in programmeCounts)) {
         programmeCounts[progId] = {}
 
-        Object.keys(bachelors.tableStats).forEach(year => (programmeCounts[progId][year] = [0, 0, 0]))
-        programmeNames[progId] = { ...name, code: code }
+        Object.keys(bachelors.tableStats).forEach(year => {
+          programmeCounts[progId][year] = [0, 0, 0]
+        })
+        programmeNames[progId] = { ...name, code }
       }
       programmeCounts[progId][thesisYear][0] += 1
 
@@ -94,7 +96,7 @@ const getFacultyThesisWritersForProgrammes = async (
 }
 
 const combineFacultyThesisWriters = async (faculty, facultyProgrammes, yearType, specialGroups) => {
-  let allThesisWriters = {
+  const allThesisWriters = {
     id: faculty,
     years: [],
     tableStats: [],

services/backend/src/services/faculty/facultyUpdates.js

@@ -74,7 +74,7 @@ const updateFacultyOverview = async (faculty, statsType) => {
   ]
   let allProgrammes = []
   let newProgrammes = []
-  let allProgrammeCodes = []
+  const allProgrammeCodes = []
 
   try {
     const all = await findFacultyProgrammeCodes(faculty, 'ALL_PROGRAMMES')

services/backend/src/services/graduationHelpers.js

@@ -1,5 +1,4 @@
-const Sequelize = require('sequelize')
-const { Op } = Sequelize
+const { Op } = require('sequelize')
 const moment = require('moment')
 const { Semester, SemesterEnrollment, Studyright } = require('../models')
 
@@ -39,7 +38,7 @@ const statutoryAbsences = async (studentnumber, startdate, enddate) =>
         },
       },
       where: {
-        studentnumber: studentnumber,
+        studentnumber,
         statutory_absence: true,
       },
     })

services/backend/src/services/languageCenterData.js

@@ -2,6 +2,7 @@ const { Course, Credit, Enrollment, StudyrightElement, Studyright } = require('.
 const { Op } = require('sequelize')
 const { redisClient } = require('./redis')
 const { orderBy } = require('lodash')
+
 const LANGUAGE_CENTER_REDIS_KEY = 'LANGUAGE_CENTER_DATA'
 
 const isBetween = (start, date, end) =>
@@ -25,12 +26,59 @@ const getLanguageCenterCourses = async () => {
   return courses
 }
 
-const getLanguageCenterData = async () => {
-  const dataOnRedis = await redisClient.getAsync(LANGUAGE_CENTER_REDIS_KEY)
-  if (dataOnRedis) return JSON.parse(dataOnRedis)
-  const freshData = await computeLanguageCenterData()
-  redisClient.setAsync(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
-  return freshData
+const createArrayOfCourses = async (attempts, courses) => {
+  const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
+  const semestersObject = {}
+  const facultyObject = {}
+  const semesterStatsMap = attempts.reduce((obj, cur) => {
+    const semester = cur.semestercode
+    facultyObject[cur.faculty] = true
+    semestersObject[semester] = true
+    if (!obj[cur.courseCode]) {
+      obj[cur.courseCode] = {}
+    }
+    if (!obj[cur.courseCode][semester]) {
+      obj[cur.courseCode][semester] = { ...fields }
+    }
+    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
+      obj[cur.courseCode][semester][cur.faculty] = { ...fields }
+    }
+    const allFacultiesTotal = obj[cur.courseCode][semester]
+    const allStats = [allFacultiesTotal]
+    if (cur.faculty) {
+      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
+      allStats.push(semesterFacultyStats)
+    }
+    for (const stats of allStats) {
+      if (cur.completed) {
+        stats.completions += 1
+      } else if (cur.enrolled) {
+        stats.enrollments += 1
+      } else {
+        stats.rejected += 1
+      }
+    }
+    return obj
+  }, {})
+
+  const courseList = courses
+    .map(c => ({ ...c, bySemesters: { ...semesterStatsMap[c.code], cellStats: {} } }))
+    .filter(course => course.bySemesters)
+
+  courseList.forEach(course => {
+    Object.keys(semestersObject).forEach(sem => {
+      const stats = course.bySemesters[sem]
+      if (!stats) return
+      stats.difference = getDifference(stats)
+      Object.keys(facultyObject).forEach(fac => {
+        const stats = course.bySemesters[sem]?.[fac]
+        if (!stats) return
+        stats.difference = getDifference(stats)
+      })
+    })
+  })
+
+  return courseList
 }
 
 const computeLanguageCenterData = async () => {
@@ -162,7 +210,7 @@ const computeLanguageCenterData = async () => {
     if (item.faculty) {
       if (isOpenUni(item.faculty)) {
         item.faculty = 'OPEN'
-      } else if (isMisc(item.faculty) || !item.faculty.substring(0, 3).match(`^H\\d`)) {
+      } else if (isMisc(item.faculty) || !item.faculty.substring(0, 3).match('^H\\d')) {
         item.faculty = 'OTHER'
       }
       return
@@ -174,7 +222,7 @@ const computeLanguageCenterData = async () => {
     const faculty = getFaculty(item.studentNumber, item.date)
     if (isOpenUni(faculty)) {
       item.faculty = 'OPEN'
-    } else if (faculty?.substring(0, 3).match(`^H\\d`)) {
+    } else if (faculty?.substring(0, 3).match('^H\\d')) {
       item.faculty = !isMisc(faculty) ? faculty : 'OTHER'
     } else {
       item.faculty = 'OTHER'
@@ -190,61 +238,12 @@ const computeLanguageCenterData = async () => {
   return { tableData, faculties }
 }
 
-const createArrayOfCourses = async (attempts, courses) => {
-  const fields = { completions: 0, enrollments: 0, difference: 0, rejected: 0 }
-  const semestersObject = {}
-  const facultyObject = {}
-  const semesterStatsMap = attempts.reduce((obj, cur) => {
-    const semester = cur.semestercode
-    facultyObject[cur.faculty] = true
-    semestersObject[semester] = true
-    if (!obj[cur.courseCode]) {
-      obj[cur.courseCode] = {}
-    }
-    if (!obj[cur.courseCode][semester]) {
-      obj[cur.courseCode][semester] = { ...fields }
-    }
-    if (cur.faculty && !obj[cur.courseCode][semester][cur.faculty]) {
-      obj[cur.courseCode][semester][cur.faculty] = { ...fields }
-    }
-    const allFacultiesTotal = obj[cur.courseCode][semester]
-    const allStats = [allFacultiesTotal]
-    if (cur.faculty) {
-      const semesterFacultyStats = obj[cur.courseCode][semester][cur.faculty]
-      allStats.push(semesterFacultyStats)
-    }
-    for (const stats of allStats) {
-      if (cur.completed) {
-        stats.completions += 1
-      } else {
-        if (cur.enrolled) {
-          stats.enrollments += 1
-        } else {
-          stats.rejected += 1
-        }
-      }
-    }
-    return obj
-  }, {})
-
-  const courseList = courses
-    .map(c => ({ ...c, bySemesters: { ...semesterStatsMap[c.code], cellStats: {} } }))
-    .filter(course => course.bySemesters)
-
-  courseList.forEach(course => {
-    Object.keys(semestersObject).forEach(sem => {
-      const stats = course.bySemesters[sem]
-      if (!stats) return
-      stats.difference = getDifference(stats)
-      Object.keys(facultyObject).forEach(fac => {
-        const stats = course.bySemesters[sem]?.[fac]
-        if (!stats) return
-        stats.difference = getDifference(stats)
-      })
-    })
-  })
-
-  return courseList
+const getLanguageCenterData = async () => {
+  const dataOnRedis = await redisClient.getAsync(LANGUAGE_CENTER_REDIS_KEY)
+  if (dataOnRedis) return JSON.parse(dataOnRedis)
+  const freshData = await computeLanguageCenterData()
+  redisClient.setAsync(LANGUAGE_CENTER_REDIS_KEY, JSON.stringify(freshData))
+  return freshData
 }
 
 module.exports = {

services/backend/src/services/mailservice.js

@@ -66,17 +66,16 @@ const sendNotificationAboutNewUser = ({ userId, userFullName }) =>
 
 const accessMessageSubject =
   'Käyttöoikeutesi Oodikoneeseen on päivitetty – Your privileges in Oodikone have been updated'
-const accessMessageText = `<p>Sinulle on annettu oikeudet käyttää <a href="https://oodikone.helsinki.fi">Oodikonetta</a> tai käyttöoikeutesi Oodikoneeseen on päivitetty.</p>
-<p>Jotta muutokset astuvat voimaan, saatat joutua kirjautumaan ulos ja takaisin sisään.</p>
-<p>HUOM! Jos automaattinen uloskirjautuminen kaikista palveluista ei onnistunut kirjautuessasi ulos Oodikoneesta, voit kirjautua ulos jokaisesta palvelusta yksitellen tai sulkea kaikki selainikkunasi.</p>
-<p>Jos olet määritellyt selaimen palauttamaan edellisen istuntosi käynnistyksen yhteydessä, saatat joutua selainikkunoiden sulkemisen lisäksi poistamaan myös listattujen palvelujen evästeet.</p>
-<p>Katso ohjeet selaimen välimuistin tyhjentämiseksi: <a href="https://helpdesk.it.helsinki.fi/help/1002">https://helpdesk.it.helsinki.fi/help/1002</a></p>
-<br/>
-<p>You've been authorized to use <a href="https://oodikone.helsinki.fi">Oodikone</a> or your privileges in Oodikone have changed.</p>
-<p>You may need to log out and back in before your changes take place.</p>
-<p>NOTE! If the automatic logging out from all services was not successful when logging out of oodikone, you can log out one by one or close all of your browser windows.</p>
-<p>If you have defined your browser to restore your previous session in connection with starting the browser, logging out may also require deleting the cookies of the listed services in addition to closing the browser windows.</p>
-<p>See the instructions for clearing the browser cache: <a href="https://helpdesk.it.helsinki.fi/en/help/1002">https://helpdesk.it.helsinki.fi/en/help/1002</a></p>`
+const accessMessageText = `Sinulle on annettu oikeudet käyttää <a href="https://oodikone.helsinki.fi">Oodikonetta</a> tai käyttöoikeutesi Oodikoneeseen on päivitetty.<br/>\n
+Jotta muutokset astuvat voimaan, saatat joutua kirjautumaan ulos ja takaisin sisään.<br/>\n
+HUOM! Jos automaattinen uloskirjautuminen kaikista palveluista ei onnistunut kirjautuessasi ulos Oodikoneesta, voit kirjautua ulos jokaisesta palvelusta yksitellen tai sulkea kaikki selainikkunasi.<br/>\n
+Jos olet määritellyt selaimen palauttamaan edellisen istuntosi käynnistyksen yhteydessä, saatat joutua selainikkunoiden sulkemisen lisäksi poistamaan myös listattujen palvelujen evästeet.<br/>\n
+Katso ohjeet selaimen välimuistin tyhjentämiseksi: <a href="https://helpdesk.it.helsinki.fi/help/1002">https://helpdesk.it.helsinki.fi/help/1002</a><br/><br/>\n
+You've been authorized to use <a href="https://oodikone.helsinki.fi">Oodikone</a> or your privileges in Oodikone have changed.<br/>\n
+You may need to log out and back in before your changes take place.<br/>\n
+NOTE! If the automatic logging out from all services was not successful when logging out of oodikone, you can log out one by one or close all of your browser windows.<br/>\n
+If you have defined your browser to restore your previous session in connection with starting the browser, logging out may also require deleting the cookies of the listed services in addition to closing the browser windows.<br/>\n
+See the instructions for clearing the browser cache: <a href="https://helpdesk.it.helsinki.fi/en/help/1002">https://helpdesk.it.helsinki.fi/en/help/1002</a><br/>\n`
 
 const previewNotificationAboutAccessToUser = () => ({ accessMessageSubject, accessMessageText })
 

services/backend/src/services/openUni/openUniSearches.js

@@ -17,7 +17,6 @@ const getCredits = async (courseCodes, startdate) =>
         course_code: {
           [Op.in]: courseCodes,
         },
-        is_open: true,
         attainment_date: {
           [Op.gte]: startdate,
         },
@@ -80,6 +79,9 @@ const getStudyRights = async students =>
         student_studentnumber: {
           [Op.in]: students.length > 0 ? students : { [Op.not]: null },
         },
+        extentcode: {
+          [Op.in]: [1, 2, 3, 4],
+        },
       },
     })
   ).map(mapStudyRights)
@@ -103,8 +105,8 @@ const createOpenUniPopulationSearch = async (userId, name, courseCodes) => {
 const updateOpenUniPopulationSearch = async (userId, id, courseCodes) => {
   const searchToUpdate = await OpenUniPopulationSearch.findOne({
     where: {
-      userId: userId,
-      id: id,
+      userId,
+      id,
     },
   })
 
@@ -115,8 +117,8 @@ const updateOpenUniPopulationSearch = async (userId, id, courseCodes) => {
 const deleteOpenUniSearch = async (userId, id) => {
   return await OpenUniPopulationSearch.destroy({
     where: {
-      userId: userId,
-      id: id,
+      userId,
+      id,
     },
   })
 }

services/backend/src/services/openUni/openUniStats.js

@@ -3,7 +3,7 @@ const { getCredits, getStudyRights, getEnrollments, getStudentInfo, getCourseNam
 
 const uniq = objects => [...new Set(objects)]
 
-const calulateTotalsForStudent = async (studentStats, studentnumber) => {
+const calculateTotalsForStudent = async (studentStats, studentnumber) => {
   Object.keys(studentStats[studentnumber].courseInfo).forEach(course => {
     if (studentStats[studentnumber].courseInfo[course].status.passed) {
       studentStats[studentnumber].totals.passed += 1
@@ -15,7 +15,7 @@ const calulateTotalsForStudent = async (studentStats, studentnumber) => {
   })
 }
 const getCustomOpenUniCourses = async (courseCodes, startdate, enddate) => {
-  const ayCourseCodes = courseCodes.map(courseCode => 'AY' + courseCode)
+  const ayCourseCodes = courseCodes.map(courseCode => `AY${courseCode}`)
   const allCourseCodes = courseCodes.concat(ayCourseCodes)
   const allCredits = await getCredits(allCourseCodes, startdate)
   const allEnrollments = await getEnrollments(allCourseCodes, startdate, enddate)
@@ -60,7 +60,7 @@ const getCustomOpenUniCourses = async (courseCodes, startdate, enddate) => {
             (acc, code) => ({ ...acc, [code.replace('AY', '')]: getEmptyCourseInfo() }),
             {}
           ),
-          email: email,
+          email,
           secondaryEmail: secondary_email,
           totals: { passed: 0, failed: 0, unfinished: 0 },
         }
@@ -100,10 +100,10 @@ const getCustomOpenUniCourses = async (courseCodes, startdate, enddate) => {
           }
         }
       }
-      await calulateTotalsForStudent(studentStats, studentnumber)
+      await calculateTotalsForStudent(studentStats, studentnumber)
     }
   }
-  const openUniStats = { students: studentStats, courses: courses }
+  const openUniStats = { students: studentStats, courses }
   return openUniStats
 }
 

services/backend/src/services/organisations.js

@@ -1,196 +1,7 @@
-const Sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const { Organization } = require('../models')
-const Op = Sequelize.Op
 const { dbConnections } = require('../database/connection')
 
-// temp refactoring faculty <-> userservice mappings away from userservice
-const facultiesAndProgrammesInUserService = [
-  ['H10', 'MH10_001'],
-  ['H10', 'MH40_011'],
-  ['H10', 'KH10_001'],
-  ['H20', 'KH20_001'],
-  ['H20', 'MH20_001'],
-  ['H20', 'MH20_003'],
-  ['H20', 'MH20_002'],
-  ['H30', 'KH30_001'],
-  ['H30', 'KH30_002'],
-  ['H30', 'MH30_002'],
-  ['H30', 'MH30_003'],
-  ['H30', 'MH30_004'],
-  ['H30', 'MH30_005'],
-  ['H30', 'MH30_001'],
-  ['H40', 'KH40_001'],
-  ['H40', 'KH40_002'],
-  ['H40', 'KH40_003'],
-  ['H40', 'KH40_004'],
-  ['H40', 'KH40_005'],
-  ['H40', 'KH40_006'],
-  ['H40', 'MH40_001'],
-  ['H40', 'MH40_002'],
-  ['H40', 'MH40_003'],
-  ['H40', 'MH40_004'],
-  ['H40', 'MH40_005'],
-  ['H40', 'MH40_006'],
-  ['H40', 'MH40_007'],
-  ['H40', 'MH40_008'],
-  ['H40', 'MH40_009'],
-  ['H40', 'MH40_010'],
-  ['H40', 'MH40_011'],
-  ['H40', 'MH40_012'],
-  ['H40', 'MH40_013'],
-  ['H40', 'MH40_014'],
-  ['H40', 'MH40_015'],
-  ['H40', 'MH70_001'],
-  ['H40', 'MH70_006'],
-  ['H50', 'KH50_001'],
-  ['H50', 'KH50_002'],
-  ['H50', 'KH50_003'],
-  ['H50', 'KH50_004'],
-  ['H50', 'KH50_005'],
-  ['H50', 'KH50_006'],
-  ['H50', 'KH50_007'],
-  ['H50', 'KH50_008'],
-  ['H50', 'MH50_001'],
-  ['H50', 'MH50_002'],
-  ['H50', 'MH50_003'],
-  ['H50', 'MH50_004'],
-  ['H50', 'MH50_005'],
-  ['H50', 'MH50_006'],
-  ['H50', 'MH50_007'],
-  ['H50', 'MH50_008'],
-  ['H50', 'MH50_009'],
-  ['H50', 'MH50_010'],
-  ['H50', 'MH50_011'],
-  ['H50', 'MH50_012'],
-  ['H50', 'MH50_013'],
-  ['H50', 'MH50_014'],
-  ['H55', 'KH55_001'],
-  ['H55', 'MH55_001'],
-  ['H55', 'MH55_002'],
-  ['H57', 'KH57_001'],
-  ['H57', 'KH57_002'],
-  ['H57', 'KH57_003'],
-  ['H57', 'MH50_002'],
-  ['H57', 'MH50_013'],
-  ['H57', 'MH57_001'],
-  ['H57', 'MH57_002'],
-  ['H57', 'MH57_003'],
-  ['H57', 'MH57_004'],
-  ['H57', 'MH57_005'],
-  ['H57', 'MH80_007'],
-  ['H60', 'KH60_001'],
-  ['H60', 'MH60_001'],
-  ['H60', 'MH60_002'],
-  ['H70', 'KH40_001'],
-  ['H70', 'KH50_001'],
-  ['H70', 'KH70_001'],
-  ['H70', 'KH70_002'],
-  ['H70', 'KH70_003'],
-  ['H70', 'KH70_004'],
-  ['H70', 'MH50_001'],
-  ['H70', 'MH57_005'],
-  ['H70', 'MH70_001'],
-  ['H70', 'MH70_002'],
-  ['H70', 'MH70_003'],
-  ['H70', 'MH70_004'],
-  ['H70', 'MH70_005'],
-  ['H70', 'MH70_006'],
-  ['H70', 'MH70_007'],
-  ['H70', 'MH70_008'],
-  ['H70', 'MH70_009'],
-  ['H70', 'MH70_010'],
-  ['H70', 'MH70_011'],
-  ['H74', 'KH74_001'],
-  ['H80', 'KH57_002'],
-  ['H80', 'KH80_001'],
-  ['H80', 'KH80_002'],
-  ['H80', 'KH80_003'],
-  ['H80', 'KH80_004'],
-  ['H80', 'MH57_002'],
-  ['H80', 'MH57_003'],
-  ['H80', 'MH57_005'],
-  ['H80', 'MH80_001'],
-  ['H80', 'MH80_002'],
-  ['H80', 'MH80_003'],
-  ['H80', 'MH80_004'],
-  ['H80', 'MH80_005'],
-  ['H80', 'MH80_006'],
-  ['H80', 'MH80_007'],
-  ['H80', 'MH80_008'],
-  ['H90', 'KH90_001'],
-  ['H90', 'MH90_001'],
-  ['H92', 'T920101'],
-  ['H92', 'T920102'],
-  ['H92', 'T920103'],
-  ['H92', 'T920104'],
-  ['H92', 'T920105'],
-  ['H92', 'T920106'],
-  ['H92', 'T920107'],
-  ['H92', 'T920108'],
-  ['H92', 'T920109'],
-  ['H92', 'T920110'],
-  ['H92', 'T920111'],
-  ['H92', 'T921101'],
-  ['H92', 'T921102'],
-  ['H92', 'T921103'],
-  ['H92', 'T921104'],
-  ['H92', 'T921105'],
-  ['H92', 'T921106'],
-  ['H92', 'T921107'],
-  ['H92', 'T921108'],
-  ['H92', 'T921109'],
-  ['H92', 'T922101'],
-  ['H92', 'T922102'],
-  ['H92', 'T922103'],
-  ['H92', 'T922104'],
-  ['H92', 'T922105'],
-  ['H92', 'T922106'],
-  ['H92', 'T923101'],
-  ['H92', 'T923102'],
-  ['H92', 'T923103'],
-  ['H92', 'T923104'],
-  ['H92', 'T923105'],
-  ['H92', 'T923106'],
-  ['H92', 'T923107'],
-  ['H30', '447000'],
-  ['H30', '447001'],
-  ['H30', '447002'],
-  ['H60', 'EDUK730'],
-  ['H60', 'K-MUUT-ERIL'],
-  ['H60', 'ED400'],
-  ['H60', 'ED200'],
-  ['H60', 'ED700'],
-  ['H60', 'ED100'],
-  ['H60', 'ED300'],
-  ['H60', 'ED600'],
-  ['H60', '8009'],
-  ['H60', '8118'],
-  ['H60', '8600'],
-  ['H60', 'DUK-PED'],
-  ['H60', '65'],
-  ['H60', '1102'],
-  ['H60', '20018'],
-  ['H60', '8601'],
-  ['H60', '9290'],
-  ['H60', '8172'],
-  ['H60', '83600'],
-  ['H60', 'DUM-PED'],
-  ['H60', 'ED500'],
-  ['H60', '83601'],
-  ['H60', '0130'],
-  ['H60', '20005-e'],
-  ['H60', '1040'],
-  ['H60', '8703'],
-  ['H60', '0005'],
-  ['H60', '7900'],
-  ['H60', '0152 '],
-]
-
-const facultiesAndProgrammesForTrends = facultiesAndProgrammesInUserService.map(f => ({
-  faculty_code: f[0],
-  programme_code: f[1],
-}))
-
 // Have facultyfetching to work like it worked during oodi-db time
 const facultiesInOodi = [
   'H10',
@@ -240,5 +51,4 @@ module.exports = {
   faculties,
   isFaculty,
   providersOfFaculty,
-  facultiesAndProgrammesForTrends,
 }

services/backend/src/services/populations/bottlenecksOf.js

@@ -12,94 +12,88 @@ const { encrypt } = require('../encrypt')
 const { CourseStatsCounter } = require('../courses/course_stats_counter')
 const { getPassingSemester } = require('../../util/semester')
 
-const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
-  const isValidRequest = async (query, params) => {
-    const { studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents } = params
+const isValidRequest = async (query, params) => {
+  const { studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents } = params
 
-    if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
-      return { error: 'Semester should be either SPRING OR FALL' }
-    }
-    if (
-      query.studentStatuses &&
-      !query.studentStatuses.every(
-        status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
-      )
-    ) {
-      return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
-    }
-    if (query.selectedStudents) {
-      const allStudents = await studentnumbersWithAllStudyrightElements({
+  if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
+    return { error: 'Semester should be either SPRING OR FALL' }
+  }
+  if (
+    query.studentStatuses &&
+    !query.studentStatuses.every(status => status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED')
+  ) {
+    return { error: 'Student status should be either EXCHANGE or NONDEGREE or TRANSFERRED' }
+  }
+  if (query.selectedStudents) {
+    const allStudents = await studentnumbersWithAllStudyrightElements({
+      studyRights,
+      startDate,
+      endDate,
+      exchangeStudents,
+      nondegreeStudents,
+      transferredOutStudents: transferredStudents,
+      tag: query.tag,
+      transferredToStudents: true,
+      graduatedStudents: true,
+    })
+    const disallowedRequest = checkThatSelectedStudentsAreUnderRequestedStudyright(query.selectedStudents, allStudents)
+    if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
+  }
+  return null
+}
+
+const getStudentsAndCourses = async (params, selectedStudents, studentnumberlist, courseCodes) => {
+  if (!studentnumberlist) {
+    const { months, studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
+      params
+    const studentnumbers =
+      selectedStudents ||
+      (await studentnumbersWithAllStudyrightElements({
         studyRights,
         startDate,
         endDate,
         exchangeStudents,
         nondegreeStudents,
         transferredOutStudents: transferredStudents,
-        tag: query.tag,
+        tag,
         transferredToStudents: true,
         graduatedStudents: true,
-      })
-      const disallowedRequest = checkThatSelectedStudentsAreUnderRequestedStudyright(
-        query.selectedStudents,
-        allStudents
-      )
-      if (disallowedRequest) return { error: 'Trying to request unauthorized students data' }
-    }
-    return null
+      }))
+
+    const allstudents = studentnumbers.reduce((numbers, num) => {
+      numbers[num] = true
+      return numbers
+    }, {})
+    const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months), courseCodes)
+    const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+    const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
+
+    const courseEnrollements = await findCourseEnrollments(
+      studentnumbers,
+      dateMonthsFromNow(startDate, months),
+      filteredCourseCodes
+    )
+    return [allstudents, courses, courseEnrollements]
   }
+  const { months, startDate } = params
+  const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
+  const allstudents = studentnumberlist.reduce((numbers, num) => {
+    numbers[num] = true
+    return numbers
+  }, {})
+  const courses = await findCourses(studentnumberlist, beforeDate, courseCodes)
+  const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
+  const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
 
-  const getStudentsAndCourses = async (selectedStudents, studentnumberlist, courseCodes) => {
-    if (!studentnumberlist) {
-      const { months, studyRights, startDate, endDate, exchangeStudents, nondegreeStudents, transferredStudents, tag } =
-        params
-      const studentnumbers =
-        selectedStudents ||
-        (await studentnumbersWithAllStudyrightElements({
-          studyRights,
-          startDate,
-          endDate,
-          exchangeStudents,
-          nondegreeStudents,
-          transferredOutStudents: transferredStudents,
-          tag,
-          transferredToStudents: true,
-          graduatedStudents: true,
-        }))
-
-      const allstudents = studentnumbers.reduce((numbers, num) => {
-        numbers[num] = true
-        return numbers
-      }, {})
-      const courses = await findCourses(studentnumbers, dateMonthsFromNow(startDate, months), courseCodes)
-      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
-      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
-
-      const courseEnrollements = await findCourseEnrollments(
-        studentnumbers,
-        dateMonthsFromNow(startDate, months),
-        filteredCourseCodes
-      )
-      return [allstudents, courses, courseEnrollements]
-    } else {
-      const { months, startDate } = params
-      const beforeDate = months && startDate ? dateMonthsFromNow(startDate, months) : new Date()
-      const allstudents = studentnumberlist.reduce((numbers, num) => {
-        numbers[num] = true
-        return numbers
-      }, {})
-      const courses = await findCourses(studentnumberlist, beforeDate, courseCodes)
-      const foundCourseCodes = Object.keys(keyBy(courses, 'code'))
-      const filteredCourseCodes = courseCodes?.filter(code => !foundCourseCodes.includes(code))
-
-      const courseEnrollements = await findCourseEnrollments(studentnumberlist, beforeDate, filteredCourseCodes)
-      return [allstudents, courses, courseEnrollements]
-    }
-  }
+  const courseEnrollements = await findCourseEnrollments(studentnumberlist, beforeDate, filteredCourseCodes)
+  return [allstudents, courses, courseEnrollements]
+}
 
+const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
   const encryptStudentnumbers = bottlenecks => {
-    for (const course in bottlenecks.coursestatistics) {
+    for (const course of Object.keys(bottlenecks.coursestatistics)) {
       const encryptedStudentStats = {}
-      for (const data in bottlenecks.coursestatistics[course].students) {
+      for (const data of Object.keys(bottlenecks.coursestatistics[course].students)) {
         encryptedStudentStats[data] = {}
         const studentnumbers = Object.keys(bottlenecks.coursestatistics[course].students[data])
         studentnumbers.forEach(studentnumber => {
@@ -121,8 +115,8 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
   // To fix failed and enrolled, no grade filter options some not so clean and nice solutions were added
   // Get the data with actual 1. courses and filtered students. 2. all students by year, if provided.
   const [[allstudents, courses, courseEnrollements], [, allCourses], error] = await Promise.all([
-    getStudentsAndCourses(query.selectedStudents, studentnumberlist, query.courses),
-    getStudentsAndCourses(allStudentsByYears, null, query.courses),
+    getStudentsAndCourses(params, query.selectedStudents, studentnumberlist, query.courses),
+    getStudentsAndCourses(params, allStudentsByYears, null, query.courses),
     isValidRequest(query, params),
   ])
 
@@ -158,11 +152,11 @@ const bottlenecksOf = async (query, studentnumberlist, encryptdata = false) => {
 
   const coursesByCode = keyBy(coursesToLoop, 'code')
   for (const course of coursesToLoop) {
-    let { course_type } = course
+    const { course_type } = course
     let maincourse = course
 
     if (course.main_course_code && course.main_course_code !== course.code) {
-      let newmain = coursesByCode[course.main_course_code]
+      const newmain = coursesByCode[course.main_course_code]
 
       if (newmain) {
         maincourse = newmain

services/backend/src/services/populations/closeToGraduation.js

@@ -0,0 +1,176 @@
+const { Op } = require('sequelize')
+
+const {
+  ElementDetail,
+  Student,
+  Studyplan,
+  Studyright,
+  StudyrightElement,
+  Credit,
+  Course,
+  Organization,
+} = require('../../models')
+const { mapToProviders } = require('../../util/utils')
+const { redisClient } = require('../redis')
+
+const CLOSE_TO_GRADUATION_REDIS_KEY = 'CLOSE_TO_GRADUATION_DATA'
+
+const findThesisAndLatestAttainments = (student, providerCode) => {
+  let thesisData
+  const latestAttainmentDates = {}
+
+  for (const attainment of student.credits) {
+    if (
+      attainment.course?.course_unit_type === 'urn:code:course-unit-type:bachelors-thesis' &&
+      attainment.course.organizations.some(org => org.code === providerCode)
+    ) {
+      thesisData = attainment
+    }
+    if (!latestAttainmentDates.total) {
+      latestAttainmentDates.total = attainment.attainment_date
+    }
+    if (!latestAttainmentDates.hops && student.studyplans[0].included_courses.includes(attainment.course?.code)) {
+      latestAttainmentDates.hops = attainment.attainment_date
+    }
+    if (thesisData && latestAttainmentDates.total && latestAttainmentDates.hops) {
+      break
+    }
+  }
+
+  return { latestAttainmentDates, thesisData }
+}
+
+const formatStudent = student => {
+  const {
+    studentnumber: studentNumber,
+    abbreviatedname: name,
+    sis_person_id,
+    email,
+    phone_number: phoneNumber,
+    secondary_email: secondaryEmail,
+  } = student
+  const semesterEnrollments = {}
+  for (const enrollment of student.studyplans[0].studyright.semester_enrollments) {
+    semesterEnrollments[enrollment.semestercode] = {
+      type: enrollment.enrollmenttype,
+      statutoryAbsence: enrollment.statutoryAbsence,
+    }
+  }
+  const { studyright_elements: studyrightElements, startdate: startOfStudyright } = student.studyplans[0].studyright
+  const programmeCode = student.studyplans[0].programme_code
+  const programme = studyrightElements?.find(
+    element => element?.element_detail.type === 20 && element.code === programmeCode
+  )?.element_detail
+  const programmeCodeToProviderCode = mapToProviders([programmeCode])[0]
+  const { latestAttainmentDates, thesisData } = findThesisAndLatestAttainments(student, programmeCodeToProviderCode)
+  const studyTrack = studyrightElements?.find(element => element?.element_detail.type === 30)?.element_detail
+
+  return {
+    student: { studentNumber, name, sis_person_id, email, phoneNumber, secondaryEmail },
+    studyright: {
+      startDate: startOfStudyright,
+      semesterEnrollments: student.studyplans[0].studyright.semester_enrollments,
+    },
+    thesisInfo: thesisData
+      ? {
+          grade: thesisData.grade,
+          attainmentDate: thesisData.attainment_date,
+          courseCode: thesisData.course.code,
+        }
+      : null,
+    programme: {
+      code: programme?.code,
+      name: programme?.name,
+      studyTrack,
+    },
+    latestAttainmentDates,
+    credits: {
+      hops: student.studyplans[0].completed_credits,
+      all: student.creditcount,
+    },
+  }
+}
+
+const findStudentsCloseToGraduation = async () =>
+  (
+    await Student.findAll({
+      attributes: [
+        'abbreviatedname',
+        'creditcount',
+        'email',
+        'phone_number',
+        'secondary_email',
+        'sis_person_id',
+        'studentnumber',
+      ],
+      include: [
+        {
+          model: Studyplan,
+          attributes: ['completed_credits', 'included_courses', 'programme_code'],
+          where: {
+            completed_credits: {
+              [Op.gte]: 160,
+            },
+            programme_code: {
+              [Op.like]: 'KH%',
+            },
+          },
+          include: [
+            {
+              model: Studyright,
+              attributes: ['semester_enrollments', 'startdate', 'studyrightid'],
+              where: {
+                graduated: 0,
+                cancelled: false,
+              },
+              include: [
+                {
+                  model: StudyrightElement,
+                  attributes: ['code'],
+                  include: [
+                    {
+                      model: ElementDetail,
+                      attributes: ['code', 'name', 'type'],
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+        {
+          model: Credit,
+          attributes: ['attainment_date', 'grade'],
+          where: {
+            credittypecode: 4,
+          },
+          include: {
+            model: Course,
+            attributes: ['code', 'course_unit_type'],
+            include: {
+              model: Organization,
+              attributes: ['code'],
+              through: {
+                attributes: [],
+              },
+            },
+          },
+        },
+      ],
+      order: [[{ model: Credit }, 'attainment_date', 'DESC']],
+    })
+  ).map(student => formatStudent(student.toJSON()))
+
+const getCloseToGraduationData = async () => {
+  const dataOnRedis = await redisClient.getAsync(CLOSE_TO_GRADUATION_REDIS_KEY)
+  if (dataOnRedis) return JSON.parse(dataOnRedis)
+  const freshData = await findStudentsCloseToGraduation()
+  redisClient.setAsync(CLOSE_TO_GRADUATION_REDIS_KEY, JSON.stringify(freshData))
+  return freshData
+}
+
+module.exports = {
+  findStudentsCloseToGraduation,
+  getCloseToGraduationData,
+  CLOSE_TO_GRADUATION_REDIS_KEY,
+}

services/backend/src/services/populations/getStudentsIncludeCoursesBetween.js

@@ -1,18 +1,18 @@
 const { Op } = require('sequelize')
 const { Tag, TagStudent } = require('../../models/models_kone')
 const {
-  Studyplan,
   Course,
   Credit,
+  ElementDetail,
   Enrollment,
   Student,
-  Transfer,
+  Studyplan,
   Studyright,
   StudyrightElement,
-  ElementDetail,
-  SemesterEnrollment,
-  Semester,
   StudyrightExtent,
+  Semester,
+  SemesterEnrollment,
+  Transfer,
 } = require('../../models')
 const moment = require('moment')
 const {
@@ -56,7 +56,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
   })
   const studyPlanCourses = Array.from(new Set([...studyPlans.map(plan => plan.included_courses)].flat()))
 
-  const creditsOfStudentOther = {
+  const creditsOfStudent = {
     [Op.or]: [
       {
         attainment_date: {
@@ -64,7 +64,10 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         },
       },
       {
-        course_code: studyPlanCourses,
+        // takes into account possible progress tests taken earlier than the start date
+        course_code: ['320001', 'MH30_001'].includes(studyright[0])
+          ? ['375063', '339101', ...studyPlanCourses]
+          : studyPlanCourses,
       },
     ],
     student_studentnumber: {
@@ -72,35 +75,12 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
     },
   }
 
-  // takes into accout possible progress tests taken earlier than the start date
-  const creditsOfStudentLaakis = {
-    student_studentnumber: {
-      [Op.in]: studentnumbers,
-    },
-    [Op.or]: [
-      {
-        attainment_date: {
-          [Op.between]: [attainmentDateFrom, endDate],
-        },
-      },
-      {
-        course_code: {
-          [Op.in]: ['375063', '339101'].concat(studyPlanCourses),
-        },
-      },
-    ],
-  }
-
-  const creditsOfStudent = ['320001', 'MH30_001'].includes(studyright[0])
-    ? creditsOfStudentLaakis
-    : creditsOfStudentOther
-
   if (studentnumbers.length === 0)
     return { students: [], enrollments: [], credits: [], extents: [], semesters: [], elementdetails: [], courses: [] }
 
   const [courses, enrollments, students, credits, extents, semesters, elementdetails] = await Promise.all([
     Course.findAll({
-      attributes: [sequelize.literal('DISTINCT ON("code") code'), 'name', 'coursetypecode'],
+      attributes: [sequelize.literal('DISTINCT ON("code") code'), 'name'],
       include: [
         {
           model: Credit,
@@ -111,7 +91,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       raw: true,
     }),
     Enrollment.findAll({
-      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber', 'semestercode'],
+      attributes: ['course_code', 'state', 'enrollment_date_time', 'studentnumber', 'semestercode', 'studyright_id'],
       where: {
         enrollment_date_time: {
           [Op.between]: [attainmentDateFrom, endDate],
@@ -129,6 +109,8 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'lastname',
         'studentnumber',
         'home_country_en',
+        'home_country_fi',
+        'home_country_sv',
         'dateofuniversityenrollment',
         'creditcount',
         'abbreviatedname',
@@ -162,6 +144,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             'cancelled',
             'is_ba_ma',
             'semester_enrollments',
+            'actual_studyrightid',
           ],
           separate: true,
           include: [
@@ -187,7 +170,13 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         },
         {
           model: Studyplan,
-          attributes: ['included_courses', 'programme_code', 'completed_credits', 'studyrightid'],
+          attributes: [
+            'included_courses',
+            'programme_code',
+            'completed_credits',
+            'studyrightid',
+            'curriculum_period_id',
+          ],
           separate: true,
         },
       ],
@@ -207,6 +196,7 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'student_studentnumber',
         'course_code',
         'language',
+        'studyright_id',
       ],
       where: creditsOfStudent,
       raw: true,
@@ -258,10 +248,9 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
       raw: true,
     }),
     sequelize.query(
-      `
-SELECT DISTINCT ON (code) code, name, type FROM element_details WHERE
-EXISTS (SELECT 1 FROM transfers WHERE studentnumber IN (:studentnumbers) AND (code = sourcecode OR code = targetcode)) OR
-EXISTS (SELECT 1 FROM studyright_elements WHERE studentnumber IN (:studentnumbers))`,
+      `SELECT DISTINCT ON (code) code, name, type FROM element_details WHERE
+      EXISTS (SELECT 1 FROM transfers WHERE studentnumber IN (:studentnumbers) AND (code = sourcecode OR code = targetcode)) OR
+      EXISTS (SELECT 1 FROM studyright_elements WHERE studentnumber IN (:studentnumbers) AND element_details.code = studyright_elements.code)`,
       {
         replacements: { studentnumbers },
         type: sequelize.QueryTypes.SELECT,

services/backend/src/services/populations/index.js

@@ -3,6 +3,7 @@ const { getStudentsIncludeCoursesBetween } = require('./getStudentsIncludeCourse
 const { optimizedStatisticsOf } = require('./optimizedStatisticsOf')
 const { studentnumbersWithAllStudyrightElements } = require('./studentnumbersWithAllStudyrightElements')
 const { getEarliestYear } = require('./shared')
+const { getStudentsCloseToGraduation } = require('./closeToGraduation')
 
 module.exports = {
   bottlenecksOf,
@@ -10,4 +11,5 @@ module.exports = {
   optimizedStatisticsOf,
   studentnumbersWithAllStudyrightElements,
   getEarliestYear,
+  getStudentsCloseToGraduation,
 }

services/backend/src/services/populations/optimizedStatisticsOf.js

@@ -31,19 +31,19 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
   // db startdate is formatted to utc so need to change it when querying
   const formattedStartDate = new Date(moment.tz(startDate, 'Europe/Helsinki').format()).toUTCString()
 
-  const studentnumbers = studentnumberlist
-    ? studentnumberlist
-    : await studentnumbersWithAllStudyrightElements({
-        studyRights,
-        startDate: formattedStartDate,
-        endDate,
-        exchangeStudents,
-        nondegreeStudents,
-        transferredOutStudents: transferredStudents,
-        tag: null,
-        transferredToStudents: true,
-        graduatedStudents: true,
-      })
+  const studentnumbers =
+    studentnumberlist ||
+    (await studentnumbersWithAllStudyrightElements({
+      studyRights,
+      startDate: formattedStartDate,
+      endDate,
+      exchangeStudents,
+      nondegreeStudents,
+      transferredOutStudents: transferredStudents,
+      tag: null,
+      transferredToStudents: true,
+      graduatedStudents: true,
+    }))
 
   const code = studyRights[0] || ''
   let optionData = {}

services/backend/src/services/populations/shared.js

@@ -1,5 +1,4 @@
-const Sequelize = require('sequelize')
-const { Op } = Sequelize
+const { Op } = require('sequelize')
 const moment = require('moment')
 const { Credit, Course, Studyright, ElementDetail, StudyrightElement } = require('../../models')
 const {
@@ -65,6 +64,16 @@ const updateCreditcriteriaInfo = (criteria, criteriaYear, criteriaChecked, yearT
   }
 }
 
+const getCurriculumVersion = curriculumPeriodId => {
+  if (!curriculumPeriodId) return null
+  const versionNumber = parseInt(curriculumPeriodId.slice(-2), 10)
+  const year = versionNumber + 1949
+  const startYear = Math.floor((year - 1) / 3) * 3 + 1
+  const endYear = startYear + 3
+  const curriculumVersion = `${startYear}-${endYear}`
+  return curriculumVersion
+}
+
 const formatStudentForPopulationStatistics = (
   {
     firstnames,
@@ -87,6 +96,9 @@ const formatStudentForPopulationStatistics = (
     option,
     birthdate,
     sis_person_id,
+    home_country_fi,
+    home_country_sv,
+    home_country_en,
   },
   enrollments,
   credits,
@@ -97,7 +109,16 @@ const formatStudentForPopulationStatistics = (
   code,
   currentSemester
 ) => {
-  const toCourse = ({ grade, attainment_date, credits, course_code, credittypecode, isStudyModule, language }) => {
+  const toCourse = ({
+    grade,
+    attainment_date,
+    credits,
+    course_code,
+    credittypecode,
+    isStudyModule,
+    language,
+    studyright_id,
+  }) => {
     const attainment_date_normailized =
       attainment_date < startDate ? startDateMoment.clone().add(1, 'day').toISOString() : attainment_date
     const passed = Credit.passed({ credittypecode })
@@ -111,16 +132,21 @@ const formatStudentForPopulationStatistics = (
       isStudyModuleCredit: isStudyModule,
       credittypecode,
       language,
+      studyright_id,
     }
   }
   const criteriaCoursesBySubstitutions = criteria?.allCourses
-    ? Object.keys(criteria.allCourses).reduce((acc, code) => {
-        acc[code] = code
-        criteria.allCourses[code].map(subst => (acc[subst] = code))
+    ? Object.keys(criteria.allCourses).reduce((acc, courseCode) => {
+        acc[courseCode] = courseCode
+        criteria.allCourses[courseCode].forEach(substitutionCode => {
+          acc[substitutionCode] = courseCode
+        })
         return acc
       }, {})
     : {}
 
+  const correctStudyplan = studyplans ? studyplans.filter(plan => plan.programme_code === code) : []
+
   const toProgressCriteria = () => {
     const criteriaChecked = {
       year1: createEmptyCriteriaYear(criteria, 'yearOne'),
@@ -130,7 +156,7 @@ const formatStudentForPopulationStatistics = (
       year5: createEmptyCriteriaYear(criteria, 'yearFive'),
       year6: createEmptyCriteriaYear(criteria, 'yearSix'),
     }
-    const correctStudyplan = studyplans ? studyplans.filter(plan => plan.programme_code === code) : []
+
     const academicYears = { first: 0, second: 0, third: 0, fourth: 0, fifth: 0, sixth: 0 }
     if (criteria.courses || criteria.credits) {
       const courses = credits[studentnumber] ? credits[studentnumber].map(toCourse) : []
@@ -193,7 +219,11 @@ const formatStudentForPopulationStatistics = (
     birthdate,
     studyplans,
     sis_person_id,
+    home_country_en,
+    home_country_fi,
+    home_country_sv,
     criteriaProgress: toProgressCriteria(),
+    curriculumVersion: getCurriculumVersion(correctStudyplan[0]?.curriculum_period_id),
   }
 }
 
@@ -264,7 +294,7 @@ const getOptionsForStudents = async (students, code, level) => {
     currentExtent = 2
     optionExtent = 1
   } else {
-    throw new Error('Invalid study level ' + level)
+    throw new Error(`Invalid study level ${level}`)
   }
 
   const programmes = await getAllProgrammes()
@@ -277,7 +307,7 @@ const getOptionsForStudents = async (students, code, level) => {
           studentnumber: {
             [Op.in]: students,
           },
-          code: code,
+          code,
         },
       },
     ],
@@ -380,8 +410,12 @@ const formatStudentsForApi = async (
         stats.transfers.targets[transfer.targetcode] = target
         stats.transfers.sources[transfer.sourcecode] = source
       })
-      if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
-      else student.option = null
+
+      if (student.studentnumber in optionData) {
+        student.option = optionData[student.studentnumber]
+      } else {
+        student.option = null
+      }
 
       stats.students.push(
         formatStudentForPopulationStatistics(
@@ -440,7 +474,7 @@ const formatStudentsForApi = async (
 }
 
 const formatQueryParamArrays = (query, params) => {
-  let res = { ...query }
+  const res = { ...query }
   params.forEach(p => {
     if (!res[p]) return
     res[p] = Array.isArray(res[p]) ? res[p] : [res[p]]
@@ -452,6 +486,7 @@ const getSubstitutions = async codes => {
   const courses = await Course.findAll({ where: { code: codes }, attributes: ['code', 'substitutions'], raw: true })
   return [...new Set(courses.map(({ code, substitutions }) => [code, ...substitutions]).flat())]
 }
+
 // This duplicate code is added here to ensure that we get the enrollments in cases no credits found for the selected students.
 const findCourseEnrollments = async (studentnumbers, beforeDate, courses = []) => {
   const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
@@ -494,6 +529,7 @@ const findCourseEnrollments = async (studentnumbers, beforeDate, courses = []) =
   )
   return res
 }
+
 const findCourses = async (studentnumbers, beforeDate, courses = []) => {
   const courseCodes = courses.length > 0 ? await getSubstitutions(courses) : ['DUMMY']
   const res = await sequelize.query(

services/backend/src/services/populations/studentnumbersWithAllStudyrightElements.js

@@ -26,7 +26,7 @@ const studentnumbersWithAllStudyrightElements = async ({
     filteredExtents.push(6, 9, 13, 14, 18, 22, 23, 99)
   }
 
-  let studyrightWhere = {
+  const studyrightWhere = {
     extentcode: {
       [Op.notIn]: filteredExtents,
     },
@@ -35,7 +35,7 @@ const studentnumbersWithAllStudyrightElements = async ({
     },
   }
 
-  let studentWhere = {}
+  const studentWhere = {}
   if (tag) {
     const taggedStudentnumbers = await TagStudent.findAll({
       attributes: ['studentnumber'],
@@ -70,7 +70,7 @@ const studentnumbersWithAllStudyrightElements = async ({
     where: {
       [Op.or]: [
         {
-          ['$studyright_elements->element_detail.type$']: {
+          '$studyright_elements->element_detail.type$': {
             [Op.ne]: 20,
           },
         },
@@ -92,7 +92,7 @@ const studentnumbersWithAllStudyrightElements = async ({
     raw: true,
   })
 
-  let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+  const studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
   // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
   const rights = await Studyright.findAll({
     attributes: ['studyrightid'],

services/backend/src/services/programmeModules.js

@@ -1,8 +1,7 @@
-const Sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const { ExcludedCourse } = require('../models/models_kone')
-const { Op } = Sequelize
 const logger = require('../util/logger')
-const { combinedStudyprogrammes } = require('./studyprogramme/studyprogrammeHelpers.js')
+const { combinedStudyprogrammes } = require('./studyprogramme/studyprogrammeHelpers')
 const { dbConnections } = require('../database/connection')
 const { ProgrammeModule } = require('../models')
 
@@ -11,8 +10,8 @@ const getCurriculumVersions = async code => {
     const result = await ProgrammeModule.findAll({ where: { code } })
     if (!result) return
     const modified = result.map(r => ({
-      ...r,
-      curriculum_period_ids: r.curriculum_period_ids.map(id => parseInt(id.substring(6)) + 1949),
+      ...r.toJSON(),
+      curriculum_period_ids: r.curriculum_period_ids.map(id => parseInt(id.substring(6), 10) + 1949),
     }))
     return modified
   } catch (e) {
@@ -66,6 +65,19 @@ const modifyParent = (course, moduleMap) => {
   return { ...course, parent_id: parent.id, parent_code: parent.code, parent_name: parent.name }
 }
 
+const labelProgammes = (modules, excludedCourses) => {
+  return modules.map(module => {
+    const label = {
+      id: module.parent_name.fi,
+      label: `${module.parent_code}\n${module.parent_name.fi}`,
+      orderNumber: module.module_order,
+    }
+    const foundCourse = excludedCourses.find(course => course.course_code === module.code)
+    const visible = { visibility: !foundCourse, id: foundCourse?.id ?? null }
+    return { ...module, label, visible }
+  })
+}
+
 const getCoursesAndModulesForProgramme = async (code, period_ids) => {
   if (!period_ids) return {}
   const result = await recursivelyGetModuleAndChildren(
@@ -107,17 +119,4 @@ const getCoursesAndModules = async (code, period_ids) => {
   return { defaultProgrammeCourses, secondProgrammeCourses: { courses: [], modules: [] } }
 }
 
-const labelProgammes = (modules, excludedCourses) => {
-  return modules.map(module => {
-    const label = {
-      id: module.parent_name.fi,
-      label: `${module.parent_code}\n${module.parent_name.fi}`,
-      orderNumber: module.module_order,
-    }
-    const foundCourse = excludedCourses.find(course => course.course_code === module.code)
-    const visible = { visibility: !foundCourse, id: foundCourse?.id ?? null }
-    return { ...module, label, visible }
-  })
-}
-
 module.exports = { getCoursesAndModules, getCurriculumVersions }

services/backend/src/services/providerCredits.js

@@ -28,19 +28,24 @@ const { getCourseCodesOfProvider } = require('./providers')
   'separate'
   Erillisoikeus: opintosuoritukset, joiden suorituksen luokittelu on "erillisellä opiskeluoikeudella" tai 
   "opettajankoulutuksen erillisellä opiskeluikeudella" suoritettu. Ei sisällä kansainvälisten vaihto-opiskelijoiden suorituksia.
-  Ulkomailta hyväksiluetut opintopisteet: ulkomailta hyväksilutetut opintopisteet.
 
   'transferred'
+  Ulkomailta hyväksiluetut opintopisteet: ulkomailta hyväksilutetut opintopisteet.
   Muut hyväksiluetut opintopisteet: kotimaassa suoritetut, hyväksiluetut opintopisteet.
+
+  Possible ambiguities / mistakes:
+  Rapo may use createddate instead of attainment_date to determine if student had active studyright.
+  This is probably very rare to cause differences, only one such was found in MH60_001
+
 */
 
-const getCategory = (extent, hadDegreeStudyright) => {
-  if ([1, 2, 3, 4].includes(extent) || hadDegreeStudyright) return 'basic' // Rapo-kategoria: Perustutkinto-opiskelijat
-  if (extent === 7 || extent === 34) return 'incoming-exchange' // Saapuvat vaihto-opiskelijat
-  if (extent === 9 && !hadDegreeStudyright) return 'open-uni' // Avoimen opiskeluoikeus, ei tutkinto-opiskelija
-  if (extent === 14) return 'agreement' // Korkeakoulujen väliset yhteistyöopinnot
-  if (extent === 99) return 'separate' // Erillisoikeus
-  return 'other'
+const getCategory = (extent, degreeStudyright) => {
+  if ([7, 34].includes(extent)) return 'incoming-exchange' // Saapuvat vaihto-opiskelijat
+  if ([9, 31].includes(extent) && !degreeStudyright) return 'open-uni' // Avoimen opiskeluoikeus, ei tutkinto-opiskelija
+  if ([14, 16].includes(extent)) return 'agreement' // Korkeakoulujen väliset yhteistyöopinnot
+  if ([13, 18, 22, 23, 99].includes(extent)) return 'separate' // Erillisoikeus
+  if ([1, 2].includes(extent) || (degreeStudyright && [1, 2].includes(degreeStudyright.extentcode))) return 'basic' // Rapo-kategoria: Perustutkinto-opiskelijat
+  return 'other' // TODO: These still happen, but not a lot. Should be investigated which go here and where they should go instead
 }
 
 /*
@@ -59,32 +64,36 @@ const findRelevantStudyright = (attainmentDate, studyrights) => {
   })
 }
 
-// At the given date, student had SOME degree-studyright.
-const isDegreeStudent = (studyrights, date) =>
-  studyrights?.some(sr => {
+const getDegreeStudyright = (studyrights, date, semestercode) =>
+  studyrights?.find(sr => {
     const rightExtentCode = [1, 2, 3, 4].includes(sr.extentcode)
     const rightDates =
       new Date(sr.startdate).getTime() <= new Date(date).getTime() &&
       new Date(date).getTime() <= new Date(sr.enddate).getTime()
-    // it is possible to also check if student had active semester enrollment to the studyright here.
-    // However, it is unsure if rapo requires this (seems like maybe not). shouldn't make a huge difference.
-    return rightExtentCode && rightDates
+    const enrolled = sr.semesterEnrollments?.some(e => e.semestercode === semestercode && e.enrollmenttype === 1)
+    return rightExtentCode && rightDates && enrolled
   })
 
-const getCreditsProduced = async (provider, isAcademicYear, specialIncluded = true) => {
-  let data = await getCreditStats(provider, isAcademicYear, specialIncluded)
-  if (data) return data
-  data = await computeCreditsProduced(provider, isAcademicYear, specialIncluded)
-  await setCreditStats(data, isAcademicYear, specialIncluded)
-  return data
-}
-
 /* Calculates credits produced by provider (programme or faculty) */
 const computeCreditsProduced = async (providerCode, isAcademicYear, specialIncluded = true) => {
   const since = new Date('2017-01-01')
-  const providercode = mapToProviders([providerCode])[0]
-  const courses = await getCourseCodesOfProvider(providercode)
-  const credits = await getCreditsForProvider(providercode, courses, since)
+  const rapoFormattedProviderCode = mapToProviders([providerCode])[0]
+  const courses = await getCourseCodesOfProvider(rapoFormattedProviderCode)
+  const credits = await getCreditsForProvider(rapoFormattedProviderCode, courses, since)
+
+  // This part also adds oikis vaasa which is provided by different organization.
+  // Uknown if there are other similar cases!
+  const vaasaCodes = {
+    '200-K001': '200-K0012',
+    '200-M001': '200-M0012',
+  }
+  const vaasaProvider = vaasaCodes[rapoFormattedProviderCode]
+  if (vaasaProvider) {
+    const courses = await getCourseCodesOfProvider(vaasaProvider)
+    const vaasaCredits = await getCreditsForProvider(vaasaProvider, courses, since)
+    credits.push(...vaasaCredits)
+  }
+
   const students = [...new Set(credits.map(({ studentNumber }) => studentNumber))]
 
   const transfers = (await allTransfers(providerCode, since)).map(t => t.studyrightid)
@@ -107,13 +116,17 @@ const computeCreditsProduced = async (providerCode, isAcademicYear, specialInclu
 
   const stats = {}
 
-  credits.forEach(({ attainmentDate, studentNumber, credits, studyrightId }) => {
+  credits.forEach(({ attainmentDate, studentNumber, credits, studyrightId, semestercode }) => {
     const relevantStudyright = studyrightId
       ? studyrightIdToStudyrightMap[studyrightId]
       : findRelevantStudyright(attainmentDate, studentNumberToStudyrightsMap[studentNumber])
     const attainmentYear = defineYear(attainmentDate, isAcademicYear)
-    const hadDegreeStudyright = isDegreeStudent(studentNumberToStudyrightsMap[studentNumber], attainmentDate)
-    const category = relevantStudyright ? getCategory(relevantStudyright.extentcode, hadDegreeStudyright) : 'other'
+    const degreeStudyright = getDegreeStudyright(
+      studentNumberToStudyrightsMap[studentNumber],
+      attainmentDate,
+      semestercode
+    )
+    const category = relevantStudyright ? getCategory(relevantStudyright.extentcode, degreeStudyright) : 'other'
     if (!stats[attainmentYear]) stats[attainmentYear] = { transferred: 0 }
     stats[attainmentYear].total += credits
     if (!stats[attainmentYear][category]) {
@@ -122,10 +135,13 @@ const computeCreditsProduced = async (providerCode, isAcademicYear, specialInclu
     stats[attainmentYear][category] += credits
   })
 
-  const transferredCredits = await getTransferredCredits(providercode, since)
-
-  transferredCredits.forEach(({ attainment_date, credits }) => {
-    const attainmentYear = defineYear(attainment_date, isAcademicYear)
+  const transferredCredits = await getTransferredCredits(rapoFormattedProviderCode, since)
+  if (vaasaProvider) {
+    const vaasaTransferredCredits = await getTransferredCredits(vaasaProvider, since)
+    transferredCredits.push(...vaasaTransferredCredits)
+  }
+  transferredCredits.forEach(({ createdate, credits }) => {
+    const attainmentYear = defineYear(createdate, isAcademicYear)
     if (!stats[attainmentYear]) stats[attainmentYear] = { transferred: 0 }
     stats[attainmentYear].transferred += credits || 0
     // Transferred not counted in total
@@ -134,6 +150,14 @@ const computeCreditsProduced = async (providerCode, isAcademicYear, specialInclu
   return { stats, id: providerCode }
 }
 
+const getCreditsProduced = async (provider, isAcademicYear, specialIncluded = true) => {
+  let data = await getCreditStats(provider, isAcademicYear, specialIncluded)
+  if (data) return data
+  data = await computeCreditsProduced(provider, isAcademicYear, specialIncluded)
+  await setCreditStats(data, isAcademicYear, specialIncluded)
+  return data
+}
+
 module.exports = {
   getCreditsProduced,
   computeCreditsProduced,

services/backend/src/services/redis.js

@@ -1,6 +1,6 @@
 const redis = require('redis')
 const bluebird = require('bluebird')
-const conf = require('../conf-backend.js')
+const conf = require('../conf-backend')
 const logger = require('../util/logger')
 
 bluebird.promisifyAll(redis.RedisClient.prototype)

services/backend/src/services/semesters.js

@@ -1,7 +1,6 @@
 const moment = require('moment')
-const sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const { Semester } = require('../models')
-const Op = sequelize.Op
 
 const getCurrentSemester = async () => {
   const today = new Date()

services/backend/src/services/sisUpdaterService.js

@@ -1,7 +1,8 @@
 const axios = require('axios')
 const { SIS_UPDATER_URL, SECRET_TOKEN } = require('../conf-backend')
-const { Studyplan } = require('../models')
+const { Studyplan, Student, StudyrightElement } = require('../models')
 const { Op } = require('sequelize')
+const logger = require('../util/logger')
 
 const client = axios.create({ baseURL: SIS_UPDATER_URL })
 
@@ -38,6 +39,37 @@ const updateSISRedisCache = async () => {
   const response = await client.get('/v1/rediscache', params)
   return response.data
 }
+const delay = time => {
+  return new Promise(res => {
+    setTimeout(res, time)
+  })
+}
+
+const updateStudentsIndividually = async () => {
+  try {
+    const studentNumbers = await Student.findAll({
+      attributes: ['studentnumber'],
+      include: {
+        model: StudyrightElement,
+        where: {
+          code: { [Op.regexp]: '(KH*|MH*)' },
+        },
+      },
+    })
+    const uniqueStudents = [...new Set(studentNumbers.map(s => s.studentnumber))]
+    const chunkSize = 4000
+    for (let from = 0; from < uniqueStudents.length - chunkSize; from += chunkSize) {
+      await client.post(
+        'v1/students',
+        { studentnumbers: uniqueStudents.slice(from, from + chunkSize), individualMode: true },
+        params
+      )
+      await delay(200)
+    }
+  } catch (e) {
+    logger.error(e)
+  }
+}
 
 const studyplansUpdate = async days => {
   const limitDate = new Date()
@@ -67,4 +99,5 @@ module.exports = {
   abort,
   updateCoursesByCourseCode,
   studyplansUpdate,
+  updateStudentsIndividually,
 }

services/backend/src/services/students.js

@@ -1,4 +1,4 @@
-const Sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const {
   dbConnections: { sequelize },
 } = require('../database/connection')
@@ -16,7 +16,6 @@ const {
   Transfer,
 } = require('../models')
 const { TagStudent, Tag } = require('../models/models_kone')
-const Op = Sequelize.Op
 const logger = require('../util/logger')
 
 const createStudent = student => Student.create(student)
@@ -288,7 +287,7 @@ const formatStudent = async ({
     updatedAt: updatedAt || createdAt,
     studyplans,
     tags,
-    sis_person_id: sis_person_id,
+    sis_person_id,
   }
 }
 
@@ -321,13 +320,12 @@ const nameLike = terms => {
   const [first, second] = terms
   if (!second) {
     return columnLike('abbreviatedname', first)
-  } else {
-    return {
-      [Op.or]: [
-        columnLike('abbreviatedname', `%${first}%${second}%`),
-        columnLike('abbreviatedname', `%${second}%${first}%`),
-      ],
-    }
+  }
+  return {
+    [Op.or]: [
+      columnLike('abbreviatedname', `%${first}%${second}%`),
+      columnLike('abbreviatedname', `%${second}%${first}%`),
+    ],
   }
 }
 

services/backend/src/services/studyGuidanceGroups.js

@@ -1,4 +1,5 @@
 const { getImporterClient } = require('../util/importerClient')
+
 const importerClient = getImporterClient()
 const { StudyGuidanceGroupTag } = require('../models/models_kone')
 const logger = require('../util/logger')
@@ -38,9 +39,16 @@ const getAllStudentsUserHasInGroups = async sisPersonId =>
       .map(member => member.personStudentNumber)
   )
 
+const getTagToUpdate = (studyProgramme, year) => {
+  if (studyProgramme || studyProgramme === null) {
+    return { studyProgramme }
+  }
+  return { year }
+}
+
 const changeGroupTags = async ({ groupId, tags }) => {
   const { studyProgramme, year } = tags
-  const tagToUpdate = studyProgramme ? { studyProgramme } : { year }
+  const tagToUpdate = getTagToUpdate(studyProgramme, year)
 
   const [result] = await StudyGuidanceGroupTag.upsert(
     {

services/backend/src/services/studyprogramme/creditGetters.js

@@ -1,56 +1,93 @@
 const { Op } = require('sequelize')
-const { Credit, Course, Organization } = require('../../models')
+const { Credit, Course, Organization, CourseProvider } = require('../../models')
 const { whereStudents } = require('.')
-const {
-  dbConnections: { sequelize },
-} = require('../../database/connection')
 const { formatCredit } = require('./studyprogrammeHelpers')
 
-const getCreditsForProvider = async (provider, codes, since) =>
-  (
-    await sequelize.query(
-      `
-      SELECT 
-        cr.id,
-        cr.course_code, 
-        cr.attainment_date, 
-        cr.student_studentnumber, 
-        cr.credits * COALESCE(share_element.share::numeric, 1) AS credits,
-        cr.studyright_id,
-        cr.semestercode
-      FROM 
-        credit cr
-      JOIN 
-        course_providers cp ON cr.course_id = cp.coursecode
-      JOIN 
-        organization o ON cp.organizationcode = o.id
-      LEFT JOIN LATERAL 
-        (
-          SELECT (jae ->> 'share')::numeric AS share
-          FROM jsonb_array_elements(cp.shares) AS jae
-          WHERE (cr.attainment_date >= (jae ->> 'startDate')::date OR jae ->> 'startDate' IS NULL)
-            AND (cr.attainment_date <= (jae ->> 'endDate')::date OR jae ->> 'endDate' IS NULL)
-          ORDER BY 
-            (jae ->> 'startDate')::date DESC NULLS LAST,
-            (jae ->> 'endDate')::date DESC NULLS LAST
-          LIMIT 1
-        ) AS share_element ON TRUE
-      WHERE 
-        (o.code = :provider)
-        AND cr.attainment_date >= :since
-        AND cr.credittypecode = 4
-        AND cr."isStudyModule" IS NOT TRUE
-      `,
-      {
-        type: sequelize.QueryTypes.SELECT,
-        replacements: { provider, codes, since },
-      }
-    )
-  ).map(formatCredit)
+const getCreditsForProvider = async (provider, codes, since) => {
+  const credits = await Credit.findAll({
+    where: {
+      course_code: { [Op.in]: codes },
+      attainment_date: { [Op.gte]: since },
+      isStudyModule: { [Op.not]: true },
+      credittypecode: 4,
+    },
+    raw: true,
+  })
+
+  const courseIds = credits.map(cr => cr.course_id)
+
+  const providers = await CourseProvider.findAll({
+    where: {
+      coursecode: { [Op.in]: courseIds },
+    },
+    raw: true,
+  })
+  const organizationIds = providers.map(p => p.organizationcode)
+  const organizations = await Organization.findAll({
+    where: {
+      id: {
+        [Op.in]: organizationIds,
+      },
+    },
+    raw: true,
+  })
+
+  const organizationIdToCodeMap = organizations.reduce((obj, org) => {
+    obj[org.id] = org.code
+    return obj
+  }, {})
+
+  const courseIdToShareMap = providers.reduce((obj, provider) => {
+    if (!obj[provider.coursecode]) obj[provider.coursecode] = []
+    obj[provider.coursecode].push(provider)
+    return obj
+  }, {})
+
+  const courseIdToShare = (courseId, attainmentDate) => {
+    const providers = courseIdToShareMap[courseId]
+    const relevantProvider = providers.find(p => organizationIdToCodeMap[p.organizationcode] === provider)
+    if (!relevantProvider?.shares) return 0
+    const relevantShare = relevantProvider.shares
+      .filter(share => {
+        const startMatches = new Date(share.startDate) <= attainmentDate || !share.startDate
+        const endMatches = new Date(share.endDate) >= attainmentDate || !share.endDate
+        const bothArentNull = share.startDate || share.endDate
+        return startMatches && endMatches && bothArentNull
+      })
+      // The next mess is for cases where there are multiple shares with only startdate or enddate.
+      // This isn't pretty and it may not be 100% correct but because the
+      // data is all over the place, this is best approximation for now.
+      .sort((a, b) => {
+        if (!b.startDate || !a.startDate) return 0
+        const bTime = b.startDate ? new Date(b.startDate).getTime() : 0
+        const aTime = a.startDate ? new Date(a.startDate).getTime() : 0
+        return bTime - aTime
+      })
+      .sort((a, b) => {
+        if (!b.endDate || !a.endDate) return 0
+        const bTime = b.endDate ? new Date(b.endDate).getTime() : 0
+        const aTime = a.endDate ? new Date(a.endDate).getTime() : 0
+        return bTime - aTime
+      })[0]
+
+    // Odd logic, but if there are multiple providers for same course code, if no fitting dates are
+    // not found for our relevant provider, we can assume the share of that date is of some other provider.
+    // But if only 1 provider exists, we can assume it has share of 1.
+    if (!relevantShare) {
+      if (providers.length > 1) return 0
+      return 1
+    }
+    return relevantShare.share
+  }
+
+  return credits
+    .map(cr => ({ ...cr, credits: cr.credits * courseIdToShare(cr.course_id, cr.attainment_date, cr.id) }))
+    .map(formatCredit)
+}
 
 const getTransferredCredits = async (provider, since) =>
   await Credit.findAll({
-    attributes: ['id', 'course_code', 'credits', 'attainment_date'],
+    attributes: ['id', 'course_code', 'credits', 'attainment_date', 'createdate'],
     include: {
       model: Course,
       attributes: ['code'],

services/backend/src/services/studyprogramme/index.js

@@ -1,15 +1,14 @@
 const _ = require('lodash')
-const Sequelize = require('sequelize')
+const { Op } = require('sequelize')
 const {
   dbConnections: { sequelize },
 } = require('../../database/connection')
-const { Op } = Sequelize
 const { Credit, Course, Organization, ProgrammeModule, Transfer, Enrollment } = require('../../models')
 const { formatTransfer } = require('./studyprogrammeHelpers')
 const logger = require('../../util/logger')
 
 const whereStudents = studentnumbers => {
-  return studentnumbers ? studentnumbers : { [Op.not]: null }
+  return studentnumbers || { [Op.not]: null }
 }
 
 const sinceDate = since => {

services/backend/src/services/studyprogramme/studentGetters.js

@@ -142,8 +142,8 @@ const getStudentsForProgrammeCourses = async (from, to, programmeCourses) => {
     const result = res.map(course => ({
       code: course.code,
       name: course.course_name,
-      totalPassed: parseInt(course.total_students),
-      totalAllcredits: parseInt(course.total_credits),
+      totalPassed: parseInt(course.total_students, 10),
+      totalAllcredits: parseInt(course.total_credits, 10),
       type: 'passed',
       isStudyModule: course.isStudyModule,
     }))
@@ -182,8 +182,8 @@ const getOwnStudentsForProgrammeCourses = async (from, to, programmeCourses, stu
     const res = {
       code: course.code,
       name: course.course_name,
-      totalProgrammeStudents: parseInt(course.total_students),
-      totalProgrammeCredits: parseInt(course.total_credits),
+      totalProgrammeStudents: parseInt(course.total_students, 10),
+      totalProgrammeCredits: parseInt(course.total_credits, 10),
       type: 'ownProgramme',
     }
     return res
@@ -226,8 +226,8 @@ const getOtherStudentsForProgrammeCourses = async (from, to, programmeCourses, s
   return res.map(course => ({
     code: course.code,
     name: course.course_name,
-    totalOtherProgrammeStudents: parseInt(course.total_students),
-    totalOtherProgrammeCredits: parseInt(course.total_credits),
+    totalOtherProgrammeStudents: parseInt(course.total_students, 10),
+    totalOtherProgrammeCredits: parseInt(course.total_credits, 10),
     type: 'otherProgramme',
     isStudyModule: course.isStudyModule,
   }))
@@ -257,8 +257,8 @@ const getTransferStudentsForProgrammeCourses = async (from, to, programmeCourses
   return res.map(course => ({
     code: course.code,
     name: course.course_name,
-    totalTransferStudents: parseInt(course.total_students),
-    totalTransferCredits: parseInt(course.total_credits),
+    totalTransferStudents: parseInt(course.total_students, 10),
+    totalTransferCredits: parseInt(course.total_credits, 10),
     isStudyModule: course.isStudyModule,
     type: 'transfer',
   }))
@@ -293,8 +293,8 @@ const getStudentsWithoutStudyrightForProgrammeCourses = async (from, to, program
   return res.map(course => ({
     code: course.code,
     name: course.course_name,
-    totalWithoutStudyrightStudents: parseInt(course.total_students),
-    totalWithoutStudyrightCredits: parseInt(course.total_credits),
+    totalWithoutStudyrightStudents: parseInt(course.total_students, 10),
+    totalWithoutStudyrightCredits: parseInt(course.total_credits, 10),
     type: 'noStudyright',
   }))
 }

services/backend/src/services/studyprogramme/studyProgrammeCriteria.js

@@ -6,7 +6,7 @@ const getCriteriaByStudyProgramme = async code => {
   if (code === '') return null
   return await ProgressCriteria.findOne({
     where: {
-      code: code,
+      code,
     },
   })
 }
@@ -71,22 +71,22 @@ const saveYearlyCreditCriteria = async (studyProgramme, credits) => {
   if (!studyProgrammeToUpdate) {
     const courseObj = { year1: [], year2: [], year3: [], year4: [], year5: [], year6: [] }
     const creditsObj = {
-      year1: parseInt(credits.year1),
-      year2: parseInt(credits.year2),
-      year3: parseInt(credits.year3),
-      year4: parseInt(credits.year4),
-      year5: parseInt(credits.year5),
-      year6: parseInt(credits.year6),
+      year1: parseInt(credits.year1, 10),
+      year2: parseInt(credits.year2, 10),
+      year3: parseInt(credits.year3, 10),
+      year4: parseInt(credits.year4, 10),
+      year5: parseInt(credits.year5, 10),
+      year6: parseInt(credits.year6, 10),
     }
     return await createCriteria(studyProgramme, courseObj, creditsObj)
   }
   const yearlyCredits = {
-    creditsYearOne: parseInt(credits.year1),
-    creditsYearTwo: parseInt(credits.year2),
-    creditsYearThree: parseInt(credits.year3),
-    creditsYearFour: parseInt(credits.year4),
-    creditsYearFive: parseInt(credits.year5),
-    creditsYearSix: parseInt(credits.year6),
+    creditsYearOne: parseInt(credits.year1, 10),
+    creditsYearTwo: parseInt(credits.year2, 10),
+    creditsYearThree: parseInt(credits.year3, 10),
+    creditsYearFour: parseInt(credits.year4, 10),
+    creditsYearFive: parseInt(credits.year5, 10),
+    creditsYearSix: parseInt(credits.year6, 10),
   }
   try {
     const updatedData = await studyProgrammeToUpdate.update({ ...yearlyCredits })

services/backend/src/services/studyprogramme/studyprogrammeBasics.js

@@ -223,7 +223,7 @@ const getBasicStatsForStudytrack = async ({ studyprogramme, combinedProgramme, s
 
   const reversedYears = getYearsArray(since.getFullYear(), isAcademicYear).reverse()
   const key = includeAllSpecials ? 'SPECIAL_INCLUDED' : 'SPECIAL_EXCLUDED'
-  const titles = tableTitles['basics'][combinedProgramme ? `${key}_COMBINED_PROGRAMME` : key]
+  const titles = tableTitles.basics[combinedProgramme ? `${key}_COMBINED_PROGRAMME` : key]
   const tableStats = reversedYears.map(year =>
     initializeTableStats(
       year,

services/backend/src/services/studyprogramme/studyprogrammeCourses.js

@@ -24,8 +24,8 @@ const getAllStudyprogrammeCourses = async studyprogramme => {
   const normalCourses = await getAllProgrammeCourses(providerCode)
   return normalCourses.reduce((acc, curr) => {
     acc.push(curr.code)
-    if (curr.substitutions && curr.substitutions.includes('AY' + curr.code)) {
-      acc.push('AY' + curr.code)
+    if (curr.substitutions && curr.substitutions.includes(`AY${curr.code}`)) {
+      acc.push(`AY${curr.code}`)
     }
     return acc
   }, [])
@@ -34,7 +34,6 @@ const getAllStudyprogrammeCourses = async studyprogramme => {
 const makeYearlyPromises = (years, academicYear, type, programmeCourses, studyprogramme) => {
   return years.map(
     year =>
-      // eslint-disable-next-line no-async-promise-executor
       new Promise(async res => {
         const from = academicYear === 'ACADEMIC_YEAR' ? new Date(year, 7, 1, 0, 0, 0) : new Date(year, 0, 1, 0, 0, 0)
         const to =
@@ -66,7 +65,7 @@ const makeYearlyPromises = (years, academicYear, type, programmeCourses, studypr
 
         res(
           result.map(c => {
-            c['year'] = year
+            c.year = year
             return c
           })
         )
@@ -227,8 +226,8 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
       }
     }
 
-    if (!acc[curr.code]['years'][curr.year]) {
-      acc[curr.code]['years'][curr.year] = {
+    if (!acc[curr.code].years[curr.year]) {
+      acc[curr.code].years[curr.year] = {
         totalAllStudents: 0,
         totalPassed: 0,
         totalNotCompleted: 0,
@@ -246,30 +245,32 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
     }
     switch (curr.type) {
       case 'passed':
-        acc[curr.code]['years'][curr.year]['totalPassed'] += curr.totalPassed
-        acc[curr.code]['years'][curr.year]['totalAllStudents'] += acc[curr.code]['years'][curr.year]['totalPassed']
-        acc[curr.code]['years'][curr.year]['totalAllCredits'] += curr.totalAllcredits
+        acc[curr.code].years[curr.year].totalPassed += curr.totalPassed
+        acc[curr.code].years[curr.year].totalAllStudents += acc[curr.code].years[curr.year].totalPassed
+        acc[curr.code].years[curr.year].totalAllCredits += curr.totalAllcredits
         break
       case 'notCompleted':
-        acc[curr.code]['years'][curr.year]['totalNotCompleted'] += curr.totalNotCompleted
-        acc[curr.code]['years'][curr.year]['totalAllStudents'] +=
-          acc[curr.code]['years'][curr.year]['totalNotCompleted']
+        acc[curr.code].years[curr.year].totalNotCompleted += curr.totalNotCompleted
+        acc[curr.code].years[curr.year].totalAllStudents += acc[curr.code].years[curr.year].totalNotCompleted
         break
       case 'ownProgramme':
-        acc[curr.code]['years'][curr.year]['totalProgrammeStudents'] += curr.totalProgrammeStudents
-        acc[curr.code]['years'][curr.year]['totalProgrammeCredits'] += curr.totalProgrammeCredits
+        acc[curr.code].years[curr.year].totalProgrammeStudents += curr.totalProgrammeStudents
+        acc[curr.code].years[curr.year].totalProgrammeCredits += curr.totalProgrammeCredits
         break
       case 'otherProgramme':
-        acc[curr.code]['years'][curr.year]['totalOtherProgrammeStudents'] += curr.totalOtherProgrammeStudents
-        acc[curr.code]['years'][curr.year]['totalOtherProgrammeCredits'] += curr.totalOtherProgrammeCredits
+        acc[curr.code].years[curr.year].totalOtherProgrammeStudents += curr.totalOtherProgrammeStudents
+        acc[curr.code].years[curr.year].totalOtherProgrammeCredits += curr.totalOtherProgrammeCredits
         break
       case 'noStudyright':
-        acc[curr.code]['years'][curr.year]['totalWithoutStudyrightStudents'] += curr.totalWithoutStudyrightStudents
-        acc[curr.code]['years'][curr.year]['totalWithoutStudyrightCredits'] += curr.totalWithoutStudyrightCredits
+        acc[curr.code].years[curr.year].totalWithoutStudyrightStudents += curr.totalWithoutStudyrightStudents
+        acc[curr.code].years[curr.year].totalWithoutStudyrightCredits += curr.totalWithoutStudyrightCredits
         break
       case 'transfer':
-        acc[curr.code]['years'][curr.year]['totalTransferStudents'] += curr.totalTransferStudents
-        acc[curr.code]['years'][curr.year]['totalTransferCredits'] += curr.totalTransferCredits
+        acc[curr.code].years[curr.year].totalTransferStudents += curr.totalTransferStudents
+        acc[curr.code].years[curr.year].totalTransferCredits += curr.totalTransferCredits
+        break
+      default:
+        break
     }
 
     return acc
@@ -294,15 +295,15 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
 
     if (allCourses[normCode]) {
       const mergedCourse = {}
-      mergedCourse['code'] = allCourses[normCode].code
-      mergedCourse['name'] = allCourses[normCode].name
-      mergedCourse['years'] = {}
+      mergedCourse.code = allCourses[normCode].code
+      mergedCourse.name = allCourses[normCode].name
+      mergedCourse.years = {}
 
       yearRange
         .filter(year => year <= maxYear)
         .forEach(year => {
-          if (!allCourses[normCode]['years'][year]) {
-            mergedCourse['years'][year] = {
+          if (!allCourses[normCode].years[year]) {
+            mergedCourse.years[year] = {
               totalAllStudents: 0,
               totalPassed: 0,
               totalNotCompleted: 0,
@@ -317,13 +318,12 @@ const getStudyprogrammeCoursesForStudytrack = async (unixMillis, studyprogramme,
               totalTransferStudents: 0,
             }
           } else {
-            mergedCourse['years'][year] = { ...allCourses[normCode]['years'][year] }
+            mergedCourse.years[year] = { ...allCourses[normCode].years[year] }
           }
 
-          if (allCourses[ayCourse]['years'][year]) {
+          if (allCourses[ayCourse].years[year]) {
             properties.forEach(prop => {
-              mergedCourse['years'][year][prop] =
-                mergedCourse['years'][year][prop] + allCourses[ayCourse]['years'][year][prop]
+              mergedCourse.years[year][prop] += allCourses[ayCourse].years[year][prop]
             })
           }
         })

services/backend/src/services/studyprogramme/studyprogrammeHelpers.js

@@ -333,7 +333,7 @@ const getGoal = programme => {
   if (programme.startsWith('LI')) {
     return 78
   }
-  return 48 //unknown, likely old doctor or licentiate
+  return 48 // unknown, likely old doctor or licentiate
 }
 
 module.exports = {

services/backend/src/services/studyprogramme/studytrackStats.js

@@ -23,11 +23,12 @@ const { studytrackStudents, enrolledStudents, absentStudents } = require('./stud
 
 const getUnique = studentnumbers => [...new Set(studentnumbers)]
 const getStudentData = (startDate, students) => {
-  let data = { female: 0, male: 0, otherUnkown: 0, finnish: 0, otherCountries: 0, otherCountriesCounts: {} }
+  const data = { female: 0, male: 0, otherUnkown: 0, finnish: 0, otherCountries: 0, otherCountriesCounts: {} }
   const creditCounts = []
   students.forEach(({ genderCode, homeCountryEn, credits }) => {
     const creditcount = credits
       .filter(credit => moment(credit.attainment_date).isAfter(startDate))
+      // eslint-disable-next-line no-return-assign
       .reduce((prev, curr) => (prev += curr.credits), 0)
     creditCounts.push(creditcount)
 
@@ -103,7 +104,7 @@ const getGraduationTimeStats = async ({
     ...graduationTimes[track].medians.combo,
     {
       y: medianCombo,
-      amount: graduationAmounts[track]['combo'][year],
+      amount: graduationAmounts[track].combo[year],
       name: year,
       statistics: statisticsCombo,
       classSize,
@@ -112,7 +113,7 @@ const getGraduationTimeStats = async ({
 
   graduationTimes[track].medians.basic = [
     ...graduationTimes[track].medians.basic,
-    { y: median, amount: graduationAmounts[track]['basic'][year], name: year, statistics, classSize },
+    { y: median, amount: graduationAmounts[track].basic[year], name: year, statistics, classSize },
   ]
 }
 
@@ -288,11 +289,12 @@ const getStudytrackDataForTheYear = async ({
           return acc
         }, {})
 
-        otherCountriesCount[track]['Total'] = countryStatsFromYears
+        otherCountriesCount[track].Total = countryStatsFromYears
       }
 
       // If the track has no stats for that year, it should be removed from the table and dropdown options
       if (all.length === 0) {
+        // eslint-disable-next-line no-unused-expressions
         emptyTracks.has(track) ? emptyTracks.set(track, emptyTracks.get(track) + 1) : emptyTracks.set(track, 1)
         return
       }
@@ -319,7 +321,7 @@ const getStudytrackDataForTheYear = async ({
       // Count stats for the main studytrack table grouped by year
       mainStatsByYear[year] = [
         [
-          studytrackNames[track]?.name['fi'] ? `${studytrackNames[track]?.name['fi']}, ${track}` : year,
+          studytrackNames[track]?.name.fi ? `${studytrackNames[track]?.name.fi}, ${track}` : year,
           ...getStats(
             all,
             started,
@@ -484,7 +486,7 @@ const getStudytrackStatsForStudyprogramme = async ({ studyprogramme, combinedPro
   const graduatedTitles = combinedProgramme ? getCorrectCombinedTitles() : tableTitles.studytracksBasic
   return {
     id: combinedProgramme ? `${studyprogramme}-${combinedProgramme}` : studyprogramme,
-    years: years,
+    years,
     ...data,
     doCombo,
     studytrackOptions,

services/backend/src/services/studyrights.js

@@ -3,10 +3,11 @@ const {
 } = require('../database/connection')
 const { Studyright, StudyrightElement, ElementDetail, Transfer } = require('../models')
 const moment = require('moment')
-const { redisClient } = require('../services/redis')
+const { redisClient } = require('./redis')
 const _ = require('lodash')
 const { Op, col, where, fn } = require('sequelize')
 const logger = require('../util/logger')
+
 const REDIS_KEY = 'STUDYRIGHT_ASSOCIATIONS_V2'
 
 const byStudent = studentNumber => {
@@ -107,9 +108,8 @@ const allUniqueStudyrightCodeArrays = () =>
 const uniqueStudyrightAssocations = elementcodes => {
   if (elementcodes === undefined) {
     return allUniqueStudyrightCodeArrays()
-  } else {
-    return uniqueStudyrightCodeArrays(elementcodes)
   }
+  return uniqueStudyrightCodeArrays(elementcodes)
 }
 
 const getAssociatedStudyrights = async elementcodes => {
@@ -199,7 +199,7 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
     return 'FALL'
   }
   const getEnrollmentStartYear = momentstartdate => {
-    if (getSemester(momentstartdate) == 'SPRING') return momentstartdate.utc(2).year() - 1
+    if (getSemester(momentstartdate) === 'SPRING') return momentstartdate.utc(2).year() - 1
     return momentstartdate.year()
   }
   const total = await Studyright.count()
@@ -221,15 +221,15 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
           }
         } else if (type === StudyRightType.PROGRAMME) {
           associations.programmes[code] = associations.programmes[code] || {
-            type: type,
-            name: name,
-            code: code,
+            type,
+            name,
+            code,
             enrollmentStartYears: {},
             studytracks: [],
           }
           const momentstartdate = moment(startdate)
           const enrollment = momentstartdate.isValid() ? getEnrollmentStartYear(momentstartdate) : null
-          const enrollmentStartYears = associations.programmes[code].enrollmentStartYears
+          const { enrollmentStartYears } = associations.programmes[code]
           if (!enrollmentStartYears[enrollment])
             enrollmentStartYears[enrollment] = {
               studyTracks: {},
@@ -269,9 +269,9 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
                   programmes: {},
                 }
                 associations.studyTracks[e.code].programmes[code] = {
-                  type: type,
-                  name: name,
-                  code: code,
+                  type,
+                  name,
+                  code,
                 }
               }
             })
@@ -304,9 +304,8 @@ const getAssociations = async (doRefresh = false) => {
     const associations = await calculateAssociationsFromDb()
     await saveAssociationsToRedis(associations)
     return associations
-  } else {
-    return studyrights
   }
+  return studyrights
 }
 
 const getFilteredAssociations = async codes => {

services/backend/src/services/tags.js

@@ -1,16 +1,5 @@
-const Sequelize = require('sequelize')
-const { Tag } = require('../models/models_kone')
-
-const Op = Sequelize.Op
-
-const findTagById = tag_id =>
-  Tag.findOne({
-    where: {
-      tag_id,
-    },
-  })
-
-const findTags = () => Tag.findAll({})
+const { Op } = require('sequelize')
+const { Tag, TagStudent } = require('../models/models_kone')
 
 const findTagsByStudytrack = async studytrack => {
   return Tag.findAll({
@@ -34,7 +23,7 @@ const findTagsFromStudytrackById = async (studytrack, tag_ids) => {
 }
 
 const createNewTag = async tag => {
-  if (isNaN(tag.year)) {
+  if (Number.isNaN(tag.year)) {
     const newTag = {
       ...tag,
       year: null,
@@ -54,11 +43,43 @@ const deleteTag = async tag => {
   })
 }
 
+const getStudentTagsByStudytrack = studytrack => {
+  return TagStudent.findAll({
+    include: {
+      model: Tag,
+      attributes: ['tag_id', 'tagname', 'personal_user_id'],
+      where: {
+        studytrack: {
+          [Op.eq]: studytrack,
+        },
+      },
+    },
+  })
+}
+
+const createMultipleStudentTags = async tags => {
+  return TagStudent.bulkCreate(tags, { ignoreDuplicates: true })
+}
+
+const deleteMultipleStudentTags = async (tagId, studentnumbers) => {
+  return TagStudent.destroy({
+    where: {
+      tag_id: {
+        [Op.eq]: tagId,
+      },
+      studentnumber: {
+        [Op.in]: studentnumbers,
+      },
+    },
+  })
+}
+
 module.exports = {
-  findTagById,
   createNewTag,
-  findTags,
   findTagsByStudytrack,
   findTagsFromStudytrackById,
   deleteTag,
+  getStudentTagsByStudytrack,
+  createMultipleStudentTags,
+  deleteMultipleStudentTags,
 }

services/backend/src/services/tagstudent.js

@@ -1,74 +0,0 @@
-const Sequelize = require('sequelize')
-const { TagStudent, Tag } = require('../models/models_kone')
-
-const Op = Sequelize.Op
-
-const getStudentTags = () => TagStudent.findAll({})
-
-const getStudentTagsByStudytrack = studytrack => {
-  return TagStudent.findAll({
-    include: {
-      model: Tag,
-      attributes: ['tag_id', 'tagname', 'personal_user_id'],
-      where: {
-        studytrack: {
-          [Op.eq]: studytrack,
-        },
-      },
-    },
-  })
-}
-
-const getStudentTagsByStudentnumber = async studentnumber => {
-  return TagStudent.findAll({
-    where: {
-      studentnumber: {
-        [Op.eq]: studentnumber,
-      },
-    },
-    include: {
-      model: Tag,
-      attributes: ['tag_id', 'tagname', 'personal_user_id'],
-    },
-  })
-}
-
-const createStudentTag = async tag => {
-  return TagStudent.create(tag)
-}
-
-const createMultipleStudentTags = async tags => {
-  return TagStudent.bulkCreate(tags, { ignoreDuplicates: true })
-}
-
-const deleteMultipleStudentTags = async (tagId, studentnumbers) => {
-  return TagStudent.destroy({
-    where: {
-      tag_id: {
-        [Op.eq]: tagId,
-      },
-      studentnumber: {
-        [Op.in]: studentnumbers,
-      },
-    },
-  })
-}
-
-const deleteStudentTag = async (studentnumber, tag_id) => {
-  return TagStudent.destroy({
-    where: {
-      tag_id,
-      studentnumber,
-    },
-  })
-}
-
-module.exports = {
-  getStudentTags,
-  getStudentTagsByStudentnumber,
-  getStudentTagsByStudytrack,
-  createStudentTag,
-  createMultipleStudentTags,
-  deleteStudentTag,
-  deleteMultipleStudentTags,
-}

services/backend/src/services/teachers.js

@@ -76,7 +76,8 @@ const markCredit = (stats, passed, failed, credits, transferred) => {
       passed: stats.passed + 1,
       transferred: transferred ? stats.transferred + credits : stats.transferred,
     }
-  } else if (failed) {
+  }
+  if (failed) {
     return {
       ...stats,
       failed: stats.failed + 1,
@@ -134,6 +135,8 @@ const markCreditForCourse = (courses, credit) => {
   }
 }
 
+const isRegularCourse = credit => !credit.isStudyModule
+
 const teacherStats = async teacherid => {
   const teacher = await findTeacherCredits(teacherid)
   const statistics = teacher.credits.filter(isRegularCourse).reduce(
@@ -230,8 +233,6 @@ const getCredits = (teacherIds, semestercodeStart, semestercodeEnd) =>
     },
   })
 
-const isRegularCourse = credit => !credit.isStudyModule
-
 const calculateCreditStatistics = credits =>
   credits.reduce(
     (stats, credit) => {

services/backend/src/services/topteachers.js

@@ -1,4 +1,4 @@
-const { redisClient } = require('../services/redis')
+const { redisClient } = require('./redis')
 const { getSemestersAndYears, getCurrentSemester } = require('./semesters')
 const { Teacher, Semester, Credit, Course } = require('../models')
 const { Op } = require('sequelize')
@@ -78,14 +78,14 @@ const updatedStats = (statistics, teacher, passed, failed, credits, transferred)
       credits: transferred ? stats.credits : stats.credits + credits,
       transferred: transferred ? stats.transferred + credits : stats.transferred,
     }
-  } else if (failed) {
+  }
+  if (failed) {
     return {
       ...stats,
       failed: stats.failed + 1,
     }
-  } else {
-    return stats
   }
+  return stats
 }
 
 const isRegularCourse = credit => !credit.isStudyModule
@@ -130,19 +130,19 @@ const findTopTeachers = async yearcode => {
   }
 }
 
+const findAndSaveTopTeachers = async yearcode => {
+  const { all, openuni } = await findTopTeachers(yearcode)
+  await setTeacherStats(ID.OPENUNI, yearcode, openuni)
+  await setTeacherStats(ID.ALL, yearcode, all)
+}
+
 const findAndSaveTeachers = async (startcode = 1, to) => {
   const endcode = to || (await getCurrentSemester()).getDataValue('yearcode')
   for (let code = startcode; code <= endcode; code++) {
     await findAndSaveTopTeachers(code)
     logger.info(`Teacher leaderboard for yearcode ${code} calculated`)
   }
-  logger.info(`Teacher leaderboard calculations done`)
-}
-
-const findAndSaveTopTeachers = async yearcode => {
-  const { all, openuni } = await findTopTeachers(yearcode)
-  await setTeacherStats(ID.OPENUNI, yearcode, openuni)
-  await setTeacherStats(ID.ALL, yearcode, all)
+  logger.info('Teacher leaderboard calculations done')
 }
 
 module.exports = {

services/backend/src/services/userService.js

@@ -3,12 +3,12 @@ const LRU = require('lru-cache')
 const _ = require('lodash')
 
 const { getStudentnumbersByElementdetails } = require('./students')
-const { facultiesAndProgrammesForTrends } = require('../services/organisations')
-const { sendNotificationAboutNewUser } = require('../services/mailservice')
-const { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } = require('../services/studyGuidanceGroups')
-const { User, UserElementDetails, AccessGroup, UserFaculties, sequelizeUser } = require('../models/models_user')
+const { sendNotificationAboutNewUser } = require('./mailservice')
+const { checkStudyGuidanceGroupsAccess, getAllStudentsUserHasInGroups } = require('./studyGuidanceGroups')
+const { User, UserElementDetails, AccessGroup } = require('../models/models_user')
 const { getUserIams, getAllUserAccess, getUserIamAccess } = require('../util/jami')
-const { createLocaleComparator } = require('../util/utils')
+const { createLocaleComparator, getFullStudyProgrammeRights } = require('../util/utils')
+const { sequelizeUser } = require('../database/connection')
 
 const courseStatisticsGroup = 'grp-oodikone-basic-users'
 const facultyStatisticsGroup = 'grp-oodikone-users'
@@ -32,11 +32,6 @@ const userIncludes = [
     as: 'accessgroup',
     attributes: ['id', 'group_code', 'group_info'],
   },
-  {
-    separate: true,
-    model: UserFaculties,
-    as: 'faculty',
-  },
 ]
 
 const accessGroupsByCodes = codes =>
@@ -64,13 +59,33 @@ const byId = async id =>
     include: userIncludes,
   })
 
-// Crud things from old userservice
+const deleteOutdatedUsers = async () =>
+  await sequelizeUser.transaction(async t => {
+    const outdatedUsersCTE = `
+      WITH outdated_users AS (
+        SELECT id FROM users
+        WHERE last_login IS NULL OR last_login < CURRENT_DATE - INTERVAL '18 months'
+      )`
 
-// some fields that frontend needs
-const formatUserForAdminView = user => ({
-  ...user.get(),
-  elementdetails: user.programme.map(p => p.elementDetailCode),
-})
+    await sequelizeUser.query(
+      `${outdatedUsersCTE} DELETE FROM user_faculties WHERE "userId" IN (SELECT id FROM outdated_users)`,
+      { transaction: t }
+    )
+    await sequelizeUser.query(
+      `${outdatedUsersCTE} DELETE FROM user_accessgroup WHERE "userId" IN (SELECT id FROM outdated_users)`,
+      { transaction: t }
+    )
+    await sequelizeUser.query(
+      `${outdatedUsersCTE} DELETE FROM user_elementdetails WHERE "userId" IN (SELECT id FROM outdated_users)`,
+      { transaction: t }
+    )
+    const [results] = await sequelizeUser.query(
+      `${outdatedUsersCTE} DELETE FROM users WHERE id IN (SELECT id FROM outdated_users) RETURNING id`,
+      { transaction: t }
+    )
+
+    return results.length
+  })
 
 const addProgrammes = async (id, codes) => {
   for (const code of codes) {
@@ -96,6 +111,7 @@ const modifyRights = async (username, rights) => {
       if (val === true) {
         return code
       }
+      return undefined
     })
     .filter(code => code)
   const rightsToRemove = Object.entries(rights)
@@ -103,6 +119,7 @@ const modifyRights = async (username, rights) => {
       if (val === false) {
         return code
       }
+      return undefined
     })
     .filter(code => code)
 
@@ -119,33 +136,22 @@ const modifyAccess = async body => {
   const { username, accessgroups } = body
   await modifyRights(username, accessgroups)
   userDataCache.del(username)
-  return formatUserForAdminView(await byUsername(username))
+  const updatedUser = await byUsername(username)
+  return updatedUser
 }
 
 const enableElementDetails = async (id, codes) => {
   await addProgrammes(id, codes)
   const user = await byId(id)
   userDataCache.del(user.username)
-  return formatUserForAdminView(user)
-}
-
-const setFaculties = async (id, faculties) => {
-  await sequelizeUser.transaction(async transaction => {
-    await UserFaculties.destroy({ where: { userId: id }, transaction })
-    for (const faculty of faculties) {
-      await UserFaculties.create({ userId: id, faculty_code: faculty }, { transaction })
-    }
-  })
-  const user = await byId(id)
-  userDataCache.del(user.username)
-  return formatUserForAdminView(user)
+  return user
 }
 
 const removeElementDetails = async (id, codes) => {
   await removeProgrammes(id, codes)
   const user = await byId(id)
   userDataCache.del(user.userId)
-  return formatUserForAdminView(user)
+  return user
 }
 
 const updateUser = async (username, fields) => {
@@ -158,21 +164,87 @@ const updateUser = async (username, fields) => {
 
   userDataCache.del(username)
 
-  return formatUserForAdminView(await byUsername(username))
+  const updatedUser = await byUsername(username)
+  return updatedUser
 }
 
 const getAccessGroups = async () => await AccessGroup.findAll()
 
+const getStudyProgrammeRights = (iamAccess, specialGroup, userProgrammes) => {
+  const iamBasedProgrammes = []
+
+  for (const [programmeCode, rights] of Object.entries(iamAccess || {})) {
+    if (specialGroup.kosu || rights.admin) {
+      iamBasedProgrammes.push({ code: programmeCode, limited: false, isIamBased: true })
+    } else {
+      iamBasedProgrammes.push({ code: programmeCode, limited: true, isIamBased: true })
+    }
+  }
+
+  return [
+    ...iamBasedProgrammes,
+    ...userProgrammes.map(({ elementDetailCode }) => ({
+      code: elementDetailCode,
+      limited: false,
+      isIamBased: false,
+    })),
+  ]
+}
+
+const formatUser = async (userFromDb, extraRights, programmeRights, getStudentAccess = true) => {
+  const accessGroups = userFromDb.accessgroup.map(entry => entry.group_code)
+
+  const fullStudyProgrammeRights = getFullStudyProgrammeRights(programmeRights)
+
+  const {
+    dataValues: {
+      id,
+      username: userId,
+      full_name: name,
+      email,
+      language,
+      sisu_person_id: sisPersonId,
+      iam_groups: iamGroups,
+    },
+  } = userFromDb
+
+  const studentsUserCanAccess = getStudentAccess
+    ? _.uniqBy(
+        (
+          await Promise.all([
+            getStudentnumbersByElementdetails(fullStudyProgrammeRights),
+            getAllStudentsUserHasInGroups(sisPersonId),
+          ])
+        ).flat()
+      )
+    : []
+
+  // attribute naming is a bit confusing, but it's used so widely in oodikone, that it's not probably worth changing
+  return {
+    id, // our internal id, not the one from sis, (e.g. 101)
+    userId, // acually username (e.g. mluukkai)
+    name,
+    language,
+    sisPersonId, //
+    iamGroups,
+    email,
+    roles: accessGroups,
+    studentsUserCanAccess, // studentnumbers used in various parts of backend. For admin this is usually empty, since no programmes / faculties are set.
+    isAdmin: accessGroups.includes('admin'),
+    programmeRights,
+  }
+}
+
 const updateAccessGroups = async (username, iamGroups = [], specialGroup = {}, sisId, user) => {
-  const { jory, hyOne, superAdmin, openUni, katselmusViewer } = specialGroup
+  const { jory, hyOne, superAdmin, openUni, katselmusViewer, fullSisuAccess } = specialGroup
 
   const userFromDb = user || (await byUsername(username))
-  const formattedUser = await formatUser(userFromDb, [], Boolean(sisId))
+  const formattedUser = await formatUser(userFromDb, [], [], Boolean(sisId))
 
   const { roles: currentAccessGroups } = formattedUser
 
   // Modify accessgroups based on current groups and iamGroups
-  let newAccessGroups = []
+  const newAccessGroups = []
   if (currentAccessGroups.includes('studyGuidanceGroups') || (sisId && (await checkStudyGuidanceGroupsAccess(sisId))))
     newAccessGroups.push('studyGuidanceGroups')
   if (iamGroups.includes(courseStatisticsGroup)) newAccessGroups.push('courseStatistics')
@@ -181,6 +253,7 @@ const updateAccessGroups = async (username, iamGroups = [], specialGroup = {}, s
   if (superAdmin || currentAccessGroups.includes('admin')) newAccessGroups.push('admin')
   if (openUni) newAccessGroups.push('openUniSearch')
   if (katselmusViewer) newAccessGroups.push('katselmusViewer')
+  if (fullSisuAccess) newAccessGroups.push('fullSisuAccess')
 
   const accessGroups = await AccessGroup.findAll({
     where: {
@@ -202,16 +275,14 @@ const updateAccessGroups = async (username, iamGroups = [], specialGroup = {}, s
   return { newAccessGroups, accessGroups }
 }
 
-// newer logic
-const enrichProgrammesFromFaculties = faculties =>
-  facultiesAndProgrammesForTrends.filter(f => faculties.includes(f.faculty_code)).map(f => f.programme_code)
-
 const findAll = async () => {
   const allUsers = await User.findAll({
     include: userIncludes,
   })
 
-  const userAccess = await getAllUserAccess()
+  const userIds = allUsers.map(user => user.dataValues.sisu_person_id)
+
+  const userAccess = await getAllUserAccess(userIds)
 
   const userAccessMap = userAccess.reduce((obj, cur) => {
     obj[cur.id] = cur
@@ -220,18 +291,15 @@ const findAll = async () => {
 
   const formattedUsers = await Promise.all(
     allUsers.map(async user => {
-      const { iamGroups, specialGroup } = userAccessMap[user.sisu_person_id] || {}
+      const { iamGroups, specialGroup, access } = userAccessMap[user.sisu_person_id] || {}
 
       const { accessGroups } = await updateAccessGroups(user.username, iamGroups, specialGroup, null, user)
 
       return {
         ...user.get(),
         iam_groups: iamGroups || [],
-        elementdetails: _.uniqBy([
-          ...user.programme.map(p => p.elementDetailCode),
-          ...enrichProgrammesFromFaculties(user.faculty.map(f => f.faculty_code)),
-        ]),
         accessgroup: accessGroups,
+        programmeRights: getStudyProgrammeRights(access, specialGroup, user.programme),
       }
     })
   )
@@ -248,63 +316,15 @@ const findOne = async id => {
 
   user.iamGroups = userIams
 
-  const { iamAccess, specialGroup } = await getUserIamAccess(user)
+  const { iamAccess, specialGroup } = await getUserIamAccess(user.sisu_person_id, user.iamGroups)
 
   const { accessGroups } = await updateAccessGroups(user.username, userIams, specialGroup)
 
   return {
     ...user.get(),
     iam_groups: iamAccess || [],
-    elementdetails: _.uniqBy([
-      ...user.programme.map(p => p.elementDetailCode),
-      ...enrichProgrammesFromFaculties(user.faculty.map(f => f.faculty_code)),
-    ]),
     accessgroup: accessGroups,
-  }
-}
-
-const formatUser = async (userFromDb, extraRights, getStudentAccess = true) => {
-  const [accessGroups, programmes, faculties] = [
-    ['accessgroup', 'group_code'],
-    ['programme', 'elementDetailCode'],
-    ['faculty', 'faculty_code'],
-  ].map(([field, code]) => userFromDb[field].map(entry => entry[code]))
-
-  const rights = _.uniqBy([...programmes, ...enrichProgrammesFromFaculties(faculties)]).concat(extraRights)
-
-  const {
-    dataValues: {
-      id,
-      username: userId,
-      full_name: name,
-      email,
-      language,
-      sisu_person_id: sisPersonId,
-      iam_groups: iamGroups,
-    },
-  } = userFromDb
-
-  const studentsUserCanAccess = getStudentAccess
-    ? _.uniqBy(
-        (
-          await Promise.all([getStudentnumbersByElementdetails(rights), getAllStudentsUserHasInGroups(sisPersonId)])
-        ).flat()
-      )
-    : []
-
-  // attribute naming is a bit confusing, but it's used so widely in oodikone, that it's not probably worth changing
-  return {
-    id, // our internal id, not the one from sis, (e.g. 101)
-    userId, // acually username (e.g. mluukkai)
-    name,
-    language,
-    sisPersonId, //
-    iamGroups,
-    email,
-    rights,
-    roles: accessGroups,
-    studentsUserCanAccess, // studentnumbers used in various parts of backend. For admin this is usually empty, since no programmes / faculties are set.
-    isAdmin: accessGroups.includes('admin'),
+    programmeRights: getStudyProgrammeRights(iamAccess, specialGroup, user.programme),
   }
 }
 
@@ -313,7 +333,7 @@ const getOrganizationAccess = async user => {
 
   if (user.iamGroups.length === 0) return {}
 
-  const { iamAccess, specialGroup } = await getUserIamAccess(user)
+  const { iamAccess, specialGroup } = await getUserIamAccess(user.sisu_person_id ?? user.id, user.iamGroups)
 
   const access = {}
   if (_.isObject(iamAccess)) {
@@ -345,7 +365,9 @@ const getMockedUser = async ({ userToMock, mockedBy }) => {
 
   const iamRights = Object.keys(access)
 
-  const formattedUser = await formatUser(userFromDb, specialGroup.kosu ? iamRights : [])
+  const programmeRights = getStudyProgrammeRights(access, specialGroup, userFromDb.programme)
+
+  const formattedUser = await formatUser(userFromDb, specialGroup.kosu ? iamRights : [], programmeRights)
 
   const toReturn = {
     ...formattedUser,
@@ -359,7 +381,7 @@ const getMockedUser = async ({ userToMock, mockedBy }) => {
   return toReturn
 }
 
-const getUser = async ({ username, name, email, iamGroups, iamRights, specialGroup, sisId }) => {
+const getUser = async ({ username, name, email, iamGroups, iamRights, specialGroup, sisId, access }) => {
   if (userDataCache.has(username)) {
     return userDataCache.get(username)
   }
@@ -378,7 +400,8 @@ const getUser = async ({ username, name, email, iamGroups, iamRights, specialGro
 
   const userFromDb = await byUsername(username)
   userFromDb.iamGroups = await getUserIams(userFromDb.sisu_person_id)
-  const formattedUser = await formatUser(userFromDb, specialGroup.kosu ? iamRights : [])
+  const programmeRights = getStudyProgrammeRights(access, specialGroup, userFromDb.programme)
+  const formattedUser = await formatUser(userFromDb, specialGroup.kosu ? iamRights : [], programmeRights)
 
   const { newAccessGroups } = await updateAccessGroups(username, iamGroups, specialGroup, sisId)
 
@@ -402,8 +425,8 @@ module.exports = {
   findOne,
   modifyAccess,
   getAccessGroups,
-  setFaculties,
   getUser,
   getMockedUser,
   getOrganizationAccess,
+  deleteOutdatedUsers,
 }

services/backend/src/util/jami.js

@@ -11,15 +11,14 @@ const jamiClient = axios.create({
   },
 })
 
-const getUserIamAccess = async (user, attempt = 1) => {
-  if (user.iamGroups.length === 0) return {}
-
-  const { id, iamGroups } = user
+const getUserIamAccess = async (sisPersonId, iamGroups, attempt = 1) => {
+  if (iamGroups.length === 0) return {}
 
   try {
     const { data: iamAccess } = await jamiClient.post('/', {
-      userId: id,
+      userId: sisPersonId,
       iamGroups,
+      getSisuAccess: true,
     })
 
     const { specialGroup } = iamAccess
@@ -34,7 +33,7 @@ const getUserIamAccess = async (user, attempt = 1) => {
       return {}
     }
 
-    return getUserIamAccess(user, attempt + 1)
+    return getUserIamAccess(sisPersonId, iamGroups, attempt + 1)
   }
 }
 
@@ -53,8 +52,8 @@ const getUserIams = async userId => {
   }
 }
 
-const getAllUserAccess = async () => {
-  const { data } = await jamiClient.get('/all-access')
+const getAllUserAccess = async userIds => {
+  const { data } = await jamiClient.post('access-and-special-groups', { userIds })
 
   return data
 }

services/backend/src/util/logger.js

@@ -1,12 +1,12 @@
 const os = require('os')
-
 const winston = require('winston')
 const { WinstonGelfTransporter } = require('winston-gelf-transporter')
 
 const { isProduction } = require('../conf-backend')
+
 const { combine, timestamp, printf, splat } = winston.format
 
-let transports = []
+const transports = []
 
 const formatDate = timestamp => {
   if (!isProduction) return new Date(timestamp).toLocaleTimeString('fi-FI').replace(' klo', '')