Merge pull request #2563 from UniversityOfHelsinkiCS/trunk

Update Sentry conf

.circleci/config.yml

@@ -94,11 +94,11 @@ jobs:
       - deploy:
           command: |
             if [ "${DEPLOY}" == "true" ]; then
-              docker exec --env SENTRY_URL --env SENTRY_AUTH_TOKEN --env SENTRY_ORG=sentry --env SENTRY_PROJECT=oodifront frontend \
+              docker exec --env SENTRY_URL --env SENTRY_AUTH_TOKEN --env SENTRY_ORG=sentry --env SENTRY_PROJECT=oodikone frontend \
                 sh +e -c 'npm run sentry-cli -- releases info "$(cat /SENTRY_RELEASE_VERSION)" && \
                 npm run sentry-cli -- releases delete "$(cat /SENTRY_RELEASE_VERSION)" || true'
 
-              docker exec --env SENTRY_URL --env SENTRY_AUTH_TOKEN --env SENTRY_ORG=sentry --env SENTRY_PROJECT=oodifront frontend \
+              docker exec --env SENTRY_URL --env SENTRY_AUTH_TOKEN --env SENTRY_ORG=sentry --env SENTRY_PROJECT=oodikone frontend \
                 sh -c 'npm run sentry-cli -- releases new --finalize "$(cat /SENTRY_RELEASE_VERSION)" && \
                 npm run sentry-cli -- releases files "$(cat /SENTRY_RELEASE_VERSION)" upload-sourcemaps --rewrite dist'
             fi

.gitignore

@@ -1,3 +1,4 @@
+**/node_modules
 data
 pgdata
 pgdata_kone

README.md

@@ -19,7 +19,7 @@ To run Oodikone locally, you will need the following:
 Launch the CLI and follow the instructions:
 
 ```bash
-./oodikone.sh
+npm run cli
 ```
 
 *Please use a terminal at least 80 characters wide, the CLI is a bit rudimentary* 😊
@@ -57,8 +57,6 @@ The development environment is entirely configured in the docker-compose.yml fil
 
 The mapping of ports and environment variables are also defined in the docker-compose file. You can start, stop and manage the development environment by running the following commands from a terminal shell in this directory.
 
-## Usage
-
 ### Start the development environment
 
 With anonymized data:
@@ -175,6 +173,28 @@ npm run cypress:run
 npm run cypress:open
 ```
 
+## How to Updater
+
+Before running updater you must run `importer-db` setup via the CLI (see [Installation](#Installation)).
+
+If you want to use the control panel UI, first start Oodikone in real data mode:
+
+```bash
+npm run docker:up:real
+```
+
+Followed by acual updater:
+
+```bash
+npm run updater:dev:up
+```
+
+When you are done, close hanging containers with:
+
+```bash
+npm run updater:dev:down
+```
+
 ## Other commands
 
 ### How to `scp` backup files from oodikone via melkki-proxy

cypress/integration/Course_population.js

@@ -10,7 +10,7 @@ describe('Course population tests', () => {
   it('Can find course population', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by entering a course code']").type('TKT20003')
+    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
     cy.contains("tr", "TKT20003").click()
     cy.contains("Fetch statistics").should('be.enabled').click()
 

cypress/integration/Course_statistics.js

@@ -10,7 +10,7 @@ describe('Course Statistics tests', () => {
   it('Searching single course having duplicate mappings shows course statistics', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by entering a course code']").type('TKT20003')
+    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
     cy.contains("tr", "TKT20003").click()
     cy.contains("Fetch statistics").should('be.enabled').click()
     cy.contains("Search for courses").should('not.exist')
@@ -40,7 +40,7 @@ describe('Course Statistics tests', () => {
   it('Searching multiple courses having duplicate mappings shows course statistics', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by entering a course code']").type('TKT')
+    cy.get("input[placeholder='Search by a course code']").type('TKT')
     cy.contains("td", "TKT20003").click()
     cy.contains("td", /^TKT10002/).click()
     cy.contains("Fetch statistics").should('be.enabled').click()
@@ -62,10 +62,10 @@ describe('Course Statistics tests', () => {
     cy.contains("Search for courses")
   })
 
-  it('Consecutive searches should not crash', () => {
+  it('On consecutive searches should not crash and search should work', () => {
     cy.url().should('include', '/coursestatistics')
     cy.contains("Search for courses")
-    cy.get("input[placeholder='Search by entering a course code']").type('TKT20003')
+    cy.get("input[placeholder='Search by a course code']").type('TKT20003')
     cy.contains("td", "TKT20003").click()
     cy.contains("Fetch statistics").should('be.enabled').click()
     cy.contains("Show population")
@@ -74,11 +74,25 @@ describe('Course Statistics tests', () => {
     cy.contains("Course statistics").click()
     cy.contains("Search for courses")
     //cy.wait(500)
-    cy.get("input[placeholder='Search by entering a course code']").type('TKT10001')
-    cy.contains("td", "TKT10001").click()
+    cy.get("input[placeholder='Search by a course code']").type('TKT20001')
+    cy.contains("td", "TKT20001").click()
     cy.contains("Fetch statistics").should('be.enabled').click()
     cy.contains("Show population")
-    cy.contains("TKT10001")
+    cy.contains("TKT20001")
+  })
+
+  it('Shows statistics when separating by semester', () => {
+    const courseCode = 'DATA11002'
+    cy.url().should('include', '/coursestatistics')
+    cy.contains("Search for courses")
+    cy.get("input[placeholder='Search by a course code']").type(courseCode)
+    cy.contains("td", courseCode).click()
+    cy.contains("Separate statistics for Spring and Fall semesters").click()
+    cy.contains("Fetch statistics").should('be.enabled').click()
+    cy.contains("Kevät 2018")
+    cy.contains("Syksy 2017")
+    cy.get("tbody").contains("19")
+    cy.contains(courseCode)
   })
 
   describe('When searching unified course stats', () => {
@@ -86,7 +100,7 @@ describe('Course Statistics tests', () => {
       cy.url().should('include', '/coursestatistics')
       cy.contains("Search for courses")
       cy.get("input[name='unifyOpenUniCourses']").parent().click()
-      cy.get("input[placeholder='Search by entering a course code']").type('TKT10002')
+      cy.get("input[placeholder='Search by a course code']").type('TKT10002')
       cy.contains("td", "TKT10002, AYTKT10002").click()
       cy.contains("Fetch statistics").should('be.enabled').click()
       cy.contains("Search for courses").should('not.exist')

cypress/integration/Population_statistics.js

@@ -1,187 +1,167 @@
 /// <reference types="Cypress" />
 
 const setPopStatsUntil = (until, includeSettings = []) => {
-  cy.contains("Advanced settings").siblings().get('[data-cy=advanced-toggle]').click()
-  includeSettings.forEach(setting => {
-    cy.contains("Advanced settings").parent().siblings().contains(setting).click()
-  })
-  cy.get(".adv-stats-until > .form-control").click().clear().type(until)
-  cy.contains("Fetch population with new settings").click()
   cy.contains("Advanced settings")
-}
+    .siblings()
+    .get("[data-cy=advanced-toggle]")
+    .click();
+  includeSettings.forEach((setting) => {
+    cy.contains("Advanced settings")
+      .parent()
+      .siblings()
+      .contains(setting)
+      .click();
+  });
+  cy.get(".adv-stats-until > .form-control").click().clear().type(until);
+  cy.contains("Fetch population with new settings").click();
+  cy.contains("Advanced settings");
+};
 
-describe('Population Statistics tests', () => {
+describe("Population Statistics tests", () => {
   beforeEach(() => {
-    cy.init()
-  })
+    cy.init();
+  });
 
   const checkAmountOfStudents = (assertion) => {
-    let students = 0
-    cy.contains("Credit accumulation").click().invoke('text').then((text) => {
-      students = Number(text.match(/\d+/g)[0])
-      expect(students).to.equal(assertion)
-    })
-    cy.contains("Credit accumulation").click().siblings().within(() => {
-      cy.get(".highcharts-series-group").find("path").should('have.length', students ? (students * 2) + 2 : 0) // For each student there should be 2 paths in the graph + 2 for the scrollbar
-    })
-  }
+    let students = 0;
+    cy.contains("Credit accumulation")
+      .click()
+      .invoke("text")
+      .then((text) => {
+        students = Number(text.match(/\d+/g)[0]);
+        expect(students).to.equal(assertion);
+      });
+  };
 
-  it('Population statistics search form is usable', () => {
-    cy.cs("navbar-studyProgramme").click()
-    cy.cs("navbar-class").click()
-    cy.contains("See population").should('be.disabled')
-    cy.url().should('include', '/populations')
-    cy.contains("Search for population")
-    cy.contains("Class of").parent().within(() => {
-      cy.get(".form-control").as("enrollmentSelect")
-    })
+  it("Population statistics search form is usable", () => {
+    cy.cs("navbar-studyProgramme").click();
+    cy.cs("navbar-class").click();
+    cy.contains("See population").should("be.disabled");
+    cy.url().should("include", "/populations");
+    cy.contains("Search for population");
+    cy.contains("Class of")
+      .parent()
+      .within(() => {
+        cy.get(".form-control").as("enrollmentSelect");
+      });
 
-    cy.get("@enrollmentSelect").its(`${[0]}.value`).then((beforeVal) => {
-      cy.get("@enrollmentSelect").click()
-      // go back to 2010-2019
-      cy.get(".yearSelectInput .rdtPrev").click({ force: true })
-      cy.get(".yearSelectInput table").contains("2014-2015").click({ force: true })
-      cy.get("@enrollmentSelect").should('not.have.value', beforeVal)
-    })
+    cy.get("@enrollmentSelect")
+      .its(`${[0]}.value`)
+      .then((beforeVal) => {
+        cy.get("@enrollmentSelect").click();
+        // go back to 2010-2019
+        cy.get(".yearSelectInput .rdtPrev").click({ force: true });
+        cy.get(".yearSelectInput table")
+          .contains("2014-2015")
+          .click({ force: true });
+        cy.get("@enrollmentSelect").should("not.have.value", beforeVal);
+      });
 
-    cy.contains("Select study programme").click().siblings().contains("Tietojenkäsittelytieteen koulutusohjelma").click()
-    cy.contains("Select degree").click().siblings().contains("Luonnontieteiden kandidaatti")
-  })
+    cy.contains("Select study programme")
+      .click()
+      .siblings()
+      .contains("Tietojenkäsittelytieteen koulutusohjelma")
+      .click();
+    cy.contains("Select degree")
+      .click()
+      .siblings()
+      .contains("Luonnontieteiden kandidaatti");
+  });
 
-  it('Population statistics is usable on general level', () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma")
-    setPopStatsUntil('September 2019')
+  // FIXME: re-enable when CI has mandatory courses in the db
+  it.skip("Population statistics is usable on general level", () => {
+    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
+    setPopStatsUntil("September 2019");
 
     cy.get(".card").within(() => {
-      cy.contains("Tietojenkäsittelytieteen maisteriohjelma")
-      cy.contains("Sample size: 29 students")
-      cy.contains("Excludes exchange students")
-      cy.contains("Excludes students who haven't enrolled present nor absent")
-    })
-    cy.contains("Courses of population").click({ force: true })
-    cy.contains("Courses of population").parent().within(() => {
-      cy.get("tr").its('length').should('be.gte', 10)
-      cy.route('/api/v3/courseyearlystats**').as('coursePage')
-      cy.contains("Laskennan mallit")
-      cy.get(':nth-child(2) > .iconCell > p > .item > .level').click({ force: true })
-      cy.wait('@coursePage')
-      cy.url().should('include', '/coursestatistics')
-    })
-    cy.contains("TKT20005")
-    cy.go("back")
-    cy.contains("Ohjelmoinnin perusteet")
-    cy.contains("Courses of population").click().parent().within(() => {
-      cy.contains("Ohjelmoinnin perusteet").siblings().eq(4).should("have.text", "15")
-    })
+      cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
+      cy.contains("Sample size: 29 students");
+      cy.contains("Excludes exchange students");
+      cy.contains("Excludes students who haven't enrolled present nor absent");
+    });
+    cy.contains("Courses of population").click({ force: true });
 
-    checkAmountOfStudents(29)
+    cy.route("/api/v3/courseyearlystats**").as("coursePage");
+    cy.wait(150);
+    cy.cs("expand-CSM10000").click();
+    cy.cs("coursestats-link-CSM12101").click();
+    cy.wait("@coursePage");
+    cy.url().should("include", "/coursestatistics");
+    cy.contains("CSM12101");
+  });
 
-    let filteredStudents = 1328493
-    cy.contains("Credit statistics").click()
-    cy.contains("Credits Gained During First").parentsUntil(".tab").get("table").within(() => {
-      cy.get("tr").eq(2).find("td").eq(2).invoke("text").then(text => filteredStudents = Number(text))
-      cy.route('POST', '/api/v2/populationstatistics/courses**').as('courseData')
-      cy.get("tr").eq(2).find('.filter').click()
-      cy.wait('@courseData')
-    }).then(() => {
-      checkAmountOfStudents(filteredStudents)
-    })
-    cy.contains("Courses of population").click({ force: true })
-    cy.contains("Courses of population").parent().within(() => {
-      cy.contains("Ohjelmoinnin perusteet").siblings().eq(5).should("have.text", "0")
-    })
-
-    cy.contains("Students (1)").click()
-    cy.contains("Student names hidden").click()
-    cy.contains("Luoto").siblings().eq(2).click()
-    cy.contains("Luoto").invoke('text').then((text) => expect(text).to.equal('Luoto Veli-Matti, 014824094'))
-  })
-
-  it('Student list checking works as intended', () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma")
-    cy.contains("Students (29)").click()
-    cy.contains("010111264")
-    cy.contains("666666666").should('not.exist')
-    cy.contains('button', "Check studentnumbers").click()
-    cy.contains('Check for studentnumbers')
-    cy.get('textarea').type("010111264").type('{enter}').type("666666666")
-    cy.contains('button', 'check students').click()
-    cy.contains('#checkstudentsresults', 'Results').within(e => {
-      cy.contains('Student numbers in list and in oodi').click()
-      cy.contains('#found', '010111264')
-      cy.contains('Student numbers in list but not in oodi').click()
-      cy.contains('#notfound', '666666666')
-      cy.contains('Student numbers in oodi but not in list').click()
-      cy.contains('#notsearched', '010533091')
-    })
-  })
-
-  it('Population statistics wont crash course population', () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma")
-    cy.contains("Courses of population").click({ force: true })
-    cy.get(':nth-child(3) > .iconCell > p > .item > .level').click({ force: true })
-    cy.get(':nth-child(3) > :nth-child(1) > div > .item > .level').click({ force: true })
-  })
-
-  it("Empty 'mandatory courses' tab has a link to the page where mandatory courses can be added", () => {
-    cy.selectStudyProgramme("Kielten kandiohjelma")
-    cy.contains("Students (5)").click()
-    cy.get("[data-cy=student-table-tabs]").contains("Mandatory Courses").click()
-    cy.contains("No mandatory courses defined. You can define them here.").find("a").click()
-    cy.contains("Kielten kandiohjelma")
-    cy.contains("Add courses")
-  })
+  it("Student list checking works as intended", () => {
+    cy.selectStudyProgramme("Tietojenkäsittelytieteen maisteriohjelma");
+    cy.contains("Students (29)").click();
+    cy.contains("010111264");
+    cy.contains("666666666").should("not.exist");
+    cy.contains("button", "Check studentnumbers").click();
+    cy.contains("Check for studentnumbers");
+    cy.get("textarea").type("010111264").type("{enter}").type("666666666");
+    cy.contains("button", "check students").click();
+    cy.contains("#checkstudentsresults", "Results").within((e) => {
+      cy.contains("Student numbers in list and in oodi").click();
+      cy.contains("#found", "010111264");
+      cy.contains("Student numbers in list but not in oodi").click();
+      cy.contains("#notfound", "666666666");
+      cy.contains("Student numbers in oodi but not in list").click();
+      cy.contains("#notsearched", "010533091");
+    });
+  });
 
   it("Empty 'tags' tab has a link to the page where tags can be created", () => {
-    cy.selectStudyProgramme("Kielten kandiohjelma")
-    cy.contains("Students (5)").click()
-    cy.get("[data-cy=student-table-tabs]").contains("Mandatory Courses").siblings().contains("Tags").click()
-    cy.contains("No tags defined. You can define them here.").find("a").click()
-    cy.contains("Kielten kandiohjelma")
-    cy.contains("Create new tag")
-  })
-  
-  it("Advanced settings work", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma")
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.contains('Statistics until')
-    // only spring
-    cy.cs('toggle-fall').click()
-    cy.contains('Fetch population').click()
+    cy.selectStudyProgramme("Kielten kandiohjelma");
+    cy.contains("Students (5)").click();
+    cy.get("[data-cy=student-table-tabs]").contains("Tags").click();
+    cy.contains("No tags defined. You can define them here.").find("a").click();
+    cy.contains("Kielten kandiohjelma");
+    cy.contains("Create new tag");
+  });
 
-    cy.contains('Credit accumulation (for 13 students)')
+  it("Advanced settings work", () => {
+    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
+    cy.get("[data-cy=advanced-toggle]").click();
+    cy.contains("Statistics until");
+    // only spring
+    cy.cs("toggle-fall").click();
+    cy.contains("Fetch population").click();
+
+    cy.contains("Credit accumulation (for 13 students)");
 
     // only fall
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.cs('toggle-fall').click()
-    cy.cs('toggle-spring').click()
+    cy.get("[data-cy=advanced-toggle]").click();
+    cy.cs("toggle-fall").click();
+    cy.cs("toggle-spring").click();
 
-    cy.contains('Fetch population').click()
+    cy.contains("Fetch population").click();
 
     // FIXME: This fails randomly both locally and on CI.
-    cy.contains('Credit accumulation (for 206 students)')
+    cy.contains("Credit accumulation (for 206 students)");
 
     // spring + fall and include cancelled
-    cy.get('[data-cy=advanced-toggle]').click()
-    cy.cs('toggle-spring').click()
-    cy.cs('toggle-cancelled').click()
-    cy.contains('Fetch population').click()
+    cy.get("[data-cy=advanced-toggle]").click();
+    cy.cs("toggle-spring").click();
+    cy.cs("toggle-cancelled").click();
+    cy.contains("Fetch population").click();
+
+    cy.contains("Credit accumulation (for 228 students)");
+  });
 
-    cy.contains('Credit accumulation (for 228 students)')
-  })
-  
   it("Credit Statistics, Statistics pane works", () => {
-    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma")
-    cy.contains("Credit statistics").click()
-    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click()
+    cy.selectStudyProgramme("Tietojenkäsittelytieteen kandiohjelma");
+    cy.contains("Credit statistics").click();
+    cy.get("[data-cy='credit-stats-tab'] > .menu > :nth-child(2)").click();
 
-    cy.get("[data-cy='credit-stats-table-name-header']").should('contain', 'Statistic for n = 219 Students')
-    cy.get("[data-cy='credit-stats-mean']").should('contain', '44.36')
-    cy.get("[data-cy='credit-stats-stdev']").should('contain', '30.77')
-    cy.get("[data-cy='credit-stats-min']").should('contain', '0')
-    cy.get("[data-cy='credit-stats-q1']").should('contain', '17')
-    cy.get("[data-cy='credit-stats-q2']").should('contain', '46')
-    cy.get("[data-cy='credit-stats-q3']").should('contain', '66')
-    cy.get("[data-cy='credit-stats-max']").should('contain', '137')
-  })
-})
+    cy.get("[data-cy='credit-stats-table-name-header']").should(
+      "contain",
+      "Statistic for n = 219 Students"
+    );
+    cy.get("[data-cy='credit-stats-mean']").should("contain", "44.36");
+    cy.get("[data-cy='credit-stats-stdev']").should("contain", "30.77");
+    cy.get("[data-cy='credit-stats-min']").should("contain", "0");
+    cy.get("[data-cy='credit-stats-q1']").should("contain", "17");
+    cy.get("[data-cy='credit-stats-q2']").should("contain", "46");
+    cy.get("[data-cy='credit-stats-q3']").should("contain", "66");
+    cy.get("[data-cy='credit-stats-max']").should("contain", "137");
+  });
+});

cypress/integration/Student_statistics.js

@@ -1,72 +1,75 @@
 /// <reference types="Cypress" />
 
-describe('Student Statistics tests', () => {
-    beforeEach(() => {
-        cy.init()
-        cy.contains("Student statistics").click()
-        cy.contains("Student names hidden")
-    })
+describe("Student Statistics tests", () => {
+  beforeEach(() => {
+    cy.init();
+    cy.contains("Student statistics").click();
+    cy.contains("Show student names");
+  });
 
-    it('Student statistics search form is usable', () => {
-        cy.contains('Student names hidden')
-        cy.url().should('include', '/students')
-        cy.get('.prompt').type('Oinonen')
-        cy.contains('Student number')
-        cy.contains('Started')
-        cy.contains('Credits')
-        cy.contains('Heidi Eeva').should('not.exist')
+  it("Student statistics search form is usable", () => {
+    cy.contains("Show student names");
+    cy.url().should("include", "/students");
+    cy.get(".prompt").type("Oinonen");
+    cy.contains("Student number");
+    cy.contains("Started");
+    cy.contains("Credits");
+    cy.contains("Heidi Eeva").should("not.exist");
 
-        cy.get('label').click()
-        cy.contains('Heidi Eeva')
+    cy.get("label").click();
+    cy.contains("Heidi Eeva");
 
-        cy.get('label').click()
-        cy.contains('Heidi Eeva').should('not.exist')
+    cy.get("label").click();
+    cy.contains("Heidi Eeva").should("not.exist");
+  });
+  it("Can get student specific page by clicking student", () => {
+    cy.url().should("include", "/students");
+    cy.get(".prompt").type("Oinonen");
+    cy.contains("014473717").click();
+    cy.contains("Updated at 02.10.2019");
+    cy.contains("Musiikin didaktiikka (EDUK4212)");
+    cy.contains("Oinonen").should("not.exist");
 
-    })
-    it('Can get student specific page by clicking student', () => {
-        cy.url().should('include', '/students')
-        cy.get('.prompt').type('Oinonen')
-        cy.contains('014473717').click()
-        cy.contains('Updated at 02.10.2019')
-        cy.contains('Musiikin didaktiikka (EDUK4212)')
-        cy.contains('Oinonen').should('not.exist')
+    cy.get("label").click();
+    cy.contains("Oinonen");
 
-        cy.get('label').click()
-        cy.contains('Oinonen')
+    cy.get("label").click();
+    cy.contains("Oinonen").should("not.exist");
+  });
 
-        cy.get('label').click()
-        cy.contains('Oinonen').should('not.exist')
-    })
+  it("Can get back to search menu", () => {
+    cy.get(".prompt").type("Oinonen");
+    cy.contains("014473717").click();
+    cy.get(".remove").click();
+    cy.contains("Student number").should("not.exist");
+    cy.contains("Credits").should("not.exist");
+  });
 
-    it('Can get back to search menu', () => {
-        cy.get('.prompt').type('Oinonen')
-        cy.contains('014473717').click()
-        cy.get('.remove').click()
-        cy.contains('Student number').should('not.exist')
-        cy.contains('Credits').should('not.exist')
-    })
+  it("Can jump to course", () => {
+    cy.get(".prompt").type("Oinonen");
+    cy.contains("014473717").click();
+    cy.contains("Musiikin didaktiikka (EDUK4212)")
+      .siblings()
+      .within(() => {
+        cy.get(".level").click();
+      });
+    cy.url().should("include", "/coursestatistics");
+    cy.contains("Musiikin didaktiikka");
+  });
 
-    it('Can jump to course', () => {
-        cy.get('.prompt').type('Oinonen')
-        cy.contains('014473717').click()
-        cy.contains('Musiikin didaktiikka (EDUK4212)').siblings().within(() => { cy.get('.level').click() })
-        cy.url().should('include', '/coursestatistics')
-        cy.contains('Musiikin didaktiikka')
-    })
+  it("Searching with bad inputs doesnt yield results", () => {
+    cy.get(".prompt").type("Oin");
+    cy.contains("Student number").should("not.exist");
 
-    it('Searching with bad inputs doesnt yield results', () => {
-        cy.get('.prompt').type('Oin')
-        cy.contains('Student number').should('not.exist')
+    cy.get(".prompt").clear().type("01114");
+    cy.contains("Student number").should("not.exist");
+  });
 
-        cy.get('.prompt').clear().type('01114')
-        cy.contains('Student number').should('not.exist')
-    })
-
-    it('Can jump to population page', () => {
-        cy.get('.prompt').type('Veli-Matti')
-        cy.contains('014824094').click()
-        cy.get('i.level.up.alternate.icon').eq(0).click()
-        cy.contains('Population statistics')
-        cy.contains('Tietojenkäsittelytieteen maisteriohjelma')
-    })
-})
+  it("Can jump to population page", () => {
+    cy.get(".prompt").type("Veli-Matti");
+    cy.contains("014824094").click();
+    cy.get("i.level.up.alternate.icon").eq(0).click();
+    cy.contains("Population statistics");
+    cy.contains("Tietojenkäsittelytieteen maisteriohjelma");
+  });
+});

cypress/integration/Studyprogramme_overview.js

@@ -1,176 +1,161 @@
 /// <reference types="Cypress" />
 
 const deleteTag = (name) => {
-  cy.contains(name).siblings().contains('Delete').click()
-  cy.contains('Are you sure you want to delete tag')
-  cy.contains('Confirm').click()
-  cy.contains(name).should('not.exist')
-}
+  cy.contains(name).siblings().contains("Delete").click();
+  cy.contains("Are you sure you want to delete tag");
+  cy.contains("Confirm").click();
+  cy.contains(name).should("not.exist");
+};
 
-describe('Studyprogramme overview', () => {
+describe("Studyprogramme overview", () => {
   beforeEach(() => {
-    cy.init()
-    cy.contains("Study programme").click().siblings().contains("Overview").click()
-    cy.contains("Study Programme", { timeout: 100000 })
-  })
+    cy.init();
+    cy.contains("Study programme")
+      .click()
+      .siblings()
+      .contains("Overview")
+      .click();
+    cy.contains("Study Programme", { timeout: 100000 });
+  });
 
-  it('can search for course mappings', () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.contains('Code Mapper').click()
-    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').within(($tr) => { cy.get('input').type('582219') })
-    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').within(($tr) => { cy.get('.results').contains("Käyttöjärjestelmät (582219)") })
-    cy.contains('tr', 'TKT20003 Käyttöjärjestelmät').within(($tr) => { cy.contains('button', "Add") })
-  })
+  // FIXME: re-enable when CI has mandatory courses in the db
+  it.skip("can search for course mappings", () => {
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
+    cy.contains("Code Mapper").click();
+    cy.wait(150);
+    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
+      cy.get("input").type("582219");
+    });
+    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
+      cy.get(".results").contains("Käyttöjärjestelmät (582219)");
+    });
+    cy.contains("tr", "TKT20003 Käyttöjärjestelmät").within(($tr) => {
+      cy.contains("button", "Add");
+    });
+  });
 
-  it('can view course groups', () => {
-    cy.contains("Kasvatustieteiden kandiohjelma").click()
-    cy.contains('Course Groups').click()
+  it("can view course groups", () => {
+    cy.contains("Kasvatustieteiden kandiohjelma").click();
+    cy.contains("Course Groups").click();
 
-    cy.contains('tr', 'Test course group').get('i.edit').click()
-    cy.contains("Edit group")
-    cy.get('.prompt').type("Professori Pekka")
-    cy.contains("Add teacher").parent().contains("000960").click()
-    cy.contains("Teachers in group").parent().contains("000960")
+    cy.contains("tr", "Test course group").get("i.edit").click();
+    cy.contains("Edit group");
+    cy.get(".prompt").type("Professori Pekka");
+    cy.contains("Add teacher").parent().contains("000960").click();
+    cy.contains("Teachers in group").parent().contains("000960");
 
-    cy.get("i.reply.link.icon").click()
-    cy.contains('tr a', 'Test course group').click()
-    cy.contains("Total teachers")
-    cy.get("i.reply.icon").click()
-  })
+    cy.get("i.reply.link.icon").click();
+    cy.contains("tr a", "Test course group").click();
+    cy.contains("Total teachers");
+    cy.get("i.reply.icon").click();
+  });
 
-  it('renders progress and productivity tables', () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.contains("Admin").click()
-    cy.contains("productivity").click()
-    cy.contains("throughput").click()
+  it("renders progress and productivity tables", () => {
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
+    cy.contains("Admin").click();
+    cy.contains("productivity").click();
+    cy.contains("throughput").click();
 
-    cy.wait(1000)
-    cy.get('.attached > :nth-child(1)').click()
-    cy.get('table').should('have.length', 3)
-    cy.contains('Population progress')
-    cy.contains('Yearly productivity')
-    cy.contains("2017-2018").siblings().contains('228').siblings().contains('178')
+    cy.wait(1000);
+    cy.get(".attached > :nth-child(1)").click();
+    cy.get("table").should("have.length", 3);
+    cy.contains("Population progress");
+    cy.contains("Yearly productivity");
+    cy.contains("2017-2018")
+      .siblings()
+      .contains("228")
+      .siblings()
+      .contains("178");
 
-    cy.get('table').eq(1).contains('2018').siblings().contains('3023.00').siblings().contains('2631.00')
-    cy.get('table').eq(1).contains('2017').siblings().contains('3159.00').siblings().contains("2914.00")
-  })
+    cy.get("table")
+      .eq(1)
+      .contains("2018")
+      .siblings()
+      .contains("3023.00")
+      .siblings()
+      .contains("2631.00");
+    cy.get("table")
+      .eq(1)
+      .contains("2017")
+      .siblings()
+      .contains("3159.00")
+      .siblings()
+      .contains("2914.00");
+  });
 
-  it('can add and delete mandatory courses and labels', () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.contains('Mandatory Courses').click()
-    cy.get('button').contains('Add Courses').click()
-    cy.route('/api/v2/coursesmulti**').as('searchResponse')
-    cy.get('input').eq(0).type('Code Generation')
-    cy.wait('@searchResponse')
-    cy.contains('Searched courses')
-    cy.get(':nth-child(1) > .ten').click()
-    cy.contains('select label')
+  it("can open Thesis page", () => {
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
+    cy.contains("Thesis Courses").click();
+    cy.contains("Add thesis course").click();
+    cy.contains("No results");
+  });
 
-    cy.contains('Group labels').click()
-    const label = 'cypress test label'
-    cy.contains(label).should('not.exist')
-    cy.contains('button', 'Add').should('be.disabled')
-    cy.get('input').eq(0).type(label)
-    cy.contains('button', 'Add').should('be.enabled').click()
-    cy.contains('button', 'Add').should('be.disabled')
-    cy.contains('table', label)
+  it("can move to Population statistics page by clickin", () => {
+    cy.contains("Tietojenkäsittelytieteen maisteriohjelma").click();
+    cy.get("i.level.up.alternate.icon").eq(0).click();
+    cy.contains("Credit accumulation (for 29 students)");
+  });
 
-    cy.contains('Mandatory courses').click()
-    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('select label').click()
-    cy.contains(label).click()
-    cy.get('table').eq(0).contains('tr', 'Code Generation').within((el) => {
-      cy.get('div.dropdown>div').eq(0).within((el) => {
-        cy.contains('select label').should('not.exist')
-        cy.contains(label)
-      })
-      cy.get('i.clear').click()
-      cy.get('div.dropdown>div').eq(0).within((el) => {
-        cy.contains(label).should('not.exist')
-        cy.contains('select label')
-      })
-    })
-    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('select label').click()
-    cy.contains(label).click()
-    cy.get('table').eq(0).contains('tr', 'Code Generation').contains('button', 'Delete').click()
-    cy.contains('Code Generation').should('not.exist')
+  it("can create and delete tags for population", () => {
+    const name = `tag-${new Date().getTime()}`;
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
+    cy.get(".attached > :nth-child(6)").click();
+    cy.get(":nth-child(1) > .field > .ui > input").type(name);
+    cy.get(".form-control").type("2018");
+    cy.contains("Create new tag").click();
+    cy.contains(name);
+    cy.contains("2018");
+    deleteTag(name);
+  });
 
-    cy.contains('Group labels').click()
-    cy.contains('tr', label).within((el) => {
-      cy.get('i.remove').click()
-    })
-    cy.contains('button', 'Remove').click()
-    cy.contains(label).should('not.exist')
-  })
+  it("can create personal tags", () => {
+    const name = `tag-${new Date().getTime()}`;
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
+    cy.get(".attached > :nth-child(6)").click();
+    cy.get(":nth-child(1) > .field > .ui > input").type(name);
+    cy.get(".form-control").type("2018");
+    cy.get(".ui > label").click();
+    cy.contains("Create new tag").click();
+    cy.get(".purple");
+    cy.contains(name);
+    deleteTag(name);
+  });
 
-  it('can open Thesis page', () => {
-    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click()
-    cy.contains('Thesis Courses').click()
-    cy.contains('Add thesis course').click()
-    cy.contains('No results')
-  })
+  it("can add tags to students", () => {
+    const name = `tag-${new Date().getTime()}`;
 
-  it('can move to Population statistics page by clickin', () => {
-    cy.contains('Tietojenkäsittelytieteen maisteriohjelma').click()
-    cy.get('i.level.up.alternate.icon').eq(0).click()
-    cy.contains('Credit accumulation (for 29 students)')
-  })
+    cy.contains("Tietojenkäsittelytieteen kandiohjelma").click();
+    cy.get(".attached > :nth-child(6)").click();
+    cy.get(":nth-child(1) > .field > .ui > input").type(name);
+    cy.get(".form-control").type("2018");
+    cy.contains("Create new tag").click();
+    cy.contains(name);
 
-  it('can create and delete tags for population', () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached > :nth-child(6)').click()
-    cy.get(':nth-child(1) > .field > .ui > input').type(name)
-    cy.get('.form-control').type('2018')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
-    cy.contains('2018')
-    deleteTag(name)
-  })
+    cy.contains("Add tags to students").click();
+    cy.get(".form > .field > .dropdown")
+      .click()
+      .get(".ui > .search")
+      .type(name)
+      .click();
 
-  it('can create personal tags', () => {
-    const name = `tag-${new Date().getTime()}`
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached > :nth-child(6)').click()
-    cy.get(':nth-child(1) > .field > .ui > input').type(name)
-    cy.get('.form-control').type('2018')
-    cy.get('.ui > label').click()
-    cy.contains('Create new tag').click()
-    cy.get('.purple')
-    cy.contains(name)
-    deleteTag(name)
-  })
+    cy.get(".form > .field > .dropdown > .visible").contains(name).click();
 
-  it('can add tags to students', () => {
-    const name = `tag-${new Date().getTime()}`
+    cy.get("textarea").type("010623419");
+    cy.get(".positive").click();
 
-    cy.contains('Tietojenkäsittelytieteen kandiohjelma').click()
-    cy.get('.attached > :nth-child(6)').click()
-    cy.get(':nth-child(1) > .field > .ui > input').type(name)
-    cy.get('.form-control').type('2018')
-    cy.contains('Create new tag').click()
-    cy.contains(name)
+    cy.contains("Student statistics").click();
+    cy.get(".prompt").type("010623419");
+    cy.contains("10").click();
+    cy.contains(name);
 
-    cy.contains('Add tags to students').click()
-    cy.get('.form > .field > .dropdown').click().get('.ui > .search').type(name).click()
-    
-    cy.get('.form > .field > .dropdown > .visible').contains(name).click()
+    cy.go("back");
+    cy.go("back");
 
-    cy.get('textarea').type('010623419')
-    cy.get('.positive').click()
+    deleteTag(name);
 
-    cy.contains('Student statistics').click()
-    cy.get('.prompt').type('010623419')
-    cy.contains('10').click()
-    cy.contains(name)
-
-    cy.go('back')
-    cy.go('back')
-
-    deleteTag(name)
-
-    cy.contains('Student statistics').click()
-    cy.get('.prompt').type('010623419')
-    cy.contains('10').click()
-    cy.contains(name).should('not.exist')
-  })
-})
+    cy.contains("Student statistics").click();
+    cy.get(".prompt").type("010623419");
+    cy.contains("10").click();
+    cy.contains(name).should("not.exist");
+  });
+});

cypress/integration/Teachers.js

@@ -46,17 +46,18 @@ describe('Teachers page tests', () => {
     cy.contains('Name').should('not.exist')
   })
 
-  it("Check leaderboad works", () => {
-    cy.get('.borderless > :nth-child(2)').click({ force: true })
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('2017-18').click({ force: true })
-    cy.contains("Recalculate this year").click({ force: true })
-    cy.wait(5000)
-    cy.reload()
-    cy.get(':nth-child(1) > .ui > .search').click({ force: true })
-    cy.contains('2017-18').click()
-    cy.contains("Passed")
-    cy.contains("Antero")
-    cy.contains("Sassmannshausen")
-  })
+  // it("Check leaderboad works", () => {
+  //   cy.get('.borderless > :nth-child(2)').click({ force: true })
+  //   cy.get(':nth-child(1) > .ui > .search').click({ force: true })
+  //   cy.get(':nth-child(1) > .ui > .search').click({ force: true })
+  //   cy.contains('2017-18').click({ force: true })
+  //   cy.contains("Recalculate this year").click({ force: true })
+  //   cy.wait(5000)
+  //   cy.reload()
+  //   cy.get(':nth-child(1) > .ui > .search').click({ force: true })
+  //   cy.contains('2017-18').click()
+  //   cy.contains("Passed")
+  //   cy.contains("Antero")
+  //   cy.contains("Sassmannshausen")
+  // })
 })

dco.data.yml

@@ -0,0 +1,75 @@
+version: "3.2"
+
+services:
+  nats:
+    image: nats-streaming:0.15.1
+    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
+    expose:
+      - "4222"
+    ports:
+      - "8222:8222"
+      - "4222:4222"
+    container_name: nats
+    volumes:
+      - datastore:/datastore
+
+  db:
+    image: postgres:10-alpine
+    ports:
+      - "5421:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: oodi_db
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_HOST_AUTH_METHOD=trust
+
+  db_sis:
+    image: postgres:9.6.3
+    ports:
+      - "5426:5432"
+    volumes:
+      - pgdata_sis:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_sis
+    environment:
+      - POSTGRES_USER=postgres
+
+  redis:
+    image: redis
+    command: ["redis-server", "--appendonly", "yes"]
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+    container_name: redis
+
+  updater_writer:
+    build:
+      context: services/backend
+      dockerfile: updater_writer/Dockerfile
+    depends_on:
+      - nats
+      - db
+    environment:
+      NATS_URI: nats://nats:4222
+      DB_URL: postgres://postgres@db:5432/tkt_oodi
+      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_SCHEMA: updater_test
+    container_name: updater_writer
+
+  sis-importer-db:
+    image: postgres:13.1
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+    volumes:
+      - sis-importer-pgdata:/var/lib/postgresql/data
+    container_name: sis-importer-db
+
+volumes:
+  datastore:
+  pgdata:
+  pgdata_sis:
+  redis-data:
+  sis-importer-pgdata:

docker-compose.yml

@@ -1,284 +0,0 @@
-version: "3.2"
-
-services:
-  nats:
-    image: nats-streaming:0.15.1
-    command: -cid updaterNATS --file_slice_max_bytes 0 --file_slice_max_age 100h -store file -dir datastore
-    expose:
-      - "4222"
-    ports:
-      - "8222:8222"
-      - "4222:4222"
-    container_name: nats
-    volumes:
-      - datastore:/datastore
-
-  analytics_db:
-    image: postgres:9.6.3
-    ports:
-      - "5423:5432"
-    volumes:
-      - analytics_pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_analytics_db
-    environment:
-      - POSTGRES_USER=postgres
-
-  user_db:
-    image: postgres:9.6.3
-    ports:
-      - "5422:5432"
-    volumes:
-      - user_pgdata:/var/lib/postgresql/data
-    container_name: oodi_user_db
-    environment:
-      - POSTGRES_USER=postgres
-
-  usage_db:
-    image: postgres:9.6.3
-    ports:
-      - "5424:5432"
-    volumes:
-      - usage_pgdata:/var/lib/postgresql/data
-    container_name: oodi_usage_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=usage_db
-
-  db:
-    image: postgres:10-alpine
-    ports:
-      - "5421:5432"
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: oodi_db
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_HOST_AUTH_METHOD=trust
-
-  db_kone:
-    image: postgres:9.6.3
-    ports:
-      - "5425:5432"
-    volumes:
-      - pgdata_kone:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_kone
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_kone
-      - POSTGRES_USER=postgres
-
-  db_sis:
-    image: postgres:9.6.3
-    ports:
-      - "5426:5432"
-    volumes:
-      - pgdata_sis:/var/lib/postgresql/data
-      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
-    container_name: db_sis
-    environment:
-      - POSTGRES_USER=postgres
-
-  redis:
-    image: redis
-    command: ["redis-server", "--appendonly", "yes"]
-    ports:
-      - "6379:6379"
-    volumes:
-      - redis-data:/data
-    container_name: redis
-
-  backend:
-    build:
-      context: services/backend
-      dockerfile: oodikone2-backend/Dockerfile
-    ports:
-      - "8080:8080"
-    environment:
-      REDIS: redis
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      # DB_MAX_CONNECTIONS: 90 # defaults to 5
-      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
-      SIS_DB_URL: postgres://postgres@db_sis:5432/db_sis
-      # if NODE_ENV === 'test', these are used instead
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
-      # same but for NODE_ENV === 'anon'
-      # ANON_DB:
-      # ANON_DB_KONE:
-      TOKEN_SECRET: IWannaBeTheVeryBest
-      SECRET_TOKEN: dev
-      USERSERVICE_URL: http://userservice:4567
-      USAGESERVICE_URL: http://usageservice:545
-      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      USAGESERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      ANALYTICS_URL: http://analytics:4568
-      NATS_URI: nats://nats:4222
-      UPDATER_URL: http://updater_scheduler:3678
-      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
-      # Set these in production:
-      # ANALYTICS_INFLUXDB_URL
-      # ANALYTICS_INFLUXDB_DB
-      # ANALYTICS_INFLUXDB_PASSWORD
-      # ANALYTICS_INFLUXDB_USER
-      # MATOMO_SITE_ID
-      # MATOMO_URL
-    container_name: backend
-    depends_on:
-      - db
-      - redis
-      - nats
-
-  frontend:
-    build:
-      context: services/oodikone2-frontend
-    environment:
-      BACKEND_ADDR: backend
-      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
-    container_name: frontend
-
-  userservice:
-    build:
-      context: services/oodikone2-userservice
-    ports:
-      - 4567:4567
-    environment:
-      DB_URL: postgres://postgres@user_db:5432/user_db
-      TEST_DB: postgres://postgres@user_db:5432/user_db_test
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-      TOKEN_SECRET: IWannaBeTheVeryBest
-    depends_on:
-      - user_db
-    container_name: userservice
-
-  usageservice:
-    build:
-      context: services/oodikone2-usageservice
-    ports:
-      - 545:545
-    environment:
-      PORT: "545"
-      DB_URL: postgres://postgres@usage_db:5432/usage_db
-      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
-    depends_on:
-      - usage_db
-    container_name: usageservice
-
-  updater_api:
-    build:
-      context: services/updater_api
-    depends_on:
-      - nats
-    environment:
-      - NATS_URI=nats://nats:4222
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi/test
-      - TOKEN=jotaindiibadaabaa
-    container_name: updater_api
-
-  updater_writer:
-    build:
-      context: services/backend
-      dockerfile: updater_writer/Dockerfile
-    depends_on:
-      - nats
-      - db
-    environment:
-      NATS_URI: nats://nats:4222
-      DB_URL: postgres://postgres@db:5432/tkt_oodi
-      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
-      TEST_DB_SCHEMA: updater_test
-    container_name: updater_writer
-
-  scheduler_mongo:
-    image: mongo:4.0.10
-    volumes:
-      - mongodata:/data/db
-    expose:
-      - "27017"
-    container_name: scheduler_mongo
-
-  updater_scheduler:
-    build:
-      context: services/updater_scheduler
-    depends_on:
-      - nats
-      - scheduler_mongo
-    environment:
-      - NATS_URI=nats://nats:4222
-      - MONGO_URI=mongodb://scheduler_mongo:27017
-      - PORT=3678
-    container_name: updater_scheduler
-
-  analytics:
-    build:
-      context: services/oodikone2-analytics
-    ports:
-      - 4568:4568
-    environment:
-      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
-      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
-    depends_on:
-      - analytics_db
-    container_name: analytics
-
-  sis-updater-nats:
-    image: nats-streaming:0.16.2
-    command: -cid sis-updater-nats -m 8222 -p 4222 --max_age 2h --hb_fail_count 2 --hb_timeout 10s --hb_interval 10s --auth dev --file_slice_max_bytes 0 --file_slice_max_age 60s --max_msgs 0 -store file -dir datastore
-    container_name: sis-updater-nats
-    volumes:
-      - sis-updater-datastore:/datastore
-
-  sis-updater-scheduler:
-    build:
-      context: services/sis-updater-scheduler
-    environment:
-      REDIS_URI: //redis:6379
-      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
-      SIS_NATS_TOKEN: dev
-      SIS_IMPORTER_HOST: importer-db
-      SIS_IMPORTER_USER: dev
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_DATABASE: importer-db
-      SECRET_TOKEN: dev
-      # LOG_HOST: host.docker.internal
-      # LOG_HOSTNAME: sis-updater-scheduler
-      # LOG_PORT: 9501
-    
-    ports:
-      - "8082:8082"
-    container_name: sis-updater-scheduler
-    depends_on:
-      - redis
-      - sis-updater-nats
-
-  sis-updater-worker:
-    build:
-      context: services/sis-updater-worker
-    environment:
-      REDIS_URI: //redis:6379
-      DB_URL: postgres://postgres@db_sis:5432/db_sis
-      SIS_NATS_URI: sis-updater-nats://sis-updater-nats:4222
-      SIS_NATS_TOKEN: dev
-      SIS_IMPORTER_HOST: importer-db
-      SIS_IMPORTER_USER: dev
-      SIS_IMPORTER_PASSWORD: dev
-      SIS_IMPORTER_DATABASE: importer-db
-      # LOG_HOST: host.docker.internal
-      # LOG_HOSTNAME: sis-updater-worker
-      # LOG_PORT: 9501
-    depends_on:
-      - redis
-      - sis-updater-nats
-
-volumes:
-  datastore:
-  analytics_pgdata:
-  user_pgdata:
-  usage_pgdata:
-  pgdata:
-  pgdata_kone:
-  pgdata_sis:
-  redis-data:
-  mongodata:
-  sis-updater-datastore:

docker/dco.data.ci.yml

@@ -0,0 +1,15 @@
+version: '3.2'
+
+services:
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
+
+  db_sis:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_sis
+
+volumes:
+  datastore-real:
+  redis-data-real:
+  mongodata-real:

docker/dco.data.dev.real.yml

@@ -0,0 +1,14 @@
+version: '3.2'
+
+services:
+  nats:
+    volumes:
+      - datastore-real:/datastore
+
+  redis:
+    volumes:
+      - redis-data-real:/data
+
+volumes:
+  datastore-real:
+  redis-data-real:

docker/dco.data.dev.yml

@@ -0,0 +1,28 @@
+version: '3.2'
+
+services:
+  db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
+    restart: unless-stopped
+    shm_size: '2gb'
+
+  db_sis:
+    command:
+      - "postgres"
+      - "-c"
+      - "max_wal_size=8GB"
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_sis,db_sis_real
+    restart: unless-stopped
+
+  adminer:
+    image: adminer:4.6.2
+    environment:
+      - ADMINER_DESIGN=pepa-linha
+    volumes:
+      - ./adminer-plugins:/var/www/html/plugins-enabled
+    ports:
+      - 5050:8080
+    container_name: adminer
+    restart: unless-stopped

docker/docker-compose.dev.real.yml

@@ -1,40 +0,0 @@
-version: '3.2'
-
-services:
-  nats:
-    volumes:
-      - datastore-real:/datastore
-
-  backend:
-    environment:
-      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
-      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
-
-  userservice:
-    environment:
-      DB_URL: postgres://postgres@user_db:5432/user_db_real
-
-  analytics:
-    environment:
-      DB_URL: postgres://postgres@analytics_db:5432/analytics_db_real
-
-  updater_writer:
-    environment:
-      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
-  updater_api:
-    environment:
-      - OODI_ADDR=https://oodikone.cs.helsinki.fi/oodi
-      - TOKEN
-
-  redis:
-    volumes:
-      - redis-data-real:/data
-
-  scheduler_mongo:
-    volumes:
-      - mongodata-real:/data/db
-
-volumes:
-  datastore-real:
-  redis-data-real:
-  mongodata-real:

docker/docker-compose.dev.yml

@@ -1,186 +0,0 @@
-version: '3.2'
-
-services:
-  user_db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
-    restart: unless-stopped
-
-  analytics_db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=analytics_db,analytics_db_real
-    restart: unless-stopped
-
-  db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test,tkt_oodi_real
-    restart: unless-stopped
-    shm_size: '2gb'
-
-  db_sis:
-    command:
-      - "postgres"
-      - "-c"
-      - "max_wal_size=8GB"
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_sis
-    restart: unless-stopped
-
-  backend:
-    build:
-      args:
-        NODE_ENV: 'development'
-        TAG: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    environment:
-      FRONT_URL: http://localhost:8000
-    volumes:
-      - ./services/backend/oodikone2-backend/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/src/models
-      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
-    restart: unless-stopped
-
-  frontend:
-    build:
-      args:
-        NODE_ENV: 'development'
-        TAG: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run docker']
-    ports:
-      - "8081:8081"
-      - "7777:7777"
-    environment:
-      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
-      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
-      USAGE_ADMINER_URL: http://localhost:5050/?pgsql=usage_db&username=postgres
-      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
-      SIS_ADMINER_URL: http://localhost:5050/?pgsql=db_sis&username=postgres
-    volumes:
-      - ./services/oodikone2-frontend/:/usr/src/app
-    restart: unless-stopped
-
-  userservice:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/oodikone2-userservice/:/usr/src/app
-    restart: unless-stopped
-
-  usageservice:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/oodikone2-usageservice/:/usr/src/app
-    restart: unless-stopped
-
-  updater_api:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/updater_api/:/usr/src/app
-    restart: unless-stopped
-
-  updater_writer:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/backend/updater_writer/:/usr/src/app
-      - ./services/backend/shared/models:/usr/src/app/models
-      - ./services/backend/shared/migrations:/usr/src/app/database/migrations
-      - ./services/backend/shared/migrations_kone:/usr/src/app/database/migrations_kone
-    environment:
-      DB_SCHEMA_KONE: kone_data
-    restart: unless-stopped
-
-  updater_scheduler:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/updater_scheduler/:/usr/src/app
-    restart: unless-stopped
-
-  analytics:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/oodikone2-analytics/:/usr/src/app
-    restart: unless-stopped
-
-  adminer:
-    image: adminer:4.6.2
-    environment:
-      - ADMINER_DESIGN=pepa-linha
-    volumes:
-      - ./adminer-plugins:/var/www/html/plugins-enabled
-    ports:
-      - 5050:8080
-    container_name: adminer
-    restart: unless-stopped
-
-  sis-updater-scheduler:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: ['sh', '-c', 'cp -r /usr/src/cache/node_modules/. /usr/src/app/node_modules/ && npm run dev']
-    volumes:
-      - ./services/sis-updater-scheduler/:/usr/src/app
-    restart: unless-stopped
-
-  sis-updater-worker:
-    build:
-      args:
-        NODE_ENV: 'development'
-    command: 'npm run dev'
-    volumes:
-      - ./services/sis-updater-worker/:/usr/src/app
-    restart: unless-stopped
-
-networks:
-  default:
-
-  # zookeeper:
-  #   image: wurstmeister/zookeeper
-  #   ports:
-  #     - "2181:2181"
-  #   environment:
-  #     - KAFKA_BROKER_ID=1
-  #   container_name: zookeeper
-
-  # kafka:
-  #   image: wurstmeister/kafka
-  #   container_name: kafka
-  #   depends_on:
-  #     - zookeeper
-  #   ports:
-  #     - "9092:9092"
-  #   environment:
-  #     - HOSTNAME_COMMAND=route -n | awk '/UG[ \t]/{print $$2}'
-  #     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-  #     - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
-  #     - KAFKA_DELETE_TOPIC_ENABLE=true
-  #     - KAFKA_CREATE_TOPICS=studentnumbers:1:1,priority_studentnumbers:1:1
-  #     - KAFKA_LOG_DIRS=/kafka/kafka-logs-1
-  #   volumes:
-  #     - ./kafka/docker.sock:/var/run/docker.sock
-  #     - ./kafka/logs:/kafka/kafka-logs-1
-
-  # mongo_db:
-  #   image: mongo
-  #   volumes:
-  #     - ./mongodata:/data/db
-  #   ports:
-  #     - "27017:27017"
-  #   container_name: mongo_db

package-lock.json

@@ -1081,9 +1081,9 @@
       },
       "dependencies": {
         "debug": {
-          "version": "3.2.6",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
-          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+          "version": "3.2.7",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+          "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
           "dev": true,
           "requires": {
             "ms": "^2.1.1"
@@ -1100,12 +1100,6 @@
         "any-observable": "^0.3.0"
       }
     },
-    "@types/color-name": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
-      "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==",
-      "dev": true
-    },
     "@types/parse-json": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
@@ -1113,9 +1107,9 @@
       "dev": true
     },
     "@types/sinonjs__fake-timers": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.1.tgz",
-      "integrity": "sha512-yYezQwGWty8ziyYLdZjwxyMb0CZR49h8JALHGrxjQHWlqGgc8kLdHEgWrgL0uZ29DMvEVBDnHU2Wg36zKSIUtA==",
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.2.tgz",
+      "integrity": "sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg==",
       "dev": true
     },
     "@types/sizzle": {
@@ -1158,9 +1152,9 @@
       "dev": true
     },
     "ajv": {
-      "version": "6.12.5",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
-      "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
       "dev": true,
       "requires": {
         "fast-deep-equal": "^3.1.1",
@@ -1215,9 +1209,9 @@
       }
     },
     "arch": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/arch/-/arch-2.1.2.tgz",
-      "integrity": "sha512-NTBIIbAfkJeIletyABbVtdPgeKfDafR+1mZV/AyyfC1UkVkp9iUjV+wwmqtUgphHYajbI86jejBJp5e+jkGTiQ==",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz",
+      "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==",
       "dev": true
     },
     "arr-diff": {
@@ -1348,9 +1342,9 @@
       "dev": true
     },
     "aws4": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz",
-      "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==",
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz",
+      "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==",
       "dev": true
     },
     "babel-plugin-add-module-exports": {
@@ -2088,9 +2082,9 @@
       }
     },
     "commander": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
-      "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
+      "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
       "dev": true
     },
     "common-tags": {
@@ -2228,9 +2222,9 @@
       },
       "dependencies": {
         "parse-json": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz",
-          "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==",
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz",
+          "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==",
           "dev": true,
           "requires": {
             "@babel/code-frame": "^7.0.0",
@@ -2317,9 +2311,9 @@
       }
     },
     "cypress": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-5.2.0.tgz",
-      "integrity": "sha512-9S2spcrpIXrQ+CQIKHsjRoLQyRc2ehB06clJXPXXp1zyOL/uZMM3Qc20ipNki4CcNwY0nBTQZffPbRpODeGYQg==",
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-6.3.0.tgz",
+      "integrity": "sha512-Ec6TAFOxdSB2HPINNJ1f7z75pENXcfCaQkz+A9j0eGSvusFJ2NNErq650DexCbNJAnCQkPqXB4XPH9kXnSQnUA==",
       "dev": true,
       "requires": {
         "@cypress/listr-verbose-renderer": "^0.4.1",
@@ -2334,7 +2328,7 @@
         "chalk": "^4.1.0",
         "check-more-types": "^2.24.0",
         "cli-table3": "~0.6.0",
-        "commander": "^4.1.1",
+        "commander": "^5.1.0",
         "common-tags": "^1.8.0",
         "debug": "^4.1.1",
         "eventemitter2": "^6.4.2",
@@ -2352,10 +2346,10 @@
         "minimist": "^1.2.5",
         "moment": "^2.27.0",
         "ospath": "^1.2.2",
-        "pretty-bytes": "^5.3.0",
+        "pretty-bytes": "^5.4.1",
         "ramda": "~0.26.1",
         "request-progress": "^3.0.0",
-        "supports-color": "^7.1.0",
+        "supports-color": "^7.2.0",
         "tmp": "~0.2.1",
         "untildify": "^4.0.0",
         "url": "^0.11.0",
@@ -2363,12 +2357,11 @@
       },
       "dependencies": {
         "ansi-styles": {
-          "version": "4.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
-          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
           "dev": true,
           "requires": {
-            "@types/color-name": "^1.1.1",
             "color-convert": "^2.0.1"
           }
         },
@@ -2706,9 +2699,9 @@
       }
     },
     "execa": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.3.tgz",
-      "integrity": "sha512-WFDXGHckXPWZX19t1kCsXzOpqX9LWYNqn4C+HqZlk/V0imTkzJZqf87ZBhvpHaftERYknpk0fjSylnXVlVgI0A==",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
+      "integrity": "sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==",
       "dev": true,
       "requires": {
         "cross-spawn": "^7.0.0",
@@ -3028,12 +3021,12 @@
       }
     },
     "find-versions": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/find-versions/-/find-versions-3.2.0.tgz",
-      "integrity": "sha512-P8WRou2S+oe222TOCHitLy8zj+SIsVJh52VP4lvXkaFVnOFFdoWv1H1Jjvel1aI6NCFOAaeAVm8qrI0odiLcww==",
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/find-versions/-/find-versions-4.0.0.tgz",
+      "integrity": "sha512-wgpWy002tA+wgmO27buH/9KzyEOQnKsG/R0yrcjPT9BOFm0zRBVQbZ95nRGXWMywS8YR5knRbpohio0bcJABxQ==",
       "dev": true,
       "requires": {
-        "semver-regex": "^2.0.0"
+        "semver-regex": "^3.1.2"
       }
     },
     "for-in": {
@@ -3075,31 +3068,31 @@
       }
     },
     "fs-extra": {
-      "version": "9.0.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
-      "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
+      "version": "9.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
+      "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
       "dev": true,
       "requires": {
         "at-least-node": "^1.0.0",
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
-        "universalify": "^1.0.0"
+        "universalify": "^2.0.0"
       },
       "dependencies": {
         "jsonfile": {
-          "version": "6.0.1",
-          "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz",
-          "integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==",
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+          "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
           "dev": true,
           "requires": {
             "graceful-fs": "^4.1.6",
-            "universalify": "^1.0.0"
+            "universalify": "^2.0.0"
           }
         },
         "universalify": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz",
-          "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==",
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
+          "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
           "dev": true
         }
       }
@@ -3208,12 +3201,12 @@
       }
     },
     "global-dirs": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.0.1.tgz",
-      "integrity": "sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.1.0.tgz",
+      "integrity": "sha512-MG6kdOUh/xBnyo9cJFeIKkLEc1AyFq42QTU4XiX51i2NEdxLxLWXIjEjmqKeSuKR7pAZjTqUVoT2b2huxVLgYQ==",
       "dev": true,
       "requires": {
-        "ini": "^1.3.5"
+        "ini": "1.3.7"
       }
     },
     "globals": {
@@ -3393,30 +3386,29 @@
       "dev": true
     },
     "husky": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/husky/-/husky-4.3.0.tgz",
-      "integrity": "sha512-tTMeLCLqSBqnflBZnlVDhpaIMucSGaYyX6855jM4AguGeWCeSzNdb1mfyWduTZ3pe3SJVvVWGL0jO1iKZVPfTA==",
+      "version": "4.3.8",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-4.3.8.tgz",
+      "integrity": "sha512-LCqqsB0PzJQ/AlCgfrfzRe3e3+NvmefAdKQhRYpxS4u6clblBoDdzzvHi8fmxKRzvMxPY/1WZWzomPZww0Anow==",
       "dev": true,
       "requires": {
         "chalk": "^4.0.0",
         "ci-info": "^2.0.0",
         "compare-versions": "^3.6.0",
         "cosmiconfig": "^7.0.0",
-        "find-versions": "^3.2.0",
+        "find-versions": "^4.0.0",
         "opencollective-postinstall": "^2.0.2",
-        "pkg-dir": "^4.2.0",
+        "pkg-dir": "^5.0.0",
         "please-upgrade-node": "^3.2.0",
         "slash": "^3.0.0",
         "which-pm-runs": "^1.0.0"
       },
       "dependencies": {
         "ansi-styles": {
-          "version": "4.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
-          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
           "dev": true,
           "requires": {
-            "@types/color-name": "^1.1.1",
             "color-convert": "^2.0.1"
           }
         },
@@ -3469,9 +3461,9 @@
       "dev": true
     },
     "import-fresh": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
-      "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==",
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+      "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
       "dev": true,
       "requires": {
         "parent-module": "^1.0.0",
@@ -3501,9 +3493,9 @@
       "dev": true
     },
     "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "version": "1.3.7",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
+      "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
       "dev": true
     },
     "inline-source-map": {
@@ -4071,12 +4063,11 @@
       },
       "dependencies": {
         "ansi-styles": {
-          "version": "4.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
-          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
           "dev": true,
           "requires": {
-            "@types/color-name": "^1.1.1",
             "color-convert": "^2.0.1"
           }
         },
@@ -4285,18 +4276,18 @@
       }
     },
     "mime-db": {
-      "version": "1.44.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
-      "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==",
+      "version": "1.45.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz",
+      "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==",
       "dev": true
     },
     "mime-types": {
-      "version": "2.1.27",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
-      "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
+      "version": "2.1.28",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.28.tgz",
+      "integrity": "sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ==",
       "dev": true,
       "requires": {
-        "mime-db": "1.44.0"
+        "mime-db": "1.45.0"
       }
     },
     "mimic-fn": {
@@ -4408,9 +4399,9 @@
       }
     },
     "moment": {
-      "version": "2.28.0",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.28.0.tgz",
-      "integrity": "sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw==",
+      "version": "2.29.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
+      "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==",
       "dev": true
     },
     "ms": {
@@ -4769,40 +4760,49 @@
       "dev": true
     },
     "pkg-dir": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
-      "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz",
+      "integrity": "sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA==",
       "dev": true,
       "requires": {
-        "find-up": "^4.0.0"
+        "find-up": "^5.0.0"
       },
       "dependencies": {
         "find-up": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
-          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
+          "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
           "dev": true,
           "requires": {
-            "locate-path": "^5.0.0",
+            "locate-path": "^6.0.0",
             "path-exists": "^4.0.0"
           }
         },
         "locate-path": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
-          "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
+          "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
           "dev": true,
           "requires": {
-            "p-locate": "^4.1.0"
+            "p-locate": "^5.0.0"
+          }
+        },
+        "p-limit": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+          "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+          "dev": true,
+          "requires": {
+            "yocto-queue": "^0.1.0"
           }
         },
         "p-locate": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
-          "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+          "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
           "dev": true,
           "requires": {
-            "p-limit": "^2.2.0"
+            "p-limit": "^3.0.2"
           }
         },
         "path-exists": {
@@ -4829,9 +4829,9 @@
       "dev": true
     },
     "pretty-bytes": {
-      "version": "5.4.1",
-      "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.4.1.tgz",
-      "integrity": "sha512-s1Iam6Gwz3JI5Hweaz4GoCD1WUNUIyzePFy5+Js2hjwGVt2Z79wNN+ZKOZ2vB6C+Xs6njyB84Z1IthQg8d9LxA==",
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.5.0.tgz",
+      "integrity": "sha512-p+T744ZyjjiaFlMUZZv6YPC5JrkNj8maRmPaQCWFJFplUAzpIUTRaTcS+7wmZtUoFXHtESJb23ISliaWyz3SHA==",
       "dev": true
     },
     "process": {
@@ -5213,9 +5213,9 @@
       "dev": true
     },
     "semver-regex": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-2.0.0.tgz",
-      "integrity": "sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-3.1.2.tgz",
+      "integrity": "sha512-bXWyL6EAKOJa81XG1OZ/Yyuq+oT0b2YLlxx7c+mrdYPaPbnj6WgVULXhinMIeZGufuUBu/eVRqXEhiv4imfwxA==",
       "dev": true
     },
     "set-blocking": {
@@ -5954,9 +5954,9 @@
       "dev": true
     },
     "uri-js": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz",
-      "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==",
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+      "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
       "dev": true,
       "requires": {
         "punycode": "^2.1.0"
@@ -6221,6 +6221,12 @@
         "buffer-crc32": "~0.2.3",
         "fd-slicer": "~1.1.0"
       }
+    },
+    "yocto-queue": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "dev": true
     }
   }
 }

package.json

@@ -1,10 +1,10 @@
 {
   "name": "oodikone",
   "version": "1.0.0",
-  "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the docker-compose.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
+  "description": "Simple CLI tool for setting up the development environment for Oodikone. The entire development environment runs inside of a Docker network that's defined in the dco.yml file. The specifics of what this means for setting up the environment and accessing logs is discussed in a later section.",
   "main": "./scripts/run.sh",
   "scripts": {
-    "cli": "./scripts/run.sh",
+    "cli": "./oodikone.sh",
     "cypress": "cypress",
     "cypress:open": "cypress open --config baseUrl=http://localhost:8081/",
     "cypress:run": "CYPRESS_baseUrl=http://frontend:8081/ npm run docker:cypress",
@@ -17,26 +17,38 @@
     "test_services": "npm run services -- -e -c test",
     "test": "./scripts/test.sh",
     "test:smoke": "./scripts/test.sh -s",
-    "docker": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml",
-    "docker:cypress": "npm run docker -- -f ./docker/docker-compose.cypress.yml run --rm cypress",
-    "docker:real": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml -f ./docker/docker-compose.dev.real.yml",
-    "docker:ci": "docker-compose -f docker-compose.yml -f ./docker/docker-compose.lateste2e.yml",
-    "docker:ci:cypress": "npm run docker:ci -- -f ./docker/docker-compose.cypress.yml run --rm cypress",
-    "docker:build": "npm run docker -- build",
-    "docker:up": "npm run docker -- up -d",
-    "docker:up:real": "npm run docker:real -- up -d",
-    "docker:down": "npm run docker -- down",
-    "docker:logs": "npm run docker -- logs -f",
-    "docker:logs:backend": "npm run docker -- logs -f backend",
-    "docker:restart": "npm run docker -- restart",
-    "docker:restart:backend": "npm run docker -- restart backend",
+    "docker:updater:base": "docker-compose -f ./dco.data.yml -f ./updater/dco.updater.yml",
+    "docker:updater:dev": "npm run docker:updater:base -- -f ./docker/dco.data.dev.yml -f ./updater/docker/dco.dev.yml",
+    "docker:updater:real": "npm run docker:updater:dev -- -f ./docker/dco.data.dev.real.yml -f ./updater/docker/dco.dev.real.yml",
+    "docker:oodikone:base": "docker-compose -f ./dco.data.yml -f ./services/dco.oodikone.yml",
+    "docker:oodikone:dev": "npm run docker:oodikone:base -- -f ./docker/dco.data.dev.yml -f ./services/docker/dco.dev.yml",
+    "docker:oodikone:real": "npm run docker:oodikone:dev -- -f ./docker/dco.data.dev.real.yml -f ./services/docker/dco.dev.real.yml",
+    "docker:both:base": "docker-compose -f ./dco.data.yml -f ./updater/dco.updater.yml -f ./services/dco.oodikone.yml",
+    "docker:both:dev": "npm run docker:both:base -- -f ./docker/dco.data.dev.yml -f ./updater/docker/dco.dev.yml -f ./services/docker/dco.dev.yml",
+    "docker:both:real": "npm run docker:both:dev -- -f ./docker/dco.data.dev.real.yml  -f ./updater/docker/dco.dev.real.yml -f ./services/docker/dco.dev.real.yml",
+    "docker:cypress": "npm run docker:oodikone:dev -- -f ./docker/dco.cypress.yml run --rm cypress",
+    "docker:ci": "npm run docker:both:base -- -f ./docker/dco.data.ci.yml -f ./services/docker/dco.ci.yml -f ./updater/docker/dco.ci.yml",
+    "docker:ci:cypress": "npm run docker:ci -- -f ./docker/dco.cypress.yml run --rm cypress",
+    "docker:build": "npm run docker:oodikone:dev -- build",
+    "docker:mac:up": "npm run docker:oodikone:dev -- up -d backend && npm run docker:up",
+    "docker:mac:up:real": "npm run docker:oodikone:real -- up -d backend && npm run docker:up:real",
+    "docker:up": "npm run docker:oodikone:dev -- up -d",
+    "docker:up:real": "npm run docker:oodikone:real -- up -d",
+    "docker:down": "npm run docker:oodikone:dev -- down",
+    "docker:logs": "npm run docker:oodikone:dev -- logs -f",
+    "docker:logs:backend": "npm run docker:oodikone:dev -- logs -f backend",
+    "docker:restart": "npm run docker:oodikone:dev -- restart",
+    "docker:restart:backend": "npm run docker:oodikone:dev -- restart backend",
     "docker:obliterate": "docker stop $(docker ps -aq) ; docker rmi -f $(docker images -a -q) ; docker system prune -a",
     "start": "npm run docker:up && npm run docker:logs",
-    "start:real": "npm run docker:real -- up -d && npm run docker:real -- logs -f",
-    "start:updater": "npm run docker:up -- --scale sis-updater-worker=4 sis-updater-scheduler sis-updater-worker db_sis sis-updater-nats db adminer redis && npm run docker:logs -- --tail=\"1000\""
+    "start:real": "npm run docker:oodikone:real -- up -d && npm run docker:oodikone:real -- logs -f",
+    "start:updater": "npm run docker:up -- --scale sis-updater-worker=4 sis-updater-scheduler sis-updater-worker db_sis sis-updater-nats db adminer redis && npm run docker:logs -- --tail=\"1000\"",
+    "updater:dev:up": "npm run docker:updater:real -- up",
+    "updater:dev:down": "npm run docker:updater:real -- down --remove-orphans"
   },
   "husky": {
     "hooks": {
+      "pre-commit": "scripts/grep_only.sh",
       "pre-push": "npm run lint"
     }
   },
@@ -52,9 +64,9 @@
   "homepage": "https://github.com/UniversityOfHelsinkiCS/oodikone#readme",
   "devDependencies": {
     "concurrently": "^5.3.0",
-    "cypress": "^5.2.0",
+    "cypress": "^6.3.0",
     "cypress-smokify": "0.0.3",
-    "husky": "^4.3.0"
+    "husky": "^4.3.8"
   },
   "dependencies": {}
 }

scripts/assets/welcome.txt

@@ -2,7 +2,7 @@
 
 This tool helps you in managing the project configuration. If you are new to
 Oodikone development, you should probably run "Set up with anonymous data"
-which will take care of setting up and starting Oodikone for you.
+which will take care of setting up and starting Oodikone for you. 
 
 Please use the anonymous data whenever possible!
 

scripts/exec_command_for_services.sh

@@ -43,14 +43,8 @@ else
   services=(
     backend
     frontend
-    updater_writer
-    updater_api
-    updater_scheduler
     analytics
     userservice
-    usageservice
-    sis-updater-scheduler
-    oodikone_sis-updater-worker_1
   )
 fi
 

scripts/grep_only.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+RESULT=$(git grep -HEn --cached "(it|describe)\.only" $(git diff --cached --name-only) ./cypress)
+if [ ! -z "$RESULT" ]; then
+    echo -e "\e[31mDo not commit .only-tests. Did you remember to git add?\e[0m"
+    echo -e "$RESULT"
+    exit 1
+else
+    echo "No .only-tests found"
+    exit 0
+fi

scripts/populate-db.sh

@@ -46,15 +46,18 @@ export -f retry
 export -f drop_create_populate
 mkdir -p backups
 
+
+echo "Enter your Uni Helsinki username:"
+read username
 echo "Fetching latest staging SIS data & OODI demo data"
-scp -r -o ProxyCommand="ssh -W %h:%p melkinpaasi.cs.helsinki.fi" oodikone-staging:/home/tkt_oodi/backups/\{sis-updater-staging.sqz,demo-anon-staging.sqz\} $BACKUPS
+scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkinpaasi.cs.helsinki.fi" $username@oodikone-staging:/home/tkt_oodi/backups/\{sis-updater-staging.sqz,demo-anon-staging.sqz\} $BACKUPS
 
 echo "Setting up services: ${SERVICES[@]}"
 npm run docker:down --prefix $DIR_PATH
 npm run docker:up --prefix $DIR_PATH -- ${SERVICES[@]}
 
 echo "Populating DB_SIS and DB_ANON"
-echo $DB_SIS_ARGS $DB_ANON_ARGS | xargs -n 3 -P 2 sh -c 'drop_create_populate $1 $2 $3' sh
+echo $DB_SIS_ARGS $DB_ANON_ARGS | xargs -n 3 -P 2 bash -c 'drop_create_populate $1 $2 $3' bash
 
 echo "Cleaning up"
 npm run docker:down --prefix $DIR_PATH

scripts/run.sh

@@ -24,9 +24,9 @@ options=(
     "Set up with real data."
     "Reset anonymous database."
     "Reset real data database."
-    "Download latest anonynmous data."
+    "Download latest anonymous data."
     "Download latest real data."
-    "Setup SIS data."
+    "Set up importer-db."
     "Quit."
 )
 
@@ -36,6 +36,7 @@ while true; do
             "Set up with anonymous data.")
                 mopo
                 run_anon_full_setup
+                ./scripts/populate-db.sh
                 ;;
             "Set up with real data.")
                 mopo
@@ -43,18 +44,19 @@ while true; do
                 ;;
             "Reset anonymous database.")
                 reset_db
+                #./scripts/populate-db.sh
                 ;;
             "Reset real data database.")
                 reset_real_db
                 ;;
-            "Download latest anonynmous data.")
+            "Download latest anonymous data.")
                 get_anon_oodikone
                 ;;
             "Download latest real data.")
                 get_oodikone_server_backup
                 ;;
-            "Setup SIS data.")
-                ./scripts/populate-db.sh
+            "Set up importer-db.")
+                run_importer_setup
                 ;;
             "Quit.")
                 break 2

scripts/scripts.sh

@@ -13,9 +13,10 @@ PSQL_REAL_DB_BACKUP="$BACKUP_DIR/latest-pg.sqz"
 KONE_REAL_DB_BACKUP="$BACKUP_DIR/latest-kone-pg.sqz"
 USER_REAL_DB_BACKUP="$BACKUP_DIR/latest-user-pg.sqz"
 ANALYTICS_REAL_DB_BACKUP="$BACKUP_DIR/latest-analytics-pg.sqz"
+SIS_REAL_DB_BACKUP="$BACKUP_DIR/latest-sis.sqz"
 
 docker-compose-dev () {
-    docker-compose -f docker-compose.yml -f ./docker/docker-compose.dev.yml "$@"
+    npm run docker:oodikone:dev -- "$@"
 }
 
 retry () {
@@ -40,6 +41,7 @@ get_oodikone_server_backup() {
     read username
     scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkinpaasi.cs.helsinki.fi" $username@oodikone.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
     scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkinpaasi.cs.helsinki.fi" $username@svm-77.cs.helsinki.fi:/home/tkt_oodi/backups/* "$BACKUP_DIR/"
+    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkinpaasi.cs.helsinki.fi" $username@svm-96.cs.helsinki.fi:/home/updater_user/backups/* "$BACKUP_DIR/"
 }
 
 get_anon_oodikone() {
@@ -48,7 +50,11 @@ get_anon_oodikone() {
 }
 
 restore_psql_from_backup () {
+    echo ""
+    echo "Restoring database from backup ($1/$2):"
+    echo "  1. Copying dump..."
     docker cp $1 $2:/asd.sqz
+    echo "  2. Writing database..."
     docker exec $2 pg_restore -U postgres --no-owner -F c --dbname=$3 -j4 /asd.sqz
 }
 
@@ -56,13 +62,15 @@ ping_psql () {
     drop_psql $1 $2
     echo "Creating psql in container $1 with db name $2"
     retry docker exec -u postgres $1 pg_isready --dbname=$2
-    docker exec -u postgres $1 psql -c "CREATE DATABASE $2" || echo "container $1 DB $2 already exists"
+    #docker exec -u postgres $1 psql -c "CREATE DATABASE $2" || echo "container $1 DB $2 already exists"
+    docker exec -u postgres $1 createdb $2 || echo "container $1 DB $2 already exists"
 }
 
 drop_psql () {
     echo "Dropping psql in container $1 with db name $2"
     retry docker exec -u postgres $1 pg_isready --dbname=$2
-    docker exec -u postgres $1 psql -c "DROP DATABASE $2" || echo "container $1 DB $2 doesn't exists"
+    #docker exec -u postgres $1 psql -c "DROP DATABASE $2" || echo "container $1 DB $2 doesn't exists"
+    docker exec -u postgres $1 dropdb $2 || echo "container $1 DB $2 does not exist"
 }
 
 db_setup_full () {
@@ -75,13 +83,25 @@ db_setup_full () {
     restore_psql_from_backup $USER_REAL_DB_BACKUP oodi_user_db user_db_real
     ping_psql "oodi_analytics_db" "analytics_db_real"
     restore_psql_from_backup $ANALYTICS_REAL_DB_BACKUP oodi_analytics_db analytics_db_real
-    # echo "Restoring MongoDB from backup"
-    # retry restore_mongodb_from_backup
+    ping_psql "db_sis" "db_sis_real"
+    restore_psql_from_backup $SIS_REAL_DB_BACKUP db_sis db_sis_real
     echo "Database setup finished"
 }
 
+run_importer_setup () {
+    echo "Setting up importer database."
+    echo "Enter your Uni Helsinki username:"
+    read username
+    scp -r -o ProxyCommand="ssh -l $username -W %h:%p melkinpaasi.cs.helsinki.fi" $username@importer:/home/importer_user/importer-db/backup/importer-db.sqz "$BACKUP_DIR/"
+    docker-compose -f dco.data.yml up -d sis-importer-db
+    ping_psql "sis-importer-db" "importer-db"
+    restore_psql_from_backup "$BACKUP_DIR/importer-db.sqz" sis-importer-db importer-db
+    docker-compose -f dco.data.yml down
+}
+
 db_anon_setup_full () {
     echo "Restoring PostgreSQL from backup"
+    
     ping_psql "oodi_db" "tkt_oodi"
     ping_psql "oodi_db" "tkt_oodi_test"
     restore_psql_from_backup $PSQL_DB_BACKUP oodi_db tkt_oodi
@@ -95,21 +115,20 @@ db_anon_setup_full () {
 
     ping_psql "oodi_analytics_db" "analytics_db"
     restore_psql_from_backup $ANALYTICS_DB_BACKUP oodi_analytics_db analytics_db
-    # echo "Restoring MongoDB from backup"
-    # retry restore_mongodb_from_backup
+
     echo "Database setup finished"
 }
 
 reset_real_db () {
     docker-compose-dev down
-    docker-compose-dev up -d db user_db db_kone analytics_db
+    docker-compose-dev up -d db user_db db_kone analytics_db db_sis
     db_setup_full
     docker-compose-dev down
 }
 
 reset_db () {
     docker-compose-dev down
-    docker-compose-dev up -d db user_db db_kone analytics_db
+    docker-compose-dev up -d db user_db db_kone analytics_db db_sis
     db_anon_setup_full
     docker-compose-dev down
 }
@@ -124,17 +143,31 @@ reset_db_for_cypress () {
     retry curl --silent --fail localhost:8080/ping
 }
 
-install_cli_npm_packages () {
+install_local_npm_packages () {
     npm ci
+
+    # The rest is required for linting to work.
+    cd services/oodikone2-frontend
+    npm ci
+    cd ../oodikone2-analytics
+    npm ci
+    cd ../oodikone2-userservice
+    npm ci
+    cd ../backend/oodikone2-backend
+    npm ci
+    cd ../updater_writer
+    npm ci
+
+    cd ../../../
 }
 
 show_instructions () {
-    cat ./scripts/assets/instructions.txt
+    cat scripts/assets/instructions.txt
 }
 
 run_full_setup () {
     echo "Setup npm packages"
-    install_cli_npm_packages
+    install_local_npm_packages
     echo "Init dirs"
     init_dirs
     echo "Getting backups from the Oodikone server, this will prompt you for your password. "
@@ -144,16 +177,17 @@ run_full_setup () {
     echo "Building images"
     docker-compose-dev build
     echo "Setup oodikone db from dump."
-    docker-compose-dev up -d db user_db db_kone analytics_db
+    docker-compose-dev up -d db user_db db_kone analytics_db db_sis
     db_setup_full
     db_anon_setup_full
     docker-compose-dev down
+    npm run docker:up:real
     show_instructions
 }
 
 run_anon_full_setup () {
     echo "Setup npm packages"
-    install_cli_npm_packages
+    install_local_npm_packages
     echo "Init dirs"
     init_dirs
     echo "Getting anon backups from the private repository. "
@@ -170,6 +204,7 @@ run_anon_full_setup () {
     docker-compose-dev up -d db user_db db_kone analytics_db
     db_anon_setup_full
     docker-compose-dev down
+    npm run docker:up
     show_instructions
     if [[ ! -z anon_data_error ]] ; then
         tput setaf 1; echo "$anon_data_error"; tput sgr0

scripts/v2_notifier.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+OLD=$(git diff --name-only --cached | grep -Ec "(\/services\/|\/routes\/)")
+NEW=$(git diff --name-only --cached | grep -Ec "(\/servicesV2\/|\/routesV2\/)")
+if (( $OLD > $NEW )); then
+    echo "Do not update old services or routes without updating SIS"
+    exit 1
+else 
+    exit 0
+fi

services/backend/oodikone2-backend/Dockerfile

@@ -4,17 +4,14 @@ WORKDIR /usr/src/app
 EXPOSE 8080
 CMD npm start
 
-COPY ./oodikone2-backend/package.json /usr/src/app/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/app/package-lock.json
-COPY ./oodikone2-backend/package.json /usr/src/cache/package.json
-COPY ./oodikone2-backend/package-lock.json /usr/src/cache/package-lock.json
+COPY ./oodikone2-backend/package* ./
+
 RUN npm ci
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
 ARG TAG=local
 ENV TAG=${TAG}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
 COPY ./oodikone2-backend /usr/src/app
 COPY ./shared/models /usr/src/app/src/models

services/backend/oodikone2-backend/package-lock.json

@@ -3105,19 +3105,11 @@
       "dev": true
     },
     "axios": {
-      "version": "0.19.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz",
-      "integrity": "sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==",
+      "version": "0.21.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
+      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
       "requires": {
-        "follow-redirects": "1.5.10",
-        "is-buffer": "^2.0.2"
-      },
-      "dependencies": {
-        "is-buffer": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz",
-          "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw=="
-        }
+        "follow-redirects": "^1.10.0"
       }
     },
     "babel-eslint": {
@@ -4271,7 +4263,8 @@
     "console-control-strings": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
-      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
+      "optional": true
     },
     "content-disposition": {
       "version": "0.5.3",
@@ -5522,22 +5515,9 @@
       "dev": true
     },
     "follow-redirects": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
-      "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
-      "requires": {
-        "debug": "=3.1.0"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "requires": {
-            "ms": "2.0.0"
-          }
-        }
-      }
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.1.tgz",
+      "integrity": "sha512-SSG5xmZh1mkPGyKzjZP8zLjltIfpW32Y5QpdNJyjcfGxK3qo3NDDkZOZSFiGn1A6SclQxY9GzEwAHQ3dmYRWpg=="
     },
     "for-in": {
       "version": "1.0.2",
@@ -5619,7 +5599,8 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5640,12 +5621,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5660,17 +5643,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5787,7 +5773,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5799,6 +5786,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5813,6 +5801,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5820,12 +5809,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.3.5",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -5844,6 +5835,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5924,7 +5916,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5936,6 +5929,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -6021,7 +6015,8 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -6057,6 +6052,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -6076,6 +6072,7 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -6119,12 +6116,14 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },

services/backend/oodikone2-backend/package.json

@@ -14,6 +14,7 @@
     "start-debug": "node --inspect-brk=0.0.0.0:9229 --max_old_space_size=4096 index.js",
     "docker_local": "NODE_ENV=docker_local nodemon index.js",
     "dev": "NODE_ENV=dev nodemon --max_old_space_size=4096 index.js",
+    "debug": "NODE_ENV=dev nodemon --max_old_space_size=4096 inspect index.js",
     "anon": "NODE_ENV=test nodemon index.js",
     "test": "npm run lint && npm run jest",
     "jest": "NODE_ENV=test jest --runInBand --verbose",
@@ -39,7 +40,7 @@
     "@sentry/node": "^5.11.0",
     "async": "^3.1.0",
     "async-lock": "^1.2.2",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
     "bluebird": "^3.5.5",
     "body-parser": "^1.19.0",
     "cors": "^2.8.5",

services/backend/oodikone2-backend/src/app.js

@@ -24,7 +24,7 @@ initializeDatabaseConnection()
     const app = express()
 
     Sentry.init({
-      dsn: process.env.SENTRY_ADDR,
+      dsn: 'https://c55e6d020db640e889948cc25ced1c19@sentry.toska.cs.helsinki.fi/2',
       environment: process.env.TAG
     })
     app.use(Sentry.Handlers.requestHandler())

services/backend/oodikone2-backend/src/conf-backend.js

@@ -32,7 +32,6 @@ const KEY_PATH = process.env.KEY_PATH // production/staging only
 const USERSERVICE_URL = process.env.USERSERVICE_URL
 const UPDATER_URL = process.env.UPDATER_URL
 const SIS_UPDATER_URL = process.env.SIS_UPDATER_URL
-const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
 const PORT = isTest ? 8079 : 8080
 const OODI_SECRET = process.env.OODI_SECRET
@@ -100,7 +99,6 @@ module.exports = {
   ACCESS_TOKEN_HEADER_KEY,
   PORT,
   ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  USAGESERVICE_URL,
   requiredGroup,
   OODI_SECRET,
   OODI_SECRET_HEADER_KEY,

services/backend/oodikone2-backend/src/databaseV2/connection.js

@@ -31,6 +31,7 @@ class DbConnection extends EventEmitter {
         return
       }
       console.log(`Sis database connection failed! Attempt ${attempt}/${this.RETRY_ATTEMPTS}`)
+      console.log(e)
       setTimeout(() => this.connect(attempt + 1), 1000 * attempt)
     }
   }

services/backend/oodikone2-backend/src/events.js

@@ -20,7 +20,7 @@ const { productivityStatsForCode, throughputStatsForCode } = require('./services
 const {
   productivityStatsForStudytrack: productivityStatsForStudytrackV2,
   throughputStatsForStudytrack: throughputStatsForStudytrackV2
-} = require('./servicesV2/studytrack')
+} = require('./servicesV2/studyprogramme')
 
 const { calculateFacultyYearlyStats } = require('./services/faculties')
 const topteachers = require('./services/topteachers')

services/backend/oodikone2-backend/src/middleware/accesslogger.js

@@ -1,5 +1,4 @@
 const morgan = require('morgan')
-const usageService = require('../services/usageService')
 const logger = require('../util/logger')
 
 // So this appears to be a hack to get neatly formatted stats like response-time etc. from morgan
@@ -26,8 +25,10 @@ const accessLogger = morgan((tokens, req, res) => {
     'ms'
   ].join(' ')
 
-  usageService.log(message, meta).catch(e => {
-    logger.error('Failed to save usage stats!', e)
+  logger.info(message, {
+    ...meta,
+    // pass this as a custom field so we can filter by it in graylog
+    isUsageStats: true
   })
 })
 

services/backend/oodikone2-backend/src/middleware/auth.js

@@ -10,13 +10,24 @@ const TOKEN_VERSION = 1 // When token structure changes, increment in userservic
 const checkAuth = async (req, res, next) => {
   const token = req.headers[ACCESS_TOKEN_HEADER_KEY]
   const uid = req.headers['uid']
-  if (!token) return res.status(403).json({ error: 'No token in headers' })
+
+  if (!token) {
+    return res.status(403).json({ error: 'No token in headers' })
+  }
+
   jwt.verify(token, conf.TOKEN_SECRET, async (err, decoded) => {
-    if (err) return res.status(403).json(err)
-    if (decoded.version !== TOKEN_VERSION)
+    if (err) {
+      return res.status(403).json(err)
+    }
+
+    if (decoded.version !== TOKEN_VERSION) {
       return res.status(401).json({ error: 'Token needs to be refreshed - invalid version' })
-    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid)
+    }
+
+    if (decoded.mockedBy ? decoded.mockedBy !== uid : decoded.userId !== uid) {
       return res.status(403).json({ error: 'User shibboleth id and token id did not match' })
+    }
+
     const userData = await getUserDataFor(decoded.userId)
     req.decodedToken = decoded
     Object.assign(req, userData)
@@ -27,12 +38,12 @@ const checkAuth = async (req, res, next) => {
 const roles = requiredRoles => async (req, res, next) => {
   if (req.decodedToken) {
     const { roles } = req
+
     if (requiredRoles.every(r => roles.indexOf(r) >= 0) || roles.includes('admin')) {
-      next()
-      return
+      return next()
     }
   }
-  console.log(`Missing required roles ${requiredRoles}`)
+
   res.status(403).json({ error: 'missing required roles' })
 }
 
@@ -40,25 +51,28 @@ const checkRequiredGroup = async (req, res, next) => {
   const hyGroups = parseHyGroups(req.headers['hygroupcn'])
   const enabled = hasRequiredGroup(hyGroups)
   const tokenOutdated = req.decodedToken.enabled !== enabled
+
   if (tokenOutdated) {
-    res.status(401).json({
-      error: 'Token needs to be refreshed - enabled doesnt match hy-group requirement'
+    return res.status(401).json({
+      error: 'Token needs to be refreshed - enabled does not match hy-group requirement'
     })
-  } else if (!enabled) {
-    res.status(403).json({ error: 'User is not enabled' })
-  } else {
-    next()
   }
+
+  if (!enabled) {
+    return res.status(403).json({ error: 'User is not enabled' })
+  }
+  next()
 }
 
 const checkUserBlacklisting = async (req, res, next) => {
   const { userId, createdAt } = req.decodedToken
   const isBlacklisted = await blacklist.isUserBlacklisted(userId, createdAt)
+
   if (isBlacklisted) {
-    res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
-  } else {
-    next()
+    return res.status(401).json({ error: 'Token needs to be refreshed - blacklisted' })
   }
+
+  next()
 }
 
 module.exports = {

services/backend/oodikone2-backend/src/modelsV2/index.js

@@ -57,7 +57,7 @@ Credit.belongsToMany(Teacher, { through: CreditTeacher, foreignKey: 'credit_id'
 Teacher.belongsToMany(Credit, { through: CreditTeacher, foreignKey: 'teacher_id' })
 
 Organization.hasMany(Studyright, { foreignKey: 'facultyCode', sourceKey: 'code' })
-Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', sourceKey: 'code' })
+Studyright.belongsTo(Organization, { foreignKey: 'facultyCode', targetKey: 'code' })
 
 Studyright.belongsTo(Student, { foreignKey: 'studentStudentnumber', targetKey: 'studentnumber' })
 Student.hasMany(Studyright, { foreignKey: 'studentStudentnumber', sourceKey: 'studentnumber' })

services/backend/oodikone2-backend/src/routes.js

@@ -1,7 +1,6 @@
 const shibbolethHeadersFix = require('unfuck-utf8-headers-middleware')
 const accessLogger = require('./middleware/accesslogger')
 const sentryUserId = require('./middleware/sentryUserId')
-const log = require('./routes/log')
 const courses = require('./routes/courses')
 const students = require('./routes/students')
 const population = require('./routes/population')
@@ -12,7 +11,6 @@ const users = require('./routes/users')
 const elementdetails = require('./routes/elementdetails')
 const auth = require('./middleware/auth')
 const teachers = require('./routes/teachers')
-const usage = require('./routes/usage')
 const providers = require('./routes/providers')
 const faculties = require('./routes/faculties')
 const facultyStats = require('./routes/facultyStats')
@@ -36,7 +34,6 @@ module.exports = (app, url) => {
   app.use(url, ping)
   app.use(shibbolethHeadersFix(['hyGroupCn', 'SHIB_LOGOUT_URL', 'eduPersonAffiliation', 'uid', 'displayName', 'mail']))
   app.use(url, matomoInit)
-  app.use(url, log)
   app.use(url, login)
   app.use(`${url}/superlogin`, superlogin)
   app.use(auth.checkAuth, auth.checkRequiredGroup, auth.checkUserBlacklisting, accessLogger, sentryUserId)
@@ -55,7 +52,6 @@ module.exports = (app, url) => {
   app.use(`${url}/teachers`, auth.roles(['teachers']), teachers)
   app.use(`${url}/users`, auth.roles(['users']), users)
   app.use(`${url}/feedback`, feedback)
-  app.use(`${url}/usage`, auth.roles(['usage']), usage)
   app.use(`${url}/course-groups`, auth.roles(['coursegroups']), courseGroups)
   app.use(`${url}/mandatory_courses`, mandatoryCourses)
   app.use(`${url}/mandatory-course-labels`, mandatoryCourseLabels)

services/backend/oodikone2-backend/src/routes/coolDataScience.js

@@ -8,6 +8,8 @@ const {
   getStartYears,
   getGraduatedStatus
 } = require('../services/coolDataScience')
+const useSisRouter = require('../util/useSisRouter')
+const trendsV2 = require('../routesV2/trends')
 
 router.get('/start-years', async (req, res) => {
   const years = await getStartYears()
@@ -76,4 +78,4 @@ router.get(
   })
 )
 
-module.exports = router
+module.exports = useSisRouter(trendsV2, router)

services/backend/oodikone2-backend/src/routes/courses.js

@@ -71,14 +71,15 @@ router.get('/v2/courseyearlystats', async (req, res) => {
 router.get('/v3/courseyearlystats', async (req, res) => {
   try {
     const { rights, roles } = req
-    const admin = roles && roles.includes('admin')
-    if (!admin) {
-      // If user has rights to see at least one programme, then they are allowed
-      // to see all of them
-      if (rights.length <= 0) {
-        return res.status(403).json({ error: 'No programmes so no access to course stats' })
-      }
+
+    const allowedByRoles = roles && ['admin', 'courseStatistics'].some(role => roles.includes(role))
+
+    // If user has rights to see at least one programme, then they are allowed
+    // to see all of them
+    if (!allowedByRoles && rights.length < 1) {
+      return res.status(403).json({ error: 'No programmes so no access to course stats' })
     }
+
     const { codes, separate: sep, unifyOpenUniCourses: unify } = req.query
 
     const separate = !sep ? false : JSON.parse(sep)

services/backend/oodikone2-backend/src/routes/elementdetails.js

@@ -2,12 +2,7 @@ const router = require('express').Router()
 const { getStudentsUserCanAccess } = require('../services/userService')
 const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
 const MandatoryCourses = require('../services/mandatoryCourses')
-const {
-  productivityStatsForCode,
-  throughputStatsForCode,
-  bachelorData,
-  masterData
-} = require('../services/studyprogramme')
+const { productivityStatsForCode, throughputStatsForCode, optionData } = require('../services/studyprogramme')
 const { findProgrammeTheses, createThesisCourse, deleteThesisCourse } = require('../services/thesis')
 const {
   getProductivity,
@@ -92,10 +87,10 @@ router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
   const code = req.params.id
   if (code) {
     if (code.includes('MH')) {
-      const data = await bachelorData(new Date('2017-07-31'), new Date(), code)
+      const data = await optionData(new Date('2017-07-31'), new Date(), code, 'MSC')
       return res.json(data)
     } else if (code.includes('KH')) {
-      const data = await masterData(new Date('2017-07-31'), new Date(), code)
+      const data = await optionData(new Date('2017-07-31'), new Date(), code, 'BSC')
       return res.json(data)
     } else {
       return res.json([])

services/backend/oodikone2-backend/src/routes/log.js

@@ -1,9 +0,0 @@
-const router = require('express').Router()
-const logger = require('../util/logger')
-
-router.post('/log', async (req, res) => {
-  logger.error(req.body.message, req.body['full_message'])
-  res.status(200).end()
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routes/population.js

@@ -495,7 +495,7 @@ router.post('/updatedatabase', async (req, res) => {
   try {
     const response = await updateStudents(studentnumbers)
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)

services/backend/oodikone2-backend/src/routes/updater.js

@@ -12,7 +12,13 @@ const {
   updateNoStudents,
   updateDaily
 } = require('../services/updaterService')
-const { updateSISMetadata, updateSISStudents, updateSISProgrammes } = require('../services/sisUpdaterService')
+const {
+  updateSISMetadata,
+  updateSISStudents,
+  updateSISProgrammes,
+  updateStudentsByStudentNumber,
+  abort
+} = require('../services/sisUpdaterService')
 const { refreshStatistics, refreshStatisticsV2 } = require('../events')
 
 router.post('/update/oldest', async (req, res) => {
@@ -24,7 +30,7 @@ router.post('/update/oldest', async (req, res) => {
   try {
     const response = await updateOldestStudents(amount)
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -35,7 +41,7 @@ router.post('/update/all', async (req, res) => {
   try {
     const response = await updateAllStudents()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -46,7 +52,7 @@ router.post('/update/active', async (req, res) => {
   try {
     const response = await updateActiveStudents()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -57,7 +63,7 @@ router.post('/update/no_student', async (req, res) => {
   try {
     const response = await updateNoStudents()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -68,7 +74,7 @@ router.post('/update/attainment', async (req, res) => {
   try {
     const response = await updateAttainments()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -79,7 +85,7 @@ router.post('/update/meta', async (req, res) => {
   try {
     const response = await updateMetadata()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -90,7 +96,7 @@ router.post('/update/studentlist', async (req, res) => {
   try {
     const response = await updateStudentlist()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -111,7 +117,7 @@ router.post('/reschedule/scheduled', async (req, res) => {
   try {
     const response = await rescheduleScheduled()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -122,7 +128,7 @@ router.post('/reschedule/fetched', async (req, res) => {
   try {
     const response = await rescheduleFetched()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -133,7 +139,7 @@ router.post('/update/daily', async (req, res) => {
   try {
     const response = await updateDaily()
     if (response) {
-      res.status(200).json('Scheduled')
+      res.status(200).json({ ...response })
     }
   } catch (err) {
     res.status(500).json(err)
@@ -170,7 +176,22 @@ router.get('/update/v2/students', async (req, res) => {
   if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
 
   try {
-    const response = await updateSISStudents(req)
+    const response = await updateSISStudents()
+    if (response) {
+      res.status(200).json('Update SIS students scheduled')
+    }
+  } catch (err) {
+    console.log(err)
+    res.status(500).json({ error: 'error' })
+  }
+})
+
+router.post('/update/v2/students', async (req, res) => {
+  const { roles } = req
+  if (!roles.includes('dev')) return res.status(403).send('No rights, please stop')
+
+  try {
+    const response = await updateStudentsByStudentNumber(req.body)
     if (response) {
       res.status(200).json('Update SIS students scheduled')
     }
@@ -205,4 +226,14 @@ router.post('/refresh_statistic_v2', async (req, res) => {
   }
 })
 
+router.get('/abort', async (req, res) => {
+  try {
+    await abort()
+    res.status(200).json()
+  } catch (err) {
+    console.log(err)
+    res.status(500).json({ error: 'error' })
+  }
+})
+
 module.exports = router

services/backend/oodikone2-backend/src/routes/usage.js

@@ -1,14 +0,0 @@
-const router = require('express').Router()
-const UsageService = require('../services/usageService')
-
-const S_TO_MS = 1000
-
-router.get('/', async (req, res) => {
-  const from = req.query.from ? new Date(req.query.from) : new Date(1 * S_TO_MS)
-  const to = req.query.to ? new Date(req.query.to) : new Date(new Date().getTime() + 60 * S_TO_MS)
-  const resultStream = await UsageService.getStream(from, to)
-  res.contentType('json')
-  resultStream.pipe(res)
-})
-
-module.exports = router

services/backend/oodikone2-backend/src/routesV2/elementdetails.js

@@ -2,7 +2,11 @@ const router = require('express').Router()
 const { getStudentsUserCanAccess } = require('../servicesV2/userService')
 //const { getAllDegreesAndProgrammes, getAllProgrammes, getAllElementDetails } = require('../services/studyrights')
 const MandatoryCourses = require('../servicesV2/mandatoryCourses')
-const { productivityStatsForStudytrack, throughputStatsForStudytrack } = require('../servicesV2/studytrack')
+const {
+  productivityStatsForStudytrack,
+  throughputStatsForStudytrack,
+  optionData
+} = require('../servicesV2/studyprogramme')
 const { findProgrammeTheses /* , createThesisCourse, deleteThesisCourse */ } = require('../services/thesis')
 
 const {
@@ -36,6 +40,22 @@ const programmeStatsSince = new Date('2017-07-31')
   }
 }) */
 
+router.get('/v2/studyprogrammes/:id/optiondata', async (req, res) => {
+  const code = req.params.id
+
+  let level
+  if (code.includes('MH')) {
+    level = 'MSC'
+  } else if (code.includes('KH')) {
+    level = 'BSC'
+  } else {
+    return res.json([])
+  }
+
+  const data = await optionData(new Date('2017-07-31'), new Date(), code, level)
+  res.json(data)
+})
+
 router.get('/v2/studyprogrammes/:id/present_students', async (req, res) => {
   try {
     const {

services/backend/oodikone2-backend/src/routesV2/population.js

@@ -214,6 +214,7 @@ router.post('/v2/populationstatistics/coursesbystudentnumberlist', async (req, r
 router.get('/v3/populationstatistics', async (req, res) => {
   const { year, semesters, studyRights: studyRightsJSON } = req.query
   const { decodedToken } = req
+
   try {
     if (!year || !semesters || !studyRightsJSON) {
       res.status(400).json({ error: 'The query should have a year, semester and studyRights defined' })

services/backend/oodikone2-backend/src/routesV2/trends.js

@@ -0,0 +1,79 @@
+const router = require('express').Router()
+const {
+  withErr,
+  getProtoC,
+  getProtoCProgramme,
+  getStatus,
+  getUber,
+  getStartYears,
+  getGraduatedStatus
+} = require('../servicesV2/trends')
+
+router.get('/start-years', async (req, res) => {
+  const years = await getStartYears()
+  res.json(years.map(({ studystartdate }) => studystartdate))
+})
+
+router.get(
+  '/uber-data',
+  withErr(async (req, res) => {
+    const data = await getUber(req.query)
+    res.json(data)
+  })
+)
+
+router.get(
+  '/proto-c-data-programme',
+  withErr(async (req, res) => {
+    const data = await getProtoCProgramme(req.query)
+    res.json(data)
+  })
+)
+
+router.get(
+  '/proto-c-data',
+  withErr(async (req, res) => {
+    const data = await getProtoC(req.query)
+    res.json(data)
+  })
+)
+
+router.get(
+  '/status-graduated',
+  withErr(async (req, res) => {
+    const { date: unixMillis, showByYear } = req.query
+    const date = new Date(Number(unixMillis))
+    const endOfToday = new Date()
+    endOfToday.setHours(23, 59, 59, 999)
+
+    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
+      return res.status(400).json({ error: 'Invalid date' })
+    }
+
+    // End of day
+    date.setHours(23, 59, 59, 999)
+    const data = await getGraduatedStatus(date.getTime(), showByYear)
+    res.json(data)
+  })
+)
+
+router.get(
+  '/status',
+  withErr(async (req, res) => {
+    const { date: unixMillis, showByYear } = req.query
+    const date = new Date(Number(unixMillis))
+    const endOfToday = new Date()
+    endOfToday.setHours(23, 59, 59, 999)
+
+    if (isNaN(date.getTime()) || date.getTime() > endOfToday.getTime()) {
+      return res.status(400).json({ error: 'Invalid date' })
+    }
+
+    // End of day
+    date.setHours(23, 59, 59, 999)
+    const status = await getStatus(date.getTime(), showByYear)
+    res.json(status)
+  })
+)
+
+module.exports = router

services/backend/oodikone2-backend/src/services/blacklist.js

@@ -11,9 +11,9 @@ const addUserToBlacklist = async userId => {
 const isUserBlacklisted = async (userId, tokenCreatedAt) => {
   const isBlacklisted = async () => {
     const blacklistTimestamp = await redisClient.getAsync(MAKE_REDIS_HASH_KEY(userId))
-    const status = !!(tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp))
-    return status
+    return tokenCreatedAt && blacklistTimestamp && moment(tokenCreatedAt) < moment(blacklistTimestamp)
   }
+
   const timer = () => new Promise(resolve => setTimeout(() => resolve(true), 5000))
   return Promise.race([isBlacklisted(), timer()])
 }

services/backend/oodikone2-backend/src/services/courses.js

@@ -737,7 +737,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses) => {
     ...statistics,
     coursecode,
     alternatives: uniqueCodes,
-    name: course.name.fi
+    name: course.name
   }
 }
 

services/backend/oodikone2-backend/src/services/populations.js

@@ -55,7 +55,8 @@ const formatStudentForPopulationStatistics = (
     gender_sv,
     gender_en,
     tags,
-    option
+    option,
+    birthdate
   },
   credits,
   startDate,
@@ -72,13 +73,15 @@ const formatStudentForPopulationStatistics = (
             .toISOString()
         : attainment_date
     const passed = Credit.passed({ credittypecode })
+
     return {
       course_code,
       date: attainment_date_normailized,
       passed,
       grade: passed ? grade : 'Hyl.',
       credits,
-      isStudyModuleCredit: isStudyModule
+      isStudyModuleCredit: isStudyModule,
+      credittypecode
     }
   }
 
@@ -103,7 +106,8 @@ const formatStudentForPopulationStatistics = (
     tags: tags || [],
     studyrightStart: startDate,
     starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]'),
-    option
+    option,
+    birthdate
   }
 }
 
@@ -202,7 +206,8 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'gender_code',
         'gender_fi',
         'gender_sv',
-        'gender_en'
+        'gender_en',
+        'birthdate'
       ],
       include: [
         {
@@ -220,7 +225,8 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
             'graduated',
             'canceldate',
             'prioritycode',
-            'faculty_code'
+            'faculty_code',
+            'studystartdate'
           ],
           separate: true,
           include: [
@@ -550,11 +556,28 @@ const parseQueryParams = query => {
   }
 }
 
-const getMastersForStudents = async (students, code) => {
+const getOptionsForStudents = async (students, code, level) => {
   if (!code || !students.length) return {}
+
+  let graduated
+  let currentExtent
+  let optionExtent
+
+  if (level === 'BSC') {
+    graduated = { graduated: 1 }
+    currentExtent = 1
+    optionExtent = 2
+  } else if (level === 'MSC') {
+    graduated = {}
+    currentExtent = 2
+    optionExtent = 1
+  } else {
+    throw new Error('Invalid study level ' + level)
+  }
+
   const programmes = await getAllProgrammes()
 
-  const bachelors = await Studyright.findAll({
+  const currentStudyrights = await Studyright.findAll({
     include: [
       {
         model: StudyrightElement,
@@ -567,20 +590,21 @@ const getMastersForStudents = async (students, code) => {
       }
     ],
     where: {
-      graduated: 1,
-      extentcode: 1,
+      ...graduated,
+      extentcode: currentExtent,
       student_studentnumber: {
         [Op.in]: students
       }
     },
     attributes: ['student_studentnumber', 'givendate']
   })
-  const bachelorsMap = bachelors.reduce((obj, studyright) => {
+
+  const currentStudyrightsMap = currentStudyrights.reduce((obj, studyright) => {
     obj[studyright.student_studentnumber] = studyright.givendate
     return obj
   }, {})
 
-  const masters = await Studyright.findAll({
+  const options = await Studyright.findAll({
     include: [
       {
         model: StudyrightElement,
@@ -602,7 +626,7 @@ const getMastersForStudents = async (students, code) => {
       }
     ],
     where: {
-      extentcode: 2,
+      extentcode: optionExtent,
       student_studentnumber: {
         [Op.in]: students
       }
@@ -610,80 +634,10 @@ const getMastersForStudents = async (students, code) => {
     order: [[StudyrightElement, 'startdate', 'DESC']],
     attributes: ['student_studentnumber', 'givendate']
   })
-  return masters
-    .filter(m => m.student_studentnumber in bachelorsMap)
-    .filter(m => m.givendate.getTime() === bachelorsMap[m.student_studentnumber].getTime())
-    .reduce((obj, element) => {
-      obj[element.student_studentnumber] = {
-        code: element.studyright_elements[0].code,
-        name: element.studyright_elements[0].element_detail.name
-      }
-      return obj
-    }, {})
-}
 
-const getBachelorsForStudents = async (students, code) => {
-  if (!code || !students.length) return {}
-  const programmes = await getAllProgrammes()
-
-  const masters = await Studyright.findAll({
-    include: [
-      {
-        model: StudyrightElement,
-        where: {
-          studentnumber: {
-            [Op.in]: students
-          },
-          code: code
-        }
-      }
-    ],
-    where: {
-      extentcode: 2,
-      student_studentnumber: {
-        [Op.in]: students
-      }
-    },
-    attributes: ['student_studentnumber', 'givendate']
-  })
-  const mastersMap = masters.reduce((obj, studyright) => {
-    obj[studyright.student_studentnumber] = studyright.givendate
-    return obj
-  }, {})
-
-  const bachelors = await Studyright.findAll({
-    include: [
-      {
-        model: StudyrightElement,
-        where: {
-          studentnumber: {
-            [Op.in]: students
-          },
-          code: {
-            [Op.in]: programmes.map(p => p.code)
-          }
-        },
-        include: [
-          {
-            model: ElementDetails,
-            attributes: ['name']
-          }
-        ],
-        attributes: ['code', 'startdate']
-      }
-    ],
-    where: {
-      extentcode: 1,
-      student_studentnumber: {
-        [Op.in]: students
-      }
-    },
-    order: [[StudyrightElement, 'startdate', 'DESC']],
-    attributes: ['student_studentnumber', 'givendate']
-  })
-  return bachelors
-    .filter(b => b.student_studentnumber in mastersMap)
-    .filter(b => b.givendate.getTime() === mastersMap[b.student_studentnumber].getTime())
+  return options
+    .filter(m => m.student_studentnumber in currentStudyrightsMap)
+    .filter(m => m.givendate.getTime() === currentStudyrightsMap[m.student_studentnumber].getTime())
     .reduce((obj, element) => {
       obj[element.student_studentnumber] = {
         code: element.studyright_elements[0].code,
@@ -786,19 +740,15 @@ const formatQueryParamArrays = (query, params) => {
 const optimizedStatisticsOf = async (query, studentnumberlist) => {
   const formattedQueryParams = formatQueryParamArrays(query, ['semesters', 'studentStatuses'])
 
-  if (
-    !formattedQueryParams.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)
-  ) {
+  if (!formattedQueryParams.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
     return { error: 'Semester should be either SPRING OR FALL' }
   }
 
   if (
     formattedQueryParams.studentStatuses &&
-    !formattedQueryParams.studentStatuses
-      .map(
-        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
-      )
-      .every(e => e === true)
+    !formattedQueryParams.studentStatuses.every(
+      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+    )
   ) {
     return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
   }
@@ -829,12 +779,13 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
         transferredStudents
       )
   // wtf
+  // plz
   const code = studyRights[0] || ''
   let optionData = {}
   if (code.includes('MH')) {
-    optionData = await getBachelorsForStudents(studentnumbers, code)
+    optionData = await getOptionsForStudents(studentnumbers, code, 'MSC')
   } else if (code.includes('KH')) {
-    optionData = await getMastersForStudents(studentnumbers, code)
+    optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
   }
 
   const { students, credits, extents, semesters, elementdetails, courses } = await getStudentsIncludeCoursesBetween(

services/backend/oodikone2-backend/src/services/sisUpdaterService.js

@@ -8,6 +8,12 @@ const updateSISStudents = async () => {
   return response.data
 }
 
+const updateStudentsByStudentNumber = async studentnumbers => {
+  const data = { studentnumbers }
+  const response = await client.post('/v1/students', data, { params: { token: SECRET_TOKEN } })
+  return response.data
+}
+
 const updateSISMetadata = async () => {
   const response = await client.get('/v1/meta', { params: { token: SECRET_TOKEN } })
   return response.data
@@ -18,8 +24,15 @@ const updateSISProgrammes = async () => {
   return response.data
 }
 
+const abort = async () => {
+  const response = await client.get('/v1/abort', { params: { token: SECRET_TOKEN } })
+  return response.data
+}
+
 module.exports = {
   updateSISMetadata,
   updateSISStudents,
-  updateSISProgrammes
+  updateSISProgrammes,
+  updateStudentsByStudentNumber,
+  abort
 }

services/backend/oodikone2-backend/src/services/studyprogramme.js

@@ -517,11 +517,27 @@ const startedStudyright = async (studentnumbers, startDate, studytrack, endDate)
   })
 }
 
-const masterData = async (startDate, endDate, code) => {
+const optionData = async (startDate, endDate, code, level) => {
   const programmes = await getAllProgrammes()
   const students = await studentnumbersWithAllStudyrightElements([code], startDate, endDate, false, true)
 
-  const bachelors = await Studyright.findAll({
+  let graduated
+  let currentExtent
+  let optionExtent
+
+  if (level === 'BSC') {
+    graduated = { graduated: 1 }
+    currentExtent = 1
+    optionExtent = 2
+  } else if (level === 'MSC') {
+    graduated = {}
+    currentExtent = 2
+    optionExtent = 1
+  } else {
+    throw new Error('Invalid study level ' + level)
+  }
+
+  const currentStudyrights = await Studyright.findAll({
     include: [
       {
         model: StudyrightElement,
@@ -532,8 +548,8 @@ const masterData = async (startDate, endDate, code) => {
       }
     ],
     where: {
-      graduated: 1,
-      extentcode: 1,
+      ...graduated,
+      extentcode: currentExtent,
       student_studentnumber: {
         [Op.in]: students
       }
@@ -541,7 +557,7 @@ const masterData = async (startDate, endDate, code) => {
     attributes: ['student_studentnumber', 'givendate', 'studystartdate']
   })
 
-  const bachelorsMap = bachelors
+  const currentStudyrightsMap = currentStudyrights
     .filter(b => b.studystartdate)
     .reduce((obj, studyright) => {
       const acualDate = new Date(Math.max(+studyright.studystartdate, +studyright.studyright_elements[0].startdate))
@@ -549,7 +565,7 @@ const masterData = async (startDate, endDate, code) => {
       return obj
     }, {})
 
-  const masters = await Studyright.findAll({
+  const options = await Studyright.findAll({
     include: [
       {
         model: StudyrightElement,
@@ -571,7 +587,7 @@ const masterData = async (startDate, endDate, code) => {
       }
     ],
     where: {
-      extentcode: 2,
+      extentcode: optionExtent,
       student_studentnumber: {
         [Op.in]: students
       }
@@ -583,11 +599,11 @@ const masterData = async (startDate, endDate, code) => {
   const data = {}
   const years = new Set()
 
-  masters
-    .filter(m => m.student_studentnumber in bachelorsMap)
-    .filter(m => m.givendate.getTime() === bachelorsMap[m.student_studentnumber].givendate.getTime())
+  options
+    .filter(m => m.student_studentnumber in currentStudyrightsMap)
+    .filter(m => m.givendate.getTime() === currentStudyrightsMap[m.student_studentnumber].givendate.getTime())
     .forEach(b => {
-      const date = bachelorsMap[b.student_studentnumber].startdate
+      const date = currentStudyrightsMap[b.student_studentnumber].startdate
       const year =
         date.getMonth() > 6 || (date.getMonth() === 6 && date.getDate() == 31)
           ? date.getFullYear()
@@ -606,103 +622,7 @@ const masterData = async (startDate, endDate, code) => {
       data[code][year] += 1
       data[code].total += 1
     })
-  const dataAsArray = Object.keys(data).map(code => {
-    const { name, total } = data[code]
-    const output = { code, name, total }
-    years.forEach(y => (output[y] = data[code][y] || 0))
-    return output
-  })
-  return { data: dataAsArray, years: Array.from(years).sort() }
-}
 
-const bachelorData = async (startDate, endDate, code) => {
-  const programmes = await getAllProgrammes()
-  const students = await studentnumbersWithAllStudyrightElements([code], startDate, endDate, false, true)
-
-  const masters = await Studyright.findAll({
-    include: [
-      {
-        model: StudyrightElement,
-        where: {
-          code: code
-        },
-        attributes: ['startdate']
-      }
-    ],
-    where: {
-      extentcode: 2,
-      student_studentnumber: {
-        [Op.in]: students
-      }
-    },
-    attributes: ['student_studentnumber', 'givendate', 'studystartdate']
-  })
-
-  const mastersMap = masters
-    .filter(m => m.studystartdate)
-    .reduce((obj, studyright) => {
-      const acualDate = new Date(Math.max(+studyright.studystartdate, +studyright.studyright_elements[0].startdate))
-      obj[studyright.student_studentnumber] = { givendate: studyright.givendate, startdate: acualDate }
-      return obj
-    }, {})
-
-  const bachelors = await Studyright.findAll({
-    include: [
-      {
-        model: StudyrightElement,
-        where: {
-          studentnumber: {
-            [Op.in]: students
-          },
-          code: {
-            [Op.in]: programmes.map(p => p.code)
-          }
-        },
-        include: [
-          {
-            model: ElementDetails,
-            attributes: ['name']
-          }
-        ],
-        attributes: ['code']
-      }
-    ],
-    where: {
-      extentcode: 1,
-      student_studentnumber: {
-        [Op.in]: students
-      }
-    },
-    order: [[StudyrightElement, 'startdate', 'DESC']],
-    attributes: ['student_studentnumber', 'startdate', 'givendate']
-  })
-
-  const data = {}
-  const years = new Set()
-
-  bachelors
-    .filter(b => b.student_studentnumber in mastersMap)
-    .filter(b => b.givendate.getTime() === mastersMap[b.student_studentnumber].givendate.getTime())
-    .forEach(b => {
-      const date = mastersMap[b.student_studentnumber].startdate
-      const year =
-        date.getMonth() > 6 || (date.getMonth() === 6 && date.getDate() == 31)
-          ? date.getFullYear()
-          : date.getFullYear() - 1
-      years.add(year)
-      const code = b.studyright_elements[0].code
-
-      if (!data[code]) {
-        data[code] = {}
-        data[code].name = b.studyright_elements[0].element_detail.name
-        data[code].total = 0
-      }
-      if (!data[code][year]) {
-        data[code][year] = 0
-      }
-      data[code][year] += 1
-      data[code].total += 1
-    })
   const dataAsArray = Object.keys(data).map(code => {
     const { name, total } = data[code]
     const output = { code, name, total }
@@ -984,6 +904,5 @@ module.exports = {
   findProgrammeThesisCredits,
   thesisProductivityFromCredits,
   thesisProductivityForStudytrack,
-  bachelorData,
-  masterData
+  optionData
 }

services/backend/oodikone2-backend/src/services/usageService.js

@@ -1,26 +0,0 @@
-const axios = require('axios')
-const { USAGESERVICE_URL } = require('../conf-backend')
-
-/** @type {axios.AxiosInstance} */
-const client = axios.create({ baseURL: USAGESERVICE_URL, headers: { secret: process.env.USAGESERVICE_SECRET } })
-
-const ping = async () => {
-  const url = '/ping'
-  await client.get(url)
-}
-
-const log = async (message, meta) => {
-  await client.post('/log', { message, meta })
-}
-
-/** @returns {Promise<stream.Readable>} */
-const getStream = async (from, to) => {
-  const res = await client.get('/log', { params: { from, to }, responseType: 'stream' })
-  return res.data
-}
-
-module.exports = {
-  ping,
-  log,
-  getStream
-}

services/backend/oodikone2-backend/src/servicesV2/courses.js

@@ -734,7 +734,7 @@ const yearlyStatsOfNew = async (coursecode, separate, unifyOpenUniCourses) => {
     ...statistics,
     coursecode,
     alternatives: uniqueCodes,
-    name: course.name.fi
+    name: course.name
   }
 }
 

services/backend/oodikone2-backend/src/servicesV2/populations.js

@@ -1,6 +1,7 @@
 const { Op } = require('sequelize')
 const moment = require('moment')
 const { sortBy } = require('lodash')
+const { getCurrentSemester } = require('./semesters')
 
 const {
   Student,
@@ -23,6 +24,7 @@ const { Tag, TagStudent } = require('../models/models_kone')
 const { getCodeToMainCourseMap, unifyOpenUniversity } = require('./courses')
 const { CourseStatsCounter } = require('./course_stats_counter')
 const { getPassingSemester, semesterEnd, semesterStart } = require('../util/semester')
+const { getAllProgrammes } = require('./studyrights')
 
 const enrolmentDates = () => {
   const query = 'SELECT DISTINCT s.dateOfUniversityEnrollment as date FROM Student s'
@@ -55,7 +57,9 @@ const formatStudentForPopulationStatistics = (
     gender_fi,
     gender_sv,
     gender_en,
-    tags
+    tags,
+    option,
+    birthdate
   },
   credits,
   startDate,
@@ -71,14 +75,16 @@ const formatStudentForPopulationStatistics = (
             .add(1, 'day')
             .toISOString()
         : attainment_date
+    const passed = Credit.passed({ credittypecode })
 
     return {
       course_code,
       date: attainment_date_normailized,
-      passed: Credit.passed({ credittypecode }),
-      grade,
+      passed,
+      grade: passed ? grade : 'Hyl.',
       credits,
-      isStudyModuleCredit: isStudyModule
+      isStudyModuleCredit: isStudyModule,
+      credittypecode
     }
   }
 
@@ -102,7 +108,9 @@ const formatStudentForPopulationStatistics = (
     updatedAt: updatedAt || createdAt,
     tags: tags || [],
     studyrightStart: startDate,
-    starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]')
+    starting: moment(started).isBetween(startDateMoment, endDateMoment, null, '[]'),
+    option,
+    birthdate
   }
 }
 
@@ -198,7 +206,8 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
         'abbreviatedname',
         'email',
         'updatedAt',
-        'gender_code'
+        'gender_code',
+        'birthdate'
         /* 'gender_fi',
         'gender_sv',
         'gender_en' */
@@ -214,11 +223,13 @@ const getStudentsIncludeCoursesBetween = async (studentnumbers, startDate, endDa
           attributes: [
             'studyrightid',
             'startdate',
+            'enddate',
             'extentcode',
             'graduated',
             'canceldate',
             'prioritycode',
-            'faculty_code'
+            'faculty_code',
+            'studystartdate'
           ],
           separate: true,
           include: [
@@ -359,6 +370,7 @@ const studentnumbersWithAllStudyrightElements = async (
   exchangeStudents,
   cancelledStudents,
   nondegreeStudents,
+  transferredStudents,
   tag
 ) => {
   // eslint-disable-line
@@ -378,6 +390,7 @@ const studentnumbersWithAllStudyrightElements = async (
   if (!nondegreeStudents) {
     filteredExtents.push(33, 99, 14, 13)
   }
+  // this does not work in SIS, that is why we have a kludge below
   if (!cancelledStudents) {
     studyrightWhere.canceldate = null
   }
@@ -421,11 +434,16 @@ const studentnumbersWithAllStudyrightElements = async (
             [Op.ne]: 20
           }
         },
-        {
-          ['$studyright_elements.startdate$']: {
+        sequelize.where(
+          sequelize.fn(
+            'GREATEST',
+            sequelize.col('studyright_elements.startdate'),
+            sequelize.col('studyright.studystartdate')
+          ),
+          {
             [Op.between]: [formattedStartDate, endDate]
           }
-        }
+        )
       ],
       ...studyrightWhere
     },
@@ -434,13 +452,56 @@ const studentnumbersWithAllStudyrightElements = async (
     raw: true
   })
 
-  const studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+  let studentnumbers = [...new Set(students.map(s => s.student_studentnumber))]
+
+  // study right cancel does not work in SIS, but the below kludge should do the trick
+  // since canceldate != null   === student does not have any enrolment
+  if (!cancelledStudents) {
+    const { semestercode } = await getCurrentSemester()
+
+    const enrolments = await SemesterEnrollment.findAll({
+      where: {
+        studentnumber: {
+          [Op.in]: studentnumbers
+        },
+        semestercode,
+        enrollment_date: {
+          [Op.not]: null
+        }
+      }
+    })
+
+    const enrolledStudentnumbers = enrolments.map(e => e.studentnumber)
+    studentnumbers = enrolledStudentnumbers
+  }
+
+  // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
+  const rights = await Studyright.findAll({
+    attributes: ['studyrightid'],
+    where: {
+      student_studentnumber: {
+        [Op.in]: studentnumbers
+      }
+    },
+    include: {
+      attributes: [],
+      model: StudyrightElement,
+      where: {
+        code: {
+          [Op.in]: studyRights
+        }
+      }
+    },
+    group: ['studyright.studyrightid'],
+    having: count('studyright_elements.id', studyRights.length, true),
+    raw: true
+  })
 
   // bit hacky solution, but this is used to filter out studentnumbers who have since changed studytracks
   const allStudytracksForStudents = await StudyrightElement.findAll({
     where: {
-      studentnumber: {
-        [Op.in]: studentnumbers
+      studyrightid: {
+        [Op.in]: rights.map(r => r.studyrightid)
       }
     },
     include: {
@@ -460,11 +521,35 @@ const studentnumbersWithAllStudyrightElements = async (
   }, {})
 
   const filteredStudentnumbers = studentnumbers.filter(studentnumber => {
-    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], 'startdate').reverse()[0]
+    const newestStudytrack = sortBy(formattedStudytracks[studentnumber], ['enddate', 'startdate']).reverse()[0]
     if (!newestStudytrack) return false
     return studyRights.includes(newestStudytrack.code)
   })
-  return filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+  const studentnumberlist = filteredStudentnumbers.length > 0 ? filteredStudentnumbers : studentnumbers
+
+  // fetch students that have transferred out of the programme and filter out these studentnumbers
+  if (!transferredStudents) {
+    const transferredOutStudents = await Transfer.findAll({
+      attributes: ['studentnumber'],
+      where: {
+        sourcecode: {
+          [Op.in]: studyRights
+        },
+        transferdate: {
+          [Op.gt]: startDate
+        },
+        studentnumber: {
+          [Op.in]: studentnumberlist
+        }
+      },
+      raw: true
+    }).map(s => s.studentnumber)
+
+    const notTransferredStudents = studentnumberlist.filter(sn => !transferredOutStudents.includes(sn))
+    return notTransferredStudents
+  }
+
+  return studentnumberlist
 }
 
 const parseQueryParams = query => {
@@ -482,10 +567,13 @@ const parseQueryParams = query => {
   const exchangeStudents = studentStatuses && studentStatuses.includes('EXCHANGE')
   const cancelledStudents = studentStatuses && studentStatuses.includes('CANCELLED')
   const nondegreeStudents = studentStatuses && studentStatuses.includes('NONDEGREE')
+  const transferredStudents = studentStatuses && studentStatuses.includes('TRANSFERRED')
+
   return {
     exchangeStudents,
     cancelledStudents,
     nondegreeStudents,
+    transferredStudents,
     // if someone passes something falsy like null as the studyright, remove it so it doesn't break
     // the sequelize query
     studyRights: (Array.isArray(studyRights) ? studyRights : Object.values(studyRights)).filter(Boolean),
@@ -496,11 +584,103 @@ const parseQueryParams = query => {
   }
 }
 
+const getOptionsForStudents = async (students, code, level) => {
+  if (!code || !students.length) return {}
+
+  let graduated
+  let currentExtent
+  let optionExtent
+
+  if (level === 'BSC') {
+    graduated = { graduated: 1 }
+    currentExtent = 1
+    optionExtent = 2
+  } else if (level === 'MSC') {
+    graduated = {}
+    currentExtent = 2
+    optionExtent = 1
+  } else {
+    throw new Error('Invalid study level ' + level)
+  }
+
+  const programmes = await getAllProgrammes()
+
+  const currentStudyrights = await Studyright.findAll({
+    include: [
+      {
+        model: StudyrightElement,
+        where: {
+          studentnumber: {
+            [Op.in]: students
+          },
+          code: code
+        }
+      }
+    ],
+    where: {
+      ...graduated,
+      extentcode: currentExtent,
+      student_studentnumber: {
+        [Op.in]: students
+      }
+    },
+    attributes: ['studentStudentnumber', 'givendate']
+  })
+
+  const currentStudyrightsMap = currentStudyrights.reduce((obj, studyright) => {
+    obj[studyright.studentStudentnumber] = studyright.givendate
+    return obj
+  }, {})
+
+  const options = await Studyright.findAll({
+    include: [
+      {
+        model: StudyrightElement,
+        where: {
+          studentnumber: {
+            [Op.in]: students
+          },
+          code: {
+            [Op.in]: programmes.map(p => p.code)
+          }
+        },
+        include: [
+          {
+            model: ElementDetail,
+            attributes: ['name']
+          }
+        ],
+        attributes: ['code', 'startdate']
+      }
+    ],
+    where: {
+      extentcode: optionExtent,
+      student_studentnumber: {
+        [Op.in]: students
+      }
+    },
+    order: [[StudyrightElement, 'startdate', 'DESC']],
+    attributes: ['studentStudentnumber', 'givendate']
+  })
+
+  return options
+    .filter(m => m.studentStudentnumber in currentStudyrightsMap)
+    .filter(m => m.givendate.getTime() === currentStudyrightsMap[m.studentStudentnumber].getTime())
+    .reduce((obj, element) => {
+      obj[element.studentStudentnumber] = {
+        code: element.studyright_elements[0].code,
+        name: element.studyright_elements[0].element_detail.name
+      }
+      return obj
+    }, {})
+}
+
 const formatStudentsForApi = async (
   { students, credits, extents, semesters, elementdetails, courses },
   startDate,
   endDate,
-  { studyRights }
+  { studyRights },
+  optionData
 ) => {
   const startDateMoment = moment(startDate)
   const endDateMoment = moment(endDate)
@@ -528,7 +708,8 @@ const formatStudentsForApi = async (
         stats.transfers.targets[transfer.targetcode] = target
         stats.transfers.sources[transfer.sourcecode] = source
       })
-
+      if (student.studentnumber in optionData) student.option = optionData[student.studentnumber]
+      else student.option = null
       stats.students.push(
         formatStudentForPopulationStatistics(student, credits, startDate, endDate, startDateMoment, endDateMoment)
       )
@@ -587,19 +768,17 @@ const formatQueryParamArrays = (query, params) => {
 const optimizedStatisticsOf = async (query, studentnumberlist) => {
   const formattedQueryParams = formatQueryParamArrays(query, ['semesters', 'studentStatuses'])
 
-  if (
-    !formattedQueryParams.semesters.map(semester => semester === 'FALL' || semester === 'SPRING').every(e => e === true)
-  ) {
+  if (!formattedQueryParams.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
     return { error: 'Semester should be either SPRING OR FALL' }
   }
 
   if (
     formattedQueryParams.studentStatuses &&
-    !formattedQueryParams.studentStatuses
-      .map(status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE')
-      .every(e => e === true)
+    !formattedQueryParams.studentStatuses.every(
+      status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+    )
   ) {
-    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
+    return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
   }
   const {
     studyRights,
@@ -609,6 +788,7 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     exchangeStudents,
     cancelledStudents,
     nondegreeStudents,
+    transferredStudents,
     tag
   } = parseQueryParams(formattedQueryParams)
 
@@ -623,8 +803,19 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
         endDate,
         exchangeStudents,
         cancelledStudents,
-        nondegreeStudents
+        nondegreeStudents,
+        transferredStudents
       )
+  // wtf
+  // plz
+  const code = studyRights[0] || ''
+  let optionData = {}
+  if (code.includes('MH')) {
+    optionData = await getOptionsForStudents(studentnumbers, code, 'MSC')
+  } else if (code.includes('KH')) {
+    optionData = await getOptionsForStudents(studentnumbers, code, 'BSC')
+  }
+
   const { students, credits, extents, semesters, elementdetails, courses } = await getStudentsIncludeCoursesBetween(
     studentnumbers,
     startDate,
@@ -637,8 +828,10 @@ const optimizedStatisticsOf = async (query, studentnumberlist) => {
     { students, credits, extents, semesters, elementdetails, courses },
     startDate,
     endDate,
-    formattedQueryParams
+    formattedQueryParams,
+    optionData
   )
+
   return formattedStudents
 }
 
@@ -693,15 +886,25 @@ const parseCreditInfo = credit => ({
 
 const bottlenecksOf = async (query, studentnumberlist) => {
   const isValidRequest = async (query, params) => {
-    const { studyRights, startDate, endDate, exchangeStudents, cancelledStudents, nondegreeStudents } = params
+    const {
+      studyRights,
+      startDate,
+      endDate,
+      exchangeStudents,
+      cancelledStudents,
+      nondegreeStudents,
+      transferredStudents
+    } = params
     if (!query.semesters.every(semester => semester === 'FALL' || semester === 'SPRING')) {
       return { error: 'Semester should be either SPRING OR FALL' }
     }
     if (
       query.studentStatuses &&
-      !query.studentStatuses.every(status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE')
+      !query.studentStatuses.every(
+        status => status === 'CANCELLED' || status === 'EXCHANGE' || status === 'NONDEGREE' || status === 'TRANSFERRED'
+      )
     ) {
-      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE' }
+      return { error: 'Student status should be either CANCELLED or EXCHANGE or NONDEGREE or TRANSFERRED' }
     }
     if (query.selectedStudents) {
       const allStudents = await studentnumbersWithAllStudyrightElements(
@@ -711,6 +914,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
         exchangeStudents,
         cancelledStudents,
         nondegreeStudents,
+        transferredStudents,
         query.tag
       )
       const disallowedRequest = checkThatSelectedStudentsAreUnderRequestedStudyright(
@@ -731,6 +935,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
         exchangeStudents,
         cancelledStudents,
         nondegreeStudents,
+        transferredStudents,
         tag
       } = params
       const studentnumbers =
@@ -742,6 +947,7 @@ const bottlenecksOf = async (query, studentnumberlist) => {
           exchangeStudents,
           cancelledStudents,
           nondegreeStudents,
+          transferredStudents,
           tag
         ))
       const allstudents = studentnumbers.reduce((numbers, num) => {

services/backend/oodikone2-backend/src/servicesV2/programmeModules.js

@@ -6,7 +6,10 @@ const { dbConnections: sisConnections } = require('../databaseV2/connection')
 const { sequelizeKone } = require('../database/connection')
 
 const byProgrammeCode = async code => {
+  // TODO use only sis connection once sis is stable
   const connection = sisConnections.established ? sisConnections.sequelize : sequelizeKone
+
+  // recursively get a module and all of its children and all of the children of the children...
   const [result] = await connection.query(
     `
     WITH RECURSIVE children as (
@@ -30,13 +33,14 @@ const byProgrammeCode = async code => {
     }
   })
 
+  // this labels the modules to match the old system in frontend
   const labeled = result.map(module => {
     const label = {
       id: module.label_name.fi,
       label: `${module.label_code}\n${module.label_name.fi}`,
       orderNumber: module.module_order
     }
-    // bit hacky
+    // check if course is excluded, and hide if it is
     const foundCourse = excluded.find(course => course.course_code === module.code)
     const visible = { visibility: !foundCourse, id: foundCourse ? foundCourse.id : null }
 
@@ -47,10 +51,6 @@ const byProgrammeCode = async code => {
 }
 
 const addExcludedCourses = async (programmecode, coursecodes) => {
-  // return ExcludedCourse.create({
-  //   programme_code: programmecode,
-  //   course_code: coursecode
-  // })
   return ExcludedCourse.bulkCreate(coursecodes.map(c => ({ programme_code: programmecode, course_code: c })))
 }
 // just copy pasted from above since almost same query

services/backend/oodikone2-backend/src/servicesV2/studyprogramme.js

@@ -12,7 +12,7 @@ const {
   ElementDetail,
   Transfer
 } = require('../modelsV2')
-const { getAssociations } = require('./studyrights')
+const { getAssociations, getAllProgrammes } = require('./studyrights')
 const { ThesisCourse, ThesisTypeEnums } = require('../models/models_kone')
 const { studentnumbersWithAllStudyrightElements } = require('./populations')
 const { semesterStart, semesterEnd } = require('../util/semester')
@@ -506,6 +506,121 @@ const startedStudyright = async (studentnumbers, startDate, studytrack, endDate)
   })
 }
 
+const optionData = async (startDate, endDate, code, level) => {
+  const programmes = await getAllProgrammes()
+  const students = await studentnumbersWithAllStudyrightElements([code], startDate, endDate, false, true)
+
+  let graduated
+  let currentExtent
+  let optionExtent
+
+  if (level === 'BSC') {
+    graduated = { graduated: 1 }
+    currentExtent = 1
+    optionExtent = 2
+  } else if (level === 'MSC') {
+    graduated = {}
+    currentExtent = 2
+    optionExtent = 1
+  } else {
+    throw new Error('Invalid study level ' + level)
+  }
+
+  const currentStudyrights = await Studyright.findAll({
+    include: [
+      {
+        model: StudyrightElement,
+        where: {
+          code: code
+        },
+        attributes: ['startdate']
+      }
+    ],
+    where: {
+      ...graduated,
+      extentcode: currentExtent,
+      student_studentnumber: {
+        [Op.in]: students
+      }
+    },
+    attributes: ['studentStudentnumber', 'givendate', 'studystartdate']
+  })
+
+  const currentStudyrightsMap = currentStudyrights
+    .filter(b => b.studystartdate)
+    .reduce((obj, studyright) => {
+      const acualDate = new Date(Math.max(+studyright.studystartdate, +studyright.studyright_elements[0].startdate))
+      obj[studyright.studentStudentnumber] = { givendate: studyright.givendate, startdate: acualDate }
+      return obj
+    }, {})
+
+  const options = await Studyright.findAll({
+    include: [
+      {
+        model: StudyrightElement,
+        where: {
+          studentnumber: {
+            [Op.in]: students
+          },
+          code: {
+            [Op.in]: programmes.map(p => p.code)
+          }
+        },
+        include: [
+          {
+            model: ElementDetail,
+            attributes: ['name']
+          }
+        ],
+        attributes: ['code']
+      }
+    ],
+    where: {
+      extentcode: optionExtent,
+      student_studentnumber: {
+        [Op.in]: students
+      }
+    },
+    order: [[StudyrightElement, 'startdate', 'DESC']],
+    attributes: ['studentStudentnumber', 'startdate', 'givendate']
+  })
+
+  const data = {}
+  const years = new Set()
+
+  options
+    .filter(m => m.studentStudentnumber in currentStudyrightsMap)
+    .filter(m => m.givendate.getTime() === currentStudyrightsMap[m.studentStudentnumber].givendate.getTime())
+    .forEach(b => {
+      const date = currentStudyrightsMap[b.studentStudentnumber].startdate
+      const year =
+        date.getMonth() > 6 || (date.getMonth() === 6 && date.getDate() == 31)
+          ? date.getFullYear()
+          : date.getFullYear() - 1
+      years.add(year)
+      const code = b.studyright_elements[0].code
+
+      if (!data[code]) {
+        data[code] = {}
+        data[code].name = b.studyright_elements[0].element_detail.name
+        data[code].total = 0
+      }
+      if (!data[code][year]) {
+        data[code][year] = 0
+      }
+      data[code][year] += 1
+      data[code].total += 1
+    })
+
+  const dataAsArray = Object.keys(data).map(code => {
+    const { name, total } = data[code]
+    const output = { code, name, total }
+    years.forEach(y => (output[y] = data[code][y] || 0))
+    return output
+  })
+  return { data: dataAsArray, years: Array.from(years).sort() }
+}
+
 const statsForClass = async (studentnumbers, startDate, studyprogramme, endDate) => {
   return Promise.all([
     creditsAfter(studentnumbers, startDate),
@@ -784,5 +899,6 @@ module.exports = {
   throughputStatsForStudytrack,
   findProgrammeThesisCredits,
   thesisProductivityFromCredits,
-  thesisProductivityForStudytrack
+  thesisProductivityForStudytrack,
+  optionData
 }

services/backend/oodikone2-backend/src/servicesV2/studyrights.js

@@ -341,7 +341,8 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
             type: type,
             name: name,
             code: code,
-            enrollmentStartYears: {}
+            enrollmentStartYears: {},
+            studytracks: []
           }
           const momentstartdate = moment(startdate)
           const enrollment = momentstartdate.isValid() ? getEnrollmentStartYear(momentstartdate) : null
@@ -388,6 +389,8 @@ const calculateAssociationsFromDb = async (chunksize = 100000) => {
                     name: e.name,
                     code: e.code
                   }
+                  if (!associations.programmes[code].studytracks.includes(e.code))
+                    associations.programmes[code].studytracks.push(e.code)
                 }
                 associations.studyTracks[e.code] = associations.studyTracks[e.code] || {
                   type: e.type,

services/backend/updater_writer/Dockerfile

@@ -4,18 +4,15 @@ WORKDIR /usr/src/app
 CMD npm start
 
 RUN apk add postgresql-client
-COPY ./updater_writer/package.json /usr/src/app/package.json
-COPY ./updater_writer/package-lock.json /usr/src/app/package-lock.json
-COPY ./updater_writer/package.json /usr/src/cache/package.json
-COPY ./updater_writer/package-lock.json /usr/src/cache/package-lock.json
+
+COPY ./updater_writer/package* ./
+
 RUN npm ci
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
 COPY ./updater_writer /usr/src/app
 COPY ./shared/models /usr/src/app/models
 COPY ./shared/migrations /usr/src/app/database/migrations
 COPY ./shared/migrations_kone /usr/src/app/database/migrations_kone
-

services/backend/updater_writer/conf-backend.js

@@ -16,7 +16,6 @@ const TOKEN_SECRET = process.env.TOKEN_SECRET
 const CERT_PATH = process.env.CERT_PATH // production/staging only
 const KEY_PATH = process.env.KEY_PATH // production/staging only
 const USERSERVICE_URL = process.env.USERSERVICE_URL
-const USAGESERVICE_URL = process.env.USAGESERVICE_URL
 const ANALYTICS_URL = process.env.ANALYTICS_URL
 const PORT = isTest ? 8079 : 8080
 const OODI_SECRET = process.env.OODI_SECRET
@@ -71,7 +70,6 @@ module.exports = {
   ACCESS_TOKEN_HEADER_KEY,
   PORT,
   ANALYTICS_URL: formatURL(ANALYTICS_URL),
-  USAGESERVICE_URL,
   requiredGroup,
   OODI_SECRET,
   OODI_SECRET_HEADER_KEY,

services/dco.oodikone.yml

@@ -0,0 +1,114 @@
+version: "3.2"
+
+services:
+
+  frontend:
+    build:
+      context: services/oodikone2-frontend
+    environment:
+      BACKEND_ADDR: backend
+      ADMINER_URL: http://localhost:5050/?pgsql=db&username=postgres
+    container_name: frontend
+
+  backend:
+    build:
+      context: services/backend
+      dockerfile: oodikone2-backend/Dockerfile
+    ports:
+      - "8080:8080"
+    environment:
+      REDIS: redis
+      DB_URL: postgres://postgres@db:5432/tkt_oodi
+      # DB_MAX_CONNECTIONS: 90 # defaults to 5
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone
+      SIS_DB_URL: postgres://postgres@db_sis:5432/db_sis
+      # if NODE_ENV === 'test', these are used instead
+      TEST_DB: postgres://postgres@db:5432/tkt_oodi_test
+      TEST_DB_KONE: postgres://postgres@db_kone:5432/db_kone_test
+      # same but for NODE_ENV === 'anon'
+      # ANON_DB:
+      # ANON_DB_KONE:
+      TOKEN_SECRET: IWannaBeTheVeryBest
+      SECRET_TOKEN: dev
+      USERSERVICE_URL: http://userservice:4567
+      USERSERVICE_SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      ANALYTICS_URL: http://analytics:4568
+      NATS_URI: nats://nats:4222
+      UPDATER_URL: http://updater_scheduler:3678
+      SIS_UPDATER_URL: http://sis-updater-scheduler:8082
+      # Set these in production:
+      # ANALYTICS_INFLUXDB_URL
+      # ANALYTICS_INFLUXDB_DB
+      # ANALYTICS_INFLUXDB_PASSWORD
+      # ANALYTICS_INFLUXDB_USER
+      # MATOMO_SITE_ID
+      # MATOMO_URL
+    container_name: backend
+    depends_on:
+      - db
+      - redis
+      - nats
+
+  db_kone:
+    image: postgres:9.6.3
+    ports:
+      - "5425:5432"
+    volumes:
+      - pgdata_kone:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: db_kone
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=db_kone
+      - POSTGRES_USER=postgres
+
+  userservice:
+    build:
+      context: services/oodikone2-userservice
+    ports:
+      - 4567:4567
+    environment:
+      DB_URL: postgres://postgres@user_db:5432/user_db
+      TEST_DB: postgres://postgres@user_db:5432/user_db_test
+      SECRET: Luukk4inenApprov3dVitunMinttuSalainenAvain666NurmivaaraPJ2019
+      TOKEN_SECRET: IWannaBeTheVeryBest
+    depends_on:
+      - user_db
+    container_name: userservice
+
+  user_db:
+    image: postgres:9.6.3
+    ports:
+      - "5422:5432"
+    volumes:
+      - user_pgdata:/var/lib/postgresql/data
+    container_name: oodi_user_db
+    environment:
+      - POSTGRES_USER=postgres
+
+  analytics:
+    build:
+      context: services/oodikone2-analytics
+    ports:
+      - 4568:4568
+    environment:
+      - DB_URL=postgres://postgres@analytics_db:5432/analytics_db
+      - TEST_DB=postgres://postgres@analytics_db:5432/analytics_db_test
+    depends_on:
+      - analytics_db
+    container_name: analytics
+
+  analytics_db:
+    image: postgres:9.6.3
+    ports:
+      - "5423:5432"
+    volumes:
+      - analytics_pgdata:/var/lib/postgresql/data
+      - ./scripts/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro
+    container_name: oodi_analytics_db
+    environment:
+      - POSTGRES_USER=postgres
+
+volumes:
+  analytics_pgdata:
+  user_pgdata:
+  pgdata_kone:

services/docker/dco.ci.yml

@@ -5,18 +5,10 @@ services:
     environment:
       - POSTGRES_DB=user_db
 
-  db:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=tkt_oodi,tkt_oodi_test
-
   analytics_db:
     environment:
       - POSTGRES_MULTIPLE_DATABASES=analytics_db
 
-  db_sis:
-    environment:
-      - POSTGRES_MULTIPLE_DATABASES=db_sis
-
   backend:
     image: toska/oodikone2-backend:${TAG}
     build:
@@ -54,38 +46,6 @@ services:
         - toska/oodikone2-userservice:staging
         - toska/oodikone2-userservice:latest
 
-  usageservice:
-    image: toska/oodikone2-usageservice:${TAG}
-    build:
-      cache_from:
-        - toska/oodikone2-usageservice:trunk
-        - toska/oodikone2-usageservice:staging
-        - toska/oodikone2-usageservice:latest
-
-  updater_api:
-    image: toska/updater_api:${TAG}
-    build:
-      cache_from:
-        - toska/updater_api:trunk
-        - toska/updater_api:staging
-        - toska/updater_api:latest
-
-  updater_writer:
-    image: toska/updater_writer:${TAG}
-    build:
-      cache_from:
-        - toska/updater_writer:trunk
-        - toska/updater_writer:staging
-        - toska/updater_writer:latest
-
-  updater_scheduler:
-    image: toska/updater_scheduler:${TAG}
-    build:
-      cache_from:
-        - toska/updater_scheduler:trunk
-        - toska/updater_scheduler:staging
-        - toska/updater_scheduler:latest
-
   analytics:
     image: toska/oodikone2-analytics:${TAG}
     build:
@@ -94,27 +54,6 @@ services:
         - toska/oodikone2-analytics:staging
         - toska/oodikone2-analytics:latest
 
-  sis-updater-scheduler:
-    image: toska/sis-updater-scheduler:${TAG}
-    environment:
-      CI: 1
-    build:
-      cache_from:
-        - toska/sis-updater-scheduler:trunk
-        - toska/sis-updater-scheduler:staging
-        - toska/sis-updater-scheduler:latest
-
-  sis-updater-worker:
-    image: toska/sis-updater-worker:${TAG}
-    environment:
-      CI: 1
-    build:
-      cache_from:
-        - toska/sis-updater-worker:trunk
-        - toska/sis-updater-worker:staging
-        - toska/sis-updater-worker:latest
-    container_name: oodikone_sis-updater-worker_1
-
   nginx:
     image: nginx:latest
     volumes:

services/docker/dco.dev.real.yml

@@ -0,0 +1,16 @@
+version: '3.2'
+
+services:
+  backend:
+    environment:
+      DB_URL: postgres://postgres@db:5432/tkt_oodi_real
+      DB_URL_KONE: postgres://postgres@db_kone:5432/db_kone_real
+      SIS_DB_URL: postgres://postgres@db_sis:5432/db_sis_real
+
+  userservice:
+    environment:
+      DB_URL: postgres://postgres@user_db:5432/user_db_real
+
+  analytics:
+    environment:
+      DB_URL: postgres://postgres@analytics_db:5432/analytics_db_real

services/docker/dco.dev.yml

@@ -0,0 +1,73 @@
+version: '3.2'
+
+services:
+  user_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=user_db,user_db_real
+    restart: unless-stopped
+
+  analytics_db:
+    environment:
+      - POSTGRES_MULTIPLE_DATABASES=analytics_db,analytics_db_real
+    restart: unless-stopped
+
+  backend:
+    build:
+      args:
+        NODE_ENV: 'development'
+        TAG: 'development'
+    command: ['npm', 'run', 'dev']
+    environment:
+      FRONT_URL: http://localhost:8000
+    volumes:
+      - backend-nodemod:/usr/src/app/node_modules
+      - ./services/backend/oodikone2-backend/:/usr/src/app
+      - ./services/backend/shared/models:/usr/src/app/src/models
+      - ./services/backend/shared/migrations:/usr/src/app/src/database/migrations
+      - ./services/backend/shared/migrations_kone:/usr/src/app/src/database/migrations_kone
+    restart: unless-stopped
+
+  frontend:
+    build:
+      args:
+        NODE_ENV: 'development'
+        TAG: 'development'
+    command: ['npm', 'run', 'docker']
+    ports:
+      - "8081:8081"
+      - "7777:7777"
+    environment:
+      USER_ADMINER_URL: http://localhost:5050/?pgsql=user_db&username=postgres
+      ANALYTICS_ADMINER_URL: http://localhost:5050/?pgsql=analytics_db&username=postgres
+      KONE_ADMINER_URL: http://localhost:5050/?pgsql=db_kone&username=postgres
+      SIS_ADMINER_URL: http://localhost:5050/?pgsql=db_sis&username=postgres
+    volumes:
+      - frontend-nodemod:/usr/src/app/node_modules
+      - ./services/oodikone2-frontend/:/usr/src/app
+    restart: unless-stopped
+
+  userservice:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['npm', 'run', 'dev']
+    volumes:
+      - userservice-nodemod:/usr/src/app/node_modules
+      - ./services/oodikone2-userservice/:/usr/src/app
+    restart: unless-stopped
+
+  analytics:
+    build:
+      args:
+        NODE_ENV: 'development'
+    command: ['npm', 'run', 'dev']
+    volumes:
+      - analytics-nodemod:/usr/src/app/node_modules
+      - ./services/oodikone2-analytics/:/usr/src/app
+    restart: unless-stopped
+
+volumes:
+  backend-nodemod:
+  analytics-nodemod:
+  frontend-nodemod:
+  userservice-nodemod:

services/oodikone2-analytics/Dockerfile

@@ -1,18 +1,16 @@
 FROM node:10.15-alpine
 
 WORKDIR /usr/src/app
-EXPOSE 4568
-CMD npm start
 
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
-RUN npm ci
+EXPOSE 4568
 
 ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
-COPY . /usr/src/app
+COPY ./package* ./
 
+RUN npm ci
+
+COPY . .
+
+CMD npm start

services/oodikone2-analytics/package-lock.json

@@ -1669,12 +1669,6 @@
           "dev": true,
           "optional": true
         },
-        "ini": {
-          "version": "1.3.5",
-          "bundled": true,
-          "dev": true,
-          "optional": true
-        },
         "is-fullwidth-code-point": {
           "version": "1.0.0",
           "bundled": true,
@@ -2255,9 +2249,9 @@
       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
     },
     "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "version": "1.3.7",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
+      "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
       "dev": true
     },
     "inquirer": {

services/oodikone2-analytics/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "node index.js",
     "dev": "NODE_ENV=dev nodemon index.js",
-    "test": "npm run lint",
+    "test": "npm install --dev && npm run lint",
     "lint": "eslint .",
     "lintfix": "prettier --write **/*.{js,json} && eslint --fix ."
   },

services/oodikone2-frontend/Dockerfile

@@ -4,10 +4,8 @@ WORKDIR /usr/src/app
 EXPOSE 5000
 CMD serve -s -l 5000 dist
 
-COPY package.json /usr/src/app/package.json
-COPY package-lock.json /usr/src/app/package-lock.json
-COPY package.json /usr/src/cache/package.json
-COPY package-lock.json /usr/src/cache/package-lock.json
+COPY ./package* ./
+
 RUN npm ci
 
 ARG BASE_PATH=/
@@ -16,7 +14,6 @@ ARG NODE_ENV=production
 ENV NODE_ENV=${NODE_ENV}
 ARG TAG=local
 ENV TAG=${TAG}
-RUN if [ "${NODE_ENV}" != "production" ]; then cp -r /usr/src/app/node_modules /usr/src/cache/node_modules; fi
 
 COPY . /usr/src/app
 RUN if [ "${NODE_ENV}" != "development" ]; then \

services/oodikone2-frontend/package-lock.json

@@ -3229,35 +3229,11 @@
       }
     },
     "axios": {
-      "version": "0.18.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz",
-      "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==",
+      "version": "0.21.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
+      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
       "requires": {
-        "follow-redirects": "1.5.10",
-        "is-buffer": "^2.0.2"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "follow-redirects": {
-          "version": "1.5.10",
-          "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
-          "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
-          "requires": {
-            "debug": "=3.1.0"
-          }
-        },
-        "is-buffer": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz",
-          "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw=="
-        }
+        "follow-redirects": "^1.10.0"
       }
     },
     "axobject-query": {
@@ -7468,8 +7444,7 @@
     "follow-redirects": {
       "version": "1.12.0",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.12.0.tgz",
-      "integrity": "sha512-JgawlbfBQKjbKegPn8vUsvJqplE7KHJuhGO4yPcb+ZOIYKSr+xobMVlfRBToZwZUUxy7lFiKBdFNloz9ui368Q==",
-      "dev": true
+      "integrity": "sha512-JgawlbfBQKjbKegPn8vUsvJqplE7KHJuhGO4yPcb+ZOIYKSr+xobMVlfRBToZwZUUxy7lFiKBdFNloz9ui368Q=="
     },
     "for-in": {
       "version": "1.0.2",
@@ -17722,6 +17697,40 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz",
       "integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg=="
     },
+    "unfuck-spa-shibboleth-session": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/unfuck-spa-shibboleth-session/-/unfuck-spa-shibboleth-session-3.1.2.tgz",
+      "integrity": "sha512-AHDN0JCyZzDyhBE+P9R7bZfDlFqgkeT0hA7SZ05ZjWbod/1qvsWCcLod58h/jYf6zoKAlMH5130Ed2loJDqqDg==",
+      "requires": {
+        "axios": "^0.19.2"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.19.2",
+          "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz",
+          "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==",
+          "requires": {
+            "follow-redirects": "1.5.10"
+          }
+        },
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "follow-redirects": {
+          "version": "1.5.10",
+          "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
+          "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
+          "requires": {
+            "debug": "=3.1.0"
+          }
+        }
+      }
+    },
     "unherit": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/unherit/-/unherit-1.1.1.tgz",

services/oodikone2-frontend/package.json

@@ -67,7 +67,7 @@
     "@babel/preset-react": "^7.0.0",
     "@sentry/browser": "^5.9.1",
     "@sentry/cli": "^1.47.1",
-    "axios": "^0.18.1",
+    "axios": "^0.21.1",
     "babel-polyfill": "^6.26.0",
     "dotenv": "^6.2.0",
     "highcharts": "^7.2.2",
@@ -102,6 +102,7 @@
     "semantic-ui-css": "^2.4.1",
     "semantic-ui-react": "^0.88.2",
     "terser-webpack-plugin": "^1.3.0",
+    "unfuck-spa-shibboleth-session": "^3.1.2",
     "uuid": "^3.3.2",
     "xlsx": "^0.14.4"
   }

services/oodikone2-frontend/src/apiConnection/index.js

@@ -128,12 +128,6 @@ export const handleRequest = store => next => async action => {
       const res = await callApi(route, method, data, params, 0, onProgress)
       store.dispatch({ type: `${prefix}SUCCESS`, response: res.data, query })
     } catch (e) {
-      // handle stale Shibboleth session by reloading frontend
-      if (e.message.toLowerCase() === 'network error') {
-        window.location.reload(true)
-        return
-      }
-
       store.dispatch({ type: `${prefix}FAILURE`, response: e, query })
       handleError(e, store.getState().actionHistory)
     }
@@ -160,12 +154,6 @@ export const handleAuth = store => next => async action => {
       }
       store.dispatch({ type: 'LOGIN_SUCCESS', token })
     } catch (err) {
-      // handle stale Shibboleth session by reloading frontend
-      if (err.message.toLowerCase() === 'network error') {
-        window.location.reload(true)
-        return
-      }
-
       store.dispatch({ type: 'LOGIN_FAILURE' })
       handleError(err, store.getState().actionHistory)
     }
@@ -174,14 +162,8 @@ export const handleAuth = store => next => async action => {
       await logout()
       store.dispatch({ type: 'LOGOUT_SUCCESSFUL' })
     } catch (err) {
-      if (err.message.toLowerCase() === 'network error') {
-        window.location.reload(true)
-        return
-      }
       store.dispatch({ type: 'LOGOUT_FAILURE' })
       handleError(err, store.getState().actionHistory)
     }
   }
 }
-
-export const sendLog = async data => callApi('/log', 'post', data)

services/oodikone2-frontend/src/app.jsx

@@ -18,7 +18,7 @@ try {
   if (IS_PRODUCTION && BASE_PATH === '/' && ['staging', 'latest'].includes(process.env.TAG)) {
     Sentry.init({
       environment: process.env.TAG,
-      dsn: 'https://02d07bd40f404cc0965f38f06183d9fb@sentry.toska.cs.helsinki.fi/3',
+      dsn: 'https://c55e6d020db640e889948cc25ced1c19@sentry.toska.cs.helsinki.fi/2',
       release: process.env.SENTRY_RELEASE_VERSION
     })
   }

services/oodikone2-frontend/src/common/InfoToolTips/populationStatistics.js

@@ -21,9 +21,6 @@ export default {
     StartingThisSemester: `Suodattaa opiskelijat, jotka aloittivat opiskelun valittuna lukuvuonna tai sitä ennen`,
     EnrollmentStatus: 'Näyttää opiskelijat, jotka olivat läsnä- tai poissaolevana valittuina lukukausina',
     CanceledStudyright: `Suodattaa opiskelijat, jotka ovat/eivät ole ilmoittautuneet läsnä- tai poissaoleviksi`,
-    DisciplineTypes: `Asettaa annetun tyyppisille annettujen tieteenalojen
-    kursseille automaattisesti osallistumissuodattimet.
-    Voit asettaa rajan osallistumismäärälle tuodaksesi esiin vain suositut kurssit`,
     ExtentGraduated: `Suodattaa opiskelijat, jotka ovat/eivät ole valmistuneet.`,
     TransferFilter: `Voit valita opiskelijat, jotka ovat vaihtaneet annetusta koulutusohjelmasta toiseen.
       Vaihtaminen tarkoittaa sitä, kun opiskelija vaihtaa koulutusohjelmaansa kesken opinto-oikeutensa.
@@ -37,20 +34,6 @@ export default {
     CreditsBeforeStudyright: `Suodattaa pois opiskelijat joilla vähemmän opintopisteitä ennen kyseisessä ohjelmassa aloittamista`,
     StudytrackFilter: `Palauttaa opiskelijat joilla on valittu kyseinen suuntautumisvaihtoehto`
   },
-  CreditAccumulationGraph: {
-    AccordionTitle: ` **Credit accumulation graph**
-    Opiskelijoiden kumulatiiviset opintopistekertymät opiskelijakohtaisina kuvaajina.`
-  },
-  CoursesOf: {
-    AccordionTitle: `**Courses of population**
-    Tämä taulukko sisältää kaikkien populaatioon liittyvien opiskelijoiden kaikki kurssisuoritukset.`
-  },
-  Students: {
-    AccordionTitle: `**Students** Taulukko joka näyttää informaatiota opiskelijoista.`
-  },
-  CreditStatistics: {
-    AccordionTitle: `**Credit Statistics** Opintopistekertymä-kategoria skaalautuvat valitun tarkastelujakson mukaan. `
-  },
   ProgrammeDistributionCustomPopulation: `Näyttää opiskelijan viimeisimmän koulutusohjelman`,
   ProgrammeDistributionCoursePopulation: `Näyttää sen koulutusohjelman, jossa opiskelija oli suorittaessaan kurssin`,
   GradeDistributionCoursePopulation: `Näyttää korkeimman arvosanan, jonka opiskelija on saanut valitulla aikavälillä.`,

services/oodikone2-frontend/src/common/index.js

@@ -76,17 +76,32 @@ export const byCodeDesc = (a, b) => b.code.localeCompare(a.code)
 
 export const studyRightRegex = new RegExp(/.*master|bachelor|doctor|licentiate|specialist.*/)
 
-export const studyrightTypes = { degree: '10', programme: '20', speciality: '30' } // speciality???
+export const studyrightElementTypes = { degree: 10, programme: 20, speciality: 30 }
 
-export const getStudentTotalCredits = student => {
-  const passedCourses = student.courses.filter(c => c.passed && !c.isStudyModuleCredit)
+export const getStudentTotalCredits = (student, includeTransferredCredits = true) => {
+  const passedCourses = includeTransferredCredits
+    ? student.courses.filter(c => c.passed && !c.isStudyModuleCredit)
+    : student.courses.filter(c => c.passed && !c.isStudyModuleCredit && c.credittypecode !== 9)
   return passedCourses.reduce((a, b) => a + b.credits, 0)
 }
 
-export const getStudentGradeMean = student => {
-  const gradedCourses = student.courses.filter(c => Number(c.grade) && !c.isStudyModuleCredit)
-  const gradeTotal = gradedCourses.reduce((a, b) => a + Number(b.grade), 0)
-  const mean = gradeTotal / gradedCourses.length || 0
+const getGradedCourses = (student, includeTransferredCredits = true) =>
+  includeTransferredCredits
+    ? student.courses.filter(c => Number(c.grade) && !c.isStudyModuleCredit)
+    : student.courses.filter(c => Number(c.grade) && !c.isStudyModuleCredit && c.credittypecode !== 9)
+
+export const getStudentGradeMean = (student, includeTransferredCredits = true) => {
+  const courses = getGradedCourses(student, includeTransferredCredits)
+  const gradeTotal = courses.reduce((a, b) => a + Number(b.grade), 0)
+  const mean = gradeTotal / courses.length || 0
+  return mean
+}
+
+export const getStudentGradeMeanWeightedByCredits = (student, includeTransferredCredits = true) => {
+  const courses = getGradedCourses(student, includeTransferredCredits)
+  const gradeTotal = courses.reduce((a, b) => a + Number(b.grade) * Number(b.credits), 0)
+  const sumWeights = courses.reduce((a, b) => a + Number(b.credits), 0)
+  const mean = gradeTotal / sumWeights || 0
   return mean
 }
 

services/oodikone2-frontend/src/common/markdown/populationStatistics/coursesOfPopulation.info.md

@@ -6,7 +6,7 @@ Opiskelijat on mahdollista suodattaa kurssin perusteella käyttämällä kurssir
 
 Klikkaamalla kurssin nimen vieressä olevaa sinistä nuolta, siirryt ko. kurssin *Course Statistics* -näkymään.
 
-Taulukon tietoja voi suodattaa antamalla opiskelijoiden vähimmäismäärän kurssilla kenttään *Limit to courses where student number at least*. Lisäksi kurssin nimi- ja koodisarakkeissa on hakukentät *("filter here")* suodattamista varten. Suodattimet saa pois tyhjentämällä kentän ja painamalla ENTER. *Huom! Nämä suodattimet vaikuttavat vain tähän taulukkoon, eivät koko populatioon.*
+Taulukon tietoja voi suodattaa antamalla opiskelijoiden vähimmäismäärän kurssilla kenttään *Limit to courses where student number at least*. Lisäksi kurssin nimi- ja koodisarakkeissa on hakukentät *("filter here")* suodattamista varten. Suodattimet saa tyhjennettyä painamalla kentän vieressä olevaa ruksia. *Huom! Nämä suodattimet vaikuttavat vain tähän taulukkoon, eivät koko populatioon.*
 
 ## Taulukot:
 

services/oodikone2-frontend/src/common/query.js

@@ -1,13 +1,16 @@
 /**
- * Global definitions for Population Statistics HTTP GET query.
+ * Global definitions for query params.
  */
 import qs from 'qs'
 
-const parseParams = location => qs.parse(location.search, { ignoreQueryPrefix: true })
+/**
+ * All query params from given `location`.
+ * @param {object} location React-Router's `location` object.
+ */
+export const queryParamsFromUrl = location => qs.parse(location.search, { ignoreQueryPrefix: true })
 
 /**
  * Get `months` parameter from query.
  * @param {object} location React-Router's `location` object.
  */
-/* eslint-disable-next-line import/prefer-default-export */
-export const getMonths = location => parseParams(location).months
+export const getMonths = location => queryParamsFromUrl(location).months

services/oodikone2-frontend/src/common/sendEvent.js

@@ -4,12 +4,24 @@ import TSA from './tsa'
 const { sendEvent } = TSA.Matomo
 
 export const ANALYTICS_CATEGORIES = {
-  populationStudents: 'Population students'
+  populationStudents: 'Population students',
+  populationStatistics: 'Population statistics',
+  degreeCourses: 'Degree courses',
+  common: 'Common'
 }
 
-export default Object.fromEntries(
-  Object.entries(ANALYTICS_CATEGORIES).map(([catKey, catName]) => [
-    catKey,
-    (action, name, value) => sendEvent(catName, action, name, value)
-  ])
-)
+// This export could be created with Object.fromEntries, but then autocomplete wont work.
+export default {
+  populationStudents: (action, name, value) => {
+    sendEvent(ANALYTICS_CATEGORIES.populationStudents, action, name, value)
+  },
+  populationStatistics: (action, name, value) => {
+    sendEvent(ANALYTICS_CATEGORIES.populationStatistics, action, name, value)
+  },
+  degreeCourses: (action, name, value) => {
+    sendEvent(ANALYTICS_CATEGORIES.degreeCourses, action, name, value)
+  },
+  common: (action, name, value) => {
+    sendEvent(ANALYTICS_CATEGORIES.common, action, name, value)
+  }
+}

services/oodikone2-frontend/src/common/sisDestructionStyle.js

@@ -0,0 +1,7 @@
+import { getTestUserSIS } from '.'
+
+export default getTestUserSIS()
+  ? {
+      backgroundColor: '#ffe9f0'
+    }
+  : null

services/oodikone2-frontend/src/components/CoolDataScience/ProtoC.jsx

@@ -617,7 +617,7 @@ const ProtoC = ({
         />
         <Checkbox
           style={{ marginLeft: '10px' }}
-          label="Include old attainments"
+          label="Include attainments attained before the studyright start"
           onChange={handleOldAttainmentToggled}
           checked={includeOldAttainments}
         />

services/oodikone2-frontend/src/components/CoolDataScience/ProtoC2.jsx

@@ -457,7 +457,7 @@ const ProtoC = ({
             />
             <Checkbox
               style={{ marginLeft: '10px' }}
-              label="Include old attainments"
+              label="Include attainments attained before the studyright start"
               onChange={handleOldAttainmentToggled}
               checked={includeOldAttainments}
             />
@@ -520,7 +520,7 @@ const ProtoC = ({
           />
           <Checkbox
             style={{ marginLeft: '10px' }}
-            label="Include old attainments"
+            label="Include attainments attained before the studyright start"
             onChange={handleOldAttainmentToggled}
             checked={includeOldAttainments}
           />

services/oodikone2-frontend/src/components/CoolDataScience/ProtoG.jsx

@@ -217,7 +217,7 @@ const ProtoG = ({ uberdata, years, isLoading, loadingYears, getUberDispatch, get
         <h2>Kandiohjelmat: Suhteellisen tavoiteaikaerittelyn kehitys</h2>
         <Checkbox
           style={{ marginLeft: 'auto' }}
-          label="Include old attainments"
+          label="Include attainments attained before the studyright start"
           onChange={handleOldAttainmentToggled}
           checked={includeOldAttainments}
         />

services/oodikone2-frontend/src/components/CoursePopulation/index.jsx

@@ -2,20 +2,20 @@ import React, { useEffect, useState, useMemo, useRef } from 'react'
 import { withRouter } from 'react-router-dom'
 import { connect } from 'react-redux'
 import { shape, func, bool } from 'prop-types'
-import { Segment, Header, Accordion, Popup } from 'semantic-ui-react'
-import qs from 'query-string'
-import ReactMarkdown from 'react-markdown'
+import { Segment, Header, Accordion } from 'semantic-ui-react'
 import scrollToComponent from 'react-scroll-to-component'
 import { getCoursePopulation } from '../../redux/populations'
 import { getSingleCourseStats } from '../../redux/singleCourseStats'
+import { getCustomPopulationCoursesByStudentnumbers } from '../../redux/populationCourses'
 import { getFaculties } from '../../redux/faculties'
 import { getSemesters } from '../../redux/semesters'
 import PopulationStudents from '../PopulationStudents'
 import CoursePopulationGradeDist from './CoursePopulationGradeDist'
 import CoursePopulationCreditGainTable from './CoursePopulationCreditGainTable'
 import CustomPopulationProgrammeDist from '../CustomPopulation/CustomPopulationProgrammeDist'
+import CustomPopulationCourses from '../CustomPopulation/CustomPopulationCourses'
 import ProgressBar from '../ProgressBar'
-import { getStudentToTargetCourseDateMap, getUserIsAdmin } from '../../common'
+import { getStudentToTargetCourseDateMap, getUserIsAdmin, getTextIn } from '../../common'
 import { useProgress, useTitle } from '../../common/hooks'
 import infotooltips from '../../common/InfoToolTips'
 import InfoBox from '../InfoBox'
@@ -23,6 +23,7 @@ import FilterTray from '../FilterTray'
 import useFilters from '../FilterTray/useFilters'
 import { CoursePopulationFilters } from '../FilterTray/FilterSets'
 import useLanguage from '../LanguagePicker/useLanguage'
+import { queryParamsFromUrl } from '../../common/query'
 
 const CoursePopulation = ({
   getCoursePopulationDispatch,
@@ -33,17 +34,13 @@ const CoursePopulation = ({
   courseData,
   getSemestersDispatch,
   semesters,
-  getFacultiesDispatch
+  getFacultiesDispatch,
+  getCustomPopulationCoursesByStudentnumbers
 }) => {
   const { language } = useLanguage()
   const { setAllStudents, filteredStudents } = useFilters()
   const selectedStudents = filteredStudents.map(stu => stu.studentNumber)
 
-  const parseQueryFromUrl = () => {
-    const { location } = history
-    const query = qs.parse(location.search)
-    return query
-  }
   const [codes, setCodes] = useState([])
   const [headerYears, setYears] = useState('')
   const [dateFrom, setDateFrom] = useState(null)
@@ -67,6 +64,12 @@ const CoursePopulation = ({
   // Pass students to filter context.
   useEffect(() => {
     setAllStudents(studentData.students || [])
+
+    // Data fetching for courses of population tab
+    if (!studentData.students) return
+    getCustomPopulationCoursesByStudentnumbers({
+      studentnumberlist: studentData.students.map(student => student.studentNumber)
+    })
   }, [studentData.students])
 
   useEffect(() => {
@@ -74,7 +77,7 @@ const CoursePopulation = ({
   }, [])
 
   useEffect(() => {
-    if (newestIndex) {
+    if (newestIndex && refs[newestIndex]) {
       scrollToComponent(refs[newestIndex].current, { align: 'bottom' })
     }
   }, [activeIndex])
@@ -93,7 +96,7 @@ const CoursePopulation = ({
 
   useEffect(() => {
     if (semesters.years && semesters.semesters) {
-      const { coursecodes, from, to, years, separate } = parseQueryFromUrl()
+      const { coursecodes, from, to, years, years2, separate } = queryParamsFromUrl(history.location)
       const parsedCourseCodes = JSON.parse(coursecodes)
       getCoursePopulationDispatch({ coursecodes, from, to, onProgress, separate })
       getSingleCourseStatsDispatch({
@@ -103,7 +106,11 @@ const CoursePopulation = ({
         separate
       })
       setCodes(parsedCourseCodes)
-      setYears(years)
+      if (years) {
+        setYears(years)
+      } else {
+        setYears(years2)
+      }
       getFromToDates(from, to, separate)
       getFacultiesDispatch()
 
@@ -113,7 +120,7 @@ const CoursePopulation = ({
     }
   }, [semesters])
 
-  const header = courseData ? `${courseData.name} ${headerYears}` : null
+  const header = courseData ? `${getTextIn(courseData.name, language)} ${headerYears}` : null
   const subHeader = codes.join(', ')
 
   if (!dateFrom || !dateTo) return null
@@ -143,29 +150,9 @@ const CoursePopulation = ({
       key: 0,
       title: {
         content: (
-          <>
-            {activeIndex.includes(0) ? (
-              <>Grade distribution</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Grade distribution
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.GradeDistributionCoursePopulation}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Grade distribution
+          </span>
         )
       },
       onTitleClick: () => handleClick(0),
@@ -188,29 +175,9 @@ const CoursePopulation = ({
       key: 1,
       title: {
         content: (
-          <>
-            {activeIndex.includes(1) ? (
-              <>Programme distribution</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Programme distribution
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.ProgrammeDistributionCoursePopulation}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Programme distribution
+          </span>
         )
       },
       onTitleClick: () => handleClick(1),
@@ -231,32 +198,30 @@ const CoursePopulation = ({
       key: 2,
       title: {
         content: (
-          <>
-            {activeIndex.includes(2) ? (
-              <>Credit gains</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Credit gains
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.CreditDistributionCoursePopulation}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Courses of population
+          </span>
         )
       },
       onTitleClick: () => handleClick(2),
+      content: {
+        content: (
+          <div ref={programmeRef}>
+            <CustomPopulationCourses selectedStudents={selectedStudents} showFilter={false} />
+          </div>
+        )
+      }
+    },
+    {
+      key: 3,
+      title: {
+        content: (
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Credit gains
+          </span>
+        )
+      },
+      onTitleClick: () => handleClick(3),
       content: {
         content: (
           <div ref={creditGainRef}>
@@ -273,42 +238,21 @@ const CoursePopulation = ({
       }
     },
     {
-      key: 3,
+      key: 4,
       title: {
         content: (
-          <>
-            {activeIndex.includes(3) ? (
-              <>Students ({selectedStudents.length})</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Students ({selectedStudents.length})
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.Students.AccordionTitle}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Students ({selectedStudents.length})
+          </span>
         )
       },
-      onTitleClick: () => handleClick(3),
+      onTitleClick: () => handleClick(4),
       content: {
         content: (
           <div ref={studentRef}>
             <PopulationStudents
               studentToTargetCourseDateMap={studentToTargetCourseDateMap}
               filteredStudents={studentData.students.filter(stu => selectedStudents.includes(stu.studentNumber))}
-              mandatoryToggle={false}
               coursePopulation
               language={language}
             />
@@ -347,7 +291,8 @@ CoursePopulation.propTypes = {
     semesters: shape({}),
     years: shape({})
   }).isRequired,
-  getFacultiesDispatch: func.isRequired
+  getFacultiesDispatch: func.isRequired,
+  getCustomPopulationCoursesByStudentnumbers: func.isRequired
 }
 
 const mapStateToProps = ({
@@ -361,7 +306,6 @@ const mapStateToProps = ({
   return {
     studentData: populations.data,
     pending: populations.pending,
-    query: populations.query,
     courseData: singleCourseStats.stats || {},
     semesters: semesters.data,
     isAdmin: getUserIsAdmin(roles)
@@ -375,7 +319,8 @@ export default withRouter(
       getCoursePopulationDispatch: getCoursePopulation,
       getSingleCourseStatsDispatch: getSingleCourseStats,
       getSemestersDispatch: getSemesters,
-      getFacultiesDispatch: getFaculties
+      getFacultiesDispatch: getFaculties,
+      getCustomPopulationCoursesByStudentnumbers
     }
   )(CoursePopulation)
 )

services/oodikone2-frontend/src/components/CourseStatistics/CourseTable/index.jsx

@@ -5,11 +5,12 @@ import { func, arrayOf, shape, string, bool } from 'prop-types'
 import { getActiveYears } from '../courseStatisticsUtils'
 import useLanguage from '../../LanguagePicker/useLanguage'
 import './courseTable.css'
+import { getTextIn } from '../../../common'
 
-const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, mandatory = false }) => {
+const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, mandatory = false, controlIcon }) => {
   const { language } = useLanguage()
   const noContent = courses.length === 0
-  const sortCourses = courses => sortBy(courses, course => course.name[language])
+  const sortCourses = courses => sortBy(courses, course => getTextIn(course.name, language))
 
   const getEmptyListRow = () => (
     <Table.Row>
@@ -24,10 +25,11 @@ const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, ma
       onClick={() => (course.min_attainment_date || mandatory ? onSelectCourse(course) : null)}
     >
       <Table.Cell width={10}>
-        <div>{course.name[language]}</div>
+        <div>{getTextIn(course.name, language)}</div>
         <div>{getActiveYears(course)}</div>
       </Table.Cell>
       <Table.Cell content={!course.alternatives ? course.code : course.alternatives.map(a => a.code).join(', ')} />
+      {controlIcon ? <Table.Cell icon={controlIcon} /> : null}
     </Table.Row>
   )
 
@@ -39,6 +41,7 @@ const CourseTable = ({ courses, onSelectCourse, hidden, title, emptyListText, ma
             <Table.Row>
               <Table.HeaderCell content={title} />
               <Table.HeaderCell content="Code" />
+              {controlIcon ? <Table.HeaderCell /> : null}
             </Table.Row>
           </Table.Header>
           <Table.Body>{noContent ? getEmptyListRow() : sortCourses(courses).map(toCourseRow)}</Table.Body>
@@ -54,18 +57,19 @@ CourseTable.propTypes = {
   hidden: bool.isRequired,
   title: string.isRequired,
   emptyListText: string,
-  controlIcon: string.isRequired
+  controlIcon: string
 }
 
 CourseTable.defaultProps = {
-  emptyListText: 'No results.'
+  emptyListText: 'No results.',
+  controlIcon: null
 }
 
 function areEqual(prevProps, nextProps) {
   if (prevProps.courses.length !== nextProps.courses.length) {
     return false
   }
-  return true
+  return prevProps.courses.every(c1 => nextProps.courses.some(c2 => c1.code === c2.code))
 }
 
 export default React.memo(CourseTable, areEqual)

services/oodikone2-frontend/src/components/CourseStatistics/FacultyLevelStatistics/index.jsx

@@ -9,33 +9,42 @@ import useLanguage from '../../LanguagePicker/useLanguage'
 const CourseTableRow = ({ facultyCode, students, credits, facultyName }) => {
   return (
     <Table.Row>
-      <Table.Cell>
+      <Table.Cell width={13}>
         {facultyCode}, {facultyName}
       </Table.Cell>
-      <Table.Cell>{students}</Table.Cell>
-      <Table.Cell>{credits}</Table.Cell>
+      <Table.Cell width={2}>{students}</Table.Cell>
+      <Table.Cell width={1}>{credits}</Table.Cell>
     </Table.Row>
   )
 }
 
-const CourseTable = ({ course, courseInstance, language }) => {
-  const { coursecode, name } = course
+const CourseTable = ({ course, courseInstance, language, selectedYear }) => {
+  const { coursecode } = course
+  const name = getTextIn(course.name, language)
+
+  const rows = courseInstance ? (
+    Object.entries(courseInstance.faculties)
+      .sort(([facultyCodeA], [facultyCodeB]) => facultyCodeA.localeCompare(facultyCodeB))
+      .map(([facultyCode, instanceFaculty]) => (
+        <CourseTableRow
+          key={`${coursecode}-${facultyCode}`}
+          facultyCode={facultyCode}
+          facultyName={getTextIn(instanceFaculty.name, language)}
+          students={instanceFaculty.students.length}
+          credits={instanceFaculty.credits}
+        />
+      ))
+  ) : (
+    <Table.Row>
+      <Table.Cell width={13}>No data for selected year</Table.Cell>
+    </Table.Row>
+  )
 
-  const rows = Object.entries(courseInstance.faculties)
-    .sort(([facultyCodeA], [facultyCodeB]) => facultyCodeA.localeCompare(facultyCodeB))
-    .map(([facultyCode, instanceFaculty]) => (
-      <CourseTableRow
-        key={`${coursecode}-${facultyCode}`}
-        facultyCode={facultyCode}
-        facultyName={getTextIn(instanceFaculty.name, language)}
-        students={instanceFaculty.students.length}
-        credits={instanceFaculty.credits}
-      />
-    ))
   return (
     <>
       <Header>
-        {courseInstance.year} {coursecode}
+        {courseInstance ? courseInstance.year : `${1949 + Number(selectedYear)}-${1950 + Number(selectedYear)}`}{' '}
+        {coursecode}
       </Header>
       <Header size="small">
         {name} ({coursecode})
@@ -43,19 +52,27 @@ const CourseTable = ({ course, courseInstance, language }) => {
       <Table compact>
         <Table.Header>
           <Table.Row>
-            <Table.HeaderCell>Faculty</Table.HeaderCell>
-            <Table.HeaderCell>Students</Table.HeaderCell>
-            <Table.HeaderCell>Credits</Table.HeaderCell>
+            <Table.HeaderCell width={13}>Faculty</Table.HeaderCell>
+            <Table.HeaderCell width={2}>Students</Table.HeaderCell>
+            <Table.HeaderCell width={1}>Credits</Table.HeaderCell>
           </Table.Row>
         </Table.Header>
         <Table.Body>{rows}</Table.Body>
-        <Table.Footer>
-          <Table.Row>
-            <Table.HeaderCell style={{ fontWeight: '700' }}>Total</Table.HeaderCell>
-            <Table.HeaderCell style={{ fontWeight: '700' }}>{courseInstance.allStudents.length}</Table.HeaderCell>
-            <Table.HeaderCell style={{ fontWeight: '700' }}>{courseInstance.allCredits}</Table.HeaderCell>
-          </Table.Row>
-        </Table.Footer>
+        {courseInstance ? (
+          <Table.Footer>
+            <Table.Row>
+              <Table.HeaderCell width={13} style={{ fontWeight: '700' }}>
+                Total
+              </Table.HeaderCell>
+              <Table.HeaderCell width={2} style={{ fontWeight: '700' }}>
+                {courseInstance ? courseInstance.allStudents.length : 0}
+              </Table.HeaderCell>
+              <Table.HeaderCell width={1} style={{ fontWeight: '700' }}>
+                {courseInstance ? courseInstance.allCredits : 0}
+              </Table.HeaderCell>
+            </Table.Row>
+          </Table.Footer>
+        ) : null}
       </Table>
     </>
   )
@@ -80,13 +97,20 @@ const FacultyLevelStatistics = () => {
   }))
 
   const yearsCourseStats = Object.values(courseStats)
-    .filter(course => !!course.facultyStats[selectedYear])
     .map(course => ({
       course,
       courseInstance: course.facultyStats[selectedYear]
     }))
+    .sort((a, b) => (a.courseInstance === undefined) - (b.courseInstance === undefined))
+
   const courseTables = yearsCourseStats.map(({ course, courseInstance }) => (
-    <CourseTable course={course} courseInstance={courseInstance} language={language} key={course.coursecode} />
+    <CourseTable
+      course={course}
+      courseInstance={courseInstance}
+      language={language}
+      key={course.coursecode}
+      selectedYear={selectedYear}
+    />
   ))
 
   return (
@@ -115,8 +139,13 @@ CourseTableRow.propTypes = {
 
 CourseTable.propTypes = {
   course: shape({}).isRequired,
-  courseInstance: shape({}).isRequired,
-  language: string.isRequired
+  courseInstance: shape({}),
+  language: string.isRequired,
+  selectedYear: string.isRequired
+}
+
+CourseTable.defaultProps = {
+  courseInstance: null
 }
 
 export default FacultyLevelStatistics

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/Tables/grades.jsx

@@ -36,18 +36,24 @@ const getTableData = (stats, isGradeSeries, isRelative) =>
     }
   })
 
-const getGradeColumns = isGradeSeries =>
-  isGradeSeries
-    ? [
-        getSortableColumn('0', '0', s => s['0']),
-        getSortableColumn('1', '1', s => s['1']),
-        getSortableColumn('2', '2', s => s['2']),
-        getSortableColumn('3', '3', s => s['3']),
-        getSortableColumn('4', '4', s => s['4']),
-        getSortableColumn('5', '5', s => s['5']),
-        getSortableColumn('OTHER_PASSED', 'Other passed', s => s['Hyv.'])
-      ]
-    : THESIS_GRADE_KEYS.map(k => getSortableColumn(k, k, s => s[k]))
+const includesHTOrTT = stats =>
+  stats.some(({ cumulative }) => ['HT', 'TT'].some(grade => Object.keys(cumulative.grades).includes(grade)))
+
+const getGradeColumns = (isGradeSeries, addHTAndTT) => {
+  if (!isGradeSeries) return THESIS_GRADE_KEYS.map(k => getSortableColumn(k, k, s => s[k]))
+  const columns = [
+    getSortableColumn('0', '0', s => s['0']),
+    getSortableColumn('1', '1', s => s['1']),
+    getSortableColumn('2', '2', s => s['2']),
+    getSortableColumn('3', '3', s => s['3']),
+    getSortableColumn('4', '4', s => s['4']),
+    getSortableColumn('5', '5', s => s['5']),
+    getSortableColumn('OTHER_PASSED', 'Other passed', s => s['Hyv.'])
+  ]
+  if (addHTAndTT)
+    columns.splice(6, 0, getSortableColumn('HT', 'HT', s => s.HT), getSortableColumn('TT', 'TT', s => s.TT))
+  return columns
+}
 
 const GradesTable = ({ stats, name, alternatives, separate, isRelative }) => {
   const {
@@ -84,7 +90,7 @@ const GradesTable = ({ stats, name, alternatives, separate, isRelative }) => {
       )
     ),
     getSortableColumn('ATTEMPTS', 'Attempts', s => s.attempts),
-    ...getGradeColumns(isGradeSeries)
+    ...getGradeColumns(isGradeSeries, includesHTOrTT(stats))
   ]
 
   const data = getTableData(stats, isGradeSeries, isRelative)

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/distribution.jsx

@@ -33,7 +33,9 @@ const getGradeSeries = series => {
           getDataObject('3', newSeries[3], 'd'),
           getDataObject('4', newSeries[4], 'e'),
           getDataObject('5', newSeries[5], 'f'),
-          getDataObject('Hyv.', newSeries['Hyv.'], 'g')
+          getDataObject('HT', newSeries.HT, 'g'),
+          getDataObject('TT', newSeries.TT, 'h'),
+          getDataObject('Hyv.', newSeries['Hyv.'], 'i')
         ],
         relative: [
           getDataObject('0', newSeries[0].map(absoluteToRelative(sumAll)), 'a'),
@@ -42,7 +44,9 @@ const getGradeSeries = series => {
           getDataObject('3', newSeries[3].map(absoluteToRelative(sumAll)), 'd'),
           getDataObject('4', newSeries[4].map(absoluteToRelative(sumAll)), 'e'),
           getDataObject('5', newSeries[5].map(absoluteToRelative(sumAll)), 'f'),
-          getDataObject('Hyv.', newSeries['Hyv.'].map(absoluteToRelative(sumAll)), 'g')
+          getDataObject('HT', newSeries.HT.map(absoluteToRelative(sumAll)), 'g'),
+          getDataObject('TT', newSeries.TT.map(absoluteToRelative(sumAll)), 'h'),
+          getDataObject('Hyv.', newSeries['Hyv.'].map(absoluteToRelative(sumAll)), 'i')
         ]
       }
     : {

services/oodikone2-frontend/src/components/CourseStatistics/ResultTabs/Panes/util.js

@@ -82,6 +82,8 @@ export const getGradeSpread = (series, isRelative) => {
     3: [],
     4: [],
     5: [],
+    HT: [],
+    TT: [],
     'Hyv.': []
   }
   const newSeries = series.reduce(

services/oodikone2-frontend/src/components/CourseStatistics/SearchForm/index.jsx

@@ -9,7 +9,7 @@ import { clearCourses, findCoursesV2, toggleUnifyOpenUniCourses } from '../../..
 import { getCourseStats, clearCourseStats } from '../../../redux/coursestats'
 import { getCourseSearchResults } from '../../../selectors/courses'
 import { useSearchHistory, usePrevious } from '../../../common/hooks'
-import { validateInputLength } from '../../../common'
+import { validateInputLength, getTextIn } from '../../../common'
 import TSA from '../../../common/tsa'
 import { mergeCourses } from '../courseStatisticsUtils'
 import AutoSubmitSearchInput from '../../AutoSubmitSearchInput'
@@ -138,7 +138,7 @@ const SearchForm = props => {
       separate,
       unifyOpenUniCourses: props.unifyOpenUniCourses
     }
-    const searchHistoryText = codes.map(code => `${selectedCourses[code].name[language]} ${code}`)
+    const searchHistoryText = codes.map(code => `${getTextIn(selectedCourses[code].name, language)} ${code}`)
     addItemToSearchHistory({
       text: searchHistoryText.join(', '),
       params
@@ -179,14 +179,33 @@ const SearchForm = props => {
   const selected = Object.values(selectedCourses).map(course => ({ ...course, selected: true }))
   const noSelectedCourses = selected.length === 0
 
+  const addAllCourses = () => {
+    courses.forEach(course => {
+      course.selected = true
+    })
+
+    const newSelectedCourses = courses.reduce((newSelected, course) => {
+      newSelected[course.code] = { ...course }
+      return newSelected
+    }, {})
+
+    setState({
+      ...state,
+      selectedCourses: {
+        ...selectedCourses,
+        ...newSelectedCourses
+      }
+    })
+  }
+
   return (
     <React.Fragment>
       <Segment loading={isLoading}>
         <Form>
           <Header content="Search for courses" />
           <div style={{ marginBottom: '15px' }}>
-            <Form.Group widths="equal">
-              <Form.Field>
+            <Form.Group>
+              <Form.Field width={8}>
                 <label>Name:</label>
                 <AutoSubmitSearchInput
                   doSearch={fetchCourses}
@@ -197,11 +216,11 @@ const SearchForm = props => {
                   minSearchLength={0}
                 />
               </Form.Field>
-              <Form.Field>
+              <Form.Field width={3}>
                 <label>Code:</label>
                 <AutoSubmitSearchInput
                   doSearch={fetchCourses}
-                  placeholder="Search by entering a course code"
+                  placeholder="Search by a course code"
                   value={courseCode}
                   onChange={cc => setState({ ...state, courseCode: cc })}
                   loading={props.coursesLoading}
@@ -209,7 +228,7 @@ const SearchForm = props => {
                   data-cy="course-code-input"
                 />
               </Form.Field>
-              <Form.Field style={{ display: 'flex', flexDirection: 'column' }}>
+              <Form.Field width={4} style={{ display: 'flex', flexDirection: 'column' }}>
                 <label>Unify open university courses:</label>
                 <span style={{ flex: 1, display: 'flex', alignItems: 'center' }}>
                   <Form.Checkbox
@@ -225,7 +244,7 @@ const SearchForm = props => {
               hidden={noSelectedCourses}
               courses={selected}
               onSelectCourse={onSelectCourse}
-              controlIcon="remove"
+              controlIcon="trash alternate outline"
             />
             {!noSelectedCourses && (
               <Fragment>
@@ -254,8 +273,19 @@ const SearchForm = props => {
               courses={courses}
               title="Searched courses"
               onSelectCourse={onSelectCourse}
-              controlIcon="plus"
             />
+            {courses.length ? (
+              <div className="select-all-container">
+                <Form.Button
+                  type="button"
+                  size="large"
+                  content="Select all search results"
+                  basic
+                  color="green"
+                  onClick={addAllCourses}
+                />
+              </div>
+            ) : null}
           </div>
         </Form>
       </Segment>

services/oodikone2-frontend/src/components/CourseStatistics/SingleCourseStats/index.jsx

@@ -143,7 +143,7 @@ const SingleCourseStats = ({
     const filteredSemesters = semesters.filter(timeFilter)
     const filteredYears = years.filter(timeFilter)
     return separate
-      ? filteredSemesters.find(year => year.texts.includes(name))
+      ? filteredSemesters.find(year => year.texts.includes(getTextIn(name, language)))
       : filteredYears.find(year => year.text === name)
   }
 
@@ -161,8 +161,10 @@ const SingleCourseStats = ({
           grades: countFilteredStudents(allstudents.grades, filter),
           categories: countFilteredStudents(allstudents.classes, filter)
         }
-        return { code, name, cumulative, students, coursecode }
+        const parsedName = separate ? getTextIn(name, language) : name
+        return { code, name: parsedName, cumulative, students, coursecode }
       })
+
     const totals = progStats.reduce(
       (acc, curr) => {
         const passed = acc.cumulative.categories.passed + curr.cumulative.categories.passed
@@ -331,7 +333,7 @@ const SingleCourseStats = ({
   if (stats.statistics.length < 1) return <Segment>No data for selected course</Segment>
 
   const options = filteredProgrammes
-    .map(({ text, ...rest }) => ({ text: typeof text === 'string' ? text : text[language], ...rest }))
+    .map(({ text, ...rest }) => ({ text: typeof text === 'string' ? text : getTextIn(text, language), ...rest }))
     .map(prog => ({ ...prog, name: prog.text }))
 
   return (
@@ -417,7 +419,7 @@ SingleCourseStats.propTypes = {
         )
       })
     ),
-    name: string,
+    name: shape({ fi: string, en: string, sv: string }),
     coursecode: string
   }).isRequired,
   programmes: arrayOf(shape({})).isRequired,

services/oodikone2-frontend/src/components/CourseStatistics/SingleCourseTab/index.jsx

@@ -1,12 +1,19 @@
-import React, { useState } from 'react'
+import React, { useState, useEffect } from 'react'
 import { connect } from 'react-redux'
 import { Segment, Label, Header, Divider, Form } from 'semantic-ui-react'
 import { shape, arrayOf, oneOfType, number, string } from 'prop-types'
 import SingleCourseStats from '../SingleCourseStats'
+import useLanguage from '../../LanguagePicker/useLanguage'
 import selectors from '../../../selectors/courseStats'
+import { getTextIn } from '../../../common'
 
 const SingleCourseTab = ({ selected, stats, courses }) => {
   const [selection, setSelection] = useState(selected)
+  const { language } = useLanguage()
+
+  useEffect(() => {
+    setSelection(selected)
+  }, [selected])
 
   if (!stats[selection]) return null
   return (
@@ -29,7 +36,10 @@ const SingleCourseTab = ({ selected, stats, courses }) => {
           <Label.Group>
             <Label
               key={stats[selection].coursecode}
-              content={`${stats[selection].alternatives.map(code => ` ${code}`)} ${stats[selection].name} `}
+              content={`${stats[selection].alternatives.map(code => ` ${code}`)} ${getTextIn(
+                stats[selection].name,
+                language
+              )} `}
             />
           </Label.Group>
         </Form>

services/oodikone2-frontend/src/components/CourseStatistics/SummaryTab/index.jsx

@@ -8,6 +8,7 @@ import { fields, setValue } from '../../../redux/coursesSummaryForm'
 import CumulativeTable from '../CumulativeTable'
 import ProgrammeDropdown from '../ProgrammeDropdown'
 import useLanguage from '../../LanguagePicker/useLanguage'
+import { getTextIn } from '../../../common'
 
 const SummaryTab = ({ form, setValue, statistics, programmes, queryInfo, onClickCourse }) => {
   const { language } = useLanguage()
@@ -24,7 +25,7 @@ const SummaryTab = ({ form, setValue, statistics, programmes, queryInfo, onClick
     const { passed, failed, passrate } = summary
     return {
       id: coursecode,
-      category: name,
+      category: getTextIn(name, language),
       passed,
       failed,
       passrate,
@@ -35,7 +36,7 @@ const SummaryTab = ({ form, setValue, statistics, programmes, queryInfo, onClick
   const options = programmes
     .map(e => ({ ...e, size: new Set(flatten(Object.values(e.students))).size }))
     .filter(e => e.size > 0)
-    .map(({ text, ...rest }) => ({ text: typeof text === 'string' ? text : text[language], ...rest }))
+    .map(({ text, ...rest }) => ({ text: typeof text === 'string' ? text : getTextIn(text, language), ...rest }))
     .map(prog => ({ ...prog, name: prog.text }))
 
   return (
@@ -69,7 +70,11 @@ SummaryTab.propTypes = {
   statistics: arrayOf(
     shape({
       coursecode: oneOfType([number, string]),
-      name: string,
+      name: shape({
+        fi: string,
+        en: string,
+        sv: string
+      }),
       summary: shape({
         failed: number,
         passed: number,

services/oodikone2-frontend/src/components/CourseStatistics/courseStatistics.css

@@ -31,3 +31,9 @@
 .yearControlButton {
   font-size: 8px !important;
 }
+
+.select-all-container {
+  width: 100%;
+  display: flex;
+  justify-content: space-around;
+}

services/oodikone2-frontend/src/components/CourseStatistics/courseStatisticsUtils.js

@@ -49,42 +49,68 @@ const sortAlternatives = alternatives =>
     ['asc', 'desc', 'desc']
   )
 
-const mergeCourses = (groups, courses, groupMeta, unifyOpenUniCourses = false) => {
-  const mergedCourses = {}
+const isAvoin = code => !!code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
 
-  courses.forEach(course => {
-    const avoinRegex = !!course.code.match(/^AY?(.+?)(?:en|fi|sv)?$/)
-    const isAvoin = avoinRegex
+const fixGroups = groupsArgs => {
+  const notAvoin = new Set()
+  const groups = {}
 
-    const groupId = isAvoin && !unifyOpenUniCourses ? course.code : groups[course.code]
+  Object.entries(groupsArgs).forEach(([code, parentCode]) => {
+    groups[code] = parentCode
 
-    if (!mergedCourses[groupId]) {
-      mergedCourses[groupId] = {
-        ...course,
-        code: (groupMeta[groupId] && groupMeta[groupId].code) || course.code,
-        name: (groupMeta[groupId] && groupMeta[groupId].name) || course.name,
-        alternatives: [{ code: course.code, latestInstanceDate: new Date(course.latest_instance_date) }],
-        min_attainment_date: new Date(course.min_attainment_date),
-        max_attainment_date: new Date(course.max_attainment_date)
-      }
-    } else {
-      const mergedCourse = mergedCourses[groupId]
-      mergedCourse.min_attainment_date = new Date(
-        Math.min(mergedCourse.min_attainment_date, new Date(course.min_attainment_date))
-      )
-      mergedCourse.max_attainment_date = new Date(
-        Math.max(mergedCourse.max_attainment_date, new Date(course.max_attainment_date))
-      )
-      mergedCourse.alternatives.push({
-        code: course.code,
-        latestInstanceDate: new Date(course.latest_instance_date)
-      })
+    // handle special case where the course's actual code starts with "A" so it is mistakenly taken as Open Uni course
+    if (groupsArgs[`AY${code}`] && code.startsWith('A')) {
+      groups[code] = code
+      notAvoin.add(code)
     }
   })
 
+  return { groups, notAvoin }
+}
+
+const mergeCourses = (groupsArgs, courses, groupMeta, unifyOpenUniCourses = false) => {
+  const mergedCourses = {}
+
+  const { groups, notAvoin } = fixGroups(groupsArgs)
+
+  courses
+    // Sort the codes so that we know non-Open Uni courses come up first
+    .sort((a, b) => a.code.length - b.code.length)
+    .forEach(course => {
+      const groupId =
+        isAvoin(course.code) && !notAvoin.has(course.code) && !unifyOpenUniCourses ? course.code : groups[course.code]
+
+      if (!mergedCourses[groupId]) {
+        mergedCourses[groupId] = {
+          ...course,
+          code: (groupMeta[groupId] && groupMeta[groupId].code) || course.code,
+          name: (groupMeta[groupId] && groupMeta[groupId].name) || course.name,
+          alternatives: [{ code: course.code, latestInstanceDate: new Date(course.latest_instance_date) }],
+          min_attainment_date: new Date(course.min_attainment_date),
+          max_attainment_date: new Date(course.max_attainment_date)
+        }
+      } else {
+        const mergedCourse = mergedCourses[groupId]
+        mergedCourse.min_attainment_date = new Date(
+          Math.min(mergedCourse.min_attainment_date, new Date(course.min_attainment_date))
+        )
+        mergedCourse.max_attainment_date = new Date(
+          Math.max(mergedCourse.max_attainment_date, new Date(course.max_attainment_date))
+        )
+        mergedCourse.alternatives.push({
+          code: course.code,
+          latestInstanceDate: new Date(course.latest_instance_date)
+        })
+      }
+    })
+
   return Object.values(mergedCourses).map(c => ({
     ...c,
-    alternatives: sortAlternatives(c.alternatives)
+    // If special case of Open Uni course that starts with A let's just sort them length, that
+    // should be good enough in this case.
+    alternatives: c.alternatives.some(course => notAvoin.has(course.code))
+      ? c.alternatives.sort((a, b) => a.code.length - b.code.length)
+      : sortAlternatives(c.alternatives)
   }))
 }
 

services/oodikone2-frontend/src/components/CourseStatistics/index.jsx

@@ -41,6 +41,10 @@ const CourseStatistics = props => {
   const { onProgress, progress } = useProgress(loading)
   useTitle('Course statistics')
 
+  useEffect(() => {
+    setSelected(initCourseCode)
+  }, [initCourseCode])
+
   useEffect(() => {
     if (statsIsEmpty) {
       setSelected(initCourseCode)

services/oodikone2-frontend/src/components/CreditAccumulationGraphHighCharts/index.jsx

@@ -13,7 +13,7 @@ import './creditAccumulationGraphHC.css'
 import CreditGraphTooltip from '../CreditGraphTooltip'
 import { clearLoading } from '../../redux/graphSpinner'
 import { setChartHeight } from '../../redux/settings'
-import { reformatDate } from '../../common'
+import { reformatDate, getTextIn } from '../../common'
 import { DISPLAY_DATE_FORMAT, API_DATE_FORMAT } from '../../constants'
 
 // boost canvas needed because tests break with large population
@@ -126,7 +126,7 @@ class CreditAccumulationGraphHighCharts extends Component {
         payload: {
           ...targetCourse,
           date: reformatDate(targetCourse.date, DISPLAY_DATE_FORMAT),
-          title: targetCourse.course.name[language]
+          title: getTextIn(targetCourse.course.name, language)
         }
       }
     ]

services/oodikone2-frontend/src/components/CustomPopulation/CustomPopulationCourses/index.jsx

@@ -1,14 +1,14 @@
 import React, { useEffect } from 'react'
 import { connect } from 'react-redux'
 import { shape, arrayOf, string, bool } from 'prop-types'
-import PopulationCourseStats from '../../PopulationCourseStats'
+import PopulationCourseStatsFlat from '../../PopulationCourseStats/PopulationCourseStatsFlat'
 import useCourseFilter from '../../FilterTray/filters/Courses/useCourseFilter'
 
-const CustomPopulationCourses = ({ courses, pending, selectedStudents, query, error }) => {
+const CustomPopulationCourses = ({ courses, pending, selectedStudents, query, error, showFilter }) => {
   const { setCoursesOnce, resetCourses } = useCourseFilter()
 
   useEffect(() => {
-    if (!pending && !error) {
+    if (!pending && !error && courses.coursestatistics) {
       setCoursesOnce(courses.coursestatistics)
     }
   }, [courses])
@@ -19,12 +19,13 @@ const CustomPopulationCourses = ({ courses, pending, selectedStudents, query, er
   }, [])
 
   return (
-    <PopulationCourseStats
+    <PopulationCourseStatsFlat
       courses={courses}
       query={query}
       pending={pending}
       selectedStudents={selectedStudents}
       customPopulation
+      showFilter={showFilter}
     />
   )
 }
@@ -34,7 +35,12 @@ CustomPopulationCourses.propTypes = {
   courses: shape([]).isRequired,
   selectedStudents: arrayOf(string).isRequired,
   query: shape({}).isRequired,
-  error: bool.isRequired
+  error: bool.isRequired,
+  showFilter: bool
+}
+
+CustomPopulationCourses.defaultProps = {
+  showFilter: false
 }
 
 const mapStateToProps = ({ populationCourses }) => ({

services/oodikone2-frontend/src/components/CustomPopulation/CustomPopulationSearch.jsx

@@ -0,0 +1,188 @@
+import React, { useState, useEffect } from 'react'
+import { connect } from 'react-redux'
+import { Button, Modal, Form, TextArea } from 'semantic-ui-react'
+import { shape, func, arrayOf, bool } from 'prop-types'
+import { useProgress, useTitle } from '../../common/hooks'
+import { getCustomPopulation } from '../../redux/populations'
+import {
+  getCustomPopulationSearches,
+  saveCustomPopulationSearch,
+  updateCustomPopulationSearch,
+  selectCustomPopulationSearch,
+  deleteCustomPopulationSearch
+} from '../../redux/customPopulationSearch'
+import { getCustomPopulationCoursesByStudentnumbers } from '../../redux/populationCourses'
+import SearchHistory from '../SearchHistory'
+
+const CustomPopulationSearch = ({
+  getCustomPopulationDispatch,
+  getCustomPopulationCoursesByStudentnumbers,
+  getCustomPopulationSearchesDispatch,
+  saveCustomPopulationSearchDispatch,
+  updateCustomPopulationSearchDispatch,
+  selectCustomPopulationSearchDispatch,
+  deleteCustomPopulationSearchDispatch,
+  loading,
+  customPopulationSearches,
+  customPopulationSearchSaving
+}) => {
+  const [modal, setModal] = useState(false)
+  const [input, setInput] = useState('')
+  const [name, setName] = useState('')
+  const [selectedSearchId, setSelectedSearchId] = useState('')
+
+  const { onProgress } = useProgress(loading)
+
+  useTitle('Custom population')
+
+  useEffect(() => {
+    getCustomPopulationSearchesDispatch()
+  }, [])
+
+  const handleNameChange = e => {
+    setName(e.target.value)
+  }
+
+  const clearForm = () => {
+    setName('')
+    setInput('')
+    setSelectedSearchId('')
+  }
+
+  const handleClose = () => {
+    setModal(false)
+    clearForm()
+  }
+
+  const onSave = () => {
+    const studentnumberlist = input.match(/[0-9]+/g)
+    if (selectedSearchId) {
+      updateCustomPopulationSearchDispatch({ id: selectedSearchId, studentnumberlist })
+    } else {
+      saveCustomPopulationSearchDispatch({ name, studentnumberlist })
+    }
+  }
+
+  const onDelete = () => {
+    if (selectedSearchId) {
+      deleteCustomPopulationSearchDispatch({ id: selectedSearchId })
+      clearForm()
+    }
+  }
+
+  const onSelectSearch = selected => {
+    if (!selected) {
+      clearForm()
+      return
+    }
+
+    const { id: selectedSearchId } = selected
+    const selectedSearch = customPopulationSearches.find(({ id }) => id === selectedSearchId)
+    if (selectedSearch) {
+      setInput(selectedSearch.students.join('\n'))
+      setName(selectedSearch.name)
+      setSelectedSearchId(selectedSearch.id)
+    }
+  }
+
+  const onClicker = e => {
+    e.preventDefault()
+    const studentnumbers = input.match(/[0-9]+/g)
+    getCustomPopulationDispatch({ studentnumberlist: studentnumbers, onProgress })
+    getCustomPopulationCoursesByStudentnumbers({ studentnumberlist: studentnumbers })
+    selectCustomPopulationSearchDispatch(selectedSearchId || null)
+    handleClose()
+  }
+
+  return (
+    <Modal
+      trigger={
+        <Button size="small" color="blue" onClick={() => setModal(true)} data-cy="custom-pop-search-button">
+          Custom population
+        </Button>
+      }
+      open={modal}
+      onClose={handleClose}
+      size="small"
+    >
+      <Modal.Content>
+        <Form>
+          <h2> Custom population new</h2>
+          <Form.Field>
+            <em> Insert name for this custom population if you wish to save it </em>
+            <Form.Input disabled={!!selectedSearchId} value={name} placeholder="name" onChange={handleNameChange} />
+          </Form.Field>
+          <Form.Field>
+            <em> Insert studentnumbers you wish to use for population here </em>
+            <TextArea
+              value={input}
+              placeholder="011111111"
+              onChange={e => setInput(e.target.value)}
+              data-cy="student-no-input"
+            />
+          </Form.Field>
+        </Form>
+        <SearchHistory
+          header="Saved populations"
+          items={customPopulationSearches.map(s => ({
+            ...s,
+            text: s.name,
+            timestamp: s.updatedAt,
+            params: { id: s.id }
+          }))}
+          updateItem={() => null}
+          handleSearch={onSelectSearch}
+        />
+      </Modal.Content>
+      <Modal.Actions>
+        <Button
+          disabled={!name || customPopulationSearchSaving}
+          loading={customPopulationSearchSaving}
+          floated="left"
+          icon="save"
+          onClick={onSave}
+          content="Save"
+        />
+        <Button disabled={!selectedSearchId} negative floated="left" icon="trash" onClick={onDelete} content="Delete" />
+        <Button onClick={handleClose}>Cancel</Button>
+        <Button positive onClick={e => onClicker(e)} data-cy="search-button">
+          Search population
+        </Button>
+      </Modal.Actions>
+    </Modal>
+  )
+}
+
+CustomPopulationSearch.propTypes = {
+  getCustomPopulationDispatch: func.isRequired,
+  getCustomPopulationCoursesByStudentnumbers: func.isRequired,
+  loading: bool.isRequired,
+  customPopulationSearches: arrayOf(shape({})).isRequired,
+  customPopulationSearchSaving: bool.isRequired,
+  saveCustomPopulationSearchDispatch: func.isRequired,
+  getCustomPopulationSearchesDispatch: func.isRequired,
+  updateCustomPopulationSearchDispatch: func.isRequired,
+  selectCustomPopulationSearchDispatch: func.isRequired,
+  deleteCustomPopulationSearchDispatch: func.isRequired
+}
+
+const mapStateToProps = ({ populations, populationCourses, customPopulationSearch }) => ({
+  loading: populations.pending,
+  custompop: populations.data.students || [],
+  courses: populationCourses.data,
+  customPopulationSearches: customPopulationSearch.customPopulationSearches,
+  customPopulationSearchSaving: customPopulationSearch.saving
+})
+
+export default connect(
+  mapStateToProps,
+  {
+    getCustomPopulationDispatch: getCustomPopulation,
+    getCustomPopulationCoursesByStudentnumbers,
+    saveCustomPopulationSearchDispatch: saveCustomPopulationSearch,
+    getCustomPopulationSearchesDispatch: getCustomPopulationSearches,
+    updateCustomPopulationSearchDispatch: updateCustomPopulationSearch,
+    selectCustomPopulationSearchDispatch: selectCustomPopulationSearch,
+    deleteCustomPopulationSearchDispatch: deleteCustomPopulationSearch
+  }
+)(CustomPopulationSearch)

services/oodikone2-frontend/src/components/CustomPopulation/UnihowDataExport.jsx

@@ -0,0 +1,45 @@
+import React from 'react'
+import moment from 'moment'
+import xlsx from 'xlsx'
+import { Dropdown } from 'semantic-ui-react'
+import { getStudentGradeMean, getStudentGradeMeanWeightedByCredits, getStudentTotalCredits } from '../../common'
+import useFilters from '../FilterTray/useFilters'
+import sendEvent from '../../common/sendEvent'
+import DataExport from '../PopulationStatistics/DataExport'
+
+const sendAnalytics = sendEvent.populationStudents
+
+export default () => {
+  const { filteredStudents } = useFilters()
+
+  const getXlsx = () => {
+    const data = filteredStudents.map(student => ({
+      'Student Number': student.studentNumber,
+      'Total Credits (no transferred credits)': getStudentTotalCredits(student, false),
+      'Grade Mean (no transferred credits)': getStudentGradeMean(student, false),
+      'Grade Mean Weighted by ECTS (no transferred credits)': getStudentGradeMeanWeightedByCredits(student, false)
+    }))
+
+    const sheet = xlsx.utils.json_to_sheet(data)
+
+    const workbook = xlsx.utils.book_new()
+    xlsx.utils.book_append_sheet(workbook, sheet)
+    return workbook
+  }
+
+  const filename = `oodikone_export_${filteredStudents.length}_students_${moment().format('YYYYMMDD-hhmmss')}.xlsx`
+
+  return (
+    <>
+      <DataExport />
+      <Dropdown.Item
+        onClick={() => {
+          xlsx.writeFile(getXlsx(), filename)
+          sendAnalytics('Download excel button clicked', 'Download excel button clicked (UniHow)')
+        }}
+        text="Excel Workbook (UniHow)"
+        icon="file excel"
+      />
+    </>
+  )
+}

services/oodikone2-frontend/src/components/CustomPopulation/index.jsx

@@ -1,22 +1,13 @@
 import React, { useState, useEffect, useRef } from 'react'
 import { connect } from 'react-redux'
-import { Button, Modal, Form, TextArea, Segment, Header, Accordion, Popup, Message } from 'semantic-ui-react'
+import { Segment, Header, Accordion, Message } from 'semantic-ui-react'
 import { shape, func, arrayOf, bool, string } from 'prop-types'
-import ReactMarkdown from 'react-markdown'
 import scrollToComponent from 'react-scroll-to-component'
 import { useProgress, useTitle } from '../../common/hooks'
 import infotooltips from '../../common/InfoToolTips'
-import { getCustomPopulation } from '../../redux/populations'
-import {
-  getCustomPopulationSearches,
-  saveCustomPopulationSearch,
-  updateCustomPopulationSearch,
-  selectCustomPopulationSearch,
-  deleteCustomPopulationSearch
-} from '../../redux/customPopulationSearch'
+import { getCustomPopulationSearches } from '../../redux/customPopulationSearch'
 import { getCustomPopulationCoursesByStudentnumbers } from '../../redux/populationCourses'
 import CreditAccumulationGraphHighCharts from '../CreditAccumulationGraphHighCharts'
-import SearchHistory from '../SearchHistory'
 import PopulationStudents from '../PopulationStudents'
 import CustomPopulationCourses from './CustomPopulationCourses'
 import CustomPopulationProgrammeDist from './CustomPopulationProgrammeDist'
@@ -27,35 +18,27 @@ import useFilters from '../FilterTray/useFilters'
 import { CustomPopulationFilters } from '../FilterTray/FilterSets'
 import useFilterTray from '../FilterTray/useFilterTray'
 import useLanguage from '../LanguagePicker/useLanguage'
+import CustomPopulationSearch from './CustomPopulationSearch'
+import UnihowDataExport from './UnihowDataExport'
 
 const CustomPopulation = ({
-  getCustomPopulationDispatch,
-  getCustomPopulationCoursesByStudentnumbers,
   getCustomPopulationSearchesDispatch,
-  saveCustomPopulationSearchDispatch,
-  updateCustomPopulationSearchDispatch,
-  selectCustomPopulationSearchDispatch,
-  deleteCustomPopulationSearchDispatch,
   custompop,
   customPopulationFlag,
   loading,
   customPopulationSearches,
   latestCreatedCustomPopulationSearchId,
-  customPopulationSearchSaving,
   searchedCustomPopulationSearchId
 }) => {
   const { language } = useLanguage()
-  const [modal, setModal] = useState(false)
-  const [input, setInput] = useState('')
-  const [name, setName] = useState('')
   const [activeIndex, setIndex] = useState([])
-  const [selectedSearchId, setSelectedSearchId] = useState('')
+  const [, setSelectedSearchId] = useState('')
   const [newestIndex, setNewest] = useState(null)
   const { setAllStudents, filteredStudents } = useFilters()
   const [trayOpen] = useFilterTray()
   const selectedStudents = filteredStudents.map(stu => stu.studentNumber)
 
-  const { onProgress, progress } = useProgress(loading)
+  const { progress } = useProgress(loading)
 
   const creditGainRef = useRef()
   const programmeRef = useRef()
@@ -86,118 +69,6 @@ const CustomPopulation = ({
     }
   }, [latestCreatedCustomPopulationSearchId])
 
-  const handleNameChange = e => {
-    setName(e.target.value)
-  }
-
-  const clearForm = () => {
-    setName('')
-    setInput('')
-    setSelectedSearchId('')
-  }
-
-  const handleClose = () => {
-    setModal(false)
-    clearForm()
-  }
-
-  const onSave = () => {
-    const studentnumberlist = input.match(/[0-9]+/g)
-    if (selectedSearchId) {
-      updateCustomPopulationSearchDispatch({ id: selectedSearchId, studentnumberlist })
-    } else {
-      saveCustomPopulationSearchDispatch({ name, studentnumberlist })
-    }
-  }
-
-  const onDelete = () => {
-    if (selectedSearchId) {
-      deleteCustomPopulationSearchDispatch({ id: selectedSearchId })
-      clearForm()
-    }
-  }
-
-  const onSelectSearch = selected => {
-    if (!selected) {
-      clearForm()
-      return
-    }
-
-    const { id: selectedSearchId } = selected
-    const selectedSearch = customPopulationSearches.find(({ id }) => id === selectedSearchId)
-    if (selectedSearch) {
-      setInput(selectedSearch.students.join('\n'))
-      setName(selectedSearch.name)
-      setSelectedSearchId(selectedSearch.id)
-    }
-  }
-
-  const onClicker = e => {
-    e.preventDefault()
-    const studentnumbers = input.match(/[0-9]+/g)
-    getCustomPopulationDispatch({ studentnumberlist: studentnumbers, onProgress })
-    getCustomPopulationCoursesByStudentnumbers({ studentnumberlist: studentnumbers })
-    selectCustomPopulationSearchDispatch(selectedSearchId || null)
-    handleClose()
-  }
-  const renderCustomPopulationSearch = () => (
-    <Modal
-      trigger={
-        <Button size="small" color="blue" onClick={() => setModal(true)} data-cy="custom-pop-search-button">
-          Custom population
-        </Button>
-      }
-      open={modal}
-      onClose={handleClose}
-      size="small"
-    >
-      <Modal.Content>
-        <Form>
-          <h2> Custom population </h2>
-          <Form.Field>
-            <em> Insert name for this custom population if you wish to save it </em>
-            <Form.Input disabled={!!selectedSearchId} value={name} placeholder="name" onChange={handleNameChange} />
-          </Form.Field>
-          <Form.Field>
-            <em> Insert studentnumbers you wish to use for population here </em>
-            <TextArea
-              value={input}
-              placeholder="011111111"
-              onChange={e => setInput(e.target.value)}
-              data-cy="student-no-input"
-            />
-          </Form.Field>
-        </Form>
-        <SearchHistory
-          header="Saved populations"
-          items={customPopulationSearches.map(s => ({
-            ...s,
-            text: s.name,
-            timestamp: s.updatedAt,
-            params: { id: s.id }
-          }))}
-          updateItem={() => null}
-          handleSearch={onSelectSearch}
-        />
-      </Modal.Content>
-      <Modal.Actions>
-        <Button
-          disabled={!name || customPopulationSearchSaving}
-          loading={customPopulationSearchSaving}
-          floated="left"
-          icon="save"
-          onClick={onSave}
-          content="Save"
-        />
-        <Button disabled={!selectedSearchId} negative floated="left" icon="trash" onClick={onDelete} content="Delete" />
-        <Button onClick={handleClose}>Cancel</Button>
-        <Button positive onClick={e => onClicker(e)} data-cy="search-button">
-          Search population
-        </Button>
-      </Modal.Actions>
-    </Modal>
-  )
-
   const handleClick = index => {
     const indexes = [...activeIndex].sort()
     if (indexes.includes(index)) {
@@ -215,29 +86,9 @@ const CustomPopulation = ({
       key: 0,
       title: {
         content: (
-          <>
-            {activeIndex.includes(0) ? (
-              <>Credit accumulation (for {selectedStudents.length} students)</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Credit accumulation (for {selectedStudents.length} students)
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.CreditAccumulationGraph.AccordionTitle}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Credit accumulation (for {selectedStudents.length} students)
+          </span>
         )
       },
       onTitleClick: () => handleClick(0),
@@ -259,29 +110,9 @@ const CustomPopulation = ({
       key: 1,
       title: {
         content: (
-          <>
-            {activeIndex.includes(1) ? (
-              <>Programme distribution</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Programme distribution
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.ProgrammeDistributionCoursePopulation}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Programme distribution
+          </span>
         )
       },
       onTitleClick: () => handleClick(1),
@@ -298,29 +129,9 @@ const CustomPopulation = ({
       key: 2,
       title: {
         content: (
-          <>
-            {activeIndex.includes(2) ? (
-              <>Courses of population</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Courses of population
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.CreditDistributionCoursePopulation}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Courses of population
+          </span>
         )
       },
       onTitleClick: () => handleClick(2),
@@ -336,29 +147,9 @@ const CustomPopulation = ({
       key: 3,
       title: {
         content: (
-          <>
-            {activeIndex.includes(3) ? (
-              <>Students ({selectedStudents.length})</>
-            ) : (
-              <Popup
-                trigger={
-                  <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
-                    Students ({selectedStudents.length})
-                  </span>
-                }
-                position="top center"
-                wide="very"
-              >
-                <Popup.Content>
-                  {' '}
-                  <ReactMarkdown
-                    source={infotooltips.PopulationStatistics.Students.AccordionTitle}
-                    escapeHtml={false}
-                  />
-                </Popup.Content>
-              </Popup>
-            )}
-          </>
+          <span style={{ paddingTop: '1vh', paddingBottom: '1vh', color: 'black', fontSize: 'large' }}>
+            Students ({selectedStudents.length})
+          </span>
         )
       },
       onTitleClick: () => handleClick(3),
@@ -367,8 +158,8 @@ const CustomPopulation = ({
           <div ref={studentRef}>
             <PopulationStudents
               language={language}
-              mandatoryToggle={false}
               filteredStudents={filteredStudents}
+              dataExport={<UnihowDataExport />}
               customPopulation
             />
           </div>
@@ -403,7 +194,7 @@ const CustomPopulation = ({
             studentnumbers you have access rights to i.e. you have rights to the programme they are in.
           </p>
         </Message>
-        {renderCustomPopulationSearch()}
+        <CustomPopulationSearch />
         {custompop.length > 0 && customPopulationFlag ? (
           <Segment className="contentSegment">{renderCustomPopulation()}</Segment>
         ) : (
@@ -424,28 +215,19 @@ CustomPopulation.defaultProps = {
 CustomPopulation.propTypes = {
   custompop: arrayOf(shape({})).isRequired,
   customPopulationFlag: bool.isRequired,
-  getCustomPopulationDispatch: func.isRequired,
-  getCustomPopulationCoursesByStudentnumbers: func.isRequired,
   loading: bool.isRequired,
   customPopulationSearches: arrayOf(shape({})).isRequired,
-  customPopulationSearchSaving: bool.isRequired,
   latestCreatedCustomPopulationSearchId: string,
   searchedCustomPopulationSearchId: string,
-  saveCustomPopulationSearchDispatch: func.isRequired,
-  getCustomPopulationSearchesDispatch: func.isRequired,
-  updateCustomPopulationSearchDispatch: func.isRequired,
-  selectCustomPopulationSearchDispatch: func.isRequired,
-  deleteCustomPopulationSearchDispatch: func.isRequired
+  getCustomPopulationSearchesDispatch: func.isRequired
 }
 
 const mapStateToProps = ({ populations, populationCourses, customPopulationSearch }) => ({
   loading: populations.pending,
   custompop: populations.data.students || [],
   courses: populationCourses.data,
-  pending: populationCourses.pending,
   customPopulationFlag: populations.customPopulationFlag,
   customPopulationSearches: customPopulationSearch.customPopulationSearches,
-  customPopulationSearchSaving: customPopulationSearch.saving,
   latestCreatedCustomPopulationSearchId: customPopulationSearch.latestCreatedCustomPopulationSearchId,
   searchedCustomPopulationSearchId: customPopulationSearch.searchedCustomPopulationSearchId
 })
@@ -453,12 +235,7 @@ const mapStateToProps = ({ populations, populationCourses, customPopulationSearc
 export default connect(
   mapStateToProps,
   {
-    getCustomPopulationDispatch: getCustomPopulation,
     getCustomPopulationCoursesByStudentnumbers,
-    saveCustomPopulationSearchDispatch: saveCustomPopulationSearch,
-    getCustomPopulationSearchesDispatch: getCustomPopulationSearches,
-    updateCustomPopulationSearchDispatch: updateCustomPopulationSearch,
-    selectCustomPopulationSearchDispatch: selectCustomPopulationSearch,
-    deleteCustomPopulationSearchDispatch: deleteCustomPopulationSearch
+    getCustomPopulationSearchesDispatch: getCustomPopulationSearches
   }
 )(CustomPopulation)