[CourseStats] Fetch coursestats based on selected years + change to use year codes

services/backend/src/routes/courses.ts

@@ -56,7 +56,7 @@ export type CourseYearlyStatsResBody = Awaited<ReturnType<typeof getCourseYearly
 router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody, CourseYearlyStatsQuery>(
   '/courseyearlystats',
   async (req, res) => {
-    const { codes, combineSubstitutions, separate, fromYear, toYear } = req.query
+    const { codes, combineSubstitutions, separate, fromYearCode, toYearCode } = req.query
 
     const courseCodes = handleQueryArrays(codes)
 
@@ -87,8 +87,8 @@ router.get<never, CanError<CourseYearlyStatsResBody>, CourseYearlyStatsReqBody,
       useSeparate,
       anonymizationSalt,
       useCombineSubstitutions,
-      fromYear,
-      toYear
+      fromYearCode,
+      toYearCode
     )
     return res.json(results)
   }

services/backend/src/services/courses/index.ts

@@ -3,7 +3,7 @@ import { Op, fn as dbFn, col as dbCol } from 'sequelize'
 
 import { Credit, Enrollment } from '@oodikone/shared/models'
 import { Name, EnrollmentState, Unification } from '@oodikone/shared/types'
-import { enrollmentTimeDateThreshold, getSemesterCodeAt } from '@oodikone/shared/util'
+import { enrollmentTimeDateThreshold, getSemesterCodeAt, yearCodeToYear, yearToYearCode } from '@oodikone/shared/util'
 import { dateIsBetween } from '@oodikone/shared/util/datetime'
 import logger from '../../../src/util/logger'
 import { CourseModel, CreditModel, EnrollmentModel, OrganizationModel, SISStudyRightElementModel } from '../../models'
@@ -332,9 +332,13 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
 
   if (lastAttainmentDate?.date == null) return 0
 
-  const newestAttainmentDate = lastAttainmentDate?.date.getFullYear()
-  const attainmentDateThreshold = new Date(newestAttainmentDate - 6, 0, 1)
+  /** Amount of years before latest attainment that the attainment count is calcucated for */
+  const yearRange = 6
 
+  const newestAttainmentYear = lastAttainmentDate?.date.getFullYear()
+  const attainmentDateThreshold = new Date(newestAttainmentYear - yearRange, 0, 1)
+
+  /** Amount of attainments between latest-attainment-year - yearRange and latest attainment date */
   const attainmentsWithinThreshold = await CreditModel.count({
     where: {
       course_code: { [Op.in]: courseCodes },
@@ -343,9 +347,9 @@ export const maxYearsToCreatePopulationFrom = async (courseCodes: string[], unif
     },
   })
 
-  // MAGIC NUMBER
-  const maxAllowedAttainments = 1_000_000 // * Lower this value to get a smaller result if necessary
-  return Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold)
+  /** Limit the allowed attainments to 15_000 during the $yearRange years */
+  const maxAllowedAttainments = 15_000 * yearRange
+  return Math.round(Math.max(1, maxAllowedAttainments / attainmentsWithinThreshold))
 }
 
 export const getCourseYearlyStats = async (
@@ -353,12 +357,12 @@ export const getCourseYearlyStats = async (
   separate: boolean,
   anonymizationSalt: string | null,
   combineSubstitutions: boolean,
-  fromYear = '1900',
-  toYear: string = new Date().getFullYear().toString()
+  fromYearCode = yearToYearCode(1950).toString(),
+  toYearCode: string = yearToYearCode(new Date().getFullYear()).toString()
 ) => {
   // Default to 1900 - currentYear+1 so that without parameters the api returns stats for all years
-  const from = new Date(`${fromYear}-08-01`) // FALL
-  const to = new Date(`${parseInt(toYear) + 1}-07-31`) // SPRING next year
+  const from = new Date(`${yearCodeToYear(fromYearCode)}-08-01`) // FALL
+  const to = new Date(`${yearCodeToYear(toYearCode) + 1}-07-31`) // SPRING next year
 
   const [credits, enrollments] = await Promise.all([
     CreditModel.findAll({

services/frontend/src/pages/CourseStatistics/CourseTab/SingleCourseStats/index.tsx

@@ -10,12 +10,14 @@ import { useLocation, useNavigate } from 'react-router'
 import { ProgrammeDropdown } from '@/components/CourseStatistics/ProgrammeDropdown'
 import { useLanguage } from '@/components/LanguagePicker/useLanguage'
 import { Section } from '@/components/Section'
+import { LoadingSkeleton } from '@/components/Section/LoadingSkeleton'
 import { useSemesters } from '@/hooks/useSemesters'
 import type { CourseSearchState } from '@/pages/CourseStatistics'
 import { countTotalStats } from '@/pages/CourseStatistics/CourseTab/SingleCourseStats/countTotalStats'
 import { ResultTabs } from '@/pages/CourseStatistics/CourseTab/SingleCourseStats/ResultTabs'
 import { YearFilter } from '@/pages/CourseStatistics/CourseTab/SingleCourseStats/YearFilter'
 import { ALL, CourseStudyProgramme } from '@/pages/CourseStatistics/util'
+import { useGetCourseStatsQuery } from '@/redux/courseStats'
 import { useAppDispatch } from '@/redux/hooks'
 import { useGetMaxYearsToCreatePopulationFromQuery } from '@/redux/populations'
 import { setSelectedCourse, clearSelectedCourse } from '@/redux/selectedCourse'
@@ -32,7 +34,7 @@ import {
 import { DropdownOption } from '@/types/dropdownOption'
 import { parseQueryParams, queryParamsToString } from '@/util/queryparams'
 import { Name } from '@oodikone/shared/types'
-import { enrollmentTimeDateThresholdYearCode } from '@oodikone/shared/util'
+import { enrollmentTimeDateThresholdYearCode, yearToYearCode } from '@oodikone/shared/util'
 
 const countFilteredStudents = (stat: Record<string, string[]>, filter: (studentNumber: string) => boolean) => {
   if (!stat) {
@@ -51,8 +53,8 @@ const countFilteredStudents = (stat: Record<string, string[]>, filter: (studentN
 }
 
 export const SingleCourseStats = ({
+  coursecode,
   availableStats,
-  stats,
   userHasAccessToAllStats,
 
   loading,
@@ -61,8 +63,8 @@ export const SingleCourseStats = ({
   programmes,
   combineSubstitutions,
 }: {
+  coursecode: string
   availableStats: AvailableStats
-  stats: CourseStat
   userHasAccessToAllStats: boolean
 
   loading: boolean
@@ -71,18 +73,50 @@ export const SingleCourseStats = ({
   programmes: CourseStudyProgramme[]
   combineSubstitutions: boolean
 }) => {
+  const [primary, setPrimary] = useState<string[]>([ALL.value])
+  const [comparison, setComparison] = useState<string[]>([])
+
+  // These are "incorrectly" reversed to allow them to be replaced with min/max when stats arrive
+  const [minFromYearCode, setMinFromYearCode] = useState(yearToYearCode(new Date().getFullYear()))
+  const [maxToYearCode, setMaxToYearCode] = useState(yearToYearCode(1950))
+
+  const [fromYearCode, setFromYearCode] = useState(yearToYearCode(1950))
+  const [toYearCode, setToYearCode] = useState(yearToYearCode(new Date().getFullYear()))
+  const [separate, setSeparate] = useState<boolean>(false)
+
   const navigate = useNavigate()
   const location = useLocation()
   const dispatch = useAppDispatch()
   const { getTextIn } = useLanguage()
-  const [primary, setPrimary] = useState<string[]>([ALL.value])
-  const [comparison, setComparison] = useState<string[]>([])
-  const [fromYear, setFromYear] = useState(0)
-  const [toYear, setToYear] = useState(0)
-  const [separate, setSeparate] = useState<boolean>(false)
-  const { coursecode } = stats
-
   const { semesters, years: semesterYears } = useSemesters()
+  const {
+    data: courseStatistics = {},
+    isFetching: isLoading,
+    isSuccess,
+  } = useGetCourseStatsQuery(
+    {
+      codes: [coursecode],
+      separate,
+      combineSubstitutions,
+      fromYearCode: fromYearCode.toString(),
+      toYearCode: toYearCode.toString(),
+    },
+    { skip: loading }
+  )
+  const uniqueCourseCodes = [
+    ...new Set(
+      [coursecode].concat(
+        courseStatistics?.[coursecode]?.[openOrRegular]?.alternatives.flatMap(group =>
+          group.flatMap(({ code }) => code)
+        ) ?? []
+      )
+    ),
+  ]
+  const { data: maxYears } = useGetMaxYearsToCreatePopulationFromQuery(
+    { courseCodes: JSON.stringify(uniqueCourseCodes) },
+    { skip: isLoading || !isSuccess }
+  )
+
   const semestersReversed = Object.values(semesters ?? [])
     .map(({ semestercode, name, yearcode }) => ({
       key: semestercode,
@@ -106,13 +140,8 @@ export const SingleCourseStats = ({
     }
   }
 
-  const uniqueCourseCodes = [
-    ...new Set([coursecode].concat(stats.alternatives.flatMap(group => group.flatMap(({ code }) => code)))),
-  ]
-
-  const { data: maxYears } = useGetMaxYearsToCreatePopulationFromQuery({
-    courseCodes: JSON.stringify(uniqueCourseCodes),
-  })
+  // NOTE: Can be undefined if query is still querying
+  const stats: CourseStat | undefined = courseStatistics[coursecode]?.[openOrRegular]
 
   let maxYearsToCreatePopulationFrom = 0
   if (maxYears) {
@@ -135,16 +164,21 @@ export const SingleCourseStats = ({
     }
     dispatch(setSelectedCourse(coursecode))
 
-    const yearCodes = stats.statistics.map(stat => stat.yearCode)
-    const initFromYear = min(yearCodes) ?? 0
-    const initToYear = max(yearCodes) ?? 0
-    setFromYear(initFromYear)
-    setToYear(initToYear)
+    const yearCodes = stats?.statistics.map(stat => stat.yearCode)
+    const initFromYear = min(yearCodes) ?? yearToYearCode(1950)
+    const initToYear = max(yearCodes) ?? yearToYearCode(new Date().getFullYear())
+    setFromYearCode(initFromYear)
+    setToYearCode(initToYear)
+
+    if (yearCodes) {
+      setMaxToYearCode(max([maxToYearCode, ...yearCodes]))
+      setMinFromYearCode(min([minFromYearCode, ...yearCodes]))
+    }
 
     return () => {
       dispatch(clearSelectedCourse())
     }
-  }, [])
+  }, [isSuccess])
 
   useEffect(() => {
     if (location.search) {
@@ -204,9 +238,9 @@ export const SingleCourseStats = ({
   }
 
   const filteredYearsAndSemesters = () => {
-    const yearCodes = stats.statistics.map(stat => stat.yearCode)
-    const from = min(yearCodes)
-    const to = max(yearCodes)
+    const from = minFromYearCode
+    const to = maxToYearCode
+
     if (from == null || to == null) {
       return {
         filteredYears: semesterYearsReversed,
@@ -221,7 +255,7 @@ export const SingleCourseStats = ({
   }
 
   const isStatInYearRange = ({ name }: { name: Name | string }) => {
-    const timeFilter = ({ value }: { value: number }) => value >= fromYear && value <= toYear
+    const timeFilter = ({ value }: { value: number }) => value >= fromYearCode && value <= toYearCode
     const filteredSemesters = semestersReversed.filter(timeFilter)
     const filteredYears = semesterYearsReversed.filter(timeFilter)
     if (separate) {
@@ -305,7 +339,7 @@ export const SingleCourseStats = ({
   }
 
   const statsForProgrammes = (programmeCodes: string[], name: string) => {
-    if (programmeCodes.length === 0) {
+    if (programmeCodes.length === 0 || !stats) {
       return undefined
     }
     const { statistics } = stats
@@ -389,15 +423,17 @@ export const SingleCourseStats = ({
 
   const handleToYearChange = event => {
     const newYear = event.target.value as number
-    if (newYear >= fromYear) {
-      setToYear(newYear)
+    if (newYear >= fromYearCode) {
+      setToYearCode(newYear)
+      setMaxToYearCode(max([maxToYearCode, newYear]))
     }
   }
 
   const handleFromYearChange = event => {
     const newYear = event.target.value as number
-    if (newYear <= toYear) {
-      setFromYear(newYear)
+    if (newYear <= toYearCode) {
+      setFromYearCode(newYear)
+      setMinFromYearCode(min([minFromYearCode, newYear]))
     }
   }
 
@@ -450,8 +486,8 @@ export const SingleCourseStats = ({
 
   const showPopulation = () => {
     const queryObject = {
-      from: fromYear,
-      to: toYear,
+      from: fromYearCode,
+      to: toYearCode,
       coursecodes: JSON.stringify([coursecode]),
       separate,
       unifyCourses: openOrRegular,
@@ -475,7 +511,7 @@ export const SingleCourseStats = ({
   const statistics = filteredProgrammeStatistics()
   const { filteredYears } = filteredYearsAndSemesters()
 
-  const timeFilter = (_, value: string) => Number(value) >= fromYear && Number(value) <= toYear
+  const timeFilter = (_, value: string) => Number(value) >= fromYearCode && Number(value) <= toYearCode
   const filteredProgrammes = programmes
     .map(programme => {
       const students = new Set(flatten(Object.values(pickBy(programme.students, timeFilter))))
@@ -483,7 +519,7 @@ export const SingleCourseStats = ({
     })
     .filter(programme => programme.size > 0)
 
-  if (stats.statistics.length < 1) {
+  if (stats?.statistics.length < 1) {
     return <Section>No data for selected course</Section>
   }
 
@@ -506,13 +542,13 @@ export const SingleCourseStats = ({
       <Section title="Statistics by time range">
         <Stack direction="row" gap={2}>
           <YearFilter
-            fromYear={fromYear}
+            fromYear={fromYearCode}
             handleFromYearChange={handleFromYearChange}
             handleToYearChange={handleToYearChange}
-            toYear={toYear}
+            toYear={toYearCode}
             years={filteredYears}
           />
-          {maxYearsToCreatePopulationFrom < toYear - fromYear + 1 ? (
+          {maxYearsToCreatePopulationFrom < toYearCode - fromYearCode + 1 ? (
             <Tooltip arrow placement="right" title={maxYearText}>
               <span>{renderShowPopulation(true)}</span>
             </Tooltip>
@@ -557,6 +593,9 @@ export const SingleCourseStats = ({
           </Stack>
         </Section>
       ) : null}
+      {isLoading ? (
+        <LoadingSkeleton />
+      ) : (
         <ResultTabs
           availableStats={availableStats}
           combineSubstitutions={combineSubstitutions}
@@ -568,6 +607,7 @@ export const SingleCourseStats = ({
           separate={separate}
           toggleOpenAndRegularCourses={toggleOpenAndRegularCourses}
         />
+      )}
     </Stack>
   )
 }

services/frontend/src/pages/CourseStatistics/CourseTab/index.tsx

@@ -49,6 +49,7 @@ export const CourseTab = ({
     name: getTextIn(name)!,
   }))
 
+  if (courses.length === 0) return null
   const multipleCourses = courses.length > 1
 
   return (
@@ -78,10 +79,10 @@ export const CourseTab = ({
       <SingleCourseStats
         availableStats={availableStats}
         combineSubstitutions={combineSubstitutions}
+        coursecode={courses.at(0)!.code}
         loading={loading}
         openOrRegular={openOrRegular}
         programmes={programmes}
-        stats={stats[selected]}
         toggleOpenAndRegularCourses={toggleOpenAndRegularCourses}
         userHasAccessToAllStats={userHasAccessToAllStats}
       />

services/frontend/src/pages/CourseStatistics/index.tsx

@@ -28,6 +28,7 @@ import { useGetAuthorizedUserQuery } from '@/redux/auth'
 import { useGetCourseStatsQuery } from '@/redux/courseStats'
 import { checkUserAccess, getFullStudyProgrammeRights, hasAccessToAllCourseStats } from '@/util/access'
 import { parseQueryParams } from '@/util/queryparams'
+import { yearToYearCode } from '@oodikone/shared/util'
 
 export type CourseSearchState = 'openStats' | 'regularStats' | 'unifyStats'
 
@@ -43,7 +44,7 @@ export const CourseStatistics = () => {
   const location = useLocation()
   const { courseCodes, separate, combineSubstitutions } = parseQueryParams(location.search)
 
-  const codes = JSON.parse(courseCodes ?? '[]')
+  const codes = JSON.parse(courseCodes ?? '[]') as string[]
   const [initialCourseCode] = codes
   const singleCourseStats = codes.length === 1
 
@@ -55,7 +56,16 @@ export const CourseStatistics = () => {
     data: courseStatsData = {},
     isFetching: isLoading,
     isSuccess,
-  } = useGetCourseStatsQuery({ codes, separate, combineSubstitutions }, { skip: skipQuery })
+  } = useGetCourseStatsQuery(
+    {
+      codes,
+      separate: separate === 'true',
+      combineSubstitutions: combineSubstitutions === 'true',
+      fromYearCode: '1',
+      toYearCode: yearToYearCode(new Date().getFullYear()).toString(),
+    },
+    { skip: skipQuery }
+  )
 
   useEffect(() => {
     setSelected(initialCourseCode)

services/frontend/src/redux/courseStats.ts

@@ -8,13 +8,17 @@ const courseStatsApi = RTKApi.injectEndpoints({
         codes,
         separate,
         combineSubstitutions,
+        fromYearCode,
+        toYearCode,
       }: {
         codes: string[]
         separate?: boolean
         combineSubstitutions?: boolean
+        fromYearCode: string
+        toYearCode: string
       }) => ({
         url: '/courseyearlystats',
-        params: { codes, separate, combineSubstitutions },
+        params: { codes, separate, combineSubstitutions, fromYearCode, toYearCode },
       }),
       transformResponse: (
         courseStats: { openStats: CourseStat; regularStats: CourseStat; unifyStats: CourseStat }[]

services/shared/routes/courses.ts

@@ -14,6 +14,6 @@ export type CourseYearlyStatsQuery = {
   codes: string[]
   separate: string
   combineSubstitutions: string
-  fromYear: string
-  toYear: string
+  fromYearCode: string
+  toYearCode: string
 }

services/shared/util/index.ts

@@ -148,3 +148,11 @@ export const enrollmentTimeDateThreshold = new Date('2021-05-31')
 /** The start of the academic year after Oodi <-> Sisu transfer */
 export const enrollmentTimeDateThresholdAcademicYear = new Date('2021-08-01')
 export const enrollmentTimeDateThresholdYearCode = 72
+
+export const yearToYearCode = (year: number | string) => {
+  return (typeof year === 'string' ? parseInt(year) : year) - 1949
+}
+
+export const yearCodeToYear = (year: number | string) => {
+  return (typeof year === 'string' ? parseInt(year) : year) + 1949
+}