From bbc11e44b9d22b1aec66d0676475d87875bf6e89 Mon Sep 17 00:00:00 2001 From: karilint Date: Mon, 15 Jun 2026 14:09:24 +0300 Subject: [PATCH] Address Darwin Core export review comments --- backend/src/routes/locality.ts | 13 +------ backend/src/routes/occurrence.ts | 2 +- backend/src/routes/species.ts | 13 +------ backend/src/routes/utils/exportFilters.ts | 11 ++++++ backend/src/services/crossSearch.ts | 10 +++-- backend/src/services/dwcDataPackageExport.ts | 39 +++++++++++++++---- backend/src/unit-tests/exportFilters.test.ts | 21 ++++++++++ .../OccurrenceDwcExportMenuItem.tsx | 1 + frontend/src/util/exportProgress.ts | 4 +- 9 files changed, 79 insertions(+), 35 deletions(-) create mode 100644 backend/src/routes/utils/exportFilters.ts create mode 100644 backend/src/unit-tests/exportFilters.test.ts diff --git a/backend/src/routes/locality.ts b/backend/src/routes/locality.ts index dc0c041b..4c18c65e 100644 --- a/backend/src/routes/locality.ts +++ b/backend/src/routes/locality.ts @@ -12,19 +12,10 @@ import { AccessError, requireOneOf } from '../middlewares/authorizer' import { deleteLocality, writeLocality } from '../services/write/locality' import { buildDwcLocalityArchiveZipBuffer } from '../services/dwcArchiveExportLocalities' import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' +import { parseNumericIds } from './utils/exportFilters' const router = Router() -const parseNumericIds = (value: unknown): number[] | undefined => { - if (value === undefined) return undefined - if (!Array.isArray(value)) throw new Error('ids must be an array.') - return value.map(id => { - const parsed = typeof id === 'number' ? id : typeof id === 'string' ? parseInt(id, 10) : NaN - if (!Number.isInteger(parsed)) throw new Error('ids must contain only integers.') - return parsed - }) -} - router.get('/all', async (req, res) => { const localities = await getAllLocalities(req.user) return res.status(200).send(fixBigInt(localities)) @@ -45,7 +36,7 @@ router.post('/export/dwc-archive', requireOneOf([Role.Admin]), async (req, res) try { return await sendDwcArchive(parseNumericIds((req.body as { ids?: unknown }).ids), res) } catch (error) { - return res.status(403).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) + return res.status(400).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) } }) diff --git a/backend/src/routes/occurrence.ts b/backend/src/routes/occurrence.ts index abadcfc4..c511535b 100644 --- a/backend/src/routes/occurrence.ts +++ b/backend/src/routes/occurrence.ts @@ -45,7 +45,7 @@ const resolveOccurrenceKeysForExport = async (req: Request): Promise { - return res.status(403).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) + return res.status(400).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) } router.get('/export/dwc-archive/progress/:exportId', requireOneOf([Role.Admin]), (req, res) => { diff --git a/backend/src/routes/species.ts b/backend/src/routes/species.ts index d9c1e3c5..369967ac 100644 --- a/backend/src/routes/species.ts +++ b/backend/src/routes/species.ts @@ -6,19 +6,10 @@ import { deleteSpecies, writeSpecies } from '../services/write/species' import { requireOneOf } from '../middlewares/authorizer' import { buildDwcArchiveZipBuffer } from '../services/dwcArchiveExport' import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' +import { parseNumericIds } from './utils/exportFilters' const router = Router() -const parseNumericIds = (value: unknown): number[] | undefined => { - if (value === undefined) return undefined - if (!Array.isArray(value)) throw new Error('ids must be an array.') - return value.map(id => { - const parsed = typeof id === 'number' ? id : typeof id === 'string' ? parseInt(id, 10) : NaN - if (!Number.isInteger(parsed)) throw new Error('ids must contain only integers.') - return parsed - }) -} - router.get('/all', async (_req, res) => { const species = await getAllSpecies() return res.status(200).send(fixBigInt(species)) @@ -44,7 +35,7 @@ router.post('/export/dwc-archive', requireOneOf([Role.Admin]), async (req, res) try { return await sendDwcArchive(parseNumericIds((req.body as { ids?: unknown }).ids), res) } catch (error) { - return res.status(403).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) + return res.status(400).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) } }) diff --git a/backend/src/routes/utils/exportFilters.ts b/backend/src/routes/utils/exportFilters.ts new file mode 100644 index 00000000..70a482d5 --- /dev/null +++ b/backend/src/routes/utils/exportFilters.ts @@ -0,0 +1,11 @@ +export const parseNumericIds = (value: unknown): number[] | undefined => { + if (value === undefined) return undefined + if (!Array.isArray(value)) throw new Error('ids must be an array.') + + return value.map(id => { + const parsed = typeof id === 'number' ? id : typeof id === 'string' && /^-?\d+$/.test(id) ? Number(id) : Number.NaN + + if (!Number.isSafeInteger(parsed)) throw new Error('ids must contain only integers.') + return parsed + }) +} diff --git a/backend/src/services/crossSearch.ts b/backend/src/services/crossSearch.ts index 84997bc1..e6977ad7 100644 --- a/backend/src/services/crossSearch.ts +++ b/backend/src/services/crossSearch.ts @@ -268,10 +268,12 @@ export const getFilteredCrossSearchOccurrenceKeys = async ( )) as Array>> const keysById = new Map() - for (const row of resultPages.flat()) { - if (typeof row.lid_now_loc !== 'number' || typeof row.species_id_com_species !== 'number') continue - const key = { lid: row.lid_now_loc, speciesId: row.species_id_com_species } - keysById.set(`${key.lid}:${key.speciesId}`, key) + for (const page of resultPages) { + for (const row of page) { + if (typeof row.lid_now_loc !== 'number' || typeof row.species_id_com_species !== 'number') continue + const key = { lid: row.lid_now_loc, speciesId: row.species_id_com_species } + keysById.set(`${key.lid}:${key.speciesId}`, key) + } } return { occurrenceKeys: [...keysById.values()] } diff --git a/backend/src/services/dwcDataPackageExport.ts b/backend/src/services/dwcDataPackageExport.ts index 4ddcc01e..24bb7d57 100644 --- a/backend/src/services/dwcDataPackageExport.ts +++ b/backend/src/services/dwcDataPackageExport.ts @@ -38,6 +38,20 @@ const occurrenceIdForRow = (lid: number, speciesId: number): string => `NOW:OCC: type LocalityForDwcDpExport = Parameters[0] type OccurrenceForDwcDpExport = Parameters[0] +const LOOKUP_EXPORT_CHUNK_SIZE = 1000 + +const chunk = (values: T[], size: number): T[][] => { + const chunks: T[][] = [] + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)) + } + return chunks +} + +const sortOccurrenceKeys = (occurrenceKeys: DwcOccurrenceKey[]): DwcOccurrenceKey[] => { + return [...occurrenceKeys].sort((a, b) => a.lid - b.lid || a.speciesId - b.speciesId) +} + export const DWC_DP_EVENT_HEADERS = [ 'eventID', 'parentEventID', @@ -334,7 +348,7 @@ const FIELD_DESCRIPTIONS: Record = { taxonID: 'Stable NOW taxon identifier; this joins to dwc-a-taxa/taxon.csv in the full export.', scientificName: 'Scientific name assembled from curated NOW taxonomic fields.', scientificNameAuthorship: 'Scientific name authorship where curated.', - taxonRank: 'Taxonomic rank when available; currently reserved for future enrichment.', + taxonRank: 'Taxonomic rank derived from curated NOW taxonomic fields when available.', identificationVerificationStatus: 'Curated identification status or qualifier.', assertionID: 'Stable assertion identifier derived from the source database field and owning event or occurrence.', verbatimAssertionType: @@ -913,15 +927,26 @@ const fetchOccurrencesForDwcDataPackageExport = async ( ): Promise => { if (occurrenceKeys && occurrenceKeys.length === 0) return [] const { nowDb } = await import('../utils/db') - const occurrences = await nowDb.now_ls.findMany({ - where: occurrenceKeys - ? { - OR: occurrenceKeys.map(key => ({ + + if (occurrenceKeys) { + const occurrences: OccurrenceForDwcDpExport[] = [] + for (const keys of chunk(sortOccurrenceKeys(occurrenceKeys), LOOKUP_EXPORT_CHUNK_SIZE)) { + const chunkOccurrences = await nowDb.now_ls.findMany({ + where: { + OR: keys.map(key => ({ lid: key.lid, species_id: key.speciesId, })), - } - : undefined, + }, + orderBy: [{ lid: 'asc' }, { species_id: 'asc' }], + select: occurrenceSelect, + }) + occurrences.push(...(chunkOccurrences as unknown as OccurrenceForDwcDpExport[])) + } + return occurrences + } + + const occurrences = await nowDb.now_ls.findMany({ orderBy: [{ lid: 'asc' }, { species_id: 'asc' }], select: occurrenceSelect, }) diff --git a/backend/src/unit-tests/exportFilters.test.ts b/backend/src/unit-tests/exportFilters.test.ts new file mode 100644 index 00000000..7c250567 --- /dev/null +++ b/backend/src/unit-tests/exportFilters.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from '@jest/globals' +import { parseNumericIds } from '../routes/utils/exportFilters' + +describe('parseNumericIds', () => { + it('accepts integer ids as numbers or strings', () => { + expect(parseNumericIds([12, '34'])).toEqual([12, 34]) + }) + + it('rejects partially numeric strings', () => { + expect(() => parseNumericIds(['12abc'])).toThrow('ids must contain only integers.') + }) + + it('rejects decimals', () => { + expect(() => parseNumericIds([12.3])).toThrow('ids must contain only integers.') + expect(() => parseNumericIds(['12.3'])).toThrow('ids must contain only integers.') + }) + + it('rejects non-array id payloads', () => { + expect(() => parseNumericIds('12')).toThrow('ids must be an array.') + }) +}) diff --git a/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx b/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx index c74228c6..4ecef9a9 100644 --- a/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx +++ b/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx @@ -39,6 +39,7 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () = progressUrl: `${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`, filename, fetchOptions: filteredFetchOptions, + progressFetchOptions: fetchOptions, notify, setNotificationMessage, startMessage: 'Generating DwC-A occurrence ZIP export...', diff --git a/frontend/src/util/exportProgress.ts b/frontend/src/util/exportProgress.ts index 89648894..7a10aa33 100644 --- a/frontend/src/util/exportProgress.ts +++ b/frontend/src/util/exportProgress.ts @@ -9,6 +9,7 @@ type DownloadExportParams = { url: string filename: string fetchOptions?: RequestInit + progressFetchOptions?: RequestInit notify: Notify setNotificationMessage: (message: string) => void startMessage: string @@ -30,6 +31,7 @@ export const downloadExportFileWithProgress = async ({ url, filename, fetchOptions = {}, + progressFetchOptions, notify, setNotificationMessage, startMessage, @@ -53,7 +55,7 @@ export const downloadExportFileWithProgress = async ({ if (!progressUrl) return false try { - const response = await fetch(progressUrl, fetchOptions) + const response = await fetch(progressUrl, progressFetchOptions ?? fetchOptions) if (!response.ok) return false const progress = (await response.json()) as ExportProgress