From 0c4e44f947fb3dd6498064345c6632c1e153f623 Mon Sep 17 00:00:00 2001 From: karilint Date: Mon, 15 Jun 2026 14:33:45 +0300 Subject: [PATCH] Address follow-up Darwin Core review comments --- .../dwcArchiveExportOccurrences.test.ts | 51 +++++++++++++++++++ backend/src/routes/occurrence.ts | 17 ++++++- backend/src/services/dwcArchiveExport.ts | 19 +++---- backend/src/services/dwcDataPackageExport.ts | 21 ++++---- backend/src/services/dwcMetadata.ts | 8 +++ 5 files changed, 95 insertions(+), 21 deletions(-) create mode 100644 backend/src/services/dwcMetadata.ts diff --git a/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts b/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts index b8ae061e..7c1e1d96 100644 --- a/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts +++ b/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts @@ -62,6 +62,57 @@ describe('DwC-A occurrence export (admin-only)', () => { expect(measurementCsv).toContain('"verbatimMeasurementType"') }) + it('returns a filtered ZIP archive for POST requests', async () => { + const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' }) + expect(loginResult.status).toEqual(200) + + const result = await request(app) + .post('/occurrence/export/dwc-archive') + .set('authorization', `bearer ${loginResult.body.token}`) + .send({ columnFilters: [{ id: 'lid_now_loc', value: '21050' }], sorting: [] }) + .buffer(true) + .parse(parseBinary) + + expect(result.status).toEqual(200) + expect(result.headers['content-type']).toMatch(/application\/zip/i) + + const zip = await JSZip.loadAsync(result.body as unknown as Buffer) + const occurrenceCsv = await zip.file('occurrence.csv')!.async('string') + expect(occurrenceCsv).toContain('NOW:OCC:21050:') + expect(occurrenceCsv).not.toContain('NOW:OCC:24750:') + }) + + it('returns an empty filtered DwC-DP ZIP archive for POST requests', async () => { + const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' }) + expect(loginResult.status).toEqual(200) + + const result = await request(app) + .post('/occurrence/export/dwc-data-package') + .set('authorization', `bearer ${loginResult.body.token}`) + .send({ columnFilters: [{ id: 'lid_now_loc', value: '9999999' }], sorting: [] }) + .buffer(true) + .parse(parseBinary) + + expect(result.status).toEqual(200) + const zip = await JSZip.loadAsync(result.body as unknown as Buffer) + const occurrenceCsv = await zip.file('occurrence.csv')!.async('string') + expect(occurrenceCsv).toContain('"occurrenceID"') + expect(occurrenceCsv).not.toContain('NOW:OCC:') + }) + + it('returns structured validation errors for invalid POST filters', async () => { + const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' }) + expect(loginResult.status).toEqual(200) + + const result = await request(app) + .post('/occurrence/export/dwc-data-package') + .set('authorization', `bearer ${loginResult.body.token}`) + .send({ columnFilters: [{ id: '', value: '21050' }], sorting: [] }) + + expect(result.status).toEqual(400) + expect(result.body).toEqual([{ name: 'Column Filters', error: 'Invalid or missing id field in filter' }]) + }) + it('rejects non-admin requests', async () => { const result = await request(app).get('/occurrence/export/dwc-archive') expect(result.status).toEqual(403) diff --git a/backend/src/routes/occurrence.ts b/backend/src/routes/occurrence.ts index c511535b..7ae0309c 100644 --- a/backend/src/routes/occurrence.ts +++ b/backend/src/routes/occurrence.ts @@ -12,6 +12,7 @@ import { buildDwcDataPackageZipBuffer, buildFullDarwinCoreExportZipBuffer } from import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' import { logger } from '../utils/logger' import { getFilteredCrossSearchOccurrenceKeys, type CrossSearchRequestParameters } from '../services/crossSearch' +import type { ValidationObject } from '../../../frontend/src/shared/validators/validator' const router = Router() @@ -31,6 +32,15 @@ const defaultCrossSearchExportFilters = { sorting: [], } satisfies CrossSearchRequestParameters +class ExportFilterValidationError extends Error { + validationErrors: ValidationObject[] + + constructor(validationErrors: ValidationObject[]) { + super('Invalid export filters.') + this.validationErrors = validationErrors + } +} + const resolveOccurrenceKeysForExport = async (req: Request): Promise => { if (req.method === 'GET') return undefined const body = req.body as Partial | undefined @@ -38,13 +48,16 @@ const resolveOccurrenceKeysForExport = async (req: Request): Promise { + if (error instanceof ExportFilterValidationError) { + return res.status(400).send(error.validationErrors) + } return res.status(400).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' }) } diff --git a/backend/src/services/dwcArchiveExport.ts b/backend/src/services/dwcArchiveExport.ts index d9f33c47..d72ff948 100644 --- a/backend/src/services/dwcArchiveExport.ts +++ b/backend/src/services/dwcArchiveExport.ts @@ -2,15 +2,16 @@ import Prisma from '../../prisma/generated/now_test_client' import JSZip from 'jszip' import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv' import { getFieldInfoText } from '../../../frontend/src/shared/fieldInfo' - -const DATASET_TITLE = 'NOW database Darwin Core export' -const DATASET_NAME = 'now-darwincore-export' -const DATASET_VERSION = '1.0.0' -const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068' -const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/' -const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International' -const DATASET_CREATOR = 'The NOW Community' -const MISSING_VALUE = '\\N' +import { + DATASET_CREATOR, + DATASET_DOI, + DATASET_LICENSE_TITLE, + DATASET_LICENSE_URL, + DATASET_NAME, + DATASET_TITLE, + DATASET_VERSION, + MISSING_VALUE, +} from './dwcMetadata' const isMeaningfulString = (value: unknown): value is string => { if (typeof value !== 'string') return false diff --git a/backend/src/services/dwcDataPackageExport.ts b/backend/src/services/dwcDataPackageExport.ts index 24bb7d57..ff6228f0 100644 --- a/backend/src/services/dwcDataPackageExport.ts +++ b/backend/src/services/dwcDataPackageExport.ts @@ -13,6 +13,16 @@ import { } from './dwcArchiveExportOccurrences' import { buildDwcArchiveZipBuffer, resolveTaxonRank, type MeasurementCsvRow } from './dwcArchiveExport' import { writeDwcCsvString } from './utils/dwcCsv' +import { + DATASET_CREATOR, + DATASET_DOI, + DATASET_LICENSE_TITLE, + DATASET_LICENSE_URL, + DATASET_NAME, + DATASET_TITLE, + DATASET_VERSION, + MISSING_VALUE, +} from './dwcMetadata' const isMeaningfulString = (value: unknown): value is string => { if (typeof value !== 'string') return false @@ -132,14 +142,6 @@ const ASSERTION_HEADERS = [ const DWC_DP_EVENT_ASSERTION_HEADERS = ['eventID', ...ASSERTION_HEADERS] as const const DWC_DP_OCCURRENCE_ASSERTION_HEADERS = ['occurrenceID', ...ASSERTION_HEADERS] as const -const DATASET_TITLE = 'NOW database Darwin Core export' -const DATASET_NAME = 'now-darwincore-export' -const DATASET_VERSION = '1.0.0' -const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068' -const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/' -const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International' -const DATASET_CREATOR = 'The NOW Community' -const MISSING_VALUE = '\\N' type AssertionHeader = (typeof ASSERTION_HEADERS)[number] type AssertionColumns = Record @@ -474,8 +476,7 @@ export const buildDwcDataPackageJson = (publicationDateIso: string): string => { 'occurrence data', 'taxon traits', ], - citation: - 'The NOW Community. NOW database Darwin Core export, version 1.0.0. https://doi.org/10.5281/zenodo.4268068. The DOI describes the NOW database generally rather than a single frozen export version; include the export date when citing a downloaded archive.', + citation: `${DATASET_CREATOR}. ${DATASET_TITLE}, version ${DATASET_VERSION}. ${DATASET_DOI}. The DOI describes the NOW database generally rather than a single frozen export version; include the export date when citing a downloaded archive.`, description: 'Production Darwin Core Data Package export from the NOW database for relational event, occurrence, geological context, and assertion data. The NOW database is a continuously curated, globally scoped fossil mammal database with Cenozoic emphasis, spanning approximately the last 66 million years.', missingValues: [MISSING_VALUE], diff --git a/backend/src/services/dwcMetadata.ts b/backend/src/services/dwcMetadata.ts new file mode 100644 index 00000000..d79596bc --- /dev/null +++ b/backend/src/services/dwcMetadata.ts @@ -0,0 +1,8 @@ +export const DATASET_TITLE = 'NOW database Darwin Core export' +export const DATASET_NAME = 'now-darwincore-export' +export const DATASET_VERSION = '1.0.0' +export const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068' +export const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/' +export const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International' +export const DATASET_CREATOR = 'The NOW Community' +export const MISSING_VALUE = '\\N'