Address follow-up Darwin Core review comments

This commit is contained in:
karilint
2026-06-15 14:33:45 +03:00
parent bbc11e44b9
commit 0c4e44f947
5 changed files with 95 additions and 21 deletions
@@ -62,6 +62,57 @@ describe('DwC-A occurrence export (admin-only)', () => {
expect(measurementCsv).toContain('"verbatimMeasurementType"')
})
it('returns a filtered ZIP archive for POST requests', async () => {
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
expect(loginResult.status).toEqual(200)
const result = await request(app)
.post('/occurrence/export/dwc-archive')
.set('authorization', `bearer ${loginResult.body.token}`)
.send({ columnFilters: [{ id: 'lid_now_loc', value: '21050' }], sorting: [] })
.buffer(true)
.parse(parseBinary)
expect(result.status).toEqual(200)
expect(result.headers['content-type']).toMatch(/application\/zip/i)
const zip = await JSZip.loadAsync(result.body as unknown as Buffer)
const occurrenceCsv = await zip.file('occurrence.csv')!.async('string')
expect(occurrenceCsv).toContain('NOW:OCC:21050:')
expect(occurrenceCsv).not.toContain('NOW:OCC:24750:')
})
it('returns an empty filtered DwC-DP ZIP archive for POST requests', async () => {
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
expect(loginResult.status).toEqual(200)
const result = await request(app)
.post('/occurrence/export/dwc-data-package')
.set('authorization', `bearer ${loginResult.body.token}`)
.send({ columnFilters: [{ id: 'lid_now_loc', value: '9999999' }], sorting: [] })
.buffer(true)
.parse(parseBinary)
expect(result.status).toEqual(200)
const zip = await JSZip.loadAsync(result.body as unknown as Buffer)
const occurrenceCsv = await zip.file('occurrence.csv')!.async('string')
expect(occurrenceCsv).toContain('"occurrenceID"')
expect(occurrenceCsv).not.toContain('NOW:OCC:')
})
it('returns structured validation errors for invalid POST filters', async () => {
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
expect(loginResult.status).toEqual(200)
const result = await request(app)
.post('/occurrence/export/dwc-data-package')
.set('authorization', `bearer ${loginResult.body.token}`)
.send({ columnFilters: [{ id: '', value: '21050' }], sorting: [] })
expect(result.status).toEqual(400)
expect(result.body).toEqual([{ name: 'Column Filters', error: 'Invalid or missing id field in filter' }])
})
it('rejects non-admin requests', async () => {
const result = await request(app).get('/occurrence/export/dwc-archive')
expect(result.status).toEqual(403)
+15 -2
View File
@@ -12,6 +12,7 @@ import { buildDwcDataPackageZipBuffer, buildFullDarwinCoreExportZipBuffer } from
import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
import { logger } from '../utils/logger'
import { getFilteredCrossSearchOccurrenceKeys, type CrossSearchRequestParameters } from '../services/crossSearch'
import type { ValidationObject } from '../../../frontend/src/shared/validators/validator'
const router = Router()
@@ -31,6 +32,15 @@ const defaultCrossSearchExportFilters = {
sorting: [],
} satisfies CrossSearchRequestParameters
class ExportFilterValidationError extends Error {
validationErrors: ValidationObject[]
constructor(validationErrors: ValidationObject[]) {
super('Invalid export filters.')
this.validationErrors = validationErrors
}
}
const resolveOccurrenceKeysForExport = async (req: Request): Promise<DwcOccurrenceKey[] | undefined> => {
if (req.method === 'GET') return undefined
const body = req.body as Partial<CrossSearchRequestParameters> | undefined
@@ -38,13 +48,16 @@ const resolveOccurrenceKeysForExport = async (req: Request): Promise<DwcOccurren
columnFilters: body?.columnFilters ?? defaultCrossSearchExportFilters.columnFilters,
sorting: body?.sorting ?? defaultCrossSearchExportFilters.sorting,
})
if ('validationErrors' in result) {
throw new Error(JSON.stringify(result.validationErrors))
if ('validationErrors' in result && Array.isArray(result.validationErrors)) {
throw new ExportFilterValidationError(result.validationErrors)
}
return result.occurrenceKeys
}
const handleExportFilterError = (error: unknown, res: Response) => {
if (error instanceof ExportFilterValidationError) {
return res.status(400).send(error.validationErrors)
}
return res.status(400).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' })
}
+10 -9
View File
@@ -2,15 +2,16 @@ import Prisma from '../../prisma/generated/now_test_client'
import JSZip from 'jszip'
import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv'
import { getFieldInfoText } from '../../../frontend/src/shared/fieldInfo'
const DATASET_TITLE = 'NOW database Darwin Core export'
const DATASET_NAME = 'now-darwincore-export'
const DATASET_VERSION = '1.0.0'
const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068'
const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/'
const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International'
const DATASET_CREATOR = 'The NOW Community'
const MISSING_VALUE = '\\N'
import {
DATASET_CREATOR,
DATASET_DOI,
DATASET_LICENSE_TITLE,
DATASET_LICENSE_URL,
DATASET_NAME,
DATASET_TITLE,
DATASET_VERSION,
MISSING_VALUE,
} from './dwcMetadata'
const isMeaningfulString = (value: unknown): value is string => {
if (typeof value !== 'string') return false
+11 -10
View File
@@ -13,6 +13,16 @@ import {
} from './dwcArchiveExportOccurrences'
import { buildDwcArchiveZipBuffer, resolveTaxonRank, type MeasurementCsvRow } from './dwcArchiveExport'
import { writeDwcCsvString } from './utils/dwcCsv'
import {
DATASET_CREATOR,
DATASET_DOI,
DATASET_LICENSE_TITLE,
DATASET_LICENSE_URL,
DATASET_NAME,
DATASET_TITLE,
DATASET_VERSION,
MISSING_VALUE,
} from './dwcMetadata'
const isMeaningfulString = (value: unknown): value is string => {
if (typeof value !== 'string') return false
@@ -132,14 +142,6 @@ const ASSERTION_HEADERS = [
const DWC_DP_EVENT_ASSERTION_HEADERS = ['eventID', ...ASSERTION_HEADERS] as const
const DWC_DP_OCCURRENCE_ASSERTION_HEADERS = ['occurrenceID', ...ASSERTION_HEADERS] as const
const DATASET_TITLE = 'NOW database Darwin Core export'
const DATASET_NAME = 'now-darwincore-export'
const DATASET_VERSION = '1.0.0'
const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068'
const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/'
const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International'
const DATASET_CREATOR = 'The NOW Community'
const MISSING_VALUE = '\\N'
type AssertionHeader = (typeof ASSERTION_HEADERS)[number]
type AssertionColumns = Record<AssertionHeader, string>
@@ -474,8 +476,7 @@ export const buildDwcDataPackageJson = (publicationDateIso: string): string => {
'occurrence data',
'taxon traits',
],
citation:
'The NOW Community. NOW database Darwin Core export, version 1.0.0. https://doi.org/10.5281/zenodo.4268068. The DOI describes the NOW database generally rather than a single frozen export version; include the export date when citing a downloaded archive.',
citation: `${DATASET_CREATOR}. ${DATASET_TITLE}, version ${DATASET_VERSION}. ${DATASET_DOI}. The DOI describes the NOW database generally rather than a single frozen export version; include the export date when citing a downloaded archive.`,
description:
'Production Darwin Core Data Package export from the NOW database for relational event, occurrence, geological context, and assertion data. The NOW database is a continuously curated, globally scoped fossil mammal database with Cenozoic emphasis, spanning approximately the last 66 million years.',
missingValues: [MISSING_VALUE],
+8
View File
@@ -0,0 +1,8 @@
export const DATASET_TITLE = 'NOW database Darwin Core export'
export const DATASET_NAME = 'now-darwincore-export'
export const DATASET_VERSION = '1.0.0'
export const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068'
export const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/'
export const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International'
export const DATASET_CREATOR = 'The NOW Community'
export const MISSING_VALUE = '\\N'