mirror of
https://github.com/nowcommunity/nowdatabase.git
synced 2026-10-12 19:13:52 +03:00
Address follow-up Darwin Core review comments
This commit is contained in:
@@ -62,6 +62,57 @@ describe('DwC-A occurrence export (admin-only)', () => {
|
||||
expect(measurementCsv).toContain('"verbatimMeasurementType"')
|
||||
})
|
||||
|
||||
it('returns a filtered ZIP archive for POST requests', async () => {
|
||||
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
|
||||
expect(loginResult.status).toEqual(200)
|
||||
|
||||
const result = await request(app)
|
||||
.post('/occurrence/export/dwc-archive')
|
||||
.set('authorization', `bearer ${loginResult.body.token}`)
|
||||
.send({ columnFilters: [{ id: 'lid_now_loc', value: '21050' }], sorting: [] })
|
||||
.buffer(true)
|
||||
.parse(parseBinary)
|
||||
|
||||
expect(result.status).toEqual(200)
|
||||
expect(result.headers['content-type']).toMatch(/application\/zip/i)
|
||||
|
||||
const zip = await JSZip.loadAsync(result.body as unknown as Buffer)
|
||||
const occurrenceCsv = await zip.file('occurrence.csv')!.async('string')
|
||||
expect(occurrenceCsv).toContain('NOW:OCC:21050:')
|
||||
expect(occurrenceCsv).not.toContain('NOW:OCC:24750:')
|
||||
})
|
||||
|
||||
it('returns an empty filtered DwC-DP ZIP archive for POST requests', async () => {
|
||||
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
|
||||
expect(loginResult.status).toEqual(200)
|
||||
|
||||
const result = await request(app)
|
||||
.post('/occurrence/export/dwc-data-package')
|
||||
.set('authorization', `bearer ${loginResult.body.token}`)
|
||||
.send({ columnFilters: [{ id: 'lid_now_loc', value: '9999999' }], sorting: [] })
|
||||
.buffer(true)
|
||||
.parse(parseBinary)
|
||||
|
||||
expect(result.status).toEqual(200)
|
||||
const zip = await JSZip.loadAsync(result.body as unknown as Buffer)
|
||||
const occurrenceCsv = await zip.file('occurrence.csv')!.async('string')
|
||||
expect(occurrenceCsv).toContain('"occurrenceID"')
|
||||
expect(occurrenceCsv).not.toContain('NOW:OCC:')
|
||||
})
|
||||
|
||||
it('returns structured validation errors for invalid POST filters', async () => {
|
||||
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
|
||||
expect(loginResult.status).toEqual(200)
|
||||
|
||||
const result = await request(app)
|
||||
.post('/occurrence/export/dwc-data-package')
|
||||
.set('authorization', `bearer ${loginResult.body.token}`)
|
||||
.send({ columnFilters: [{ id: '', value: '21050' }], sorting: [] })
|
||||
|
||||
expect(result.status).toEqual(400)
|
||||
expect(result.body).toEqual([{ name: 'Column Filters', error: 'Invalid or missing id field in filter' }])
|
||||
})
|
||||
|
||||
it('rejects non-admin requests', async () => {
|
||||
const result = await request(app).get('/occurrence/export/dwc-archive')
|
||||
expect(result.status).toEqual(403)
|
||||
|
||||
@@ -12,6 +12,7 @@ import { buildDwcDataPackageZipBuffer, buildFullDarwinCoreExportZipBuffer } from
|
||||
import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
|
||||
import { logger } from '../utils/logger'
|
||||
import { getFilteredCrossSearchOccurrenceKeys, type CrossSearchRequestParameters } from '../services/crossSearch'
|
||||
import type { ValidationObject } from '../../../frontend/src/shared/validators/validator'
|
||||
|
||||
const router = Router()
|
||||
|
||||
@@ -31,6 +32,15 @@ const defaultCrossSearchExportFilters = {
|
||||
sorting: [],
|
||||
} satisfies CrossSearchRequestParameters
|
||||
|
||||
class ExportFilterValidationError extends Error {
|
||||
validationErrors: ValidationObject[]
|
||||
|
||||
constructor(validationErrors: ValidationObject[]) {
|
||||
super('Invalid export filters.')
|
||||
this.validationErrors = validationErrors
|
||||
}
|
||||
}
|
||||
|
||||
const resolveOccurrenceKeysForExport = async (req: Request): Promise<DwcOccurrenceKey[] | undefined> => {
|
||||
if (req.method === 'GET') return undefined
|
||||
const body = req.body as Partial<CrossSearchRequestParameters> | undefined
|
||||
@@ -38,13 +48,16 @@ const resolveOccurrenceKeysForExport = async (req: Request): Promise<DwcOccurren
|
||||
columnFilters: body?.columnFilters ?? defaultCrossSearchExportFilters.columnFilters,
|
||||
sorting: body?.sorting ?? defaultCrossSearchExportFilters.sorting,
|
||||
})
|
||||
if ('validationErrors' in result) {
|
||||
throw new Error(JSON.stringify(result.validationErrors))
|
||||
if ('validationErrors' in result && Array.isArray(result.validationErrors)) {
|
||||
throw new ExportFilterValidationError(result.validationErrors)
|
||||
}
|
||||
return result.occurrenceKeys
|
||||
}
|
||||
|
||||
const handleExportFilterError = (error: unknown, res: Response) => {
|
||||
if (error instanceof ExportFilterValidationError) {
|
||||
return res.status(400).send(error.validationErrors)
|
||||
}
|
||||
return res.status(400).send({ error: error instanceof Error ? error.message : 'Invalid export filters.' })
|
||||
}
|
||||
|
||||
|
||||
@@ -2,15 +2,16 @@ import Prisma from '../../prisma/generated/now_test_client'
|
||||
import JSZip from 'jszip'
|
||||
import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv'
|
||||
import { getFieldInfoText } from '../../../frontend/src/shared/fieldInfo'
|
||||
|
||||
const DATASET_TITLE = 'NOW database Darwin Core export'
|
||||
const DATASET_NAME = 'now-darwincore-export'
|
||||
const DATASET_VERSION = '1.0.0'
|
||||
const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068'
|
||||
const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/'
|
||||
const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International'
|
||||
const DATASET_CREATOR = 'The NOW Community'
|
||||
const MISSING_VALUE = '\\N'
|
||||
import {
|
||||
DATASET_CREATOR,
|
||||
DATASET_DOI,
|
||||
DATASET_LICENSE_TITLE,
|
||||
DATASET_LICENSE_URL,
|
||||
DATASET_NAME,
|
||||
DATASET_TITLE,
|
||||
DATASET_VERSION,
|
||||
MISSING_VALUE,
|
||||
} from './dwcMetadata'
|
||||
|
||||
const isMeaningfulString = (value: unknown): value is string => {
|
||||
if (typeof value !== 'string') return false
|
||||
|
||||
@@ -13,6 +13,16 @@ import {
|
||||
} from './dwcArchiveExportOccurrences'
|
||||
import { buildDwcArchiveZipBuffer, resolveTaxonRank, type MeasurementCsvRow } from './dwcArchiveExport'
|
||||
import { writeDwcCsvString } from './utils/dwcCsv'
|
||||
import {
|
||||
DATASET_CREATOR,
|
||||
DATASET_DOI,
|
||||
DATASET_LICENSE_TITLE,
|
||||
DATASET_LICENSE_URL,
|
||||
DATASET_NAME,
|
||||
DATASET_TITLE,
|
||||
DATASET_VERSION,
|
||||
MISSING_VALUE,
|
||||
} from './dwcMetadata'
|
||||
|
||||
const isMeaningfulString = (value: unknown): value is string => {
|
||||
if (typeof value !== 'string') return false
|
||||
@@ -132,14 +142,6 @@ const ASSERTION_HEADERS = [
|
||||
|
||||
const DWC_DP_EVENT_ASSERTION_HEADERS = ['eventID', ...ASSERTION_HEADERS] as const
|
||||
const DWC_DP_OCCURRENCE_ASSERTION_HEADERS = ['occurrenceID', ...ASSERTION_HEADERS] as const
|
||||
const DATASET_TITLE = 'NOW database Darwin Core export'
|
||||
const DATASET_NAME = 'now-darwincore-export'
|
||||
const DATASET_VERSION = '1.0.0'
|
||||
const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068'
|
||||
const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/'
|
||||
const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International'
|
||||
const DATASET_CREATOR = 'The NOW Community'
|
||||
const MISSING_VALUE = '\\N'
|
||||
|
||||
type AssertionHeader = (typeof ASSERTION_HEADERS)[number]
|
||||
type AssertionColumns = Record<AssertionHeader, string>
|
||||
@@ -474,8 +476,7 @@ export const buildDwcDataPackageJson = (publicationDateIso: string): string => {
|
||||
'occurrence data',
|
||||
'taxon traits',
|
||||
],
|
||||
citation:
|
||||
'The NOW Community. NOW database Darwin Core export, version 1.0.0. https://doi.org/10.5281/zenodo.4268068. The DOI describes the NOW database generally rather than a single frozen export version; include the export date when citing a downloaded archive.',
|
||||
citation: `${DATASET_CREATOR}. ${DATASET_TITLE}, version ${DATASET_VERSION}. ${DATASET_DOI}. The DOI describes the NOW database generally rather than a single frozen export version; include the export date when citing a downloaded archive.`,
|
||||
description:
|
||||
'Production Darwin Core Data Package export from the NOW database for relational event, occurrence, geological context, and assertion data. The NOW database is a continuously curated, globally scoped fossil mammal database with Cenozoic emphasis, spanning approximately the last 66 million years.',
|
||||
missingValues: [MISSING_VALUE],
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
export const DATASET_TITLE = 'NOW database Darwin Core export'
|
||||
export const DATASET_NAME = 'now-darwincore-export'
|
||||
export const DATASET_VERSION = '1.0.0'
|
||||
export const DATASET_DOI = 'https://doi.org/10.5281/zenodo.4268068'
|
||||
export const DATASET_LICENSE_URL = 'https://creativecommons.org/licenses/by/4.0/'
|
||||
export const DATASET_LICENSE_TITLE = 'Creative Commons Attribution 4.0 International'
|
||||
export const DATASET_CREATOR = 'The NOW Community'
|
||||
export const MISSING_VALUE = '\\N'
|
||||
Reference in New Issue
Block a user