Add DwC-DP event occurrence export

This commit is contained in:
karilint
2026-05-13 11:54:16 +03:00
parent 647773d1c9
commit 4b99fe6c6b
7 changed files with 1085 additions and 1 deletions
@@ -67,4 +67,34 @@ describe('DwC-A occurrence export (admin-only)', () => {
expect(result.status).toEqual(403)
expect(result.body).toEqual(noPermError)
})
it('returns a DwC-DP ZIP archive for admins', async () => {
const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
expect(loginResult.status).toEqual(200)
const result = await request(app)
.get('/occurrence/export/dwc-data-package')
.set('authorization', `bearer ${loginResult.body.token}`)
.buffer(true)
.parse(parseBinary)
expect(result.status).toEqual(200)
expect(result.headers['content-type']).toMatch(/application\/zip/i)
expect(result.headers['content-disposition']).toMatch(/attachment;\s*filename="now_dwc_dp_test_export_/i)
const zip = await JSZip.loadAsync(result.body as unknown as Buffer)
expect(zip.file('datapackage.json')).toBeTruthy()
expect(zip.file('event.csv')).toBeTruthy()
expect(zip.file('geological-context.csv')).toBeTruthy()
expect(zip.file('occurrence.csv')).toBeTruthy()
expect(zip.file('event-assertion.csv')).toBeTruthy()
expect(zip.file('occurrence-assertion.csv')).toBeTruthy()
expect(zip.file('eml.xml')).toBeTruthy()
})
it('rejects non-admin DwC-DP requests', async () => {
const result = await request(app).get('/occurrence/export/dwc-data-package')
expect(result.status).toEqual(403)
expect(result.body).toEqual(noPermError)
})
})
+8
View File
@@ -7,6 +7,7 @@ import {
buildDwcOccurrenceArchiveZipStream,
type DwcOccurrenceExportProgress,
} from '../services/dwcArchiveExportOccurrences'
import { buildDwcDataPackageZipBuffer } from '../services/dwcDataPackageExport'
import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
import { logger } from '../utils/logger'
@@ -63,6 +64,13 @@ router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (req, res, n
})
})
router.get('/export/dwc-data-package', requireOneOf([Role.Admin]), async (_req, res) => {
const zipBuffer = await buildDwcDataPackageZipBuffer()
res.setHeader('Content-Type', 'application/zip')
res.setHeader('Content-Disposition', `attachment; filename="now_dwc_dp_test_export_${currentDateAsString()}.zip"`)
return res.status(200).send(zipBuffer)
})
router.get('/:lid/:speciesId', getOccurrenceDetail)
router.put(
'/:lid/:speciesId',
@@ -0,0 +1,681 @@
import { format } from 'fast-csv'
import { Writable } from 'stream'
import JSZip from 'jszip'
import { getContinentByCountry } from '../../../frontend/src/shared/validators/countryContinents'
import {
mapLocalityToGeologicalContextRow,
mapLocalityToMeasurementRows,
type LocalityMeasurementCsvRow,
} from './dwcArchiveExportLocalities'
import {
mapOccurrenceToMeasurementRows,
mapOccurrenceToOccurrenceRow,
type OccurrenceCsvRow,
} from './dwcArchiveExportOccurrences'
import type { MeasurementCsvRow } from './dwcArchiveExport'
const writeCsvString = async (headers: string[], rows: Array<Record<string, unknown>>): Promise<string> => {
return await new Promise((resolve, reject) => {
let output = ''
const csvStream = format({
delimiter: ',',
headers,
quoteColumns: true,
quoteHeaders: true,
includeEndRowDelimiter: true,
})
const sink = new Writable({
write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
output += typeof chunk === 'string' ? chunk : chunk.toString('utf8')
callback()
},
})
sink.on('finish', () => resolve(output))
sink.on('error', reject)
csvStream.on('error', reject)
csvStream.pipe(sink)
for (const row of rows) {
csvStream.write(row)
}
csvStream.end()
})
}
const isMeaningfulString = (value: unknown): value is string => {
if (typeof value !== 'string') return false
const trimmed = value.trim()
if (!trimmed) return false
if (trimmed === '-') return false
return true
}
const toMaybeMeaningful = (value: string | null | undefined): string => (isMeaningfulString(value) ? value.trim() : '')
const toMaybeMeaningfulNumber = (value: number | null | undefined): string => {
if (value === null || value === undefined) return ''
if (!Number.isFinite(value)) return ''
if (value === 0) return ''
return value.toString()
}
const eventIdForLocality = (lid: number): string => `NOW:EVENT:${lid}`
const locationIdForLocality = (lid: number): string => `NOW:LOC:${lid}`
const occurrenceIdForRow = (lid: number, speciesId: number): string => `NOW:OCC:${lid}:${speciesId}`
type LocalityForDwcDpExport = Parameters<typeof mapLocalityToMeasurementRows>[0]
type OccurrenceForDwcDpExport = Parameters<typeof mapOccurrenceToOccurrenceRow>[0]
export const DWC_DP_EVENT_HEADERS = [
'eventID',
'parentEventID',
'eventType',
'locationID',
'locality',
'continent',
'country',
'stateProvince',
'county',
'higherGeography',
'decimalLatitude',
'decimalLongitude',
'verbatimLatitude',
'verbatimLongitude',
'verbatimElevation',
'eventRemarks',
'geologicalContextID',
] as const
type DwcDpEventHeader = (typeof DWC_DP_EVENT_HEADERS)[number]
type DwcDpEventRow = Record<DwcDpEventHeader, string>
export const DWC_DP_GEOLOGICAL_CONTEXT_HEADERS = [
'geologicalContextID',
'lithostratigraphicTerms',
'group',
'formation',
'member',
'bed',
'earliestAgeOrLowestStage',
'latestAgeOrHighestStage',
] as const
type DwcDpGeologicalContextHeader = (typeof DWC_DP_GEOLOGICAL_CONTEXT_HEADERS)[number]
type DwcDpGeologicalContextRow = Record<DwcDpGeologicalContextHeader, string>
export const DWC_DP_OCCURRENCE_HEADERS = [
'occurrenceID',
'eventID',
'organismQuantity',
'organismQuantityType',
'occurrenceStatus',
'occurrenceRemarks',
'taxonID',
'scientificName',
'scientificNameAuthorship',
'taxonRank',
'identificationVerificationStatus',
] as const
type DwcDpOccurrenceHeader = (typeof DWC_DP_OCCURRENCE_HEADERS)[number]
type DwcDpOccurrenceRow = Record<DwcDpOccurrenceHeader, string>
const ASSERTION_HEADERS = [
'assertionID',
'verbatimAssertionType',
'assertionType',
'assertionTypeIRI',
'assertionTypeSource',
'assertionMadeDate',
'assertionEffectiveDate',
'assertionValue',
'assertionValueIRI',
'assertionValueSource',
'assertionValueNumeric',
'assertionUnit',
'assertionUnitIRI',
'assertionUnitSource',
'assertionError',
'assertionBy',
'assertionByID',
'assertionProtocols',
'assertionProtocolID',
'assertionReferences',
'assertionRemarks',
] as const
const DWC_DP_EVENT_ASSERTION_HEADERS = ['eventID', ...ASSERTION_HEADERS] as const
const DWC_DP_OCCURRENCE_ASSERTION_HEADERS = ['occurrenceID', ...ASSERTION_HEADERS] as const
type AssertionHeader = (typeof ASSERTION_HEADERS)[number]
type AssertionColumns = Record<AssertionHeader, string>
type DwcDpEventAssertionHeader = (typeof DWC_DP_EVENT_ASSERTION_HEADERS)[number]
type DwcDpOccurrenceAssertionHeader = (typeof DWC_DP_OCCURRENCE_ASSERTION_HEADERS)[number]
type DwcDpEventAssertionRow = Record<DwcDpEventAssertionHeader, string>
type DwcDpOccurrenceAssertionRow = Record<DwcDpOccurrenceAssertionHeader, string>
export const DWC_DP_TABLES = {
event: 'event.csv',
geologicalContext: 'geological-context.csv',
occurrence: 'occurrence.csv',
eventAssertion: 'event-assertion.csv',
occurrenceAssertion: 'occurrence-assertion.csv',
dataPackage: 'datapackage.json',
eml: 'eml.xml',
} as const
const numericValue = (value: string): string => {
if (!value.trim()) return ''
const parsed = Number(value)
return Number.isFinite(parsed) ? value : ''
}
const assertionColumnsFromMeasurement = ({
measurementID,
verbatimMeasurementType,
measurementType,
measurementValue,
measurementUnit,
measurementMethod,
}: Pick<
MeasurementCsvRow | LocalityMeasurementCsvRow,
| 'measurementID'
| 'verbatimMeasurementType'
| 'measurementType'
| 'measurementValue'
| 'measurementUnit'
| 'measurementMethod'
>): AssertionColumns => ({
assertionID: measurementID,
verbatimAssertionType: verbatimMeasurementType,
assertionType: measurementType,
assertionTypeIRI: '',
assertionTypeSource: '',
assertionMadeDate: '',
assertionEffectiveDate: '',
assertionValue: measurementValue,
assertionValueIRI: '',
assertionValueSource: '',
assertionValueNumeric: numericValue(measurementValue),
assertionUnit: measurementUnit,
assertionUnitIRI: '',
assertionUnitSource: '',
assertionError: '',
assertionBy: '',
assertionByID: '',
assertionProtocols: measurementMethod,
assertionProtocolID: '',
assertionReferences: '',
assertionRemarks: '',
})
export const mapLocalityToDwcDpEventRow = (locality: LocalityForDwcDpExport): DwcDpEventRow => {
const continent = getContinentByCountry(locality.country) ?? ''
const higherGeography = [
continent,
toMaybeMeaningful(locality.country),
toMaybeMeaningful(locality.state),
toMaybeMeaningful(locality.county),
toMaybeMeaningful(locality.basin),
toMaybeMeaningful(locality.subbasin),
]
.filter(Boolean)
.join('|')
return {
eventID: eventIdForLocality(locality.lid),
parentEventID: '',
eventType: 'paleontological locality',
locationID: locationIdForLocality(locality.lid),
locality: toMaybeMeaningful(locality.loc_name),
continent,
country: toMaybeMeaningful(locality.country),
stateProvince: toMaybeMeaningful(locality.state),
county: toMaybeMeaningful(locality.county),
higherGeography,
decimalLatitude: toMaybeMeaningfulNumber(locality.dec_lat),
decimalLongitude: toMaybeMeaningfulNumber(locality.dec_long),
verbatimLatitude: toMaybeMeaningful(locality.dms_lat),
verbatimLongitude: toMaybeMeaningful(locality.dms_long),
verbatimElevation: locality.altitude === null || locality.altitude === undefined ? '' : String(locality.altitude),
eventRemarks: [
toMaybeMeaningful(locality.loc_detail),
toMaybeMeaningful(locality.age_comm),
toMaybeMeaningful(locality.tax_comm),
]
.filter(Boolean)
.join(' | '),
geologicalContextID: `NOW:GEO:${locality.lid}`,
}
}
export const mapLocalityToDwcDpGeologicalContextRow = (locality: LocalityForDwcDpExport): DwcDpGeologicalContextRow => {
const geologicalContextRow = mapLocalityToGeologicalContextRow(locality)
return {
geologicalContextID: `NOW:GEO:${locality.lid}`,
lithostratigraphicTerms: geologicalContextRow.lithostratigraphicTerms,
group: geologicalContextRow.group,
formation: geologicalContextRow.formation,
member: geologicalContextRow.member,
bed: geologicalContextRow.bed,
earliestAgeOrLowestStage: geologicalContextRow.earliestAgeOrLowestStage,
latestAgeOrHighestStage: geologicalContextRow.latestAgeOrHighestStage,
}
}
export const mapOccurrenceToDwcDpOccurrenceRow = (occurrence: OccurrenceForDwcDpExport): DwcDpOccurrenceRow => {
const occurrenceRow: OccurrenceCsvRow = mapOccurrenceToOccurrenceRow(occurrence)
return {
occurrenceID: occurrenceRow.occurrenceID,
eventID: eventIdForLocality(occurrence.lid),
organismQuantity: occurrenceRow.organismQuantity,
organismQuantityType: occurrenceRow.organismQuantityType,
occurrenceStatus: occurrenceRow.occurrenceStatus,
occurrenceRemarks: occurrenceRow.occurrenceRemarks,
taxonID: occurrenceRow.taxonID,
scientificName: occurrenceRow.scientificName,
scientificNameAuthorship: toMaybeMeaningful(occurrence.com_species.sp_author),
taxonRank: '',
identificationVerificationStatus: occurrenceRow.identificationQualifier,
}
}
export const mapLocalityToDwcDpEventAssertionRows = (locality: LocalityForDwcDpExport): DwcDpEventAssertionRow[] => {
const eventID = eventIdForLocality(locality.lid)
return mapLocalityToMeasurementRows(locality).map(row => ({
eventID,
...assertionColumnsFromMeasurement({
...row,
measurementID: row.measurementID.replace(locationIdForLocality(locality.lid), eventID),
}),
}))
}
export const mapOccurrenceToDwcDpOccurrenceAssertionRows = (
occurrence: OccurrenceForDwcDpExport
): DwcDpOccurrenceAssertionRow[] => {
const occurrenceID = occurrenceIdForRow(occurrence.lid, occurrence.species_id)
return mapOccurrenceToMeasurementRows(occurrence).map(row => ({
occurrenceID,
...assertionColumnsFromMeasurement(row),
}))
}
const field = (name: string, type = 'string') => ({ name, type })
const schemaFor = ({
headers,
primaryKey,
foreignKeys = [],
}: {
headers: readonly string[]
primaryKey: string | string[]
foreignKeys?: Array<{
fields: string
reference: { resource: string; fields: string }
}>
}) => ({
fields: headers.map(header => field(header, header.endsWith('Numeric') ? 'number' : 'string')),
primaryKey,
foreignKeys,
})
export const buildDwcDataPackageJson = (publicationDateIso: string): string => {
const dataPackage = {
profile: 'data-package',
name: 'now-dwc-dp-test-export',
title: 'NOW database Darwin Core Data Package test export',
created: publicationDateIso,
homepage: 'https://nowdatabase.org/',
description:
'Admin-only test Darwin Core Data Package export for NOW localities as events and NOW locality-species rows as occurrences.',
resources: [
{
name: 'event',
path: DWC_DP_TABLES.event,
profile: 'tabular-data-resource',
schema: schemaFor({
headers: DWC_DP_EVENT_HEADERS,
primaryKey: 'eventID',
foreignKeys: [
{
fields: 'geologicalContextID',
reference: { resource: 'geological-context', fields: 'geologicalContextID' },
},
],
}),
},
{
name: 'geological-context',
path: DWC_DP_TABLES.geologicalContext,
profile: 'tabular-data-resource',
schema: schemaFor({
headers: DWC_DP_GEOLOGICAL_CONTEXT_HEADERS,
primaryKey: 'geologicalContextID',
}),
},
{
name: 'occurrence',
path: DWC_DP_TABLES.occurrence,
profile: 'tabular-data-resource',
schema: schemaFor({
headers: DWC_DP_OCCURRENCE_HEADERS,
primaryKey: 'occurrenceID',
foreignKeys: [
{
fields: 'eventID',
reference: { resource: 'event', fields: 'eventID' },
},
],
}),
},
{
name: 'event-assertion',
path: DWC_DP_TABLES.eventAssertion,
profile: 'tabular-data-resource',
schema: schemaFor({
headers: DWC_DP_EVENT_ASSERTION_HEADERS,
primaryKey: 'assertionID',
foreignKeys: [
{
fields: 'eventID',
reference: { resource: 'event', fields: 'eventID' },
},
],
}),
},
{
name: 'occurrence-assertion',
path: DWC_DP_TABLES.occurrenceAssertion,
profile: 'tabular-data-resource',
schema: schemaFor({
headers: DWC_DP_OCCURRENCE_ASSERTION_HEADERS,
primaryKey: 'assertionID',
foreignKeys: [
{
fields: 'occurrenceID',
reference: { resource: 'occurrence', fields: 'occurrenceID' },
},
],
}),
},
],
}
return `${JSON.stringify(dataPackage, null, 2)}\n`
}
export const buildDwcDataPackageEmlXml = (publicationDateIso: string): string => {
return `<?xml version="1.0" encoding="UTF-8"?>
<eml:eml
xmlns:eml="eml://ecoinformatics.org/eml-2.1.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
packageId="nowdatabase-dwc-dp-test-export"
system="nowdatabase"
xsi:schemaLocation="eml://ecoinformatics.org/eml-2.1.1 https://eml.ecoinformatics.org/eml-2.1.1/eml.xsd"
>
<!-- TODO(#1150): Replace placeholder metadata with real dataset-level EML generation. -->
<dataset>
<title>NOW database Darwin Core Data Package test export</title>
<creator>
<individualName>
<surName>NOW database</surName>
</individualName>
</creator>
<contact>
<individualName>
<surName>NOW database</surName>
</individualName>
</contact>
<pubDate>${publicationDateIso}</pubDate>
<abstract>
<para>Admin-only test Darwin Core Data Package export. Localities are modeled as events, locality geological fields are modeled as geological contexts, locality-species rows are modeled as occurrences, and non-core facts are modeled as event or occurrence assertions.</para>
</abstract>
<intellectualRights>
<para>TODO(#1150): Add rights / license information.</para>
</intellectualRights>
</dataset>
</eml:eml>
`
}
const localitySelect = {
lid: true,
loc_name: true,
basin: true,
subbasin: true,
country: true,
state: true,
county: true,
dec_lat: true,
dec_long: true,
dms_lat: true,
dms_long: true,
approx_coord: true,
altitude: true,
loc_detail: true,
chron: true,
lgroup: true,
formation: true,
member: true,
bed: true,
bfa_max: true,
bfa_min: true,
bfa_max_abs: true,
bfa_min_abs: true,
frac_max: true,
frac_min: true,
max_age: true,
min_age: true,
date_meth: true,
age_comm: true,
site_area: true,
gen_loc: true,
plate: true,
appr_num_spm: true,
num_spm: true,
true_quant: true,
complete: true,
num_quad: true,
rock_type: true,
rt_adj: true,
lith_comm: true,
depo_context1: true,
depo_context2: true,
depo_context3: true,
depo_context4: true,
depo_comm: true,
sed_env_1: true,
sed_env_2: true,
event_circum: true,
se_comm: true,
assem_fm: true,
transport: true,
trans_mod: true,
weath_trmp: true,
pt_conc: true,
size_type: true,
vert_pres: true,
plant_pres: true,
invert_pres: true,
time_rep: true,
taph_comm: true,
tax_comm: true,
datum_plane: true,
tos: true,
bos: true,
loc_status: true,
hominin_skeletal_remains: true,
climate_type: true,
biome: true,
v_ht: true,
v_struct: true,
v_envi_det: true,
disturb: true,
nutrients: true,
water: true,
seasonality: true,
seas_intens: true,
pri_prod: true,
moisture: true,
temperature: true,
estimate_precip: true,
estimate_temp: true,
estimate_npp: true,
pers_woody_cover: true,
pers_pollen_ap: true,
pers_pollen_nap: true,
pers_pollen_other: true,
stone_tool_cut_marks_on_bones: true,
bipedal_footprints: true,
stone_tool_technology: true,
technological_mode_1: true,
technological_mode_2: true,
technological_mode_3: true,
cultural_stage_1: true,
cultural_stage_2: true,
cultural_stage_3: true,
regional_culture_1: true,
regional_culture_2: true,
regional_culture_3: true,
now_time_unit_now_loc_bfa_maxTonow_time_unit: {
select: { tu_name: true, tu_display_name: true, rank: true, sequence: true },
},
now_time_unit_now_loc_bfa_minTonow_time_unit: {
select: { tu_name: true, tu_display_name: true, rank: true, sequence: true },
},
now_syn_loc: {
select: { synonym: true },
},
now_ss: {
select: { sed_struct: true },
},
now_coll_meth: {
select: { coll_meth: true },
},
now_mus: {
select: {
museum: true,
com_mlist: { select: { institution: true, alt_int_name: true, city: true, state: true, country: true } },
},
},
now_ls: {
select: {
com_species: {
select: { order_name: true, tht: true, genus_name: true },
},
},
},
} as const
const occurrenceSelect = {
lid: true,
species_id: true,
nis: true,
pct: true,
quad: true,
mni: true,
qua: true,
id_status: true,
orig_entry: true,
source_name: true,
body_mass: true,
mesowear: true,
mw_or_high: true,
mw_or_low: true,
mw_cs_sharp: true,
mw_cs_round: true,
mw_cs_blunt: true,
mw_scale_min: true,
mw_scale_max: true,
mw_value: true,
microwear: true,
dc13_mean: true,
dc13_n: true,
dc13_max: true,
dc13_min: true,
dc13_stdev: true,
do18_mean: true,
do18_n: true,
do18_max: true,
do18_min: true,
do18_stdev: true,
com_species: {
select: {
species_id: true,
class_name: true,
subclass_or_superorder_name: true,
order_name: true,
suborder_or_superfamily_name: true,
family_name: true,
subfamily_name: true,
genus_name: true,
species_name: true,
unique_identifier: true,
taxonomic_status: true,
common_name: true,
sp_author: true,
sp_comment: true,
},
},
} as const
export const buildDwcDataPackageZipBufferFromRows = async ({
localities,
occurrences,
publicationDateIso = new Date().toISOString().slice(0, 10),
}: {
localities: LocalityForDwcDpExport[]
occurrences: OccurrenceForDwcDpExport[]
publicationDateIso?: string
}): Promise<Buffer> => {
const eventRows = localities.map(mapLocalityToDwcDpEventRow)
const geologicalContextRows = localities.map(mapLocalityToDwcDpGeologicalContextRow)
const occurrenceRows = occurrences.map(mapOccurrenceToDwcDpOccurrenceRow)
const eventAssertionRows = localities.flatMap(mapLocalityToDwcDpEventAssertionRows)
const occurrenceAssertionRows = occurrences.flatMap(mapOccurrenceToDwcDpOccurrenceAssertionRows)
const zip = new JSZip()
zip.file(DWC_DP_TABLES.event, await writeCsvString([...DWC_DP_EVENT_HEADERS], eventRows))
zip.file(
DWC_DP_TABLES.geologicalContext,
await writeCsvString([...DWC_DP_GEOLOGICAL_CONTEXT_HEADERS], geologicalContextRows)
)
zip.file(DWC_DP_TABLES.occurrence, await writeCsvString([...DWC_DP_OCCURRENCE_HEADERS], occurrenceRows))
zip.file(DWC_DP_TABLES.eventAssertion, await writeCsvString([...DWC_DP_EVENT_ASSERTION_HEADERS], eventAssertionRows))
zip.file(
DWC_DP_TABLES.occurrenceAssertion,
await writeCsvString([...DWC_DP_OCCURRENCE_ASSERTION_HEADERS], occurrenceAssertionRows)
)
zip.file(DWC_DP_TABLES.dataPackage, buildDwcDataPackageJson(publicationDateIso))
zip.file(DWC_DP_TABLES.eml, buildDwcDataPackageEmlXml(publicationDateIso))
return await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 6 } })
}
export const buildDwcDataPackageZipBuffer = async (): Promise<Buffer> => {
const { nowDb } = await import('../utils/db')
const localities = (await nowDb.now_loc.findMany({
orderBy: { lid: 'asc' },
select: localitySelect,
})) as unknown as LocalityForDwcDpExport[]
const occurrences = (await nowDb.now_ls.findMany({
orderBy: [{ lid: 'asc' }, { species_id: 'asc' }],
select: occurrenceSelect,
})) as unknown as OccurrenceForDwcDpExport[]
return await buildDwcDataPackageZipBufferFromRows({
localities,
occurrences,
})
}
export type DwcDpLocalityFixture = LocalityForDwcDpExport
export type DwcDpOccurrenceFixture = OccurrenceForDwcDpExport
@@ -0,0 +1,196 @@
import { describe, expect, it } from '@jest/globals'
import JSZip from 'jszip'
import {
DWC_DP_TABLES,
buildDwcDataPackageZipBufferFromRows,
mapLocalityToDwcDpEventAssertionRows,
mapLocalityToDwcDpEventRow,
mapOccurrenceToDwcDpOccurrenceAssertionRows,
mapOccurrenceToDwcDpOccurrenceRow,
type DwcDpLocalityFixture,
type DwcDpOccurrenceFixture,
} from '../services/dwcDataPackageExport'
describe('DwC-DP export mapping', () => {
const locality = {
lid: 42,
loc_name: 'Test locality',
basin: 'Test basin',
subbasin: 'Test subbasin',
country: 'Finland',
state: 'Uusimaa',
county: 'Helsinki',
dec_lat: 60.1699,
dec_long: 24.9384,
dms_lat: null,
dms_long: null,
approx_coord: null,
altitude: 123,
loc_detail: 'Some notes',
tax_comm: null,
chron: 'Test chron',
lgroup: 'Test group',
formation: 'Test formation',
member: 'Test member',
bed: 'Test bed',
bfa_max: 'BFA_MAX',
bfa_min: 'BFA_MIN',
bfa_max_abs: null,
bfa_min_abs: null,
frac_max: null,
frac_min: null,
max_age: 12.3,
min_age: 4.5,
date_meth: 'radioisotope',
age_comm: 'Age comment',
appr_num_spm: null,
num_spm: null,
hominin_skeletal_remains: false,
stone_tool_cut_marks_on_bones: false,
bipedal_footprints: false,
stone_tool_technology: false,
now_syn_loc: [],
now_ss: [],
now_coll_meth: [],
now_mus: [],
now_ls: [],
now_time_unit_now_loc_bfa_maxTonow_time_unit: null,
now_time_unit_now_loc_bfa_minTonow_time_unit: null,
} as unknown as DwcDpLocalityFixture
const occurrence = {
lid: 42,
species_id: 21052,
nis: 7,
pct: null,
quad: null,
mni: 2,
qua: 'A',
id_status: 'confirmed',
orig_entry: 'Original occurrence note',
source_name: 'Source collection',
body_mass: BigInt(1234),
mesowear: null,
mw_or_high: null,
mw_or_low: null,
mw_cs_sharp: null,
mw_cs_round: null,
mw_cs_blunt: null,
mw_scale_min: null,
mw_scale_max: null,
mw_value: null,
microwear: null,
dc13_mean: -11.2,
dc13_n: 4,
dc13_max: null,
dc13_min: null,
dc13_stdev: null,
do18_mean: null,
do18_n: null,
do18_max: null,
do18_min: null,
do18_stdev: null,
com_species: {
species_id: 21052,
class_name: 'Mammalia',
subclass_or_superorder_name: null,
order_name: 'Rodentia',
suborder_or_superfamily_name: null,
family_name: 'Testidae',
subfamily_name: null,
genus_name: 'Simplomys',
species_name: 'simplicidens',
unique_identifier: '-',
taxonomic_status: null,
common_name: null,
sp_author: 'Test Author',
sp_comment: null,
},
} as unknown as DwcDpOccurrenceFixture
it('maps NOW localities to DwC-DP events and event assertions', () => {
const eventRow = mapLocalityToDwcDpEventRow(locality)
expect(eventRow).toEqual(
expect.objectContaining({
eventID: 'NOW:EVENT:42',
eventType: 'paleontological locality',
locationID: 'NOW:LOC:42',
locality: 'Test locality',
geologicalContextID: 'NOW:GEO:42',
})
)
const assertionRows = mapLocalityToDwcDpEventAssertionRows(locality)
expect(assertionRows).toEqual(
expect.arrayContaining([
expect.objectContaining({
eventID: 'NOW:EVENT:42',
assertionID: 'NOW:EVENT:42:max_age',
verbatimAssertionType: 'max_age',
assertionValueNumeric: '12.3',
assertionUnit: 'Ma',
}),
])
)
})
it('maps NOW locality-species rows to DwC-DP occurrences and occurrence assertions', () => {
const occurrenceRow = mapOccurrenceToDwcDpOccurrenceRow(occurrence)
expect(occurrenceRow).toEqual(
expect.objectContaining({
occurrenceID: 'NOW:OCC:42:21052',
eventID: 'NOW:EVENT:42',
taxonID: 'NOW:21052',
scientificName: 'Simplomys simplicidens Test Author',
identificationVerificationStatus: 'confirmed',
})
)
const assertionRows = mapOccurrenceToDwcDpOccurrenceAssertionRows(occurrence)
expect(assertionRows).toEqual(
expect.arrayContaining([
expect.objectContaining({
occurrenceID: 'NOW:OCC:42:21052',
assertionID: 'NOW:OCC:42:21052:now_ls.body_mass',
verbatimAssertionType: 'now_ls.body_mass',
assertionValueNumeric: '1234',
assertionUnit: 'g',
}),
])
)
})
it('generates a ZIP archive with DwC-DP files and relational metadata', async () => {
const zipBuffer = await buildDwcDataPackageZipBufferFromRows({
localities: [locality],
occurrences: [occurrence],
publicationDateIso: '2026-05-13',
})
const zip = await JSZip.loadAsync(zipBuffer)
expect(zip.file(DWC_DP_TABLES.event)).toBeTruthy()
expect(zip.file(DWC_DP_TABLES.geologicalContext)).toBeTruthy()
expect(zip.file(DWC_DP_TABLES.occurrence)).toBeTruthy()
expect(zip.file(DWC_DP_TABLES.eventAssertion)).toBeTruthy()
expect(zip.file(DWC_DP_TABLES.occurrenceAssertion)).toBeTruthy()
expect(zip.file(DWC_DP_TABLES.dataPackage)).toBeTruthy()
expect(zip.file(DWC_DP_TABLES.eml)).toBeTruthy()
const dataPackageJson = JSON.parse(await zip.file(DWC_DP_TABLES.dataPackage)!.async('string')) as {
resources: Array<{ name: string; schema: { foreignKeys?: unknown[] } }>
}
expect(dataPackageJson.resources.map(resource => resource.name)).toEqual([
'event',
'geological-context',
'occurrence',
'event-assertion',
'occurrence-assertion',
])
expect(dataPackageJson.resources.find(resource => resource.name === 'occurrence')?.schema.foreignKeys).toEqual([
{
fields: 'eventID',
reference: { resource: 'event', fields: 'eventID' },
},
])
})
})
@@ -0,0 +1,75 @@
# Darwin Core Data Package export (v1, admin-only)
Issue: `nowcommunity/nowdatabase#1150`
This repository includes an **admin-only** Darwin Core Data Package (DwC-DP) test export for relational locality and occurrence data.
DwC-DP is intentionally published as a separate format from the existing DwC-A exports. The taxon trait-heavy `com_species` export remains a DwC-A `Taxon` core with `MeasurementOrFact`, because those traits are synthesized at taxon level rather than linked to specimen/material source entities.
## Access
- Backend route: `GET /occurrence/export/dwc-data-package` (**Role.Admin only**)
- The frontend exposes this as an export option on the cross-search occurrence table for administrators.
## Output
The downloaded ZIP contains:
- `datapackage.json` (Frictionless Data Package descriptor with table schemas and foreign keys)
- `event.csv` (DwC-DP Event rows derived from `now_loc`)
- `geological-context.csv` (geological context rows referenced from `event.csv`)
- `occurrence.csv` (DwC-DP Occurrence rows derived from `now_ls`)
- `event-assertion.csv` (locality-level facts formerly expressed as locality `MeasurementOrFact`)
- `occurrence-assertion.csv` (occurrence-level facts formerly expressed as occurrence `MeasurementOrFact`)
- `eml.xml` (minimal placeholder EML metadata)
## Model
### `event.csv`
One row per `now_loc` record.
Key mappings:
- `eventID` = `NOW:EVENT:<lid>`
- `eventType` = `paleontological locality`
- `locationID` = `NOW:LOC:<lid>`
- locality, geography, coordinates, elevation, and remarks map from the corresponding locality fields
- `geologicalContextID` = `NOW:GEO:<lid>`
### `geological-context.csv`
One row per locality geological context.
Key mappings:
- `geologicalContextID` = `NOW:GEO:<lid>`
- stratigraphic terms map from `chron`, `lgroup`, `formation`, `member`, and `bed`
- earliest/latest stage terms map from `bfa_max` / `bfa_min`, using related time-unit display names when available
### `occurrence.csv`
One row per `now_ls` record.
Key mappings:
- `occurrenceID` = `NOW:OCC:<lid>:<species_id>`
- `eventID` = `NOW:EVENT:<lid>`
- `taxonID` = `NOW:<species_id>`
- `scientificName` follows the existing occurrence DwC-A mapping
- `identificationVerificationStatus` maps from `id_status`
### Assertion Tables
The assertion tables preserve the non-core facts without overloading one DwC-A `measurementorfact.csv` across multiple entity types.
- `event-assertion.csv` uses `eventID` and contains locality-level facts from `now_loc` and related locality lists.
- `occurrence-assertion.csv` uses `occurrenceID` and contains occurrence-level facts from `now_ls`.
The original database field name is preserved in `verbatimAssertionType`.
## Relationship To Existing DwC-A Exports
- Keep species traits as DwC-A `Taxon` + `MeasurementOrFact`.
- Keep locality and occurrence DwC-A exports available for current test users.
- Prefer this DwC-DP export for relational event/locality/geology/occurrence data because it avoids the DwC-A one-core star-schema limitation.
@@ -9,6 +9,7 @@ import { LocalitiesMap } from '../Map/LocalitiesMap'
import { formatWithMaxThreeDecimals } from '@/util/numberFormatting'
import { occurrenceLabels } from '@/constants/occurrenceLabels'
import { OccurrenceDwcExportMenuItem } from '@/components/Occurrence/OccurrenceDwcExportMenuItem'
import { OccurrenceDwcDpExportMenuItem } from '@/components/Occurrence/OccurrenceDwcDpExportMenuItem'
import { matchesCountryOrContinent } from '@/shared/validators/countryContinents'
export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: CrossSearch) => void }) => {
@@ -1090,7 +1091,12 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros
isCrossSearchTable={true}
isError={isError}
error={error}
renderExtraExportMenuItems={handleClose => <OccurrenceDwcExportMenuItem handleClose={handleClose} />}
renderExtraExportMenuItems={handleClose => (
<>
<OccurrenceDwcExportMenuItem handleClose={handleClose} />
<OccurrenceDwcDpExportMenuItem handleClose={handleClose} />
</>
)}
/>
</>
)
@@ -0,0 +1,88 @@
import { useState } from 'react'
import { MenuItem } from '@mui/material'
import { useNotify } from '@/hooks/notification'
import { BACKEND_URL } from '@/util/config'
import { useUser } from '@/hooks/user'
import { Role } from '@/shared/types'
import { currentDateAsString } from '@/shared/currentDateAsString'
export const OccurrenceDwcDpExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
const [loading, setLoading] = useState(false)
const { notify, setMessage: setNotificationMessage } = useNotify()
const user = useUser()
if (user.role !== Role.Admin) {
return null
}
const fetchOptions = user.token ? { headers: { Authorization: `Bearer ${user.token}` } } : {}
const filename = `now_dwc_dp_test_export_${currentDateAsString()}.zip`
const fetchZipFile = async () => {
setLoading(true)
notify('Generating DwC-DP ZIP export, please wait...', 'info', null)
try {
const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-data-package`, fetchOptions)
if (!response.ok) {
throw new Error('Server response was not OK.')
}
const reader = response.body?.getReader()
if (!reader) {
throw new Error('Missing response stream.')
}
const file: Uint8Array[] = []
let bytes = 0
let closed = false
const showDownloadProgress = () => {
if (!closed) {
setTimeout(() => {
setNotificationMessage(`Downloading DwC-DP ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
showDownloadProgress()
}, 500)
}
}
notify('Downloading DwC-DP ZIP...', 'info', null)
showDownloadProgress()
while (true) {
const { done, value } = await reader.read()
if (done) break
bytes = bytes + value.length
file.push(value)
}
closed = true
const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' }))
const downloadLink = document.createElement('a')
downloadLink.href = blobUrl
downloadLink.download = filename
document.body.appendChild(downloadLink)
downloadLink.click()
downloadLink.remove()
window.URL.revokeObjectURL(blobUrl)
notify('Download finished.')
} catch {
notify('Downloading DwC-DP export failed.', 'error')
} finally {
setLoading(false)
}
}
return (
<MenuItem
onClick={() => {
void fetchZipFile()
handleClose()
}}
disabled={loading}
>
Export DwC-DP (events + occurrences)
</MenuItem>
)
}