From 57b77ee2ed06eb2583c451ab98bb2a836a3d7fa0 Mon Sep 17 00:00:00 2001 From: karilint Date: Mon, 18 May 2026 15:55:00 +0300 Subject: [PATCH] Add full Darwin Core export progress --- .../dwcArchiveExportOccurrences.test.ts | 29 ++++ backend/src/routes/occurrence.ts | 9 +- backend/src/services/dwcArchiveExport.ts | 53 +------ .../services/dwcArchiveExportLocalities.ts | 64 +-------- .../services/dwcArchiveExportOccurrences.ts | 107 ++------------ backend/src/services/dwcDataPackageExport.ts | 126 +++++++++++------ backend/src/services/utils/dwcCsv.ts | 62 ++++++++ backend/src/unit-tests/dwcCsv.test.ts | 22 +++ .../unit-tests/dwcDataPackageExport.test.ts | 41 +++++- .../functionality/dwc_data_package.md | 27 +++- documentation/functionality/dwc_export.md | 2 + .../functionality/dwc_export_localities.md | 2 + .../functionality/dwc_export_occurrences.md | 2 + .../CrossSearch/CrossSearchExportMenuItem.tsx | 52 ++----- .../CrossSearch/CrossSearchTable.tsx | 2 + .../Locality/LocalityDwcExportMenuItem.tsx | 60 ++------ .../src/components/Locality/LocalityTable.tsx | 22 ++- .../OccurrenceDwcDpExportMenuItem.tsx | 60 ++------ .../OccurrenceDwcExportMenuItem.tsx | 101 ++------------ ...OccurrenceFullDarwinCoreExportMenuItem.tsx | 56 ++++++++ .../Species/SpeciesDwcExportMenuItem.tsx | 60 ++------ .../src/components/TableView/TableToolBar.tsx | 28 +++- .../src/components/TableView/TableView.tsx | 4 +- frontend/src/components/TableView/helpers.tsx | 1 + frontend/src/util/exportProgress.ts | 132 ++++++++++++++++++ .../components/TableToolBar.export.test.tsx | 14 +- 26 files changed, 610 insertions(+), 528 deletions(-) create mode 100644 backend/src/services/utils/dwcCsv.ts create mode 100644 backend/src/unit-tests/dwcCsv.test.ts create mode 100644 frontend/src/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem.tsx create mode 100644 frontend/src/util/exportProgress.ts diff --git a/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts b/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts index ea0eb81f..a6340e2b 100644 --- a/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts +++ b/backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts @@ -97,4 +97,33 @@ describe('DwC-A occurrence export (admin-only)', () => { expect(result.status).toEqual(403) expect(result.body).toEqual(noPermError) }) + + it('returns a full Darwin Core ZIP archive for admins', async () => { + const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' }) + expect(loginResult.status).toEqual(200) + + const result = await request(app) + .get('/occurrence/export/dwc-full-package') + .set('authorization', `bearer ${loginResult.body.token}`) + .buffer(true) + .parse(parseBinary) + + expect(result.status).toEqual(200) + expect(result.headers['content-type']).toMatch(/application\/zip/i) + expect(result.headers['content-disposition']).toMatch(/attachment;\s*filename="now_dwc_full_test_export_/i) + + const zip = await JSZip.loadAsync(result.body as unknown as Buffer) + expect(zip.file('README.txt')).toBeTruthy() + expect(zip.file('dwc-dp/datapackage.json')).toBeTruthy() + expect(zip.file('dwc-dp/occurrence.csv')).toBeTruthy() + expect(zip.file('dwc-a-taxa/taxon.csv')).toBeTruthy() + expect(zip.file('dwc-a-taxa/measurementorfact.csv')).toBeTruthy() + expect(zip.file('dwc-a-taxa/meta.xml')).toBeTruthy() + }) + + it('rejects non-admin full Darwin Core requests', async () => { + const result = await request(app).get('/occurrence/export/dwc-full-package') + expect(result.status).toEqual(403) + expect(result.body).toEqual(noPermError) + }) }) diff --git a/backend/src/routes/occurrence.ts b/backend/src/routes/occurrence.ts index 6115ba7c..3a66a8d9 100644 --- a/backend/src/routes/occurrence.ts +++ b/backend/src/routes/occurrence.ts @@ -7,7 +7,7 @@ import { buildDwcOccurrenceArchiveZipStream, type DwcOccurrenceExportProgress, } from '../services/dwcArchiveExportOccurrences' -import { buildDwcDataPackageZipBuffer } from '../services/dwcDataPackageExport' +import { buildDwcDataPackageZipBuffer, buildFullDarwinCoreExportZipBuffer } from '../services/dwcDataPackageExport' import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' import { logger } from '../utils/logger' @@ -71,6 +71,13 @@ router.get('/export/dwc-data-package', requireOneOf([Role.Admin]), async (_req, return res.status(200).send(zipBuffer) }) +router.get('/export/dwc-full-package', requireOneOf([Role.Admin]), async (_req, res) => { + const zipBuffer = await buildFullDarwinCoreExportZipBuffer() + res.setHeader('Content-Type', 'application/zip') + res.setHeader('Content-Disposition', `attachment; filename="now_dwc_full_test_export_${currentDateAsString()}.zip"`) + return res.status(200).send(zipBuffer) +}) + router.get('/:lid/:speciesId', getOccurrenceDetail) router.put( '/:lid/:speciesId', diff --git a/backend/src/services/dwcArchiveExport.ts b/backend/src/services/dwcArchiveExport.ts index ea522631..cb02b786 100644 --- a/backend/src/services/dwcArchiveExport.ts +++ b/backend/src/services/dwcArchiveExport.ts @@ -1,7 +1,6 @@ import Prisma from '../../prisma/generated/now_test_client' -import { format } from 'fast-csv' -import { Writable } from 'stream' import JSZip from 'jszip' +import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv' const isMeaningfulString = (value: unknown): value is string => { if (typeof value !== 'string') return false @@ -11,51 +10,7 @@ const isMeaningfulString = (value: unknown): value is string => { return true } -const toDwcString = (value: unknown): string => { - if (value === null || value === undefined) return '' - if (typeof value === 'bigint') return value.toString() - if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : '' - if (typeof value === 'boolean') return value ? 'true' : 'false' - if (typeof value === 'string') return value - if (typeof value === 'object') { - try { - return JSON.stringify(value) ?? '' - } catch { - return '' - } - } - return '' -} - -const writeCsvString = async (headers: string[], rows: Array>): Promise => { - return await new Promise((resolve, reject) => { - let output = '' - const csvStream = format({ - delimiter: ',', - headers, - quoteColumns: true, - quoteHeaders: true, - includeEndRowDelimiter: true, - }) - - const sink = new Writable({ - write(chunk: Buffer, _encoding, callback) { - output += chunk.toString('utf8') - callback() - }, - }) - - sink.on('finish', () => resolve(output)) - sink.on('error', reject) - csvStream.on('error', reject) - - csvStream.pipe(sink) - for (const row of rows) { - csvStream.write(row) - } - csvStream.end() - }) -} +const toDwcString = toDwcCsvString export const TAXON_HEADERS = [ 'taxonID', @@ -967,8 +922,8 @@ export const buildDwcArchiveZipBufferFromSpecies = async ( const taxonRows = speciesRows.map(mapSpeciesToTaxonRow) const measurementRows = speciesRows.flatMap(mapSpeciesToMeasurementRows) - const taxonCsv = await writeCsvString([...TAXON_HEADERS], taxonRows) - const measurementCsv = await writeCsvString([...MEASUREMENT_HEADERS], measurementRows) + const taxonCsv = writeDwcCsvString(TAXON_HEADERS, taxonRows) + const measurementCsv = writeDwcCsvString(MEASUREMENT_HEADERS, measurementRows) const metaXml = buildMetaXml() const publicationDateIso = new Date().toISOString().slice(0, 10) const emlXml = buildEmlXml(publicationDateIso) diff --git a/backend/src/services/dwcArchiveExportLocalities.ts b/backend/src/services/dwcArchiveExportLocalities.ts index 5c257a4c..0ee8abc0 100644 --- a/backend/src/services/dwcArchiveExportLocalities.ts +++ b/backend/src/services/dwcArchiveExportLocalities.ts @@ -1,8 +1,7 @@ import type { now_loc, now_time_unit } from '../../prisma/generated/now_test_client' -import { format } from 'fast-csv' -import { Writable } from 'stream' import JSZip from 'jszip' import { getContinentByCountry } from '../../../frontend/src/shared/validators/countryContinents' +import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv' const isMeaningfulString = (value: unknown): value is string => { if (typeof value !== 'string') return false @@ -12,60 +11,7 @@ const isMeaningfulString = (value: unknown): value is string => { return true } -const toDwcString = (value: unknown): string => { - if (value === null || value === undefined) return '' - if (typeof value === 'bigint') return value.toString() - if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : '' - if (typeof value === 'boolean') return value ? 'true' : 'false' - if (typeof value === 'string') return value - if (typeof value === 'object') { - if (value instanceof Date) return value.toISOString() - if (typeof (value as { toString?: unknown }).toString === 'function') { - const asString = (value as { toString: () => string }).toString() - if (asString && asString !== '[object Object]') return asString - } - try { - return JSON.stringify(value) ?? '' - } catch { - return '' - } - } - return '' -} - -const writeCsvString = async (headers: string[], rows: Array>): Promise => { - return await new Promise((resolve, reject) => { - let output = '' - const csvStream = format({ - delimiter: ',', - headers, - quoteColumns: true, - quoteHeaders: true, - includeEndRowDelimiter: true, - }) - - const sink = new Writable({ - write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) { - if (typeof chunk === 'string') { - output += chunk - } else { - output += chunk.toString('utf8') - } - callback() - }, - }) - - sink.on('finish', () => resolve(output)) - sink.on('error', reject) - csvStream.on('error', reject) - - csvStream.pipe(sink) - for (const row of rows) { - csvStream.write(row) - } - csvStream.end() - }) -} +const toDwcString = toDwcCsvString export const LOCATION_HEADERS = [ 'locationID', @@ -1381,9 +1327,9 @@ export const buildDwcLocalityArchiveZipBufferFromLocalities = async ( const geologicalContextRows = localities.map(mapLocalityToGeologicalContextRow) const measurementRows = localities.flatMap(mapLocalityToMeasurementRows) - const locationCsv = await writeCsvString([...LOCATION_HEADERS], locationRows) - const geologyCsv = await writeCsvString([...GEOLOGICAL_CONTEXT_HEADERS], geologicalContextRows) - const measurementCsv = await writeCsvString([...LOCALITY_MEASUREMENT_HEADERS], measurementRows) + const locationCsv = writeDwcCsvString(LOCATION_HEADERS, locationRows) + const geologyCsv = writeDwcCsvString(GEOLOGICAL_CONTEXT_HEADERS, geologicalContextRows) + const measurementCsv = writeDwcCsvString(LOCALITY_MEASUREMENT_HEADERS, measurementRows) const metaXml = buildLocalityMetaXml() const publicationDateIso = new Date().toISOString().slice(0, 10) diff --git a/backend/src/services/dwcArchiveExportOccurrences.ts b/backend/src/services/dwcArchiveExportOccurrences.ts index d1721359..27c601ea 100644 --- a/backend/src/services/dwcArchiveExportOccurrences.ts +++ b/backend/src/services/dwcArchiveExportOccurrences.ts @@ -1,11 +1,8 @@ import Prisma from '../../prisma/generated/now_test_client' -import { format } from 'fast-csv' -import { createReadStream, createWriteStream } from 'fs' +import { createReadStream } from 'fs' import { mkdtemp, rm } from 'fs/promises' import { tmpdir } from 'os' import path from 'path' -import { once } from 'events' -import { Writable } from 'stream' import JSZip from 'jszip' import { GEOLOGICAL_CONTEXT_HEADERS, @@ -14,40 +11,7 @@ import { mapLocalityToLocationRow, } from './dwcArchiveExportLocalities' import { MEASUREMENT_HEADERS, TAXON_HEADERS, mapSpeciesToTaxonRow, type MeasurementCsvRow } from './dwcArchiveExport' - -const writeCsvString = async (headers: string[], rows: Array>): Promise => { - if (rows.length === 0) { - return `${headers.map(header => `"${header.replace(/"/g, '""')}"`).join(',')}\n` - } - - return await new Promise((resolve, reject) => { - let output = '' - const csvStream = format({ - delimiter: ',', - headers, - quoteColumns: true, - quoteHeaders: true, - includeEndRowDelimiter: true, - }) - - const sink = new Writable({ - write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) { - output += typeof chunk === 'string' ? chunk : chunk.toString('utf8') - callback() - }, - }) - - sink.on('finish', () => resolve(output)) - sink.on('error', reject) - csvStream.on('error', reject) - - csvStream.pipe(sink) - for (const row of rows) { - csvStream.write(row) - } - csvStream.end() - }) -} +import { createDwcCsvFileWriter, toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv' const isMeaningfulString = (value: unknown): value is string => { if (typeof value !== 'string') return false @@ -57,18 +21,7 @@ const isMeaningfulString = (value: unknown): value is string => { return true } -const toDwcString = (value: unknown): string => { - if (value === null || value === undefined) return '' - if (typeof value === 'bigint') return value.toString() - if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : '' - if (typeof value === 'boolean') return value ? 'true' : 'false' - if (typeof value === 'string') return value - if (typeof value === 'object' && typeof (value as { toString?: unknown }).toString === 'function') { - const asString = (value as { toString: () => string }).toString() - return asString === '[object Object]' ? '' : asString - } - return '' -} +const toDwcString = toDwcCsvString const toMaybeMeaningful = (value: string | null | undefined): string => (isMeaningfulString(value) ? value.trim() : '') @@ -432,35 +385,6 @@ const localityLookupSelect = { const speciesLookupSelect = occurrenceSelect.com_species.select -const csvCell = (value: unknown): string => `"${toDwcString(value).replace(/"/g, '""')}"` - -const csvLine = (headers: readonly string[], row: Record): string => - `${headers.map(header => csvCell(row[header])).join(',')}\n` - -const createCsvFileWriter = async (filePath: string, headers: readonly string[]) => { - const stream = createWriteStream(filePath, { encoding: 'utf8' }) - await new Promise((resolve, reject) => { - stream.once('open', () => resolve()) - stream.once('error', reject) - }) - - const write = async (line: string): Promise => { - if (!stream.write(line)) await once(stream, 'drain') - } - - await write(`${headers.map(csvCell).join(',')}\n`) - - return { - writeRow: async (row: Record): Promise => { - await write(csvLine(headers, row)) - }, - close: async (): Promise => { - stream.end() - await once(stream, 'finish') - }, - } -} - async function* iterateOccurrenceRows(): AsyncGenerator { const { nowDb } = await import('../utils/db') let cursor: { lid: number; species_id: number } | undefined @@ -600,17 +524,14 @@ export const buildDwcOccurrenceArchiveZipBufferFromOccurrences = async ( species => species.species_id.toString() ) - const locationCsv = await writeCsvString([...LOCATION_HEADERS], localities.map(mapLocalityToLocationRow)) - const geologicalContextCsv = await writeCsvString( - [...GEOLOGICAL_CONTEXT_HEADERS], + const locationCsv = writeDwcCsvString(LOCATION_HEADERS, localities.map(mapLocalityToLocationRow)) + const geologicalContextCsv = writeDwcCsvString( + GEOLOGICAL_CONTEXT_HEADERS, localities.map(mapLocalityToGeologicalContextRow) ) - const taxonCsv = await writeCsvString([...TAXON_HEADERS], speciesRows.map(mapSpeciesToTaxonRow)) - const occurrenceCsv = await writeCsvString([...OCCURRENCE_HEADERS], occurrences.map(mapOccurrenceToOccurrenceRow)) - const measurementCsv = await writeCsvString( - [...MEASUREMENT_HEADERS], - occurrences.flatMap(mapOccurrenceToMeasurementRows) - ) + const taxonCsv = writeDwcCsvString(TAXON_HEADERS, speciesRows.map(mapSpeciesToTaxonRow)) + const occurrenceCsv = writeDwcCsvString(OCCURRENCE_HEADERS, occurrences.map(mapOccurrenceToOccurrenceRow)) + const measurementCsv = writeDwcCsvString(MEASUREMENT_HEADERS, occurrences.flatMap(mapOccurrenceToMeasurementRows)) const metaXml = buildOccurrenceMetaXml() const emlXml = buildOccurrenceEmlXml(new Date().toISOString().slice(0, 10)) @@ -635,8 +556,8 @@ const writeOccurrenceAndMeasurementFiles = async ({ measurementFilePath: string reportProgress?: DwcOccurrenceExportProgressReporter }): Promise<{ localityIds: number[]; speciesIds: number[] }> => { - const occurrenceWriter = await createCsvFileWriter(occurrenceFilePath, OCCURRENCE_HEADERS) - const measurementWriter = await createCsvFileWriter(measurementFilePath, MEASUREMENT_HEADERS) + const occurrenceWriter = await createDwcCsvFileWriter(occurrenceFilePath, OCCURRENCE_HEADERS) + const measurementWriter = await createDwcCsvFileWriter(measurementFilePath, MEASUREMENT_HEADERS) const localityIds = new Set() const speciesIds = new Set() const totalOccurrences = await countOccurrenceRows() @@ -692,8 +613,8 @@ const writeLocalityLookupFiles = async ({ reportProgress?: DwcOccurrenceExportProgressReporter }): Promise => { const { nowDb } = await import('../utils/db') - const locationWriter = await createCsvFileWriter(locationFilePath, LOCATION_HEADERS) - const geologicalContextWriter = await createCsvFileWriter(geologicalContextFilePath, GEOLOGICAL_CONTEXT_HEADERS) + const locationWriter = await createDwcCsvFileWriter(locationFilePath, LOCATION_HEADERS) + const geologicalContextWriter = await createDwcCsvFileWriter(geologicalContextFilePath, GEOLOGICAL_CONTEXT_HEADERS) let generatedLocalities = 0 reportProgress?.({ @@ -741,7 +662,7 @@ const writeTaxonLookupFile = async ({ reportProgress?: DwcOccurrenceExportProgressReporter }): Promise => { const { nowDb } = await import('../utils/db') - const taxonWriter = await createCsvFileWriter(taxonFilePath, TAXON_HEADERS) + const taxonWriter = await createDwcCsvFileWriter(taxonFilePath, TAXON_HEADERS) let generatedTaxa = 0 reportProgress?.({ diff --git a/backend/src/services/dwcDataPackageExport.ts b/backend/src/services/dwcDataPackageExport.ts index 7ecb030b..063d053d 100644 --- a/backend/src/services/dwcDataPackageExport.ts +++ b/backend/src/services/dwcDataPackageExport.ts @@ -1,5 +1,3 @@ -import { format } from 'fast-csv' -import { Writable } from 'stream' import JSZip from 'jszip' import { getContinentByCountry } from '../../../frontend/src/shared/validators/countryContinents' import { @@ -12,37 +10,8 @@ import { mapOccurrenceToOccurrenceRow, type OccurrenceCsvRow, } from './dwcArchiveExportOccurrences' -import type { MeasurementCsvRow } from './dwcArchiveExport' - -const writeCsvString = async (headers: string[], rows: Array>): Promise => { - return await new Promise((resolve, reject) => { - let output = '' - const csvStream = format({ - delimiter: ',', - headers, - quoteColumns: true, - quoteHeaders: true, - includeEndRowDelimiter: true, - }) - - const sink = new Writable({ - write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) { - output += typeof chunk === 'string' ? chunk : chunk.toString('utf8') - callback() - }, - }) - - sink.on('finish', () => resolve(output)) - sink.on('error', reject) - csvStream.on('error', reject) - - csvStream.pipe(sink) - for (const row of rows) { - csvStream.write(row) - } - csvStream.end() - }) -} +import { buildDwcArchiveZipBuffer, type MeasurementCsvRow } from './dwcArchiveExport' +import { writeDwcCsvString } from './utils/dwcCsv' const isMeaningfulString = (value: unknown): value is string => { if (typeof value !== 'string') return false @@ -642,16 +611,13 @@ export const buildDwcDataPackageZipBufferFromRows = async ({ const occurrenceAssertionRows = occurrences.flatMap(mapOccurrenceToDwcDpOccurrenceAssertionRows) const zip = new JSZip() - zip.file(DWC_DP_TABLES.event, await writeCsvString([...DWC_DP_EVENT_HEADERS], eventRows)) - zip.file( - DWC_DP_TABLES.geologicalContext, - await writeCsvString([...DWC_DP_GEOLOGICAL_CONTEXT_HEADERS], geologicalContextRows) - ) - zip.file(DWC_DP_TABLES.occurrence, await writeCsvString([...DWC_DP_OCCURRENCE_HEADERS], occurrenceRows)) - zip.file(DWC_DP_TABLES.eventAssertion, await writeCsvString([...DWC_DP_EVENT_ASSERTION_HEADERS], eventAssertionRows)) + zip.file(DWC_DP_TABLES.event, writeDwcCsvString(DWC_DP_EVENT_HEADERS, eventRows)) + zip.file(DWC_DP_TABLES.geologicalContext, writeDwcCsvString(DWC_DP_GEOLOGICAL_CONTEXT_HEADERS, geologicalContextRows)) + zip.file(DWC_DP_TABLES.occurrence, writeDwcCsvString(DWC_DP_OCCURRENCE_HEADERS, occurrenceRows)) + zip.file(DWC_DP_TABLES.eventAssertion, writeDwcCsvString(DWC_DP_EVENT_ASSERTION_HEADERS, eventAssertionRows)) zip.file( DWC_DP_TABLES.occurrenceAssertion, - await writeCsvString([...DWC_DP_OCCURRENCE_ASSERTION_HEADERS], occurrenceAssertionRows) + writeDwcCsvString(DWC_DP_OCCURRENCE_ASSERTION_HEADERS, occurrenceAssertionRows) ) zip.file(DWC_DP_TABLES.dataPackage, buildDwcDataPackageJson(publicationDateIso)) zip.file(DWC_DP_TABLES.eml, buildDwcDataPackageEmlXml(publicationDateIso)) @@ -677,5 +643,83 @@ export const buildDwcDataPackageZipBuffer = async (): Promise => { }) } +const addZipEntriesUnderPrefix = async ({ + sourceZipBuffer, + targetZip, + prefix, +}: { + sourceZipBuffer: Buffer + targetZip: JSZip + prefix: string +}): Promise => { + const sourceZip = await JSZip.loadAsync(sourceZipBuffer) + const files = Object.values(sourceZip.files).filter(file => !file.dir) + + await Promise.all( + files.map(async file => { + targetZip.file(`${prefix}/${file.name}`, await file.async('nodebuffer')) + }) + ) +} + +const buildFullDarwinCoreReadme = (): string => `NOW database Darwin Core full test export + +This ZIP is a convenience bundle. It contains two separate standards-based export +artifacts in their own folders: + +- dwc-dp/ + Darwin Core Data Package test export for relational event, geological context, + occurrence, event assertion, and occurrence assertion data. + +- dwc-a-taxa/ + Darwin Core Archive test export for taxon records and synthesized taxon-level + traits. The traits remain in DwC-A Taxon + MeasurementOrFact form because they + are not currently linked to source specimen/material records. + +Join key: + +- dwc-dp/occurrence.csv taxonID uses NOW: +- dwc-a-taxa/taxon.csv taxonID uses the same NOW: +- dwc-a-taxa/measurementorfact.csv links taxon traits by that same taxonID + +The outer ZIP is not itself a single DwC-DP or DwC-A artifact. Each subfolder is +intended to remain internally understandable as its own export format. +` + +export const buildFullDarwinCoreExportZipBufferFromArchives = async ({ + dwcDataPackageZipBuffer, + dwcTaxonArchiveZipBuffer, +}: { + dwcDataPackageZipBuffer: Buffer + dwcTaxonArchiveZipBuffer: Buffer +}): Promise => { + const zip = new JSZip() + zip.file('README.txt', buildFullDarwinCoreReadme()) + await addZipEntriesUnderPrefix({ + sourceZipBuffer: dwcDataPackageZipBuffer, + targetZip: zip, + prefix: 'dwc-dp', + }) + await addZipEntriesUnderPrefix({ + sourceZipBuffer: dwcTaxonArchiveZipBuffer, + targetZip: zip, + prefix: 'dwc-a-taxa', + }) + + return await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 6 } }) +} + +export const buildFullDarwinCoreExportZipBuffer = async (): Promise => { + const [dwcDataPackageZipBuffer, dwcTaxonArchiveZipBuffer] = await Promise.all([ + buildDwcDataPackageZipBuffer(), + buildDwcArchiveZipBuffer(), + ]) + + return await buildFullDarwinCoreExportZipBufferFromArchives({ + dwcDataPackageZipBuffer, + dwcTaxonArchiveZipBuffer, + }) +} + export type DwcDpLocalityFixture = LocalityForDwcDpExport export type DwcDpOccurrenceFixture = OccurrenceForDwcDpExport diff --git a/backend/src/services/utils/dwcCsv.ts b/backend/src/services/utils/dwcCsv.ts new file mode 100644 index 00000000..d6c8a060 --- /dev/null +++ b/backend/src/services/utils/dwcCsv.ts @@ -0,0 +1,62 @@ +import { createWriteStream } from 'fs' +import { once } from 'events' + +export const normalizeDwcCsvValue = (value: string): string => value.replace(/\r\n|\r|\n/g, ' ').replace(/[ \t]+/g, ' ') + +export const toDwcCsvString = (value: unknown): string => { + if (value === null || value === undefined) return '' + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : '' + if (typeof value === 'boolean') return value ? 'true' : 'false' + if (typeof value === 'string') return normalizeDwcCsvValue(value) + if (value instanceof Date) return value.toISOString() + if (typeof value === 'object') { + if (typeof (value as { toString?: unknown }).toString === 'function') { + const asString = (value as { toString: () => string }).toString() + if (asString && asString !== '[object Object]') return normalizeDwcCsvValue(asString) + } + try { + return normalizeDwcCsvValue(JSON.stringify(value) ?? '') + } catch { + return '' + } + } + return '' +} + +export const dwcCsvCell = (value: unknown): string => `"${toDwcCsvString(value).replace(/"/g, '""')}"` + +export const dwcCsvLine = (headers: readonly string[], row: Record): string => + `${headers.map(header => dwcCsvCell(row[header])).join(',')}\n` + +export const writeDwcCsvString = (headers: readonly string[], rows: Array>): string => { + const lines = [`${headers.map(dwcCsvCell).join(',')}\n`] + for (const row of rows) { + lines.push(dwcCsvLine(headers, row)) + } + return lines.join('') +} + +export const createDwcCsvFileWriter = async (filePath: string, headers: readonly string[]) => { + const stream = createWriteStream(filePath, { encoding: 'utf8' }) + await new Promise((resolve, reject) => { + stream.once('open', () => resolve()) + stream.once('error', reject) + }) + + const write = async (line: string): Promise => { + if (!stream.write(line)) await once(stream, 'drain') + } + + await write(`${headers.map(dwcCsvCell).join(',')}\n`) + + return { + writeRow: async (row: Record): Promise => { + await write(dwcCsvLine(headers, row)) + }, + close: async (): Promise => { + stream.end() + await once(stream, 'finish') + }, + } +} diff --git a/backend/src/unit-tests/dwcCsv.test.ts b/backend/src/unit-tests/dwcCsv.test.ts new file mode 100644 index 00000000..4d5a185d --- /dev/null +++ b/backend/src/unit-tests/dwcCsv.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from '@jest/globals' +import { dwcCsvCell, normalizeDwcCsvValue, writeDwcCsvString } from '../services/utils/dwcCsv' + +describe('DwC CSV writer', () => { + it('normalizes embedded line feeds before quoting cells', () => { + expect(normalizeDwcCsvValue('first\nsecond\r\nthird\rfourth')).toEqual('first second third fourth') + expect(dwcCsvCell('first\n"second"')).toEqual('"first ""second"""') + }) + + it('writes one physical line per row even when source values contain line breaks', () => { + const csv = writeDwcCsvString( + ['id', 'remarks'], + [ + { id: 'one', remarks: 'alpha\nbeta' }, + { id: 'two', remarks: 'gamma\r\ndelta' }, + ] + ) + + expect(csv).toEqual('"id","remarks"\n"one","alpha beta"\n"two","gamma delta"\n') + expect(csv.split('\n')).toHaveLength(4) + }) +}) diff --git a/backend/src/unit-tests/dwcDataPackageExport.test.ts b/backend/src/unit-tests/dwcDataPackageExport.test.ts index 051a82d0..b1b05fd2 100644 --- a/backend/src/unit-tests/dwcDataPackageExport.test.ts +++ b/backend/src/unit-tests/dwcDataPackageExport.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals' import JSZip from 'jszip' import { DWC_DP_TABLES, + buildFullDarwinCoreExportZipBufferFromArchives, buildDwcDataPackageZipBufferFromRows, mapLocalityToDwcDpEventAssertionRows, mapLocalityToDwcDpEventRow, @@ -161,8 +162,12 @@ describe('DwC-DP export mapping', () => { }) it('generates a ZIP archive with DwC-DP files and relational metadata', async () => { + const multilineLocality = { + ...locality, + loc_detail: 'Some notes\nwith a line feed', + } as DwcDpLocalityFixture const zipBuffer = await buildDwcDataPackageZipBufferFromRows({ - localities: [locality], + localities: [multilineLocality], occurrences: [occurrence], publicationDateIso: '2026-05-13', }) @@ -176,6 +181,10 @@ describe('DwC-DP export mapping', () => { expect(zip.file(DWC_DP_TABLES.dataPackage)).toBeTruthy() expect(zip.file(DWC_DP_TABLES.eml)).toBeTruthy() + const eventCsv = await zip.file(DWC_DP_TABLES.event)!.async('string') + expect(eventCsv).toContain('Some notes with a line feed') + expect(eventCsv.trimEnd().split('\n')).toHaveLength(2) + const dataPackageJson = JSON.parse(await zip.file(DWC_DP_TABLES.dataPackage)!.async('string')) as { resources: Array<{ name: string; schema: { foreignKeys?: unknown[] } }> } @@ -193,4 +202,34 @@ describe('DwC-DP export mapping', () => { }, ]) }) + + it('combines DwC-DP and DwC-A taxon archives into a namespaced convenience bundle', async () => { + const dwcDataPackageZip = new JSZip() + dwcDataPackageZip.file('datapackage.json', '{}\n') + dwcDataPackageZip.file('event.csv', '"eventID"\n') + dwcDataPackageZip.file('occurrence.csv', '"occurrenceID","taxonID"\n') + + const dwcTaxonArchiveZip = new JSZip() + dwcTaxonArchiveZip.file('taxon.csv', '"taxonID"\n') + dwcTaxonArchiveZip.file('measurementorfact.csv', '"taxonID","measurementID"\n') + dwcTaxonArchiveZip.file('meta.xml', '\n') + + const fullZipBuffer = await buildFullDarwinCoreExportZipBufferFromArchives({ + dwcDataPackageZipBuffer: await dwcDataPackageZip.generateAsync({ type: 'nodebuffer' }), + dwcTaxonArchiveZipBuffer: await dwcTaxonArchiveZip.generateAsync({ type: 'nodebuffer' }), + }) + + const fullZip = await JSZip.loadAsync(fullZipBuffer) + expect(fullZip.file('README.txt')).toBeTruthy() + expect(fullZip.file('dwc-dp/datapackage.json')).toBeTruthy() + expect(fullZip.file('dwc-dp/event.csv')).toBeTruthy() + expect(fullZip.file('dwc-dp/occurrence.csv')).toBeTruthy() + expect(fullZip.file('dwc-a-taxa/taxon.csv')).toBeTruthy() + expect(fullZip.file('dwc-a-taxa/measurementorfact.csv')).toBeTruthy() + expect(fullZip.file('dwc-a-taxa/meta.xml')).toBeTruthy() + + const readme = await fullZip.file('README.txt')!.async('string') + expect(readme).toContain('dwc-dp/occurrence.csv taxonID') + expect(readme).toContain('dwc-a-taxa/taxon.csv taxonID') + }) }) diff --git a/documentation/functionality/dwc_data_package.md b/documentation/functionality/dwc_data_package.md index 9aaf412e..2a25b47a 100644 --- a/documentation/functionality/dwc_data_package.md +++ b/documentation/functionality/dwc_data_package.md @@ -9,7 +9,8 @@ DwC-DP is intentionally published as a separate format from the existing DwC-A e ## Access - Backend route: `GET /occurrence/export/dwc-data-package` (**Role.Admin only**) -- The frontend exposes this as an export option on the cross-search occurrence table for administrators. +- Backend route: `GET /occurrence/export/dwc-full-package` (**Role.Admin only**) +- The frontend exposes both as export options on the cross-search occurrence table for administrators. ## Output @@ -23,6 +24,30 @@ The downloaded ZIP contains: - `occurrence-assertion.csv` (occurrence-level facts formerly expressed as occurrence `MeasurementOrFact`) - `eml.xml` (minimal placeholder EML metadata) +## Full Darwin Core Convenience Bundle + +The full export route downloads one outer ZIP for users who need both occurrence/event data and taxon traits. + +The outer ZIP contains: + +- `README.txt` +- `dwc-dp/` with the DwC-DP event + occurrence package described above +- `dwc-a-taxa/` with the existing DwC-A taxon trait archive: + - `taxon.csv` + - `measurementorfact.csv` + - `meta.xml` + - `eml.xml` + +The outer ZIP is a NOW convenience bundle, not a single hybrid standard artifact. Each subfolder remains its own internally described export format. + +For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line. + +Join key: + +- `dwc-dp/occurrence.csv` uses `taxonID = NOW:` +- `dwc-a-taxa/taxon.csv` uses the same `taxonID` +- `dwc-a-taxa/measurementorfact.csv` links synthesized taxon traits by the same `taxonID` + ## Model ### `event.csv` diff --git a/documentation/functionality/dwc_export.md b/documentation/functionality/dwc_export.md index e1462a81..de8adbed 100644 --- a/documentation/functionality/dwc_export.md +++ b/documentation/functionality/dwc_export.md @@ -4,6 +4,8 @@ Issue: `nowcommunity/nowdatabase#1150` This repository includes an **admin-only** Darwin Core Archive (DwC-A) export intended for initial testing. +For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line. + ## Access - Backend route: `GET /species/export/dwc-archive` (**Role.Admin only**) diff --git a/documentation/functionality/dwc_export_localities.md b/documentation/functionality/dwc_export_localities.md index 4606eadb..42571715 100644 --- a/documentation/functionality/dwc_export_localities.md +++ b/documentation/functionality/dwc_export_localities.md @@ -2,6 +2,8 @@ This document describes the admin-only Darwin Core Archive (DwC-A) test export for localities. +For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line. + ## Files The export ZIP contains: diff --git a/documentation/functionality/dwc_export_occurrences.md b/documentation/functionality/dwc_export_occurrences.md index fe38347c..214bf1d0 100644 --- a/documentation/functionality/dwc_export_occurrences.md +++ b/documentation/functionality/dwc_export_occurrences.md @@ -2,6 +2,8 @@ This document describes the admin-only Darwin Core Archive (DwC-A) test export for occurrence records (`now_ls`). +For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line. + ## Files The export ZIP contains: diff --git a/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx b/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx index 766b0b89..dd014cad 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx @@ -5,6 +5,7 @@ import { MenuItem } from '@mui/material' import { useUser } from '@/hooks/user' import { usePageContext } from '../Page' import { currentDateAsString } from '@/shared/currentDateAsString' +import { downloadExportFileWithProgress } from '@/util/exportProgress' export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) @@ -21,46 +22,21 @@ export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => const URIColumnFilters = encodeURIComponent(JSON.stringify(sqlColumnFilters)) const URIOrderBy = encodeURIComponent(JSON.stringify(sqlOrderBy)) - notify('Generating CSV file, please wait...', 'info', null) - const response = await fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions) - - if (!response.ok) { - throw new Error('Server response was not OK.') - } - const reader = response.body!.getReader() - const file: Uint8Array[] = [] - let bytes = 0 - let closed = false - - const showDownloadProgress = () => { - if (!closed) { - setTimeout(() => { - setNotificationMessage(`Downloading CSV file, ${Math.round((bytes / 1000000) * 10) / 10} MB`) - showDownloadProgress() - }, 500) - } - } - notify(`Downloading CSV file...`, 'info', null) - showDownloadProgress() - while (true) { - const { done, value } = await reader.read() - if (done) break - bytes = bytes + value.length - file.push(value) - } - closed = true try { - const blobUrl = window.URL.createObjectURL(new Blob(file)) - const downloadLink = document.createElement('a') - downloadLink.href = blobUrl - downloadLink.download = filename - document.body.appendChild(downloadLink) - downloadLink.click() - downloadLink.remove() - window.URL.revokeObjectURL(blobUrl) - notify('Download finished.') + await downloadExportFileWithProgress({ + url: `${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, + filename, + fetchOptions, + notify, + setNotificationMessage, + startMessage: 'Generating cross-search CSV export...', + waitingMessage: 'Waiting for cross-search rows', + downloadMessage: 'Downloading CSV file', + failureMessage: 'Downloading cross-search export failed.', + contentType: 'text/csv', + }) } catch { - notify('Downloading cross-search export failed.', 'error') + // downloadExportFileWithProgress owns the failure notification. } finally { setLoading(false) } diff --git a/frontend/src/components/CrossSearch/CrossSearchTable.tsx b/frontend/src/components/CrossSearch/CrossSearchTable.tsx index 12cfddbc..110364d6 100755 --- a/frontend/src/components/CrossSearch/CrossSearchTable.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchTable.tsx @@ -10,6 +10,7 @@ import { formatWithMaxThreeDecimals } from '@/util/numberFormatting' import { occurrenceLabels } from '@/constants/occurrenceLabels' import { OccurrenceDwcExportMenuItem } from '@/components/Occurrence/OccurrenceDwcExportMenuItem' import { OccurrenceDwcDpExportMenuItem } from '@/components/Occurrence/OccurrenceDwcDpExportMenuItem' +import { OccurrenceFullDarwinCoreExportMenuItem } from '@/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem' import { matchesCountryOrContinent } from '@/shared/validators/countryContinents' export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: CrossSearch) => void }) => { @@ -1095,6 +1096,7 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros <> + )} /> diff --git a/frontend/src/components/Locality/LocalityDwcExportMenuItem.tsx b/frontend/src/components/Locality/LocalityDwcExportMenuItem.tsx index 25f60135..b05490ee 100644 --- a/frontend/src/components/Locality/LocalityDwcExportMenuItem.tsx +++ b/frontend/src/components/Locality/LocalityDwcExportMenuItem.tsx @@ -5,6 +5,7 @@ import { BACKEND_URL } from '@/util/config' import { useUser } from '@/hooks/user' import { Role } from '@/shared/types' import { currentDateAsString } from '@/shared/currentDateAsString' +import { downloadExportFileWithProgress } from '@/util/exportProgress' export const LocalityDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) @@ -20,55 +21,22 @@ export const LocalityDwcExportMenuItem = ({ handleClose }: { handleClose: () => const fetchZipFile = async () => { setLoading(true) - notify('Generating DwC-A ZIP export, please wait...', 'info', null) try { - const response = await fetch(`${BACKEND_URL}/locality/export/dwc-archive`, fetchOptions) - if (!response.ok) { - throw new Error('Server response was not OK.') - } - - const reader = response.body?.getReader() - if (!reader) { - throw new Error('Missing response stream.') - } - - const file: Uint8Array[] = [] - let bytes = 0 - let closed = false - - const showDownloadProgress = () => { - if (!closed) { - setTimeout(() => { - setNotificationMessage(`Downloading DwC-A ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`) - showDownloadProgress() - }, 500) - } - } - - notify('Downloading DwC-A ZIP...', 'info', null) - showDownloadProgress() - - while (true) { - const { done, value } = await reader.read() - if (done) break - bytes = bytes + value.length - file.push(value) - } - closed = true - - const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' })) - const downloadLink = document.createElement('a') - downloadLink.href = blobUrl - downloadLink.download = filename - document.body.appendChild(downloadLink) - downloadLink.click() - downloadLink.remove() - window.URL.revokeObjectURL(blobUrl) - - notify('Download finished.') + await downloadExportFileWithProgress({ + url: `${BACKEND_URL}/locality/export/dwc-archive`, + filename, + fetchOptions, + notify, + setNotificationMessage, + startMessage: 'Generating DwC-A locality ZIP export...', + waitingMessage: 'Waiting for locality export rows', + downloadMessage: 'Downloading DwC-A locality ZIP', + failureMessage: 'Downloading DwC-A export failed.', + contentType: 'application/zip', + }) } catch { - notify('Downloading DwC-A export failed.', 'error') + // downloadExportFileWithProgress owns the failure notification. } finally { setLoading(false) } diff --git a/frontend/src/components/Locality/LocalityTable.tsx b/frontend/src/components/Locality/LocalityTable.tsx index efc158c9..e0734a7b 100755 --- a/frontend/src/components/Locality/LocalityTable.tsx +++ b/frontend/src/components/Locality/LocalityTable.tsx @@ -485,18 +485,16 @@ export const LocalityTable = ({ selectorFn }: { selectorFn?: (newObject: Localit a.click() } - const svgExport = (table: MRT_TableInstance) => { - void (async () => { - const rowData: Locality[] = table.getPrePaginationRowModel().rows.map(row => row.original as unknown as Locality) - const { generateSvg } = await import('../Map/generateSvg') - const dataString = generateSvg(rowData) - const blob = new Blob([dataString], { type: 'image/svg+xml' }) - const url = URL.createObjectURL(blob) - const a = document.createElement('a') - a.href = url - a.download = `localities-map-${currentDateAsString()}.svg` - a.click() - })() + const svgExport = async (table: MRT_TableInstance) => { + const rowData: Locality[] = table.getPrePaginationRowModel().rows.map(row => row.original as unknown as Locality) + const { generateSvg } = await import('../Map/generateSvg') + const dataString = generateSvg(rowData) + const blob = new Blob([dataString], { type: 'image/svg+xml' }) + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = `localities-map-${currentDateAsString()}.svg` + a.click() } const checkRowRestriction = (row: Locality) => { diff --git a/frontend/src/components/Occurrence/OccurrenceDwcDpExportMenuItem.tsx b/frontend/src/components/Occurrence/OccurrenceDwcDpExportMenuItem.tsx index f48036e5..1d240e51 100644 --- a/frontend/src/components/Occurrence/OccurrenceDwcDpExportMenuItem.tsx +++ b/frontend/src/components/Occurrence/OccurrenceDwcDpExportMenuItem.tsx @@ -5,6 +5,7 @@ import { BACKEND_URL } from '@/util/config' import { useUser } from '@/hooks/user' import { Role } from '@/shared/types' import { currentDateAsString } from '@/shared/currentDateAsString' +import { downloadExportFileWithProgress } from '@/util/exportProgress' export const OccurrenceDwcDpExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) @@ -20,55 +21,22 @@ export const OccurrenceDwcDpExportMenuItem = ({ handleClose }: { handleClose: () const fetchZipFile = async () => { setLoading(true) - notify('Generating DwC-DP ZIP export, please wait...', 'info', null) try { - const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-data-package`, fetchOptions) - if (!response.ok) { - throw new Error('Server response was not OK.') - } - - const reader = response.body?.getReader() - if (!reader) { - throw new Error('Missing response stream.') - } - - const file: Uint8Array[] = [] - let bytes = 0 - let closed = false - - const showDownloadProgress = () => { - if (!closed) { - setTimeout(() => { - setNotificationMessage(`Downloading DwC-DP ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`) - showDownloadProgress() - }, 500) - } - } - - notify('Downloading DwC-DP ZIP...', 'info', null) - showDownloadProgress() - - while (true) { - const { done, value } = await reader.read() - if (done) break - bytes = bytes + value.length - file.push(value) - } - closed = true - - const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' })) - const downloadLink = document.createElement('a') - downloadLink.href = blobUrl - downloadLink.download = filename - document.body.appendChild(downloadLink) - downloadLink.click() - downloadLink.remove() - window.URL.revokeObjectURL(blobUrl) - - notify('Download finished.') + await downloadExportFileWithProgress({ + url: `${BACKEND_URL}/occurrence/export/dwc-data-package`, + filename, + fetchOptions, + notify, + setNotificationMessage, + startMessage: 'Generating DwC-DP ZIP export...', + waitingMessage: 'Waiting for DwC-DP event and occurrence rows', + downloadMessage: 'Downloading DwC-DP ZIP', + failureMessage: 'Downloading DwC-DP export failed.', + contentType: 'application/zip', + }) } catch { - notify('Downloading DwC-DP export failed.', 'error') + // downloadExportFileWithProgress owns the failure notification. } finally { setLoading(false) } diff --git a/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx b/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx index 92b5f434..ee429b8e 100644 --- a/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx +++ b/frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx @@ -5,15 +5,7 @@ import { BACKEND_URL } from '@/util/config' import { useUser } from '@/hooks/user' import { Role } from '@/shared/types' import { currentDateAsString } from '@/shared/currentDateAsString' - -type OccurrenceExportProgress = { - message: string -} - -const createExportId = () => { - if (window.crypto?.randomUUID) return window.crypto.randomUUID() - return `${Date.now()}-${Math.random().toString(36).slice(2)}` -} +import { createExportId, downloadExportFileWithProgress } from '@/util/exportProgress' export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) @@ -30,85 +22,22 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () = const fetchZipFile = async () => { setLoading(true) const exportId = createExportId() - let generationProgressTimer: number | undefined - notify('Generating DwC-A ZIP export...', 'info', null) - - const updateGenerationProgress = async () => { - try { - const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`, fetchOptions) - if (!response.ok) return - - const progress = (await response.json()) as OccurrenceExportProgress - setNotificationMessage(progress.message) - } catch { - // The download request owns the final success/failure notification. - } - } - - const stopGenerationProgress = () => { - if (generationProgressTimer !== undefined) { - window.clearInterval(generationProgressTimer) - generationProgressTimer = undefined - } - } - try { - generationProgressTimer = window.setInterval(() => { - void updateGenerationProgress() - }, 1000) - - const response = await fetch( - `${BACKEND_URL}/occurrence/export/dwc-archive?${new URLSearchParams({ exportId })}`, - fetchOptions - ) - stopGenerationProgress() - - if (!response.ok) { - throw new Error('Server response was not OK.') - } - - const reader = response.body?.getReader() - if (!reader) { - throw new Error('Missing response stream.') - } - - const file: Uint8Array[] = [] - let bytes = 0 - let closed = false - - const showDownloadProgress = () => { - if (!closed) { - setTimeout(() => { - setNotificationMessage(`Downloading DwC-A ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`) - showDownloadProgress() - }, 500) - } - } - - notify('Downloading DwC-A ZIP...', 'info', null) - showDownloadProgress() - - while (true) { - const { done, value } = await reader.read() - if (done) break - bytes = bytes + value.length - file.push(value) - } - closed = true - - const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' })) - const downloadLink = document.createElement('a') - downloadLink.href = blobUrl - downloadLink.download = filename - document.body.appendChild(downloadLink) - downloadLink.click() - downloadLink.remove() - window.URL.revokeObjectURL(blobUrl) - - notify('Download finished.') + await downloadExportFileWithProgress({ + url: `${BACKEND_URL}/occurrence/export/dwc-archive?${new URLSearchParams({ exportId })}`, + progressUrl: `${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`, + filename, + fetchOptions, + notify, + setNotificationMessage, + startMessage: 'Generating DwC-A occurrence ZIP export...', + waitingMessage: 'Waiting for occurrence export rows', + downloadMessage: 'Downloading DwC-A occurrence ZIP', + failureMessage: 'Downloading DwC-A export failed.', + contentType: 'application/zip', + }) } catch { - stopGenerationProgress() - notify('Downloading DwC-A export failed.', 'error') + // downloadExportFileWithProgress owns the failure notification. } finally { setLoading(false) } diff --git a/frontend/src/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem.tsx b/frontend/src/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem.tsx new file mode 100644 index 00000000..127d3eb1 --- /dev/null +++ b/frontend/src/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem.tsx @@ -0,0 +1,56 @@ +import { useState } from 'react' +import { MenuItem } from '@mui/material' +import { useNotify } from '@/hooks/notification' +import { BACKEND_URL } from '@/util/config' +import { useUser } from '@/hooks/user' +import { Role } from '@/shared/types' +import { currentDateAsString } from '@/shared/currentDateAsString' +import { downloadExportFileWithProgress } from '@/util/exportProgress' + +export const OccurrenceFullDarwinCoreExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { + const [loading, setLoading] = useState(false) + const { notify, setMessage: setNotificationMessage } = useNotify() + const user = useUser() + + if (user.role !== Role.Admin) { + return null + } + + const fetchOptions = user.token ? { headers: { Authorization: `Bearer ${user.token}` } } : {} + const filename = `now_dwc_full_test_export_${currentDateAsString()}.zip` + + const fetchZipFile = async () => { + setLoading(true) + + try { + await downloadExportFileWithProgress({ + url: `${BACKEND_URL}/occurrence/export/dwc-full-package`, + filename, + fetchOptions, + notify, + setNotificationMessage, + startMessage: 'Generating full Darwin Core ZIP export...', + waitingMessage: 'Waiting for DwC-DP and taxon trait archive files', + downloadMessage: 'Downloading full Darwin Core ZIP', + failureMessage: 'Downloading full Darwin Core export failed.', + contentType: 'application/zip', + }) + } catch { + // downloadExportFileWithProgress owns the failure notification. + } finally { + setLoading(false) + } + } + + return ( + { + void fetchZipFile() + handleClose() + }} + disabled={loading} + > + Export full Darwin Core package + + ) +} diff --git a/frontend/src/components/Species/SpeciesDwcExportMenuItem.tsx b/frontend/src/components/Species/SpeciesDwcExportMenuItem.tsx index 6707f5e5..4031c80e 100644 --- a/frontend/src/components/Species/SpeciesDwcExportMenuItem.tsx +++ b/frontend/src/components/Species/SpeciesDwcExportMenuItem.tsx @@ -5,6 +5,7 @@ import { BACKEND_URL } from '@/util/config' import { useUser } from '@/hooks/user' import { Role } from '@/shared/types' import { currentDateAsString } from '@/shared/currentDateAsString' +import { downloadExportFileWithProgress } from '@/util/exportProgress' export const SpeciesDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) @@ -20,55 +21,22 @@ export const SpeciesDwcExportMenuItem = ({ handleClose }: { handleClose: () => v const fetchZipFile = async () => { setLoading(true) - notify('Generating DwC-A ZIP export, please wait...', 'info', null) try { - const response = await fetch(`${BACKEND_URL}/species/export/dwc-archive`, fetchOptions) - if (!response.ok) { - throw new Error('Server response was not OK.') - } - - const reader = response.body?.getReader() - if (!reader) { - throw new Error('Missing response stream.') - } - - const file: Uint8Array[] = [] - let bytes = 0 - let closed = false - - const showDownloadProgress = () => { - if (!closed) { - setTimeout(() => { - setNotificationMessage(`Downloading DwC-A ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`) - showDownloadProgress() - }, 500) - } - } - - notify('Downloading DwC-A ZIP...', 'info', null) - showDownloadProgress() - - while (true) { - const { done, value } = await reader.read() - if (done) break - bytes = bytes + value.length - file.push(value) - } - closed = true - - const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' })) - const downloadLink = document.createElement('a') - downloadLink.href = blobUrl - downloadLink.download = filename - document.body.appendChild(downloadLink) - downloadLink.click() - downloadLink.remove() - window.URL.revokeObjectURL(blobUrl) - - notify('Download finished.') + await downloadExportFileWithProgress({ + url: `${BACKEND_URL}/species/export/dwc-archive`, + filename, + fetchOptions, + notify, + setNotificationMessage, + startMessage: 'Generating DwC-A taxon ZIP export...', + waitingMessage: 'Waiting for taxon export rows', + downloadMessage: 'Downloading DwC-A taxon ZIP', + failureMessage: 'Downloading DwC-A export failed.', + contentType: 'application/zip', + }) } catch { - notify('Downloading DwC-A export failed.', 'error') + // downloadExportFileWithProgress owns the failure notification. } finally { setLoading(false) } diff --git a/frontend/src/components/TableView/TableToolBar.tsx b/frontend/src/components/TableView/TableToolBar.tsx index 921d16b5..2b5843b2 100644 --- a/frontend/src/components/TableView/TableToolBar.tsx +++ b/frontend/src/components/TableView/TableToolBar.tsx @@ -10,6 +10,7 @@ import { Link, useLocation } from 'react-router-dom' import AddCircleIcon from '@mui/icons-material/AddCircle' import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportMenuItem' import { usePageContext } from '../Page' +import { useNotify } from '@/hooks/notification' export type ColumnVisibilityGroup = { id: string @@ -42,8 +43,8 @@ export const TableToolBar = ({ }: { table: MRT_TableInstance tableName: string - kmlExport?: (table: MRT_TableInstance) => void - svgExport?: (table: MRT_TableInstance) => void + kmlExport?: (table: MRT_TableInstance) => void | Promise + svgExport?: (table: MRT_TableInstance) => void | Promise isCrossSearchTable?: boolean selectorFn?: (id: T) => void showNewButton?: boolean @@ -52,6 +53,7 @@ export const TableToolBar = ({ renderExtraExportMenuItems?: (handleClose: () => void) => ReactNode }) => { const { previousTableUrls, setPreviousTableUrls } = usePageContext() + const { notify, setMessage: setNotificationMessage } = useNotify() const location = useLocation() const [anchorEl, setAnchorEl] = useState(null) const open = Boolean(anchorEl) @@ -183,6 +185,22 @@ export const TableToolBar = ({ column.toggleVisibility?.(!column.getIsVisible?.()) } + const exportVisibleTableRows = () => { + notify(`Preparing ${tableName} table CSV export...`, 'info', null) + const rowCount = exportRows(table, tableName) + setNotificationMessage(`Generated ${tableName} table CSV with ${rowCount} rows. Saving file...`) + notify('Download finished.') + } + + const runMapExport = async (label: string, exportFn: (table: MRT_TableInstance) => void | Promise) => { + notify(`Preparing ${label} export...`, 'info', null) + const rowCount = table.getPrePaginationRowModel().rows.length + setNotificationMessage(`Generating ${label} export from ${rowCount} rows...`) + await exportFn(table) + setNotificationMessage(`Saving ${label} export...`) + notify('Download finished.') + } + return (
{!selectorFn && !hideLeftButtons && ( @@ -291,7 +309,7 @@ export const TableToolBar = ({ ) : ( { - exportRows(table, tableName) + exportVisibleTableRows() handleClose() }} > @@ -304,7 +322,7 @@ export const TableToolBar = ({ {kmlExport && ( { - kmlExport(table) + void runMapExport('KML', kmlExport) handleClose() }} > @@ -315,7 +333,7 @@ export const TableToolBar = ({ {svgExport && ( { - svgExport(table) + void runMapExport('SVG map', svgExport) handleClose() }} > diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index 05723e1f..ebb92b8f 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -104,8 +104,8 @@ export const TableView = ({ getDetailPath?: (row: T) => string url?: string title: string - kmlExport?: (table: MRT_TableInstance) => void - svgExport?: (table: MRT_TableInstance) => void + kmlExport?: (table: MRT_TableInstance) => void | Promise + svgExport?: (table: MRT_TableInstance) => void | Promise isCrossSearchTable?: boolean clickableRows?: boolean enableColumnFilterModes?: boolean diff --git a/frontend/src/components/TableView/helpers.tsx b/frontend/src/components/TableView/helpers.tsx index 01502b9a..c374f992 100755 --- a/frontend/src/components/TableView/helpers.tsx +++ b/frontend/src/components/TableView/helpers.tsx @@ -29,4 +29,5 @@ export const exportRows = (table: MRT_TableInstance, t }) const csv = generateCsv(csvConfig)(rowData) download(csvConfig)(csv) + return rowData.length } diff --git a/frontend/src/util/exportProgress.ts b/frontend/src/util/exportProgress.ts new file mode 100644 index 00000000..89648894 --- /dev/null +++ b/frontend/src/util/exportProgress.ts @@ -0,0 +1,132 @@ +import { Severity } from '@/components/Notification' + +type Notify = (msg: string, severity?: Severity, timeoutValue?: number | null) => void +type ExportProgress = { + message: string +} + +type DownloadExportParams = { + url: string + filename: string + fetchOptions?: RequestInit + notify: Notify + setNotificationMessage: (message: string) => void + startMessage: string + waitingMessage: string + downloadMessage: string + failureMessage: string + contentType?: string + progressUrl?: string +} + +export const createExportId = () => { + if (window.crypto?.randomUUID) return window.crypto.randomUUID() + return `${Date.now()}-${Math.random().toString(36).slice(2)}` +} + +const formatMb = (bytes: number) => Math.round((bytes / 1000000) * 10) / 10 + +export const downloadExportFileWithProgress = async ({ + url, + filename, + fetchOptions = {}, + notify, + setNotificationMessage, + startMessage, + waitingMessage, + downloadMessage, + failureMessage, + contentType, + progressUrl, +}: DownloadExportParams) => { + let progressTimer: number | undefined + let waitingSeconds = 0 + + const stopProgress = () => { + if (progressTimer !== undefined) { + window.clearInterval(progressTimer) + progressTimer = undefined + } + } + + const updateServerProgress = async () => { + if (!progressUrl) return false + + try { + const response = await fetch(progressUrl, fetchOptions) + if (!response.ok) return false + + const progress = (await response.json()) as ExportProgress + setNotificationMessage(progress.message) + return true + } catch { + return false + } + } + + notify(startMessage, 'info', null) + + try { + progressTimer = window.setInterval(() => { + waitingSeconds += 1 + void (async () => { + const hasServerProgress = await updateServerProgress() + if (!hasServerProgress) { + setNotificationMessage(`${waitingMessage} (${waitingSeconds}s)`) + } + })() + }, 1000) + + const response = await fetch(url, fetchOptions) + stopProgress() + + if (!response.ok) { + throw new Error('Server response was not OK.') + } + + const reader = response.body?.getReader() + if (!reader) { + throw new Error('Missing response stream.') + } + + const file: Uint8Array[] = [] + let bytes = 0 + let closed = false + + const showDownloadProgress = () => { + if (!closed) { + setTimeout(() => { + setNotificationMessage(`${downloadMessage}, ${formatMb(bytes)} MB`) + showDownloadProgress() + }, 500) + } + } + + notify(`${downloadMessage}...`, 'info', null) + showDownloadProgress() + + while (true) { + const { done, value } = await reader.read() + if (done) break + bytes = bytes + value.length + file.push(value) + } + closed = true + setNotificationMessage(`Saving ${filename} (${formatMb(bytes)} MB)`) + + const blobUrl = window.URL.createObjectURL(new Blob(file, contentType ? { type: contentType } : undefined)) + const downloadLink = document.createElement('a') + downloadLink.href = blobUrl + downloadLink.download = filename + document.body.appendChild(downloadLink) + downloadLink.click() + downloadLink.remove() + window.URL.revokeObjectURL(blobUrl) + + notify('Download finished.') + } catch (error) { + stopProgress() + notify(failureMessage, 'error') + throw error + } +} diff --git a/frontend/tests/components/TableToolBar.export.test.tsx b/frontend/tests/components/TableToolBar.export.test.tsx index e5a40b0f..1aa5058d 100644 --- a/frontend/tests/components/TableToolBar.export.test.tsx +++ b/frontend/tests/components/TableToolBar.export.test.tsx @@ -23,19 +23,29 @@ jest.mock('../../src/components/Page', () => ({ }), })) +jest.mock('../../src/hooks/notification', () => ({ + useNotify: () => ({ + notify: jest.fn(), + setMessage: jest.fn(), + }), +})) + jest.mock('material-react-table', () => ({ MRT_ShowHideColumnsButton: () => , })) describe('TableToolBar export menu', () => { it('omits the species export option while keeping other exports available', async () => { + const kmlExport: (table: MRT_TableInstance) => void = jest.fn(() => undefined) + const svgExport: (table: MRT_TableInstance) => void = jest.fn(() => undefined) + const { container } = render( } tableName="Localities" - kmlExport={jest.fn()} - svgExport={jest.fn()} + kmlExport={kmlExport} + svgExport={svgExport} /> )