Add full Darwin Core export progress

backend/src/api-tests/occurrence/dwcArchiveExportOccurrences.test.ts

@@ -97,4 +97,33 @@ describe('DwC-A occurrence export (admin-only)', () => {
     expect(result.status).toEqual(403)
     expect(result.body).toEqual(noPermError)
   })
+
+  it('returns a full Darwin Core ZIP archive for admins', async () => {
+    const loginResult = await send<{ token: string }>('user/login', 'POST', { username: 'testSu', password: 'test' })
+    expect(loginResult.status).toEqual(200)
+
+    const result = await request(app)
+      .get('/occurrence/export/dwc-full-package')
+      .set('authorization', `bearer ${loginResult.body.token}`)
+      .buffer(true)
+      .parse(parseBinary)
+
+    expect(result.status).toEqual(200)
+    expect(result.headers['content-type']).toMatch(/application\/zip/i)
+    expect(result.headers['content-disposition']).toMatch(/attachment;\s*filename="now_dwc_full_test_export_/i)
+
+    const zip = await JSZip.loadAsync(result.body as unknown as Buffer)
+    expect(zip.file('README.txt')).toBeTruthy()
+    expect(zip.file('dwc-dp/datapackage.json')).toBeTruthy()
+    expect(zip.file('dwc-dp/occurrence.csv')).toBeTruthy()
+    expect(zip.file('dwc-a-taxa/taxon.csv')).toBeTruthy()
+    expect(zip.file('dwc-a-taxa/measurementorfact.csv')).toBeTruthy()
+    expect(zip.file('dwc-a-taxa/meta.xml')).toBeTruthy()
+  })
+
+  it('rejects non-admin full Darwin Core requests', async () => {
+    const result = await request(app).get('/occurrence/export/dwc-full-package')
+    expect(result.status).toEqual(403)
+    expect(result.body).toEqual(noPermError)
+  })
 })

backend/src/routes/occurrence.ts

@@ -7,7 +7,7 @@ import {
   buildDwcOccurrenceArchiveZipStream,
   type DwcOccurrenceExportProgress,
 } from '../services/dwcArchiveExportOccurrences'
-import { buildDwcDataPackageZipBuffer } from '../services/dwcDataPackageExport'
+import { buildDwcDataPackageZipBuffer, buildFullDarwinCoreExportZipBuffer } from '../services/dwcDataPackageExport'
 import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
 import { logger } from '../utils/logger'
 
@@ -71,6 +71,13 @@ router.get('/export/dwc-data-package', requireOneOf([Role.Admin]), async (_req,
   return res.status(200).send(zipBuffer)
 })
 
+router.get('/export/dwc-full-package', requireOneOf([Role.Admin]), async (_req, res) => {
+  const zipBuffer = await buildFullDarwinCoreExportZipBuffer()
+  res.setHeader('Content-Type', 'application/zip')
+  res.setHeader('Content-Disposition', `attachment; filename="now_dwc_full_test_export_${currentDateAsString()}.zip"`)
+  return res.status(200).send(zipBuffer)
+})
+
 router.get('/:lid/:speciesId', getOccurrenceDetail)
 router.put(
   '/:lid/:speciesId',

backend/src/services/dwcArchiveExport.ts

@@ -1,7 +1,6 @@
 import Prisma from '../../prisma/generated/now_test_client'
-import { format } from 'fast-csv'
-import { Writable } from 'stream'
 import JSZip from 'jszip'
+import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv'
 
 const isMeaningfulString = (value: unknown): value is string => {
   if (typeof value !== 'string') return false
@@ -11,51 +10,7 @@ const isMeaningfulString = (value: unknown): value is string => {
   return true
 }
 
-const toDwcString = (value: unknown): string => {
+const toDwcString = toDwcCsvString
-  if (value === null || value === undefined) return ''
-  if (typeof value === 'bigint') return value.toString()
-  if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : ''
-  if (typeof value === 'boolean') return value ? 'true' : 'false'
-  if (typeof value === 'string') return value
-  if (typeof value === 'object') {
-    try {
-      return JSON.stringify(value) ?? ''
-    } catch {
-      return ''
-    }
-  }
-  return ''
-}
-
-const writeCsvString = async (headers: string[], rows: Array<Record<string, unknown>>): Promise<string> => {
-  return await new Promise((resolve, reject) => {
-    let output = ''
-    const csvStream = format({
-      delimiter: ',',
-      headers,
-      quoteColumns: true,
-      quoteHeaders: true,
-      includeEndRowDelimiter: true,
-    })
-
-    const sink = new Writable({
-      write(chunk: Buffer, _encoding, callback) {
-        output += chunk.toString('utf8')
-        callback()
-      },
-    })
-
-    sink.on('finish', () => resolve(output))
-    sink.on('error', reject)
-    csvStream.on('error', reject)
-
-    csvStream.pipe(sink)
-    for (const row of rows) {
-      csvStream.write(row)
-    }
-    csvStream.end()
-  })
-}
 
 export const TAXON_HEADERS = [
   'taxonID',
@@ -967,8 +922,8 @@ export const buildDwcArchiveZipBufferFromSpecies = async (
   const taxonRows = speciesRows.map(mapSpeciesToTaxonRow)
   const measurementRows = speciesRows.flatMap(mapSpeciesToMeasurementRows)
 
-  const taxonCsv = await writeCsvString([...TAXON_HEADERS], taxonRows)
+  const taxonCsv = writeDwcCsvString(TAXON_HEADERS, taxonRows)
-  const measurementCsv = await writeCsvString([...MEASUREMENT_HEADERS], measurementRows)
+  const measurementCsv = writeDwcCsvString(MEASUREMENT_HEADERS, measurementRows)
   const metaXml = buildMetaXml()
   const publicationDateIso = new Date().toISOString().slice(0, 10)
   const emlXml = buildEmlXml(publicationDateIso)

backend/src/services/dwcArchiveExportLocalities.ts

@@ -1,8 +1,7 @@
 import type { now_loc, now_time_unit } from '../../prisma/generated/now_test_client'
-import { format } from 'fast-csv'
-import { Writable } from 'stream'
 import JSZip from 'jszip'
 import { getContinentByCountry } from '../../../frontend/src/shared/validators/countryContinents'
+import { toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv'
 
 const isMeaningfulString = (value: unknown): value is string => {
   if (typeof value !== 'string') return false
@@ -12,60 +11,7 @@ const isMeaningfulString = (value: unknown): value is string => {
   return true
 }
 
-const toDwcString = (value: unknown): string => {
+const toDwcString = toDwcCsvString
-  if (value === null || value === undefined) return ''
-  if (typeof value === 'bigint') return value.toString()
-  if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : ''
-  if (typeof value === 'boolean') return value ? 'true' : 'false'
-  if (typeof value === 'string') return value
-  if (typeof value === 'object') {
-    if (value instanceof Date) return value.toISOString()
-    if (typeof (value as { toString?: unknown }).toString === 'function') {
-      const asString = (value as { toString: () => string }).toString()
-      if (asString && asString !== '[object Object]') return asString
-    }
-    try {
-      return JSON.stringify(value) ?? ''
-    } catch {
-      return ''
-    }
-  }
-  return ''
-}
-
-const writeCsvString = async (headers: string[], rows: Array<Record<string, unknown>>): Promise<string> => {
-  return await new Promise((resolve, reject) => {
-    let output = ''
-    const csvStream = format({
-      delimiter: ',',
-      headers,
-      quoteColumns: true,
-      quoteHeaders: true,
-      includeEndRowDelimiter: true,
-    })
-
-    const sink = new Writable({
-      write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
-        if (typeof chunk === 'string') {
-          output += chunk
-        } else {
-          output += chunk.toString('utf8')
-        }
-        callback()
-      },
-    })
-
-    sink.on('finish', () => resolve(output))
-    sink.on('error', reject)
-    csvStream.on('error', reject)
-
-    csvStream.pipe(sink)
-    for (const row of rows) {
-      csvStream.write(row)
-    }
-    csvStream.end()
-  })
-}
 
 export const LOCATION_HEADERS = [
   'locationID',
@@ -1381,9 +1327,9 @@ export const buildDwcLocalityArchiveZipBufferFromLocalities = async (
   const geologicalContextRows = localities.map(mapLocalityToGeologicalContextRow)
   const measurementRows = localities.flatMap(mapLocalityToMeasurementRows)
 
-  const locationCsv = await writeCsvString([...LOCATION_HEADERS], locationRows)
+  const locationCsv = writeDwcCsvString(LOCATION_HEADERS, locationRows)
-  const geologyCsv = await writeCsvString([...GEOLOGICAL_CONTEXT_HEADERS], geologicalContextRows)
+  const geologyCsv = writeDwcCsvString(GEOLOGICAL_CONTEXT_HEADERS, geologicalContextRows)
-  const measurementCsv = await writeCsvString([...LOCALITY_MEASUREMENT_HEADERS], measurementRows)
+  const measurementCsv = writeDwcCsvString(LOCALITY_MEASUREMENT_HEADERS, measurementRows)
   const metaXml = buildLocalityMetaXml()
 
   const publicationDateIso = new Date().toISOString().slice(0, 10)

backend/src/services/dwcArchiveExportOccurrences.ts

@@ -1,11 +1,8 @@
 import Prisma from '../../prisma/generated/now_test_client'
-import { format } from 'fast-csv'
+import { createReadStream } from 'fs'
-import { createReadStream, createWriteStream } from 'fs'
 import { mkdtemp, rm } from 'fs/promises'
 import { tmpdir } from 'os'
 import path from 'path'
-import { once } from 'events'
-import { Writable } from 'stream'
 import JSZip from 'jszip'
 import {
   GEOLOGICAL_CONTEXT_HEADERS,
@@ -14,40 +11,7 @@ import {
   mapLocalityToLocationRow,
 } from './dwcArchiveExportLocalities'
 import { MEASUREMENT_HEADERS, TAXON_HEADERS, mapSpeciesToTaxonRow, type MeasurementCsvRow } from './dwcArchiveExport'
-
+import { createDwcCsvFileWriter, toDwcCsvString, writeDwcCsvString } from './utils/dwcCsv'
-const writeCsvString = async (headers: string[], rows: Array<Record<string, unknown>>): Promise<string> => {
-  if (rows.length === 0) {
-    return `${headers.map(header => `"${header.replace(/"/g, '""')}"`).join(',')}\n`
-  }
-
-  return await new Promise((resolve, reject) => {
-    let output = ''
-    const csvStream = format({
-      delimiter: ',',
-      headers,
-      quoteColumns: true,
-      quoteHeaders: true,
-      includeEndRowDelimiter: true,
-    })
-
-    const sink = new Writable({
-      write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
-        output += typeof chunk === 'string' ? chunk : chunk.toString('utf8')
-        callback()
-      },
-    })
-
-    sink.on('finish', () => resolve(output))
-    sink.on('error', reject)
-    csvStream.on('error', reject)
-
-    csvStream.pipe(sink)
-    for (const row of rows) {
-      csvStream.write(row)
-    }
-    csvStream.end()
-  })
-}
 
 const isMeaningfulString = (value: unknown): value is string => {
   if (typeof value !== 'string') return false
@@ -57,18 +21,7 @@ const isMeaningfulString = (value: unknown): value is string => {
   return true
 }
 
-const toDwcString = (value: unknown): string => {
+const toDwcString = toDwcCsvString
-  if (value === null || value === undefined) return ''
-  if (typeof value === 'bigint') return value.toString()
-  if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : ''
-  if (typeof value === 'boolean') return value ? 'true' : 'false'
-  if (typeof value === 'string') return value
-  if (typeof value === 'object' && typeof (value as { toString?: unknown }).toString === 'function') {
-    const asString = (value as { toString: () => string }).toString()
-    return asString === '[object Object]' ? '' : asString
-  }
-  return ''
-}
 
 const toMaybeMeaningful = (value: string | null | undefined): string => (isMeaningfulString(value) ? value.trim() : '')
 
@@ -432,35 +385,6 @@ const localityLookupSelect = {
 
 const speciesLookupSelect = occurrenceSelect.com_species.select
 
-const csvCell = (value: unknown): string => `"${toDwcString(value).replace(/"/g, '""')}"`
-
-const csvLine = (headers: readonly string[], row: Record<string, unknown>): string =>
-  `${headers.map(header => csvCell(row[header])).join(',')}\n`
-
-const createCsvFileWriter = async (filePath: string, headers: readonly string[]) => {
-  const stream = createWriteStream(filePath, { encoding: 'utf8' })
-  await new Promise<void>((resolve, reject) => {
-    stream.once('open', () => resolve())
-    stream.once('error', reject)
-  })
-
-  const write = async (line: string): Promise<void> => {
-    if (!stream.write(line)) await once(stream, 'drain')
-  }
-
-  await write(`${headers.map(csvCell).join(',')}\n`)
-
-  return {
-    writeRow: async (row: Record<string, unknown>): Promise<void> => {
-      await write(csvLine(headers, row))
-    },
-    close: async (): Promise<void> => {
-      stream.end()
-      await once(stream, 'finish')
-    },
-  }
-}
-
 async function* iterateOccurrenceRows(): AsyncGenerator<OccurrenceForExport> {
   const { nowDb } = await import('../utils/db')
   let cursor: { lid: number; species_id: number } | undefined
@@ -600,17 +524,14 @@ export const buildDwcOccurrenceArchiveZipBufferFromOccurrences = async (
     species => species.species_id.toString()
   )
 
-  const locationCsv = await writeCsvString([...LOCATION_HEADERS], localities.map(mapLocalityToLocationRow))
+  const locationCsv = writeDwcCsvString(LOCATION_HEADERS, localities.map(mapLocalityToLocationRow))
-  const geologicalContextCsv = await writeCsvString(
+  const geologicalContextCsv = writeDwcCsvString(
-    [...GEOLOGICAL_CONTEXT_HEADERS],
+    GEOLOGICAL_CONTEXT_HEADERS,
     localities.map(mapLocalityToGeologicalContextRow)
   )
-  const taxonCsv = await writeCsvString([...TAXON_HEADERS], speciesRows.map(mapSpeciesToTaxonRow))
+  const taxonCsv = writeDwcCsvString(TAXON_HEADERS, speciesRows.map(mapSpeciesToTaxonRow))
-  const occurrenceCsv = await writeCsvString([...OCCURRENCE_HEADERS], occurrences.map(mapOccurrenceToOccurrenceRow))
+  const occurrenceCsv = writeDwcCsvString(OCCURRENCE_HEADERS, occurrences.map(mapOccurrenceToOccurrenceRow))
-  const measurementCsv = await writeCsvString(
+  const measurementCsv = writeDwcCsvString(MEASUREMENT_HEADERS, occurrences.flatMap(mapOccurrenceToMeasurementRows))
-    [...MEASUREMENT_HEADERS],
-    occurrences.flatMap(mapOccurrenceToMeasurementRows)
-  )
   const metaXml = buildOccurrenceMetaXml()
   const emlXml = buildOccurrenceEmlXml(new Date().toISOString().slice(0, 10))
 
@@ -635,8 +556,8 @@ const writeOccurrenceAndMeasurementFiles = async ({
   measurementFilePath: string
   reportProgress?: DwcOccurrenceExportProgressReporter
 }): Promise<{ localityIds: number[]; speciesIds: number[] }> => {
-  const occurrenceWriter = await createCsvFileWriter(occurrenceFilePath, OCCURRENCE_HEADERS)
+  const occurrenceWriter = await createDwcCsvFileWriter(occurrenceFilePath, OCCURRENCE_HEADERS)
-  const measurementWriter = await createCsvFileWriter(measurementFilePath, MEASUREMENT_HEADERS)
+  const measurementWriter = await createDwcCsvFileWriter(measurementFilePath, MEASUREMENT_HEADERS)
   const localityIds = new Set<number>()
   const speciesIds = new Set<number>()
   const totalOccurrences = await countOccurrenceRows()
@@ -692,8 +613,8 @@ const writeLocalityLookupFiles = async ({
   reportProgress?: DwcOccurrenceExportProgressReporter
 }): Promise<void> => {
   const { nowDb } = await import('../utils/db')
-  const locationWriter = await createCsvFileWriter(locationFilePath, LOCATION_HEADERS)
+  const locationWriter = await createDwcCsvFileWriter(locationFilePath, LOCATION_HEADERS)
-  const geologicalContextWriter = await createCsvFileWriter(geologicalContextFilePath, GEOLOGICAL_CONTEXT_HEADERS)
+  const geologicalContextWriter = await createDwcCsvFileWriter(geologicalContextFilePath, GEOLOGICAL_CONTEXT_HEADERS)
   let generatedLocalities = 0
 
   reportProgress?.({
@@ -741,7 +662,7 @@ const writeTaxonLookupFile = async ({
   reportProgress?: DwcOccurrenceExportProgressReporter
 }): Promise<void> => {
   const { nowDb } = await import('../utils/db')
-  const taxonWriter = await createCsvFileWriter(taxonFilePath, TAXON_HEADERS)
+  const taxonWriter = await createDwcCsvFileWriter(taxonFilePath, TAXON_HEADERS)
   let generatedTaxa = 0
 
   reportProgress?.({

backend/src/services/dwcDataPackageExport.ts

@@ -1,5 +1,3 @@
-import { format } from 'fast-csv'
-import { Writable } from 'stream'
 import JSZip from 'jszip'
 import { getContinentByCountry } from '../../../frontend/src/shared/validators/countryContinents'
 import {
@@ -12,37 +10,8 @@ import {
   mapOccurrenceToOccurrenceRow,
   type OccurrenceCsvRow,
 } from './dwcArchiveExportOccurrences'
-import type { MeasurementCsvRow } from './dwcArchiveExport'
+import { buildDwcArchiveZipBuffer, type MeasurementCsvRow } from './dwcArchiveExport'
-
+import { writeDwcCsvString } from './utils/dwcCsv'
-const writeCsvString = async (headers: string[], rows: Array<Record<string, unknown>>): Promise<string> => {
-  return await new Promise((resolve, reject) => {
-    let output = ''
-    const csvStream = format({
-      delimiter: ',',
-      headers,
-      quoteColumns: true,
-      quoteHeaders: true,
-      includeEndRowDelimiter: true,
-    })
-
-    const sink = new Writable({
-      write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
-        output += typeof chunk === 'string' ? chunk : chunk.toString('utf8')
-        callback()
-      },
-    })
-
-    sink.on('finish', () => resolve(output))
-    sink.on('error', reject)
-    csvStream.on('error', reject)
-
-    csvStream.pipe(sink)
-    for (const row of rows) {
-      csvStream.write(row)
-    }
-    csvStream.end()
-  })
-}
 
 const isMeaningfulString = (value: unknown): value is string => {
   if (typeof value !== 'string') return false
@@ -642,16 +611,13 @@ export const buildDwcDataPackageZipBufferFromRows = async ({
   const occurrenceAssertionRows = occurrences.flatMap(mapOccurrenceToDwcDpOccurrenceAssertionRows)
 
   const zip = new JSZip()
-  zip.file(DWC_DP_TABLES.event, await writeCsvString([...DWC_DP_EVENT_HEADERS], eventRows))
+  zip.file(DWC_DP_TABLES.event, writeDwcCsvString(DWC_DP_EVENT_HEADERS, eventRows))
-  zip.file(
+  zip.file(DWC_DP_TABLES.geologicalContext, writeDwcCsvString(DWC_DP_GEOLOGICAL_CONTEXT_HEADERS, geologicalContextRows))
-    DWC_DP_TABLES.geologicalContext,
+  zip.file(DWC_DP_TABLES.occurrence, writeDwcCsvString(DWC_DP_OCCURRENCE_HEADERS, occurrenceRows))
-    await writeCsvString([...DWC_DP_GEOLOGICAL_CONTEXT_HEADERS], geologicalContextRows)
+  zip.file(DWC_DP_TABLES.eventAssertion, writeDwcCsvString(DWC_DP_EVENT_ASSERTION_HEADERS, eventAssertionRows))
-  )
-  zip.file(DWC_DP_TABLES.occurrence, await writeCsvString([...DWC_DP_OCCURRENCE_HEADERS], occurrenceRows))
-  zip.file(DWC_DP_TABLES.eventAssertion, await writeCsvString([...DWC_DP_EVENT_ASSERTION_HEADERS], eventAssertionRows))
   zip.file(
     DWC_DP_TABLES.occurrenceAssertion,
-    await writeCsvString([...DWC_DP_OCCURRENCE_ASSERTION_HEADERS], occurrenceAssertionRows)
+    writeDwcCsvString(DWC_DP_OCCURRENCE_ASSERTION_HEADERS, occurrenceAssertionRows)
   )
   zip.file(DWC_DP_TABLES.dataPackage, buildDwcDataPackageJson(publicationDateIso))
   zip.file(DWC_DP_TABLES.eml, buildDwcDataPackageEmlXml(publicationDateIso))
@@ -677,5 +643,83 @@ export const buildDwcDataPackageZipBuffer = async (): Promise<Buffer> => {
   })
 }
 
+const addZipEntriesUnderPrefix = async ({
+  sourceZipBuffer,
+  targetZip,
+  prefix,
+}: {
+  sourceZipBuffer: Buffer
+  targetZip: JSZip
+  prefix: string
+}): Promise<void> => {
+  const sourceZip = await JSZip.loadAsync(sourceZipBuffer)
+  const files = Object.values(sourceZip.files).filter(file => !file.dir)
+
+  await Promise.all(
+    files.map(async file => {
+      targetZip.file(`${prefix}/${file.name}`, await file.async('nodebuffer'))
+    })
+  )
+}
+
+const buildFullDarwinCoreReadme = (): string => `NOW database Darwin Core full test export
+
+This ZIP is a convenience bundle. It contains two separate standards-based export
+artifacts in their own folders:
+
+- dwc-dp/
+  Darwin Core Data Package test export for relational event, geological context,
+  occurrence, event assertion, and occurrence assertion data.
+
+- dwc-a-taxa/
+  Darwin Core Archive test export for taxon records and synthesized taxon-level
+  traits. The traits remain in DwC-A Taxon + MeasurementOrFact form because they
+  are not currently linked to source specimen/material records.
+
+Join key:
+
+- dwc-dp/occurrence.csv taxonID uses NOW:<species_id>
+- dwc-a-taxa/taxon.csv taxonID uses the same NOW:<species_id>
+- dwc-a-taxa/measurementorfact.csv links taxon traits by that same taxonID
+
+The outer ZIP is not itself a single DwC-DP or DwC-A artifact. Each subfolder is
+intended to remain internally understandable as its own export format.
+`
+
+export const buildFullDarwinCoreExportZipBufferFromArchives = async ({
+  dwcDataPackageZipBuffer,
+  dwcTaxonArchiveZipBuffer,
+}: {
+  dwcDataPackageZipBuffer: Buffer
+  dwcTaxonArchiveZipBuffer: Buffer
+}): Promise<Buffer> => {
+  const zip = new JSZip()
+  zip.file('README.txt', buildFullDarwinCoreReadme())
+  await addZipEntriesUnderPrefix({
+    sourceZipBuffer: dwcDataPackageZipBuffer,
+    targetZip: zip,
+    prefix: 'dwc-dp',
+  })
+  await addZipEntriesUnderPrefix({
+    sourceZipBuffer: dwcTaxonArchiveZipBuffer,
+    targetZip: zip,
+    prefix: 'dwc-a-taxa',
+  })
+
+  return await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 6 } })
+}
+
+export const buildFullDarwinCoreExportZipBuffer = async (): Promise<Buffer> => {
+  const [dwcDataPackageZipBuffer, dwcTaxonArchiveZipBuffer] = await Promise.all([
+    buildDwcDataPackageZipBuffer(),
+    buildDwcArchiveZipBuffer(),
+  ])
+
+  return await buildFullDarwinCoreExportZipBufferFromArchives({
+    dwcDataPackageZipBuffer,
+    dwcTaxonArchiveZipBuffer,
+  })
+}
+
 export type DwcDpLocalityFixture = LocalityForDwcDpExport
 export type DwcDpOccurrenceFixture = OccurrenceForDwcDpExport

backend/src/services/utils/dwcCsv.ts

@@ -0,0 +1,62 @@
+import { createWriteStream } from 'fs'
+import { once } from 'events'
+
+export const normalizeDwcCsvValue = (value: string): string => value.replace(/\r\n|\r|\n/g, ' ').replace(/[ \t]+/g, ' ')
+
+export const toDwcCsvString = (value: unknown): string => {
+  if (value === null || value === undefined) return ''
+  if (typeof value === 'bigint') return value.toString()
+  if (typeof value === 'number') return Number.isFinite(value) ? value.toString() : ''
+  if (typeof value === 'boolean') return value ? 'true' : 'false'
+  if (typeof value === 'string') return normalizeDwcCsvValue(value)
+  if (value instanceof Date) return value.toISOString()
+  if (typeof value === 'object') {
+    if (typeof (value as { toString?: unknown }).toString === 'function') {
+      const asString = (value as { toString: () => string }).toString()
+      if (asString && asString !== '[object Object]') return normalizeDwcCsvValue(asString)
+    }
+    try {
+      return normalizeDwcCsvValue(JSON.stringify(value) ?? '')
+    } catch {
+      return ''
+    }
+  }
+  return ''
+}
+
+export const dwcCsvCell = (value: unknown): string => `"${toDwcCsvString(value).replace(/"/g, '""')}"`
+
+export const dwcCsvLine = (headers: readonly string[], row: Record<string, unknown>): string =>
+  `${headers.map(header => dwcCsvCell(row[header])).join(',')}\n`
+
+export const writeDwcCsvString = (headers: readonly string[], rows: Array<Record<string, unknown>>): string => {
+  const lines = [`${headers.map(dwcCsvCell).join(',')}\n`]
+  for (const row of rows) {
+    lines.push(dwcCsvLine(headers, row))
+  }
+  return lines.join('')
+}
+
+export const createDwcCsvFileWriter = async (filePath: string, headers: readonly string[]) => {
+  const stream = createWriteStream(filePath, { encoding: 'utf8' })
+  await new Promise<void>((resolve, reject) => {
+    stream.once('open', () => resolve())
+    stream.once('error', reject)
+  })
+
+  const write = async (line: string): Promise<void> => {
+    if (!stream.write(line)) await once(stream, 'drain')
+  }
+
+  await write(`${headers.map(dwcCsvCell).join(',')}\n`)
+
+  return {
+    writeRow: async (row: Record<string, unknown>): Promise<void> => {
+      await write(dwcCsvLine(headers, row))
+    },
+    close: async (): Promise<void> => {
+      stream.end()
+      await once(stream, 'finish')
+    },
+  }
+}

backend/src/unit-tests/dwcCsv.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from '@jest/globals'
+import { dwcCsvCell, normalizeDwcCsvValue, writeDwcCsvString } from '../services/utils/dwcCsv'
+
+describe('DwC CSV writer', () => {
+  it('normalizes embedded line feeds before quoting cells', () => {
+    expect(normalizeDwcCsvValue('first\nsecond\r\nthird\rfourth')).toEqual('first second third fourth')
+    expect(dwcCsvCell('first\n"second"')).toEqual('"first ""second"""')
+  })
+
+  it('writes one physical line per row even when source values contain line breaks', () => {
+    const csv = writeDwcCsvString(
+      ['id', 'remarks'],
+      [
+        { id: 'one', remarks: 'alpha\nbeta' },
+        { id: 'two', remarks: 'gamma\r\ndelta' },
+      ]
+    )
+
+    expect(csv).toEqual('"id","remarks"\n"one","alpha beta"\n"two","gamma delta"\n')
+    expect(csv.split('\n')).toHaveLength(4)
+  })
+})

backend/src/unit-tests/dwcDataPackageExport.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'
 import JSZip from 'jszip'
 import {
   DWC_DP_TABLES,
+  buildFullDarwinCoreExportZipBufferFromArchives,
   buildDwcDataPackageZipBufferFromRows,
   mapLocalityToDwcDpEventAssertionRows,
   mapLocalityToDwcDpEventRow,
@@ -161,8 +162,12 @@ describe('DwC-DP export mapping', () => {
   })
 
   it('generates a ZIP archive with DwC-DP files and relational metadata', async () => {
+    const multilineLocality = {
+      ...locality,
+      loc_detail: 'Some notes\nwith a line feed',
+    } as DwcDpLocalityFixture
     const zipBuffer = await buildDwcDataPackageZipBufferFromRows({
-      localities: [locality],
+      localities: [multilineLocality],
       occurrences: [occurrence],
       publicationDateIso: '2026-05-13',
     })
@@ -176,6 +181,10 @@ describe('DwC-DP export mapping', () => {
     expect(zip.file(DWC_DP_TABLES.dataPackage)).toBeTruthy()
     expect(zip.file(DWC_DP_TABLES.eml)).toBeTruthy()
 
+    const eventCsv = await zip.file(DWC_DP_TABLES.event)!.async('string')
+    expect(eventCsv).toContain('Some notes with a line feed')
+    expect(eventCsv.trimEnd().split('\n')).toHaveLength(2)
+
     const dataPackageJson = JSON.parse(await zip.file(DWC_DP_TABLES.dataPackage)!.async('string')) as {
       resources: Array<{ name: string; schema: { foreignKeys?: unknown[] } }>
     }
@@ -193,4 +202,34 @@ describe('DwC-DP export mapping', () => {
       },
     ])
   })
+
+  it('combines DwC-DP and DwC-A taxon archives into a namespaced convenience bundle', async () => {
+    const dwcDataPackageZip = new JSZip()
+    dwcDataPackageZip.file('datapackage.json', '{}\n')
+    dwcDataPackageZip.file('event.csv', '"eventID"\n')
+    dwcDataPackageZip.file('occurrence.csv', '"occurrenceID","taxonID"\n')
+
+    const dwcTaxonArchiveZip = new JSZip()
+    dwcTaxonArchiveZip.file('taxon.csv', '"taxonID"\n')
+    dwcTaxonArchiveZip.file('measurementorfact.csv', '"taxonID","measurementID"\n')
+    dwcTaxonArchiveZip.file('meta.xml', '<archive />\n')
+
+    const fullZipBuffer = await buildFullDarwinCoreExportZipBufferFromArchives({
+      dwcDataPackageZipBuffer: await dwcDataPackageZip.generateAsync({ type: 'nodebuffer' }),
+      dwcTaxonArchiveZipBuffer: await dwcTaxonArchiveZip.generateAsync({ type: 'nodebuffer' }),
+    })
+
+    const fullZip = await JSZip.loadAsync(fullZipBuffer)
+    expect(fullZip.file('README.txt')).toBeTruthy()
+    expect(fullZip.file('dwc-dp/datapackage.json')).toBeTruthy()
+    expect(fullZip.file('dwc-dp/event.csv')).toBeTruthy()
+    expect(fullZip.file('dwc-dp/occurrence.csv')).toBeTruthy()
+    expect(fullZip.file('dwc-a-taxa/taxon.csv')).toBeTruthy()
+    expect(fullZip.file('dwc-a-taxa/measurementorfact.csv')).toBeTruthy()
+    expect(fullZip.file('dwc-a-taxa/meta.xml')).toBeTruthy()
+
+    const readme = await fullZip.file('README.txt')!.async('string')
+    expect(readme).toContain('dwc-dp/occurrence.csv taxonID')
+    expect(readme).toContain('dwc-a-taxa/taxon.csv taxonID')
+  })
 })

documentation/functionality/dwc_data_package.md

@@ -9,7 +9,8 @@ DwC-DP is intentionally published as a separate format from the existing DwC-A e
 ## Access
 
 - Backend route: `GET /occurrence/export/dwc-data-package` (**Role.Admin only**)
-- The frontend exposes this as an export option on the cross-search occurrence table for administrators.
+- Backend route: `GET /occurrence/export/dwc-full-package` (**Role.Admin only**)
+- The frontend exposes both as export options on the cross-search occurrence table for administrators.
 
 ## Output
 
@@ -23,6 +24,30 @@ The downloaded ZIP contains:
 - `occurrence-assertion.csv` (occurrence-level facts formerly expressed as occurrence `MeasurementOrFact`)
 - `eml.xml` (minimal placeholder EML metadata)
 
+## Full Darwin Core Convenience Bundle
+
+The full export route downloads one outer ZIP for users who need both occurrence/event data and taxon traits.
+
+The outer ZIP contains:
+
+- `README.txt`
+- `dwc-dp/` with the DwC-DP event + occurrence package described above
+- `dwc-a-taxa/` with the existing DwC-A taxon trait archive:
+  - `taxon.csv`
+  - `measurementorfact.csv`
+  - `meta.xml`
+  - `eml.xml`
+
+The outer ZIP is a NOW convenience bundle, not a single hybrid standard artifact. Each subfolder remains its own internally described export format.
+
+For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line.
+
+Join key:
+
+- `dwc-dp/occurrence.csv` uses `taxonID = NOW:<species_id>`
+- `dwc-a-taxa/taxon.csv` uses the same `taxonID`
+- `dwc-a-taxa/measurementorfact.csv` links synthesized taxon traits by the same `taxonID`
+
 ## Model
 
 ### `event.csv`

documentation/functionality/dwc_export.md

@@ -4,6 +4,8 @@ Issue: `nowcommunity/nowdatabase#1150`
 
 This repository includes an **admin-only** Darwin Core Archive (DwC-A) export intended for initial testing.
 
+For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line.
+
 ## Access
 
 - Backend route: `GET /species/export/dwc-archive` (**Role.Admin only**)

documentation/functionality/dwc_export_localities.md

@@ -2,6 +2,8 @@
 
 This document describes the admin-only Darwin Core Archive (DwC-A) test export for localities.
 
+For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line.
+
 ## Files
 
 The export ZIP contains:

documentation/functionality/dwc_export_occurrences.md

@@ -2,6 +2,8 @@
 
 This document describes the admin-only Darwin Core Archive (DwC-A) test export for occurrence records (`now_ls`).
 
+For spreadsheet compatibility, embedded line breaks in exported CSV cell values are normalized to spaces. This keeps each exported record on one physical CSV line.
+
 ## Files
 
 The export ZIP contains:

frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx

@@ -5,6 +5,7 @@ import { MenuItem } from '@mui/material'
 import { useUser } from '@/hooks/user'
 import { usePageContext } from '../Page'
 import { currentDateAsString } from '@/shared/currentDateAsString'
+import { downloadExportFileWithProgress } from '@/util/exportProgress'
 
 export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
@@ -21,46 +22,21 @@ export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () =>
     const URIColumnFilters = encodeURIComponent(JSON.stringify(sqlColumnFilters))
     const URIOrderBy = encodeURIComponent(JSON.stringify(sqlOrderBy))
 
-    notify('Generating CSV file, please wait...', 'info', null)
-    const response = await fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions)
-
-    if (!response.ok) {
-      throw new Error('Server response was not OK.')
-    }
-    const reader = response.body!.getReader()
-    const file: Uint8Array[] = []
-    let bytes = 0
-    let closed = false
-
-    const showDownloadProgress = () => {
-      if (!closed) {
-        setTimeout(() => {
-          setNotificationMessage(`Downloading CSV file, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
-          showDownloadProgress()
-        }, 500)
-      }
-    }
-    notify(`Downloading CSV file...`, 'info', null)
-    showDownloadProgress()
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      bytes = bytes + value.length
-      file.push(value)
-    }
-    closed = true
     try {
-      const blobUrl = window.URL.createObjectURL(new Blob(file))
+      await downloadExportFileWithProgress({
-      const downloadLink = document.createElement('a')
+        url: `${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`,
-      downloadLink.href = blobUrl
+        filename,
-      downloadLink.download = filename
+        fetchOptions,
-      document.body.appendChild(downloadLink)
+        notify,
-      downloadLink.click()
+        setNotificationMessage,
-      downloadLink.remove()
+        startMessage: 'Generating cross-search CSV export...',
-      window.URL.revokeObjectURL(blobUrl)
+        waitingMessage: 'Waiting for cross-search rows',
-      notify('Download finished.')
+        downloadMessage: 'Downloading CSV file',
+        failureMessage: 'Downloading cross-search export failed.',
+        contentType: 'text/csv',
+      })
     } catch {
-      notify('Downloading cross-search export failed.', 'error')
+      // downloadExportFileWithProgress owns the failure notification.
     } finally {
       setLoading(false)
     }

frontend/src/components/CrossSearch/CrossSearchTable.tsx

@@ -10,6 +10,7 @@ import { formatWithMaxThreeDecimals } from '@/util/numberFormatting'
 import { occurrenceLabels } from '@/constants/occurrenceLabels'
 import { OccurrenceDwcExportMenuItem } from '@/components/Occurrence/OccurrenceDwcExportMenuItem'
 import { OccurrenceDwcDpExportMenuItem } from '@/components/Occurrence/OccurrenceDwcDpExportMenuItem'
+import { OccurrenceFullDarwinCoreExportMenuItem } from '@/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem'
 import { matchesCountryOrContinent } from '@/shared/validators/countryContinents'
 
 export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: CrossSearch) => void }) => {
@@ -1095,6 +1096,7 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros
           <>
             <OccurrenceDwcExportMenuItem handleClose={handleClose} />
             <OccurrenceDwcDpExportMenuItem handleClose={handleClose} />
+            <OccurrenceFullDarwinCoreExportMenuItem handleClose={handleClose} />
           </>
         )}
       />

frontend/src/components/Locality/LocalityDwcExportMenuItem.tsx

@@ -5,6 +5,7 @@ import { BACKEND_URL } from '@/util/config'
 import { useUser } from '@/hooks/user'
 import { Role } from '@/shared/types'
 import { currentDateAsString } from '@/shared/currentDateAsString'
+import { downloadExportFileWithProgress } from '@/util/exportProgress'
 
 export const LocalityDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
@@ -20,55 +21,22 @@ export const LocalityDwcExportMenuItem = ({ handleClose }: { handleClose: () =>
 
   const fetchZipFile = async () => {
     setLoading(true)
-    notify('Generating DwC-A ZIP export, please wait...', 'info', null)
 
     try {
-      const response = await fetch(`${BACKEND_URL}/locality/export/dwc-archive`, fetchOptions)
+      await downloadExportFileWithProgress({
-      if (!response.ok) {
+        url: `${BACKEND_URL}/locality/export/dwc-archive`,
-        throw new Error('Server response was not OK.')
+        filename,
-      }
+        fetchOptions,
-
+        notify,
-      const reader = response.body?.getReader()
+        setNotificationMessage,
-      if (!reader) {
+        startMessage: 'Generating DwC-A locality ZIP export...',
-        throw new Error('Missing response stream.')
+        waitingMessage: 'Waiting for locality export rows',
-      }
+        downloadMessage: 'Downloading DwC-A locality ZIP',
-
+        failureMessage: 'Downloading DwC-A export failed.',
-      const file: Uint8Array[] = []
+        contentType: 'application/zip',
-      let bytes = 0
+      })
-      let closed = false
-
-      const showDownloadProgress = () => {
-        if (!closed) {
-          setTimeout(() => {
-            setNotificationMessage(`Downloading DwC-A ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
-            showDownloadProgress()
-          }, 500)
-        }
-      }
-
-      notify('Downloading DwC-A ZIP...', 'info', null)
-      showDownloadProgress()
-
-      while (true) {
-        const { done, value } = await reader.read()
-        if (done) break
-        bytes = bytes + value.length
-        file.push(value)
-      }
-      closed = true
-
-      const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' }))
-      const downloadLink = document.createElement('a')
-      downloadLink.href = blobUrl
-      downloadLink.download = filename
-      document.body.appendChild(downloadLink)
-      downloadLink.click()
-      downloadLink.remove()
-      window.URL.revokeObjectURL(blobUrl)
-
-      notify('Download finished.')
     } catch {
-      notify('Downloading DwC-A export failed.', 'error')
+      // downloadExportFileWithProgress owns the failure notification.
     } finally {
       setLoading(false)
     }

frontend/src/components/Locality/LocalityTable.tsx

@@ -485,18 +485,16 @@ export const LocalityTable = ({ selectorFn }: { selectorFn?: (newObject: Localit
     a.click()
   }
 
-  const svgExport = <T extends MRT_RowData>(table: MRT_TableInstance<T>) => {
+  const svgExport = async <T extends MRT_RowData>(table: MRT_TableInstance<T>) => {
-    void (async () => {
+    const rowData: Locality[] = table.getPrePaginationRowModel().rows.map(row => row.original as unknown as Locality)
-      const rowData: Locality[] = table.getPrePaginationRowModel().rows.map(row => row.original as unknown as Locality)
+    const { generateSvg } = await import('../Map/generateSvg')
-      const { generateSvg } = await import('../Map/generateSvg')
+    const dataString = generateSvg(rowData)
-      const dataString = generateSvg(rowData)
+    const blob = new Blob([dataString], { type: 'image/svg+xml' })
-      const blob = new Blob([dataString], { type: 'image/svg+xml' })
+    const url = URL.createObjectURL(blob)
-      const url = URL.createObjectURL(blob)
+    const a = document.createElement('a')
-      const a = document.createElement('a')
+    a.href = url
-      a.href = url
+    a.download = `localities-map-${currentDateAsString()}.svg`
-      a.download = `localities-map-${currentDateAsString()}.svg`
+    a.click()
-      a.click()
-    })()
   }
 
   const checkRowRestriction = (row: Locality) => {

frontend/src/components/Occurrence/OccurrenceDwcDpExportMenuItem.tsx

@@ -5,6 +5,7 @@ import { BACKEND_URL } from '@/util/config'
 import { useUser } from '@/hooks/user'
 import { Role } from '@/shared/types'
 import { currentDateAsString } from '@/shared/currentDateAsString'
+import { downloadExportFileWithProgress } from '@/util/exportProgress'
 
 export const OccurrenceDwcDpExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
@@ -20,55 +21,22 @@ export const OccurrenceDwcDpExportMenuItem = ({ handleClose }: { handleClose: ()
 
   const fetchZipFile = async () => {
     setLoading(true)
-    notify('Generating DwC-DP ZIP export, please wait...', 'info', null)
 
     try {
-      const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-data-package`, fetchOptions)
+      await downloadExportFileWithProgress({
-      if (!response.ok) {
+        url: `${BACKEND_URL}/occurrence/export/dwc-data-package`,
-        throw new Error('Server response was not OK.')
+        filename,
-      }
+        fetchOptions,
-
+        notify,
-      const reader = response.body?.getReader()
+        setNotificationMessage,
-      if (!reader) {
+        startMessage: 'Generating DwC-DP ZIP export...',
-        throw new Error('Missing response stream.')
+        waitingMessage: 'Waiting for DwC-DP event and occurrence rows',
-      }
+        downloadMessage: 'Downloading DwC-DP ZIP',
-
+        failureMessage: 'Downloading DwC-DP export failed.',
-      const file: Uint8Array[] = []
+        contentType: 'application/zip',
-      let bytes = 0
+      })
-      let closed = false
-
-      const showDownloadProgress = () => {
-        if (!closed) {
-          setTimeout(() => {
-            setNotificationMessage(`Downloading DwC-DP ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
-            showDownloadProgress()
-          }, 500)
-        }
-      }
-
-      notify('Downloading DwC-DP ZIP...', 'info', null)
-      showDownloadProgress()
-
-      while (true) {
-        const { done, value } = await reader.read()
-        if (done) break
-        bytes = bytes + value.length
-        file.push(value)
-      }
-      closed = true
-
-      const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' }))
-      const downloadLink = document.createElement('a')
-      downloadLink.href = blobUrl
-      downloadLink.download = filename
-      document.body.appendChild(downloadLink)
-      downloadLink.click()
-      downloadLink.remove()
-      window.URL.revokeObjectURL(blobUrl)
-
-      notify('Download finished.')
     } catch {
-      notify('Downloading DwC-DP export failed.', 'error')
+      // downloadExportFileWithProgress owns the failure notification.
     } finally {
       setLoading(false)
     }

frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx

@@ -5,15 +5,7 @@ import { BACKEND_URL } from '@/util/config'
 import { useUser } from '@/hooks/user'
 import { Role } from '@/shared/types'
 import { currentDateAsString } from '@/shared/currentDateAsString'
-
+import { createExportId, downloadExportFileWithProgress } from '@/util/exportProgress'
-type OccurrenceExportProgress = {
-  message: string
-}
-
-const createExportId = () => {
-  if (window.crypto?.randomUUID) return window.crypto.randomUUID()
-  return `${Date.now()}-${Math.random().toString(36).slice(2)}`
-}
 
 export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
@@ -30,85 +22,22 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () =
   const fetchZipFile = async () => {
     setLoading(true)
     const exportId = createExportId()
-    let generationProgressTimer: number | undefined
-    notify('Generating DwC-A ZIP export...', 'info', null)
-
-    const updateGenerationProgress = async () => {
-      try {
-        const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`, fetchOptions)
-        if (!response.ok) return
-
-        const progress = (await response.json()) as OccurrenceExportProgress
-        setNotificationMessage(progress.message)
-      } catch {
-        // The download request owns the final success/failure notification.
-      }
-    }
-
-    const stopGenerationProgress = () => {
-      if (generationProgressTimer !== undefined) {
-        window.clearInterval(generationProgressTimer)
-        generationProgressTimer = undefined
-      }
-    }
-
     try {
-      generationProgressTimer = window.setInterval(() => {
+      await downloadExportFileWithProgress({
-        void updateGenerationProgress()
+        url: `${BACKEND_URL}/occurrence/export/dwc-archive?${new URLSearchParams({ exportId })}`,
-      }, 1000)
+        progressUrl: `${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`,
-
+        filename,
-      const response = await fetch(
+        fetchOptions,
-        `${BACKEND_URL}/occurrence/export/dwc-archive?${new URLSearchParams({ exportId })}`,
+        notify,
-        fetchOptions
+        setNotificationMessage,
-      )
+        startMessage: 'Generating DwC-A occurrence ZIP export...',
-      stopGenerationProgress()
+        waitingMessage: 'Waiting for occurrence export rows',
-
+        downloadMessage: 'Downloading DwC-A occurrence ZIP',
-      if (!response.ok) {
+        failureMessage: 'Downloading DwC-A export failed.',
-        throw new Error('Server response was not OK.')
+        contentType: 'application/zip',
-      }
+      })
-
-      const reader = response.body?.getReader()
-      if (!reader) {
-        throw new Error('Missing response stream.')
-      }
-
-      const file: Uint8Array[] = []
-      let bytes = 0
-      let closed = false
-
-      const showDownloadProgress = () => {
-        if (!closed) {
-          setTimeout(() => {
-            setNotificationMessage(`Downloading DwC-A ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
-            showDownloadProgress()
-          }, 500)
-        }
-      }
-
-      notify('Downloading DwC-A ZIP...', 'info', null)
-      showDownloadProgress()
-
-      while (true) {
-        const { done, value } = await reader.read()
-        if (done) break
-        bytes = bytes + value.length
-        file.push(value)
-      }
-      closed = true
-
-      const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' }))
-      const downloadLink = document.createElement('a')
-      downloadLink.href = blobUrl
-      downloadLink.download = filename
-      document.body.appendChild(downloadLink)
-      downloadLink.click()
-      downloadLink.remove()
-      window.URL.revokeObjectURL(blobUrl)
-
-      notify('Download finished.')
     } catch {
-      stopGenerationProgress()
+      // downloadExportFileWithProgress owns the failure notification.
-      notify('Downloading DwC-A export failed.', 'error')
     } finally {
       setLoading(false)
     }

frontend/src/components/Occurrence/OccurrenceFullDarwinCoreExportMenuItem.tsx

@@ -0,0 +1,56 @@
+import { useState } from 'react'
+import { MenuItem } from '@mui/material'
+import { useNotify } from '@/hooks/notification'
+import { BACKEND_URL } from '@/util/config'
+import { useUser } from '@/hooks/user'
+import { Role } from '@/shared/types'
+import { currentDateAsString } from '@/shared/currentDateAsString'
+import { downloadExportFileWithProgress } from '@/util/exportProgress'
+
+export const OccurrenceFullDarwinCoreExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
+  const [loading, setLoading] = useState(false)
+  const { notify, setMessage: setNotificationMessage } = useNotify()
+  const user = useUser()
+
+  if (user.role !== Role.Admin) {
+    return null
+  }
+
+  const fetchOptions = user.token ? { headers: { Authorization: `Bearer ${user.token}` } } : {}
+  const filename = `now_dwc_full_test_export_${currentDateAsString()}.zip`
+
+  const fetchZipFile = async () => {
+    setLoading(true)
+
+    try {
+      await downloadExportFileWithProgress({
+        url: `${BACKEND_URL}/occurrence/export/dwc-full-package`,
+        filename,
+        fetchOptions,
+        notify,
+        setNotificationMessage,
+        startMessage: 'Generating full Darwin Core ZIP export...',
+        waitingMessage: 'Waiting for DwC-DP and taxon trait archive files',
+        downloadMessage: 'Downloading full Darwin Core ZIP',
+        failureMessage: 'Downloading full Darwin Core export failed.',
+        contentType: 'application/zip',
+      })
+    } catch {
+      // downloadExportFileWithProgress owns the failure notification.
+    } finally {
+      setLoading(false)
+    }
+  }
+
+  return (
+    <MenuItem
+      onClick={() => {
+        void fetchZipFile()
+        handleClose()
+      }}
+      disabled={loading}
+    >
+      Export full Darwin Core package
+    </MenuItem>
+  )
+}

frontend/src/components/Species/SpeciesDwcExportMenuItem.tsx

@@ -5,6 +5,7 @@ import { BACKEND_URL } from '@/util/config'
 import { useUser } from '@/hooks/user'
 import { Role } from '@/shared/types'
 import { currentDateAsString } from '@/shared/currentDateAsString'
+import { downloadExportFileWithProgress } from '@/util/exportProgress'
 
 export const SpeciesDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
@@ -20,55 +21,22 @@ export const SpeciesDwcExportMenuItem = ({ handleClose }: { handleClose: () => v
 
   const fetchZipFile = async () => {
     setLoading(true)
-    notify('Generating DwC-A ZIP export, please wait...', 'info', null)
 
     try {
-      const response = await fetch(`${BACKEND_URL}/species/export/dwc-archive`, fetchOptions)
+      await downloadExportFileWithProgress({
-      if (!response.ok) {
+        url: `${BACKEND_URL}/species/export/dwc-archive`,
-        throw new Error('Server response was not OK.')
+        filename,
-      }
+        fetchOptions,
-
+        notify,
-      const reader = response.body?.getReader()
+        setNotificationMessage,
-      if (!reader) {
+        startMessage: 'Generating DwC-A taxon ZIP export...',
-        throw new Error('Missing response stream.')
+        waitingMessage: 'Waiting for taxon export rows',
-      }
+        downloadMessage: 'Downloading DwC-A taxon ZIP',
-
+        failureMessage: 'Downloading DwC-A export failed.',
-      const file: Uint8Array[] = []
+        contentType: 'application/zip',
-      let bytes = 0
+      })
-      let closed = false
-
-      const showDownloadProgress = () => {
-        if (!closed) {
-          setTimeout(() => {
-            setNotificationMessage(`Downloading DwC-A ZIP, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
-            showDownloadProgress()
-          }, 500)
-        }
-      }
-
-      notify('Downloading DwC-A ZIP...', 'info', null)
-      showDownloadProgress()
-
-      while (true) {
-        const { done, value } = await reader.read()
-        if (done) break
-        bytes = bytes + value.length
-        file.push(value)
-      }
-      closed = true
-
-      const blobUrl = window.URL.createObjectURL(new Blob(file, { type: 'application/zip' }))
-      const downloadLink = document.createElement('a')
-      downloadLink.href = blobUrl
-      downloadLink.download = filename
-      document.body.appendChild(downloadLink)
-      downloadLink.click()
-      downloadLink.remove()
-      window.URL.revokeObjectURL(blobUrl)
-
-      notify('Download finished.')
     } catch {
-      notify('Downloading DwC-A export failed.', 'error')
+      // downloadExportFileWithProgress owns the failure notification.
     } finally {
       setLoading(false)
     }

frontend/src/components/TableView/TableToolBar.tsx

@@ -10,6 +10,7 @@ import { Link, useLocation } from 'react-router-dom'
 import AddCircleIcon from '@mui/icons-material/AddCircle'
 import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportMenuItem'
 import { usePageContext } from '../Page'
+import { useNotify } from '@/hooks/notification'
 
 export type ColumnVisibilityGroup = {
   id: string
@@ -42,8 +43,8 @@ export const TableToolBar = <T extends MRT_RowData>({
 }: {
   table: MRT_TableInstance<T>
   tableName: string
-  kmlExport?: (table: MRT_TableInstance<T>) => void
+  kmlExport?: (table: MRT_TableInstance<T>) => void | Promise<void>
-  svgExport?: (table: MRT_TableInstance<T>) => void
+  svgExport?: (table: MRT_TableInstance<T>) => void | Promise<void>
   isCrossSearchTable?: boolean
   selectorFn?: (id: T) => void
   showNewButton?: boolean
@@ -52,6 +53,7 @@ export const TableToolBar = <T extends MRT_RowData>({
   renderExtraExportMenuItems?: (handleClose: () => void) => ReactNode
 }) => {
   const { previousTableUrls, setPreviousTableUrls } = usePageContext<T>()
+  const { notify, setMessage: setNotificationMessage } = useNotify()
   const location = useLocation()
   const [anchorEl, setAnchorEl] = useState<null | HTMLElement>(null)
   const open = Boolean(anchorEl)
@@ -183,6 +185,22 @@ export const TableToolBar = <T extends MRT_RowData>({
     column.toggleVisibility?.(!column.getIsVisible?.())
   }
 
+  const exportVisibleTableRows = () => {
+    notify(`Preparing ${tableName} table CSV export...`, 'info', null)
+    const rowCount = exportRows(table, tableName)
+    setNotificationMessage(`Generated ${tableName} table CSV with ${rowCount} rows. Saving file...`)
+    notify('Download finished.')
+  }
+
+  const runMapExport = async (label: string, exportFn: (table: MRT_TableInstance<T>) => void | Promise<void>) => {
+    notify(`Preparing ${label} export...`, 'info', null)
+    const rowCount = table.getPrePaginationRowModel().rows.length
+    setNotificationMessage(`Generating ${label} export from ${rowCount} rows...`)
+    await exportFn(table)
+    setNotificationMessage(`Saving ${label} export...`)
+    notify('Download finished.')
+  }
+
   return (
     <div className="table-tool-bar">
       {!selectorFn && !hideLeftButtons && (
@@ -291,7 +309,7 @@ export const TableToolBar = <T extends MRT_RowData>({
           ) : (
             <MenuItem
               onClick={() => {
-                exportRows(table, tableName)
+                exportVisibleTableRows()
                 handleClose()
               }}
             >
@@ -304,7 +322,7 @@ export const TableToolBar = <T extends MRT_RowData>({
           {kmlExport && (
             <MenuItem
               onClick={() => {
-                kmlExport(table)
+                void runMapExport('KML', kmlExport)
                 handleClose()
               }}
             >
@@ -315,7 +333,7 @@ export const TableToolBar = <T extends MRT_RowData>({
           {svgExport && (
             <MenuItem
               onClick={() => {
-                svgExport(table)
+                void runMapExport('SVG map', svgExport)
                 handleClose()
               }}
             >

frontend/src/components/TableView/TableView.tsx

@@ -104,8 +104,8 @@ export const TableView = <T extends MRT_RowData>({
   getDetailPath?: (row: T) => string
   url?: string
   title: string
-  kmlExport?: (table: MRT_TableInstance<T>) => void
+  kmlExport?: (table: MRT_TableInstance<T>) => void | Promise<void>
-  svgExport?: (table: MRT_TableInstance<T>) => void
+  svgExport?: (table: MRT_TableInstance<T>) => void | Promise<void>
   isCrossSearchTable?: boolean
   clickableRows?: boolean
   enableColumnFilterModes?: boolean

frontend/src/components/TableView/helpers.tsx

@@ -29,4 +29,5 @@ export const exportRows = <T extends MRT_RowData>(table: MRT_TableInstance<T>, t
   })
   const csv = generateCsv(csvConfig)(rowData)
   download(csvConfig)(csv)
+  return rowData.length
 }

frontend/src/util/exportProgress.ts

@@ -0,0 +1,132 @@
+import { Severity } from '@/components/Notification'
+
+type Notify = (msg: string, severity?: Severity, timeoutValue?: number | null) => void
+type ExportProgress = {
+  message: string
+}
+
+type DownloadExportParams = {
+  url: string
+  filename: string
+  fetchOptions?: RequestInit
+  notify: Notify
+  setNotificationMessage: (message: string) => void
+  startMessage: string
+  waitingMessage: string
+  downloadMessage: string
+  failureMessage: string
+  contentType?: string
+  progressUrl?: string
+}
+
+export const createExportId = () => {
+  if (window.crypto?.randomUUID) return window.crypto.randomUUID()
+  return `${Date.now()}-${Math.random().toString(36).slice(2)}`
+}
+
+const formatMb = (bytes: number) => Math.round((bytes / 1000000) * 10) / 10
+
+export const downloadExportFileWithProgress = async ({
+  url,
+  filename,
+  fetchOptions = {},
+  notify,
+  setNotificationMessage,
+  startMessage,
+  waitingMessage,
+  downloadMessage,
+  failureMessage,
+  contentType,
+  progressUrl,
+}: DownloadExportParams) => {
+  let progressTimer: number | undefined
+  let waitingSeconds = 0
+
+  const stopProgress = () => {
+    if (progressTimer !== undefined) {
+      window.clearInterval(progressTimer)
+      progressTimer = undefined
+    }
+  }
+
+  const updateServerProgress = async () => {
+    if (!progressUrl) return false
+
+    try {
+      const response = await fetch(progressUrl, fetchOptions)
+      if (!response.ok) return false
+
+      const progress = (await response.json()) as ExportProgress
+      setNotificationMessage(progress.message)
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  notify(startMessage, 'info', null)
+
+  try {
+    progressTimer = window.setInterval(() => {
+      waitingSeconds += 1
+      void (async () => {
+        const hasServerProgress = await updateServerProgress()
+        if (!hasServerProgress) {
+          setNotificationMessage(`${waitingMessage} (${waitingSeconds}s)`)
+        }
+      })()
+    }, 1000)
+
+    const response = await fetch(url, fetchOptions)
+    stopProgress()
+
+    if (!response.ok) {
+      throw new Error('Server response was not OK.')
+    }
+
+    const reader = response.body?.getReader()
+    if (!reader) {
+      throw new Error('Missing response stream.')
+    }
+
+    const file: Uint8Array[] = []
+    let bytes = 0
+    let closed = false
+
+    const showDownloadProgress = () => {
+      if (!closed) {
+        setTimeout(() => {
+          setNotificationMessage(`${downloadMessage}, ${formatMb(bytes)} MB`)
+          showDownloadProgress()
+        }, 500)
+      }
+    }
+
+    notify(`${downloadMessage}...`, 'info', null)
+    showDownloadProgress()
+
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      bytes = bytes + value.length
+      file.push(value)
+    }
+    closed = true
+    setNotificationMessage(`Saving ${filename} (${formatMb(bytes)} MB)`)
+
+    const blobUrl = window.URL.createObjectURL(new Blob(file, contentType ? { type: contentType } : undefined))
+    const downloadLink = document.createElement('a')
+    downloadLink.href = blobUrl
+    downloadLink.download = filename
+    document.body.appendChild(downloadLink)
+    downloadLink.click()
+    downloadLink.remove()
+    window.URL.revokeObjectURL(blobUrl)
+
+    notify('Download finished.')
+  } catch (error) {
+    stopProgress()
+    notify(failureMessage, 'error')
+    throw error
+  }
+}

frontend/tests/components/TableToolBar.export.test.tsx

@@ -23,19 +23,29 @@ jest.mock('../../src/components/Page', () => ({
   }),
 }))
 
+jest.mock('../../src/hooks/notification', () => ({
+  useNotify: () => ({
+    notify: jest.fn(),
+    setMessage: jest.fn(),
+  }),
+}))
+
 jest.mock('material-react-table', () => ({
   MRT_ShowHideColumnsButton: () => <button type="button">Show/Hide</button>,
 }))
 
 describe('TableToolBar export menu', () => {
   it('omits the species export option while keeping other exports available', async () => {
+    const kmlExport: (table: MRT_TableInstance<MRT_RowData>) => void = jest.fn(() => undefined)
+    const svgExport: (table: MRT_TableInstance<MRT_RowData>) => void = jest.fn(() => undefined)
+
     const { container } = render(
       <MemoryRouter>
         <TableToolBar
           table={{} as MRT_TableInstance<MRT_RowData>}
           tableName="Localities"
-          kmlExport={jest.fn()}
+          kmlExport={kmlExport}
-          svgExport={jest.fn()}
+          svgExport={svgExport}
         />
       </MemoryRouter>
     )