diff --git a/backend/src/routes/occurrence.ts b/backend/src/routes/occurrence.ts index 036af5fd..2d2e8328 100644 --- a/backend/src/routes/occurrence.ts +++ b/backend/src/routes/occurrence.ts @@ -3,14 +3,44 @@ import { pipeline } from 'stream' import { getOccurrenceDetail, updateOccurrenceDetail } from '../controllers/occurrenceController' import { requireOneOf } from '../middlewares/authorizer' import { Role } from '../../../frontend/src/shared/types' -import { buildDwcOccurrenceArchiveZipStream } from '../services/dwcArchiveExportOccurrences' +import { + buildDwcOccurrenceArchiveZipStream, + type DwcOccurrenceExportProgress, +} from '../services/dwcArchiveExportOccurrences' import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' import { logger } from '../utils/logger' const router = Router() -router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (_req, res, next) => { - const archive = await buildDwcOccurrenceArchiveZipStream() +const occurrenceExportProgress = new Map() + +const scheduleProgressCleanup = (exportId: string) => { + setTimeout( + () => { + occurrenceExportProgress.delete(exportId) + }, + 5 * 60 * 1000 + ) +} + +router.get('/export/dwc-archive/progress/:exportId', requireOneOf([Role.Admin]), (req, res) => { + const progress = occurrenceExportProgress.get(req.params.exportId) + if (!progress) return res.status(404).send({ message: 'Occurrence export progress not found.' }) + return res.status(200).send(progress) +}) + +router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (req, res, next) => { + const exportId = typeof req.query.exportId === 'string' ? req.query.exportId : undefined + const reportProgress = exportId + ? (progress: DwcOccurrenceExportProgress) => { + occurrenceExportProgress.set(exportId, progress) + } + : undefined + + const archive = await buildDwcOccurrenceArchiveZipStream({ reportProgress }).catch(error => { + if (exportId) scheduleProgressCleanup(exportId) + throw error + }) res.setHeader('Content-Type', 'application/zip') res.setHeader( 'Content-Disposition', @@ -20,6 +50,15 @@ router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (_req, res, archive.cleanup().catch(cleanupError => { logger.error(`Failed to clean up occurrence DwC export temp files: ${String(cleanupError)}`) }) + if (exportId) { + occurrenceExportProgress.set(exportId, { + stage: 'complete', + generated: 1, + total: 1, + message: 'DwC-A ZIP export ready.', + }) + scheduleProgressCleanup(exportId) + } if (error) next(error) }) }) diff --git a/backend/src/services/dwcArchiveExportOccurrences.ts b/backend/src/services/dwcArchiveExportOccurrences.ts index 9c4f3a32..d1721359 100644 --- a/backend/src/services/dwcArchiveExportOccurrences.ts +++ b/backend/src/services/dwcArchiveExportOccurrences.ts @@ -140,6 +140,15 @@ type DwcOccurrenceArchiveStream = { cleanup: () => Promise } +export type DwcOccurrenceExportProgress = { + stage: 'occurrences' | 'localities' | 'taxa' | 'zipping' | 'complete' + generated: number + total: number | null + message: string +} + +type DwcOccurrenceExportProgressReporter = (progress: DwcOccurrenceExportProgress) => void + const scientificNameForOccurrence = (species: SpeciesForOccurrenceExport): string => { const nameParts = [ toMaybeMeaningful(species.genus_name), @@ -475,6 +484,11 @@ async function* iterateOccurrenceRows(): AsyncGenerator { } } +const countOccurrenceRows = async (): Promise => { + const { nowDb } = await import('../utils/db') + return await nowDb.now_ls.count() +} + const chunk = (values: T[], size: number): T[][] => { const chunks: T[][] = [] for (let index = 0; index < values.length; index += size) { @@ -615,14 +629,25 @@ export const buildDwcOccurrenceArchiveZipBufferFromOccurrences = async ( const writeOccurrenceAndMeasurementFiles = async ({ occurrenceFilePath, measurementFilePath, + reportProgress, }: { occurrenceFilePath: string measurementFilePath: string + reportProgress?: DwcOccurrenceExportProgressReporter }): Promise<{ localityIds: number[]; speciesIds: number[] }> => { const occurrenceWriter = await createCsvFileWriter(occurrenceFilePath, OCCURRENCE_HEADERS) const measurementWriter = await createCsvFileWriter(measurementFilePath, MEASUREMENT_HEADERS) const localityIds = new Set() const speciesIds = new Set() + const totalOccurrences = await countOccurrenceRows() + let generatedOccurrences = 0 + + reportProgress?.({ + stage: 'occurrences', + generated: generatedOccurrences, + total: totalOccurrences, + message: `Generating occurrence rows: ${generatedOccurrences}/${totalOccurrences} generated`, + }) try { for await (const occurrence of iterateOccurrenceRows()) { @@ -633,6 +658,16 @@ const writeOccurrenceAndMeasurementFiles = async ({ for (const measurementRow of mapOccurrenceToMeasurementRows(occurrence)) { await measurementWriter.writeRow(measurementRow) } + + generatedOccurrences += 1 + if (generatedOccurrences === totalOccurrences || generatedOccurrences % OCCURRENCE_EXPORT_PAGE_SIZE === 0) { + reportProgress?.({ + stage: 'occurrences', + generated: generatedOccurrences, + total: totalOccurrences, + message: `Generating occurrence rows: ${generatedOccurrences}/${totalOccurrences} generated`, + }) + } } } finally { await occurrenceWriter.close() @@ -649,14 +684,24 @@ const writeLocalityLookupFiles = async ({ localityIds, locationFilePath, geologicalContextFilePath, + reportProgress, }: { localityIds: number[] locationFilePath: string geologicalContextFilePath: string + reportProgress?: DwcOccurrenceExportProgressReporter }): Promise => { const { nowDb } = await import('../utils/db') const locationWriter = await createCsvFileWriter(locationFilePath, LOCATION_HEADERS) const geologicalContextWriter = await createCsvFileWriter(geologicalContextFilePath, GEOLOGICAL_CONTEXT_HEADERS) + let generatedLocalities = 0 + + reportProgress?.({ + stage: 'localities', + generated: generatedLocalities, + total: localityIds.length, + message: `Generating location lookup rows: ${generatedLocalities}/${localityIds.length} generated`, + }) try { for (const ids of chunk(localityIds, LOOKUP_EXPORT_CHUNK_SIZE)) { @@ -670,7 +715,15 @@ const writeLocalityLookupFiles = async ({ const localityForExport = locality as unknown as LocalityForOccurrenceExport await locationWriter.writeRow(mapLocalityToLocationRow(localityForExport)) await geologicalContextWriter.writeRow(mapLocalityToGeologicalContextRow(localityForExport)) + generatedLocalities += 1 } + + reportProgress?.({ + stage: 'localities', + generated: generatedLocalities, + total: localityIds.length, + message: `Generating location lookup rows: ${generatedLocalities}/${localityIds.length} generated`, + }) } } finally { await locationWriter.close() @@ -681,12 +734,22 @@ const writeLocalityLookupFiles = async ({ const writeTaxonLookupFile = async ({ speciesIds, taxonFilePath, + reportProgress, }: { speciesIds: number[] taxonFilePath: string + reportProgress?: DwcOccurrenceExportProgressReporter }): Promise => { const { nowDb } = await import('../utils/db') const taxonWriter = await createCsvFileWriter(taxonFilePath, TAXON_HEADERS) + let generatedTaxa = 0 + + reportProgress?.({ + stage: 'taxa', + generated: generatedTaxa, + total: speciesIds.length, + message: `Generating taxon lookup rows: ${generatedTaxa}/${speciesIds.length} generated`, + }) try { for (const ids of chunk(speciesIds, LOOKUP_EXPORT_CHUNK_SIZE)) { @@ -698,14 +761,26 @@ const writeTaxonLookupFile = async ({ for (const species of speciesRows) { await taxonWriter.writeRow(mapSpeciesToTaxonRow(species)) + generatedTaxa += 1 } + + reportProgress?.({ + stage: 'taxa', + generated: generatedTaxa, + total: speciesIds.length, + message: `Generating taxon lookup rows: ${generatedTaxa}/${speciesIds.length} generated`, + }) } } finally { await taxonWriter.close() } } -export const buildDwcOccurrenceArchiveZipStream = async (): Promise => { +export const buildDwcOccurrenceArchiveZipStream = async ({ + reportProgress, +}: { + reportProgress?: DwcOccurrenceExportProgressReporter +} = {}): Promise => { const tempDirectory = await mkdtemp(path.join(tmpdir(), 'now-dwc-occurrences-')) const files = { location: path.join(tempDirectory, 'location.csv'), @@ -719,13 +794,22 @@ export const buildDwcOccurrenceArchiveZipStream = async (): Promise { + if (window.crypto?.randomUUID) return window.crypto.randomUUID() + return `${Date.now()}-${Math.random().toString(36).slice(2)}` +} + export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) const { notify, setMessage: setNotificationMessage } = useNotify() @@ -20,10 +29,40 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () = const fetchZipFile = async () => { setLoading(true) - notify('Generating DwC-A ZIP export, please wait...', 'info', null) + const exportId = createExportId() + let generationProgressTimer: number | undefined + notify('Generating DwC-A ZIP export...', 'info', null) + + const updateGenerationProgress = async () => { + try { + const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`, fetchOptions) + if (!response.ok) return + + const progress = (await response.json()) as OccurrenceExportProgress + setNotificationMessage(progress.message) + } catch { + // The download request owns the final success/failure notification. + } + } + + const stopGenerationProgress = () => { + if (generationProgressTimer !== undefined) { + window.clearInterval(generationProgressTimer) + generationProgressTimer = undefined + } + } try { - const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-archive`, fetchOptions) + generationProgressTimer = window.setInterval(() => { + void updateGenerationProgress() + }, 1000) + + const response = await fetch( + `${BACKEND_URL}/occurrence/export/dwc-archive?${new URLSearchParams({ exportId })}`, + fetchOptions + ) + stopGenerationProgress() + if (!response.ok) { throw new Error('Server response was not OK.') } @@ -68,6 +107,7 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () = notify('Download finished.') } catch { + stopGenerationProgress() notify('Downloading DwC-A export failed.', 'error') } finally { setLoading(false)