Show DwC occurrence export progress

backend/src/routes/occurrence.ts

@@ -3,14 +3,44 @@ import { pipeline } from 'stream'
 import { getOccurrenceDetail, updateOccurrenceDetail } from '../controllers/occurrenceController'
 import { requireOneOf } from '../middlewares/authorizer'
 import { Role } from '../../../frontend/src/shared/types'
-import { buildDwcOccurrenceArchiveZipStream } from '../services/dwcArchiveExportOccurrences'
+import {
+  buildDwcOccurrenceArchiveZipStream,
+  type DwcOccurrenceExportProgress,
+} from '../services/dwcArchiveExportOccurrences'
 import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
 import { logger } from '../utils/logger'
 
 const router = Router()
 
-router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (_req, res, next) => {
+const occurrenceExportProgress = new Map<string, DwcOccurrenceExportProgress>()
-  const archive = await buildDwcOccurrenceArchiveZipStream()
+
+const scheduleProgressCleanup = (exportId: string) => {
+  setTimeout(
+    () => {
+      occurrenceExportProgress.delete(exportId)
+    },
+    5 * 60 * 1000
+  )
+}
+
+router.get('/export/dwc-archive/progress/:exportId', requireOneOf([Role.Admin]), (req, res) => {
+  const progress = occurrenceExportProgress.get(req.params.exportId)
+  if (!progress) return res.status(404).send({ message: 'Occurrence export progress not found.' })
+  return res.status(200).send(progress)
+})
+
+router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (req, res, next) => {
+  const exportId = typeof req.query.exportId === 'string' ? req.query.exportId : undefined
+  const reportProgress = exportId
+    ? (progress: DwcOccurrenceExportProgress) => {
+        occurrenceExportProgress.set(exportId, progress)
+      }
+    : undefined
+
+  const archive = await buildDwcOccurrenceArchiveZipStream({ reportProgress }).catch(error => {
+    if (exportId) scheduleProgressCleanup(exportId)
+    throw error
+  })
   res.setHeader('Content-Type', 'application/zip')
   res.setHeader(
     'Content-Disposition',
@@ -20,6 +50,15 @@ router.get('/export/dwc-archive', requireOneOf([Role.Admin]), async (_req, res,
     archive.cleanup().catch(cleanupError => {
       logger.error(`Failed to clean up occurrence DwC export temp files: ${String(cleanupError)}`)
     })
+    if (exportId) {
+      occurrenceExportProgress.set(exportId, {
+        stage: 'complete',
+        generated: 1,
+        total: 1,
+        message: 'DwC-A ZIP export ready.',
+      })
+      scheduleProgressCleanup(exportId)
+    }
     if (error) next(error)
   })
 })

backend/src/services/dwcArchiveExportOccurrences.ts

@@ -140,6 +140,15 @@ type DwcOccurrenceArchiveStream = {
   cleanup: () => Promise<void>
 }
 
+export type DwcOccurrenceExportProgress = {
+  stage: 'occurrences' | 'localities' | 'taxa' | 'zipping' | 'complete'
+  generated: number
+  total: number | null
+  message: string
+}
+
+type DwcOccurrenceExportProgressReporter = (progress: DwcOccurrenceExportProgress) => void
+
 const scientificNameForOccurrence = (species: SpeciesForOccurrenceExport): string => {
   const nameParts = [
     toMaybeMeaningful(species.genus_name),
@@ -475,6 +484,11 @@ async function* iterateOccurrenceRows(): AsyncGenerator<OccurrenceForExport> {
   }
 }
 
+const countOccurrenceRows = async (): Promise<number> => {
+  const { nowDb } = await import('../utils/db')
+  return await nowDb.now_ls.count()
+}
+
 const chunk = <T>(values: T[], size: number): T[][] => {
   const chunks: T[][] = []
   for (let index = 0; index < values.length; index += size) {
@@ -615,14 +629,25 @@ export const buildDwcOccurrenceArchiveZipBufferFromOccurrences = async (
 const writeOccurrenceAndMeasurementFiles = async ({
   occurrenceFilePath,
   measurementFilePath,
+  reportProgress,
 }: {
   occurrenceFilePath: string
   measurementFilePath: string
+  reportProgress?: DwcOccurrenceExportProgressReporter
 }): Promise<{ localityIds: number[]; speciesIds: number[] }> => {
   const occurrenceWriter = await createCsvFileWriter(occurrenceFilePath, OCCURRENCE_HEADERS)
   const measurementWriter = await createCsvFileWriter(measurementFilePath, MEASUREMENT_HEADERS)
   const localityIds = new Set<number>()
   const speciesIds = new Set<number>()
+  const totalOccurrences = await countOccurrenceRows()
+  let generatedOccurrences = 0
+
+  reportProgress?.({
+    stage: 'occurrences',
+    generated: generatedOccurrences,
+    total: totalOccurrences,
+    message: `Generating occurrence rows: ${generatedOccurrences}/${totalOccurrences} generated`,
+  })
 
   try {
     for await (const occurrence of iterateOccurrenceRows()) {
@@ -633,6 +658,16 @@ const writeOccurrenceAndMeasurementFiles = async ({
       for (const measurementRow of mapOccurrenceToMeasurementRows(occurrence)) {
         await measurementWriter.writeRow(measurementRow)
       }
+
+      generatedOccurrences += 1
+      if (generatedOccurrences === totalOccurrences || generatedOccurrences % OCCURRENCE_EXPORT_PAGE_SIZE === 0) {
+        reportProgress?.({
+          stage: 'occurrences',
+          generated: generatedOccurrences,
+          total: totalOccurrences,
+          message: `Generating occurrence rows: ${generatedOccurrences}/${totalOccurrences} generated`,
+        })
+      }
     }
   } finally {
     await occurrenceWriter.close()
@@ -649,14 +684,24 @@ const writeLocalityLookupFiles = async ({
   localityIds,
   locationFilePath,
   geologicalContextFilePath,
+  reportProgress,
 }: {
   localityIds: number[]
   locationFilePath: string
   geologicalContextFilePath: string
+  reportProgress?: DwcOccurrenceExportProgressReporter
 }): Promise<void> => {
   const { nowDb } = await import('../utils/db')
   const locationWriter = await createCsvFileWriter(locationFilePath, LOCATION_HEADERS)
   const geologicalContextWriter = await createCsvFileWriter(geologicalContextFilePath, GEOLOGICAL_CONTEXT_HEADERS)
+  let generatedLocalities = 0
+
+  reportProgress?.({
+    stage: 'localities',
+    generated: generatedLocalities,
+    total: localityIds.length,
+    message: `Generating location lookup rows: ${generatedLocalities}/${localityIds.length} generated`,
+  })
 
   try {
     for (const ids of chunk(localityIds, LOOKUP_EXPORT_CHUNK_SIZE)) {
@@ -670,7 +715,15 @@ const writeLocalityLookupFiles = async ({
         const localityForExport = locality as unknown as LocalityForOccurrenceExport
         await locationWriter.writeRow(mapLocalityToLocationRow(localityForExport))
         await geologicalContextWriter.writeRow(mapLocalityToGeologicalContextRow(localityForExport))
+        generatedLocalities += 1
       }
+
+      reportProgress?.({
+        stage: 'localities',
+        generated: generatedLocalities,
+        total: localityIds.length,
+        message: `Generating location lookup rows: ${generatedLocalities}/${localityIds.length} generated`,
+      })
     }
   } finally {
     await locationWriter.close()
@@ -681,12 +734,22 @@ const writeLocalityLookupFiles = async ({
 const writeTaxonLookupFile = async ({
   speciesIds,
   taxonFilePath,
+  reportProgress,
 }: {
   speciesIds: number[]
   taxonFilePath: string
+  reportProgress?: DwcOccurrenceExportProgressReporter
 }): Promise<void> => {
   const { nowDb } = await import('../utils/db')
   const taxonWriter = await createCsvFileWriter(taxonFilePath, TAXON_HEADERS)
+  let generatedTaxa = 0
+
+  reportProgress?.({
+    stage: 'taxa',
+    generated: generatedTaxa,
+    total: speciesIds.length,
+    message: `Generating taxon lookup rows: ${generatedTaxa}/${speciesIds.length} generated`,
+  })
 
   try {
     for (const ids of chunk(speciesIds, LOOKUP_EXPORT_CHUNK_SIZE)) {
@@ -698,14 +761,26 @@ const writeTaxonLookupFile = async ({
 
       for (const species of speciesRows) {
         await taxonWriter.writeRow(mapSpeciesToTaxonRow(species))
+        generatedTaxa += 1
       }
+
+      reportProgress?.({
+        stage: 'taxa',
+        generated: generatedTaxa,
+        total: speciesIds.length,
+        message: `Generating taxon lookup rows: ${generatedTaxa}/${speciesIds.length} generated`,
+      })
     }
   } finally {
     await taxonWriter.close()
   }
 }
 
-export const buildDwcOccurrenceArchiveZipStream = async (): Promise<DwcOccurrenceArchiveStream> => {
+export const buildDwcOccurrenceArchiveZipStream = async ({
+  reportProgress,
+}: {
+  reportProgress?: DwcOccurrenceExportProgressReporter
+} = {}): Promise<DwcOccurrenceArchiveStream> => {
   const tempDirectory = await mkdtemp(path.join(tmpdir(), 'now-dwc-occurrences-'))
   const files = {
     location: path.join(tempDirectory, 'location.csv'),
@@ -719,13 +794,22 @@ export const buildDwcOccurrenceArchiveZipStream = async (): Promise<DwcOccurrenc
     const { localityIds, speciesIds } = await writeOccurrenceAndMeasurementFiles({
       occurrenceFilePath: files.occurrence,
       measurementFilePath: files.measurement,
+      reportProgress,
     })
     await writeLocalityLookupFiles({
       localityIds,
       locationFilePath: files.location,
       geologicalContextFilePath: files.geologicalContext,
+      reportProgress,
+    })
+    await writeTaxonLookupFile({ speciesIds, taxonFilePath: files.taxon, reportProgress })
+
+    reportProgress?.({
+      stage: 'zipping',
+      generated: 0,
+      total: null,
+      message: 'Compressing DwC-A ZIP...',
     })
-    await writeTaxonLookupFile({ speciesIds, taxonFilePath: files.taxon })
 
     const zip = new JSZip()
     zip.file('location.csv', createReadStream(files.location))

frontend/src/components/Occurrence/OccurrenceDwcExportMenuItem.tsx

@@ -6,6 +6,15 @@ import { useUser } from '@/hooks/user'
 import { Role } from '@/shared/types'
 import { currentDateAsString } from '@/shared/currentDateAsString'
 
+type OccurrenceExportProgress = {
+  message: string
+}
+
+const createExportId = () => {
+  if (window.crypto?.randomUUID) return window.crypto.randomUUID()
+  return `${Date.now()}-${Math.random().toString(36).slice(2)}`
+}
+
 export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
   const { notify, setMessage: setNotificationMessage } = useNotify()
@@ -20,10 +29,40 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () =
 
   const fetchZipFile = async () => {
     setLoading(true)
-    notify('Generating DwC-A ZIP export, please wait...', 'info', null)
+    const exportId = createExportId()
+    let generationProgressTimer: number | undefined
+    notify('Generating DwC-A ZIP export...', 'info', null)
+
+    const updateGenerationProgress = async () => {
+      try {
+        const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-archive/progress/${exportId}`, fetchOptions)
+        if (!response.ok) return
+
+        const progress = (await response.json()) as OccurrenceExportProgress
+        setNotificationMessage(progress.message)
+      } catch {
+        // The download request owns the final success/failure notification.
+      }
+    }
+
+    const stopGenerationProgress = () => {
+      if (generationProgressTimer !== undefined) {
+        window.clearInterval(generationProgressTimer)
+        generationProgressTimer = undefined
+      }
+    }
 
     try {
-      const response = await fetch(`${BACKEND_URL}/occurrence/export/dwc-archive`, fetchOptions)
+      generationProgressTimer = window.setInterval(() => {
+        void updateGenerationProgress()
+      }, 1000)
+
+      const response = await fetch(
+        `${BACKEND_URL}/occurrence/export/dwc-archive?${new URLSearchParams({ exportId })}`,
+        fetchOptions
+      )
+      stopGenerationProgress()
+
       if (!response.ok) {
         throw new Error('Server response was not OK.')
       }
@@ -68,6 +107,7 @@ export const OccurrenceDwcExportMenuItem = ({ handleClose }: { handleClose: () =
 
       notify('Download finished.')
     } catch {
+      stopGenerationProgress()
       notify('Downloading DwC-A export failed.', 'error')
     } finally {
       setLoading(false)