Merge pull request #839 from nowcommunity/cs-export-issues

Fix and improve cross search export

backend/src/routes/crossSearch.ts

@@ -5,13 +5,33 @@ import {
   parseAndValidateCrossSearchRouteParameters,
 } from '../services/crossSearch'
 import { fixBigInt } from '../utils/common'
-import { format } from '@fast-csv/format'
+import { format, FormatterRow, FormatterRowTransformFunction } from 'fast-csv'
 import { pipeline } from 'stream'
 import { logger } from '../utils/logger'
 import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
+import { CrossSearch } from '../../../frontend/src/shared/types'
+import { once } from 'events'
 
 const router = Router()
 
+const transformFunction = (row: CrossSearch & { full_count?: number }) => {
+  const transformedRow: { [key: string]: string | number | boolean | null } = {}
+  const keys = Object.keys(row) as Array<keyof (CrossSearch & { full_count?: number })>
+  for (const key of keys) {
+    if (key === 'full_count') {
+      delete row['full_count']
+      continue
+    }
+    const value = row[key]
+    if (typeof value === 'string') {
+      transformedRow[key] = value.replace(/[\r\n]+/g, ' ')
+    } else {
+      transformedRow[key] = row[key]
+    }
+  }
+  return transformedRow
+}
+
 router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => {
   let validatedValues
   try {
@@ -90,36 +110,58 @@ router.get(`/export/:columnfilters/:sorting`, async (req, res) => {
     return res.status(403).send('Unknown error')
   }
 
-  let data
+  let dataArray: Partial<CrossSearch[][]>
   try {
-    data = await getCrossSearchRawSql(
+    dataArray = (await getCrossSearchRawSql(
       req.user,
       undefined,
       undefined,
       validatedValues.validatedColumnFilters,
       validatedValues.validatedSorting
-    )
+    )) as Partial<CrossSearch[][]>
   } catch (error) {
     if (error instanceof Error) return res.status(403).send({ error: error.message })
     return res.status(403).send('Unknown error')
   }
 
-  res.attachment(`cross_search${currentDateAsString()}.csv`) // filename will get overwritten in frontend when fetching data from this route
   res.on('finish', () => {
     logger.info('Cross search export sent.')
   })
+  res.on('error', () => {
+    logger.info('ERROR!')
+  })
+  res.attachment(`cross_search${currentDateAsString()}.csv`) // filename will get overwritten in frontend when fetching data from this route
 
   // quoteColumns is needed to make sure linebreaks do not mess up the data
-  const stream = format({ headers: true, quoteColumns: true })
+  const stream = format({ headers: true, quoteColumns: true }).transform(
+    transformFunction as FormatterRowTransformFunction<FormatterRow, FormatterRow>
+  )
+
   pipeline(stream, res, err => {
     if (err) {
       logger.error(`Error in crosssearch/export pipeline: ${err.message}`)
+    } else {
+      logger.info('Cross search pipeline finished.')
     }
   })
 
-  for (const row of data) {
-    stream.write(row)
+  try {
+    for (const data of dataArray) {
+      if (data) {
+        for (const row of data) {
+          const ok = stream.write(row)
+          if (!ok) {
+            await once(stream, 'drain')
+          }
+        }
+      }
+    }
+  } catch (error) {
+    logger.error(
+      'Error in crosssearch/export: Could not write to stream, user might have left the page or refreshed it.'
+    )
   }
+
   return stream.end()
 })
 

backend/src/services/crossSearch.ts

@@ -113,6 +113,29 @@ export const getCrossSearchRawSql = async (
   const showAll = user ? [Role.Admin, Role.EditUnrestricted].includes(user.role) : false
   const allowedLocalities = user ? await getAllowedLocalities(user) : []
 
+  if (!limit) {
+    // this is only ran when exporting the cross-search table
+    limit = 20000
+    offset = 0
+    const results = []
+    while (true) {
+      const sql = generateCrossSearchSql(
+        showAll,
+        allowedLocalities,
+        limit,
+        offset,
+        convertedColumnFilters,
+        convertedOrderBy,
+        descendingOrder
+      )
+      const result: Partial<CrossSearch>[] = await nowDb.$queryRaw(sql)
+      if (result.length === 0) break
+      results.push(result)
+      offset = offset + limit
+    }
+    return results
+  }
+
   const sql = generateCrossSearchSql(
     showAll,
     allowedLocalities,

frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx

@@ -9,46 +9,66 @@ import { currentDateAsString } from '@/shared/currentDateAsString'
 export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
   const [loading, setLoading] = useState(false)
   const { sqlColumnFilters, sqlOrderBy } = usePageContext()
-  const notify = useNotify()
+  const { notify, setMessage: setNotificationMessage } = useNotify()
 
   const token = useUser().token
   const fetchOptions = token ? { headers: { Authorization: `Bearer ${token}` } } : {}
 
   const filename = `cross_search${currentDateAsString()}.csv`
 
-  const fetchCSVFile = () => {
+  const fetchCSVFile = async () => {
     setLoading(true)
-    notify('Downloading CSV file, please wait...', 'info', null)
     const URIColumnFilters = encodeURIComponent(JSON.stringify(sqlColumnFilters))
     const URIOrderBy = encodeURIComponent(JSON.stringify(sqlOrderBy))
 
-    fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions)
-      .then(response => {
-        if (!response.ok) {
-          throw new Error('Server response was not OK.')
-        }
-        return response.blob()
-      })
-      .then(blob => {
-        const blobUrl = window.URL.createObjectURL(blob)
-        const downloadLink = document.createElement('a')
-        downloadLink.href = blobUrl
-        downloadLink.download = filename
-        document.body.appendChild(downloadLink)
-        downloadLink.click()
-        downloadLink.remove()
-        window.URL.revokeObjectURL(blobUrl)
-        notify('Download finished.')
-      })
-      .catch(_ => {
-        notify('Downloading cross-search export failed.', 'error')
-      })
-      .finally(() => setLoading(false))
+    notify('Generating CSV file, please wait...', 'info', null)
+    const response = await fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions)
+
+    if (!response.ok) {
+      throw new Error('Server response was not OK.')
+    }
+    const reader = response.body!.getReader()
+    const file: Uint8Array[] = []
+    let bytes = 0
+    let closed = false
+
+    const showDownloadProgress = () => {
+      if (!closed) {
+        setTimeout(() => {
+          setNotificationMessage(`Downloading CSV file, ${Math.round((bytes / 1000000) * 10) / 10} MB`)
+          showDownloadProgress()
+        }, 500)
+      }
+    }
+    notify(`Downloading CSV file...`, 'info', null)
+    showDownloadProgress()
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      bytes = bytes + value.length
+      file.push(value)
+    }
+    closed = true
+    try {
+      const blobUrl = window.URL.createObjectURL(new Blob(file))
+      const downloadLink = document.createElement('a')
+      downloadLink.href = blobUrl
+      downloadLink.download = filename
+      document.body.appendChild(downloadLink)
+      downloadLink.click()
+      downloadLink.remove()
+      window.URL.revokeObjectURL(blobUrl)
+      notify('Download finished.')
+    } catch {
+      notify('Downloading cross-search export failed.', 'error')
+    } finally {
+      setLoading(false)
+    }
   }
   return (
     <MenuItem
       onClick={() => {
-        fetchCSVFile()
+        void fetchCSVFile()
         handleClose()
       }}
       disabled={loading}

frontend/src/components/DetailView/common/ContactForm.tsx

@@ -43,7 +43,7 @@ export const ContactForm = <T extends object>({
   const [sendEmailMutation] = useEmailMutation()
   const { trigger, getValues, register, setValue, control, formState } = useForm<ContactFormValues>()
   const { errors } = formState
-  const notify = useNotify()
+  const { notify } = useNotify()
   const user = useUser()
   const { data: personData, isLoading: personQueryloading } = useGetPersonDetailsQuery(user.initials ?? skipToken)
 

frontend/src/components/Locality/LocalityDetails.tsx

@@ -35,7 +35,7 @@ export const LocalityDetails = () => {
     skip: isNew,
   })
 
-  const notify = useNotify()
+  const { notify } = useNotify()
   const [deleteMutation, { isSuccess: deleteSuccess, isError: deleteError }] = useDeleteLocalityMutation()
 
   useEffect(() => {

frontend/src/components/Locality/LocalityTable.tsx

@@ -25,7 +25,7 @@ export const LocalityTable = ({ selectorFn }: { selectorFn?: (newObject: Localit
   const [getLocalitySpeciesList, { isLoading }] = useGetLocalitySpeciesListMutation()
   const [filteredLocalities, setFilteredLocalities] = useState<SimplifiedLocality[]>()
   const columnFilters = usePageContext()
-  const notify = useNotify()
+  const { notify } = useNotify()
 
   const handleLocalityRowActionClick = (row: Locality) => {
     setSelectedLocality(row.lid.toString())

frontend/src/components/Login.tsx

@@ -17,7 +17,7 @@ export const Login = () => {
   const location = useLocation()
   const search = new URLSearchParams(location.search)
   const loginHadExpired = search.get('expired')
-  const notify = useNotify()
+  const { notify } = useNotify()
 
   document.title = 'Login'
 

frontend/src/components/NavBar.tsx

@@ -16,7 +16,7 @@ export const NavBar = () => {
   const user = useUser()
   const navigate = useNavigate()
   const dispatch = useDispatch()
-  const notify = useNotify()
+  const { notify } = useNotify()
   const pages: LinkDefinition[] = [
     { title: 'Localities', url: '/locality' },
     { title: 'Species', url: '/species' },

frontend/src/components/Notification.tsx

@@ -8,6 +8,7 @@ export type NotificationContext = {
   open: boolean
   setOpen: (open: boolean) => void
   message: string
+  setMessage: (message: string) => void
   severity?: Severity
   timeoutValue?: number | null
 }
@@ -36,7 +37,7 @@ export const NotificationContextProvider = ({ children }: { children: ReactNode
     }, 200)
   }
   return (
-    <NotificationContext.Provider value={{ open, setOpen, message, severity, timeoutValue, notify }}>
+    <NotificationContext.Provider value={{ open, setOpen, message, setMessage, severity, timeoutValue, notify }}>
       {children}
     </NotificationContext.Provider>
   )

frontend/src/components/Person/PersonDetails.tsx

@@ -18,7 +18,7 @@ export const PersonDetails = () => {
   const { id: idFromUrl } = useParams()
   const user = useUser()
   const [editPersonRequest, { isLoading: mutationLoading }] = useEditPersonMutation()
-  const notify = useNotify()
+  const { notify } = useNotify()
   const navigate = useNavigate()
 
   // A bit convoluted, but this is to allow using the same component for user's own page

frontend/src/components/Person/Tabs/AddUserModal.tsx

@@ -119,7 +119,7 @@ export const AddUserModal = ({ isOpen, onClose, onSave, personInitials }: Props)
     userGroup: userGroups[0],
   })
   const [createUser, { isLoading }] = useCreateUserMutation()
-  const notify = useNotify()
+  const { notify } = useNotify()
 
   const save = () => {
     const userDetails: UserDetailsType = {

frontend/src/components/Person/Tabs/ChangePasswordForm.tsx

@@ -11,7 +11,7 @@ export const ChangePasswordForm = () => {
   const [newPassword, setNewPassword] = useState('')
   const [verifyPassword, setVerifyPassword] = useState('')
   const [changePasswordMutation] = useChangePasswordMutation()
-  const notify = useNotify()
+  const { notify } = useNotify()
   const sx = { display: 'flex', alignItems: 'center', width: '10em', fontWeight: 'bold' }
   const dispatch = useDispatch()
 

frontend/src/components/Person/Tabs/PersonTab.tsx

@@ -15,7 +15,7 @@ import { AddUserModal } from './AddUserModal'
 export const PersonTab = () => {
   const { textField, dropdownWithSearch, data, mode } = useDetailContext<PersonDetailsType>()
   const currentUser = useUser()
-  const notify = useNotify()
+  const { notify } = useNotify()
   const { id: idFromUrl } = useParams()
   const isNew = idFromUrl === 'new'
   const isAdmin = currentUser.role == Role.Admin

frontend/src/components/Reference/ReferenceDetails.tsx

@@ -24,7 +24,7 @@ export const ReferenceDetails = () => {
   const { isFetching, isError, data } = useGetReferenceDetailsQuery(id!, { skip: isNew })
   const [editReferenceRequest, { isLoading: mutationLoading }] = useEditReferenceMutation()
   const [deleteMutation, { isSuccess: deleteSuccess, isError: deleteError }] = useDeleteReferenceMutation()
-  const notify = useNotify()
+  const { notify } = useNotify()
   const navigate = useNavigate()
 
   useEffect(() => {

frontend/src/components/Region/RegionDetails.tsx

@@ -15,7 +15,7 @@ export const RegionDetails = () => {
   const { isLoading, isError, data } = useGetRegionDetailsQuery(id!, { skip: isNew })
   const [editRegionRequest, { isLoading: mutationLoading }] = useEditRegionMutation()
   const [deleteMutation, { isSuccess: deleteSuccess, isError: deleteError }] = useDeleteRegionMutation()
-  const notify = useNotify()
+  const { notify } = useNotify()
   const navigate = useNavigate()
 
   useEffect(() => {

frontend/src/components/Species/SpeciesDetails.tsx

@@ -25,7 +25,7 @@ export const SpeciesDetails = () => {
   }
   const [editSpeciesRequest, { isLoading: mutationLoading }] = useEditSpeciesMutation()
   const { isError, isFetching, data } = useGetSpeciesDetailsQuery(id!, { skip: isNew })
-  const notify = useNotify()
+  const { notify } = useNotify()
   const navigate = useNavigate()
   const [deleteMutation, { isSuccess: deleteSuccess, isError: deleteError }] = useDeleteSpeciesMutation()
   const [modifiedData, setModifiedData] = useState<SpeciesDetailsType | undefined>()

frontend/src/components/TimeBound/TimeBoundDetails.tsx

@@ -26,7 +26,7 @@ export const TimeBoundDetails = () => {
   })
   const [editTimeBoundRequest] = useEditTimeBoundMutation()
 
-  const notify = useNotify()
+  const { notify } = useNotify()
   const navigate = useNavigate()
   const [deleteMutation, { isSuccess: deleteSuccess, isError: deleteError }] = useDeleteTimeBoundMutation()
 

frontend/src/components/TimeUnit/TimeUnitDetails.tsx

@@ -25,7 +25,7 @@ export const TimeUnitDetails = () => {
   const { isLoading, isError, isFetching, data } = useGetTimeUnitDetailsQuery(encodeURIComponent(id!), { skip: isNew })
   const [editTimeUnitRequest] = useEditTimeUnitMutation()
 
-  const notify = useNotify()
+  const { notify } = useNotify()
   const navigate = useNavigate()
   const [deleteMutation, { isSuccess: deleteSuccess, isError: deleteError }] = useDeleteTimeUnitMutation()
 

frontend/src/hooks/notification.ts

@@ -2,6 +2,8 @@ import { NotificationContext } from '@/components/Notification'
 import { useContext } from 'react'
 
 export const useNotify = () => {
-  const { notify } = useContext(NotificationContext)
-  return notify
+  const { notify, setMessage } = useContext(NotificationContext)
+  // setMessage can be used to change a notification's message without creating a new one
+  // use this to create a progress bar etc.
+  return { notify, setMessage }
 }