Page filtered occurrence key export lookup

backend/src/services/crossSearch.ts

@@ -13,6 +13,8 @@ import { generateCrossSearchLocalitiesSql, generateCrossSearchSql } from './quer
 import { ValidationObject } from '../../../frontend/src/shared/validators/validator'
 import { validateCrossSearchRouteParams } from '../../../frontend/src/shared/validators/crossSearch'
 
+const CROSS_SEARCH_EXPORT_PAGE_SIZE = 20000
+
 export type CrossSearchRequestParameters = {
   limit?: string | number
   offset?: string | number
@@ -122,7 +124,7 @@ export const getCrossSearchRawSql = async (
 
   if (!limit) {
     // this is only ran when exporting the cross-search table
-    limit = 20000
+    limit = CROSS_SEARCH_EXPORT_PAGE_SIZE
     offset = 0
     const results = []
     while (true) {
@@ -259,21 +261,26 @@ export const getFilteredCrossSearchOccurrenceKeys = async (
     return { validationErrors }
   }
 
-  const resultPages = (await getCrossSearchRawSql(
-    user,
-    undefined,
-    undefined,
-    validatedColumnFilters,
-    validatedSorting
-  )) as Array<Array<Partial<CrossSearch>>>
-
   const keysById = new Map<string, { lid: number; speciesId: number }>()
-  for (const page of resultPages) {
+  let offset = 0
+  while (true) {
+    const page = (await getCrossSearchRawSql(
+      user,
+      CROSS_SEARCH_EXPORT_PAGE_SIZE,
+      offset,
+      validatedColumnFilters,
+      validatedSorting
+    )) as Array<Partial<CrossSearch>>
+    if (page.length === 0) break
+
     for (const row of page) {
       if (typeof row.lid_now_loc !== 'number' || typeof row.species_id_com_species !== 'number') continue
       const key = { lid: row.lid_now_loc, speciesId: row.species_id_com_species }
       keysById.set(`${key.lid}:${key.speciesId}`, key)
     }
+
+    if (page.length < CROSS_SEARCH_EXPORT_PAGE_SIZE) break
+    offset += CROSS_SEARCH_EXPORT_PAGE_SIZE
   }
 
   return { occurrenceKeys: [...keysById.values()] }

backend/src/services/dwcDataPackageExport.ts

@@ -415,9 +415,12 @@ const DWC_TERM_IRIS: Record<string, string> = {
   identificationVerificationStatus: 'http://rs.tdwg.org/dwc/terms/identificationVerificationStatus',
 }
 
+const fieldTitle = (name: string): string =>
+  name.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/^./, first => first.toUpperCase())
+
 const field = (name: string, type = 'string') => ({
   name,
-  title: name.replace(/([A-Z])/g, ' $1').replace(/^./, first => first.toUpperCase()),
+  title: fieldTitle(name),
   description: FIELD_DESCRIPTIONS[name] ?? `Curated NOW database value for ${name}.`,
   type,
   format: 'default',

backend/src/unit-tests/dwcDataPackageExport.test.ts

@@ -194,7 +194,7 @@ describe('DwC-DP export mapping', () => {
       resources: Array<{
         name: string
         mediatype?: string
-        schema: { foreignKeys?: unknown[]; missingValues?: string[] }
+        schema: { fields: Array<{ name: string; title: string }>; foreignKeys?: unknown[]; missingValues?: string[] }
       }>
     }
     expect(dataPackageJson.name).toBe('now-darwincore-export')
@@ -210,6 +210,16 @@ describe('DwC-DP export mapping', () => {
     ])
     expect(dataPackageJson.resources.every(resource => resource.mediatype === 'text/csv')).toBe(true)
     expect(dataPackageJson.resources.every(resource => resource.schema.missingValues?.includes('\\N'))).toBe(true)
+    expect(
+      dataPackageJson.resources
+        .find(resource => resource.name === 'event')
+        ?.schema.fields.find(field => field.name === 'eventID')?.title
+    ).toBe('Event ID')
+    expect(
+      dataPackageJson.resources
+        .find(resource => resource.name === 'event-assertion')
+        ?.schema.fields.find(field => field.name === 'assertionTypeIRI')?.title
+    ).toBe('Assertion Type IRI')
     expect(dataPackageJson.resources.find(resource => resource.name === 'occurrence')?.schema.foreignKeys).toEqual([
       {
         fields: 'eventID',