diff --git a/backend/src/services/crossSearch.ts b/backend/src/services/crossSearch.ts index e6977ad7..c0fc3e18 100644 --- a/backend/src/services/crossSearch.ts +++ b/backend/src/services/crossSearch.ts @@ -13,6 +13,8 @@ import { generateCrossSearchLocalitiesSql, generateCrossSearchSql } from './quer import { ValidationObject } from '../../../frontend/src/shared/validators/validator' import { validateCrossSearchRouteParams } from '../../../frontend/src/shared/validators/crossSearch' +const CROSS_SEARCH_EXPORT_PAGE_SIZE = 20000 + export type CrossSearchRequestParameters = { limit?: string | number offset?: string | number @@ -122,7 +124,7 @@ export const getCrossSearchRawSql = async ( if (!limit) { // this is only ran when exporting the cross-search table - limit = 20000 + limit = CROSS_SEARCH_EXPORT_PAGE_SIZE offset = 0 const results = [] while (true) { @@ -259,21 +261,26 @@ export const getFilteredCrossSearchOccurrenceKeys = async ( return { validationErrors } } - const resultPages = (await getCrossSearchRawSql( - user, - undefined, - undefined, - validatedColumnFilters, - validatedSorting - )) as Array>> - const keysById = new Map() - for (const page of resultPages) { + let offset = 0 + while (true) { + const page = (await getCrossSearchRawSql( + user, + CROSS_SEARCH_EXPORT_PAGE_SIZE, + offset, + validatedColumnFilters, + validatedSorting + )) as Array> + if (page.length === 0) break + for (const row of page) { if (typeof row.lid_now_loc !== 'number' || typeof row.species_id_com_species !== 'number') continue const key = { lid: row.lid_now_loc, speciesId: row.species_id_com_species } keysById.set(`${key.lid}:${key.speciesId}`, key) } + + if (page.length < CROSS_SEARCH_EXPORT_PAGE_SIZE) break + offset += CROSS_SEARCH_EXPORT_PAGE_SIZE } return { occurrenceKeys: [...keysById.values()] } diff --git a/backend/src/services/dwcDataPackageExport.ts b/backend/src/services/dwcDataPackageExport.ts index a4551d68..376fdcad 100644 --- a/backend/src/services/dwcDataPackageExport.ts +++ b/backend/src/services/dwcDataPackageExport.ts @@ -415,9 +415,12 @@ const DWC_TERM_IRIS: Record = { identificationVerificationStatus: 'http://rs.tdwg.org/dwc/terms/identificationVerificationStatus', } +const fieldTitle = (name: string): string => + name.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/^./, first => first.toUpperCase()) + const field = (name: string, type = 'string') => ({ name, - title: name.replace(/([A-Z])/g, ' $1').replace(/^./, first => first.toUpperCase()), + title: fieldTitle(name), description: FIELD_DESCRIPTIONS[name] ?? `Curated NOW database value for ${name}.`, type, format: 'default', diff --git a/backend/src/unit-tests/dwcDataPackageExport.test.ts b/backend/src/unit-tests/dwcDataPackageExport.test.ts index 28e24922..e5ccac02 100644 --- a/backend/src/unit-tests/dwcDataPackageExport.test.ts +++ b/backend/src/unit-tests/dwcDataPackageExport.test.ts @@ -194,7 +194,7 @@ describe('DwC-DP export mapping', () => { resources: Array<{ name: string mediatype?: string - schema: { foreignKeys?: unknown[]; missingValues?: string[] } + schema: { fields: Array<{ name: string; title: string }>; foreignKeys?: unknown[]; missingValues?: string[] } }> } expect(dataPackageJson.name).toBe('now-darwincore-export') @@ -210,6 +210,16 @@ describe('DwC-DP export mapping', () => { ]) expect(dataPackageJson.resources.every(resource => resource.mediatype === 'text/csv')).toBe(true) expect(dataPackageJson.resources.every(resource => resource.schema.missingValues?.includes('\\N'))).toBe(true) + expect( + dataPackageJson.resources + .find(resource => resource.name === 'event') + ?.schema.fields.find(field => field.name === 'eventID')?.title + ).toBe('Event ID') + expect( + dataPackageJson.resources + .find(resource => resource.name === 'event-assertion') + ?.schema.fields.find(field => field.name === 'assertionTypeIRI')?.title + ).toBe('Assertion Type IRI') expect(dataPackageJson.resources.find(resource => resource.name === 'occurrence')?.schema.foreignKeys).toEqual([ { fields: 'eventID',