Merge remote-tracking branch 'origin/main' into locality-details-displayed-in-sidebar

backend/package-lock.json

@@ -24,6 +24,7 @@
         "express": "^4.18.2",
         "express-async-errors": "^3.1.1",
         "express-rate-limit": "^7.5.0",
+        "fast-csv": "^5.0.2",
         "jsonwebtoken": "^9.0.2",
         "mariadb": "^3.3.0",
         "md5": "^2.3.0",
@@ -2456,6 +2457,33 @@
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       }
     },
+    "node_modules/@fast-csv/format": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-5.0.2.tgz",
+      "integrity": "sha512-fRYcWvI8vs0Zxa/8fXd/QlmQYWWkJqKZPAXM+vksnplb3owQFKTPPh9JqOtD0L3flQw/AZjjXdPkD7Kp/uHm8g==",
+      "license": "MIT",
+      "dependencies": {
+        "lodash.escaperegexp": "^4.1.2",
+        "lodash.isboolean": "^3.0.3",
+        "lodash.isequal": "^4.5.0",
+        "lodash.isfunction": "^3.0.9",
+        "lodash.isnil": "^4.0.0"
+      }
+    },
+    "node_modules/@fast-csv/parse": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.2.tgz",
+      "integrity": "sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==",
+      "license": "MIT",
+      "dependencies": {
+        "lodash.escaperegexp": "^4.1.2",
+        "lodash.groupby": "^4.6.0",
+        "lodash.isfunction": "^3.0.9",
+        "lodash.isnil": "^4.0.0",
+        "lodash.isundefined": "^3.0.1",
+        "lodash.uniq": "^4.5.0"
+      }
+    },
     "node_modules/@humanfs/core": {
       "version": "0.19.1",
       "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
@@ -6360,6 +6388,19 @@
         "express": "^4.11 || 5 || ^5.0.0-beta.1"
       }
     },
+    "node_modules/fast-csv": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-5.0.2.tgz",
+      "integrity": "sha512-CnB2zYAzzeh5Ta0UhSf32NexLy2SsEsSMY+fMWPV40k1OgaLEbm9Hf5dms3z/9fASZHBjB6i834079gVeksEqQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@fast-csv/format": "5.0.2",
+        "@fast-csv/parse": "5.0.2"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -9468,6 +9509,18 @@
       "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
       "dev": true
     },
+    "node_modules/lodash.escaperegexp": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz",
+      "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.groupby": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
+      "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==",
+      "license": "MIT"
+    },
     "node_modules/lodash.includes": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -9478,11 +9531,30 @@
       "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
       "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg=="
     },
+    "node_modules/lodash.isequal": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
+      "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==",
+      "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isfunction": {
+      "version": "3.0.9",
+      "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz",
+      "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==",
+      "license": "MIT"
+    },
     "node_modules/lodash.isinteger": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
       "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA=="
     },
+    "node_modules/lodash.isnil": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz",
+      "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==",
+      "license": "MIT"
+    },
     "node_modules/lodash.isnumber": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
@@ -9498,6 +9570,12 @@
       "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
       "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw=="
     },
+    "node_modules/lodash.isundefined": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz",
+      "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==",
+      "license": "MIT"
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -9510,6 +9588,12 @@
       "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
       "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg=="
     },
+    "node_modules/lodash.uniq": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
+      "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==",
+      "license": "MIT"
+    },
     "node_modules/lru-cache": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",

backend/package.json

@@ -53,6 +53,7 @@
     "express": "^4.18.2",
     "express-async-errors": "^3.1.1",
     "express-rate-limit": "^7.5.0",
+    "fast-csv": "^5.0.2",
     "jsonwebtoken": "^9.0.2",
     "mariadb": "^3.3.0",
     "md5": "^2.3.0",

backend/src/api-tests/crossSearch/export.test.ts

@@ -0,0 +1,67 @@
+import { beforeAll, afterAll, describe, it, expect } from '@jest/globals'
+import { resetDatabase, send, resetDatabaseTimeout } from '../utils'
+import { pool } from '../../utils/db'
+
+describe('Getting cross-search export', () => {
+  beforeAll(async () => {
+    await resetDatabase()
+  }, resetDatabaseTimeout)
+  afterAll(async () => {
+    await pool.end()
+  })
+
+  it('with invalid column filters type does not work', async () => {
+    const response = await send(`crosssearch/export/"not an array"/[]`, 'GET')
+    expect(response.status).toEqual(403)
+    expect(response.body).toEqual({ error: 'ColumnFilters is not an array.' })
+  })
+
+  it('with invalid column filters content does not work', async () => {
+    const { body: responseBody1, status: responseStatus1 } = await send(
+      `crosssearch/export/[{"id": "not_a_column", "value": "Dmanisi"}]/[]`,
+      'GET'
+    )
+    expect(responseStatus1).toEqual(403)
+    expect(responseBody1).toEqual({ error: 'columnFilters has an invalid column id.' })
+
+    const { body: responseBody2, status: responseStatus2 } = await send(
+      `crosssearch/export/[{"value": "Dmanisi"}]/[]`,
+      'GET'
+    )
+    expect(responseStatus2).toEqual(403)
+    expect(responseBody2).toEqual([{ error: 'Invalid or missing id field in filter', name: 'Column Filters' }])
+
+    const { body: responseBody3, status: responseStatus3 } = await send(
+      `crosssearch/export/[{"id": "lid_now_loc' OR 1=1 --", "value": "Dmanisi"}]/[]`,
+      'GET'
+    )
+    expect(responseStatus3).toEqual(403)
+    expect(responseBody3).toEqual({ error: 'columnFilters has an invalid column id.' })
+  })
+
+  it('with invalid sorting type does not work', async () => {
+    const response = await send(`crosssearch/export/[]/20`, 'GET')
+    expect(response.status).toEqual(403)
+    expect(response.body).toEqual({ error: 'Sorting is not an array.' })
+  })
+
+  it('with invalid sorting content does not work', async () => {
+    const { body: responseBody1, status: responseStatus1 } = await send(
+      `crosssearch/export/[]/[{"id": "not_a_column", "desc": true}]`,
+      'GET'
+    )
+    expect(responseStatus1).toEqual(403)
+    expect(responseBody1).toEqual({ error: 'orderBy was not a valid column id.' })
+
+    const { body: responseBody2, status: responseStatus2 } = await send(`crosssearch/export/[]/[{"desc": true}]`, 'GET')
+    expect(responseStatus2).toEqual(403)
+    expect(responseBody2).toEqual([{ error: 'Invalid or missing id field in sort object', name: 'Sorting' }])
+
+    const { body: responseBody3, status: responseStatus3 } = await send(
+      `crosssearch/export/[]/[{"id": "lid_now_loc' OR 1=1 --", "desc": true}]`,
+      'GET'
+    )
+    expect(responseStatus3).toEqual(403)
+    expect(responseBody3).toEqual({ error: 'orderBy was not a valid column id.' })
+  })
+})

backend/src/api-tests/region/get.test.ts

@@ -2,7 +2,7 @@ import { beforeAll, afterAll, describe, it, expect, beforeEach } from '@jest/glo
 import { login, logout, resetDatabase, send, resetDatabaseTimeout } from '../utils'
 import { pool } from '../../utils/db'
 
-describe('Deleting a region works', () => {
+describe('Getting region data works', () => {
   beforeAll(async () => {
     await resetDatabase()
   }, resetDatabaseTimeout)

backend/src/routes/crossSearch.ts

@@ -1,40 +1,93 @@
 import { Router } from 'express'
-import { getCrossSearchRawSql, validateCrossSearchRouteParameters } from '../services/crossSearch'
+import { getCrossSearchRawSql, parseAndValidateCrossSearchRouteParameters } from '../services/crossSearch'
 import { fixBigInt } from '../utils/common'
-import { ColumnFilter, SortingState } from '../services/queries/crossSearchQuery'
+import { format } from '@fast-csv/format'
+import { pipeline } from 'stream'
+import { logger } from '../utils/logger'
+import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString'
 
 const router = Router()
 
 router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => {
-  let limit
+  let validatedValues
-  let offset
-  let columnFilters
-  let sorting
   try {
-    limit = parseInt(req.params.limit)
+    const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({
-    offset = parseInt(req.params.offset)
+      limit: req.params.limit,
-    columnFilters = JSON.parse(req.params.columnfilters) as unknown
+      offset: req.params.offset,
-    sorting = JSON.parse(req.params.sorting) as unknown
+      columnFilters: req.params.columnfilters,
+      sorting: req.params.sorting,
+    })
+    validatedValues = values
+    if (validationErrors.length > 0) {
+      return res.status(403).send(validationErrors)
+    }
   } catch (error) {
-    return res.status(403).send({ error: 'Parsing URL parameters failed' })
+    if (error instanceof Error) return res.status(403).send({ error: error.message })
-  }
+    return res.status(403).send('Unknown error')
-  const validationErrors = validateCrossSearchRouteParameters({ limit, offset, columnFilters, sorting })
-  if (validationErrors.length > 0) {
-    return res.status(403).send(validationErrors)
   }
+
   try {
     const result = await getCrossSearchRawSql(
       req.user,
-      limit,
+      validatedValues.validatedLimit,
-      offset,
+      validatedValues.validatedOffset,
-      columnFilters as ColumnFilter[],
+      validatedValues.validatedColumnFilters,
-      sorting as SortingState[]
+      validatedValues.validatedSorting
     )
     return res.status(200).send(fixBigInt(result))
   } catch (error) {
-    if (error && typeof error === 'object' && 'message' in error) return res.status(403).send({ error: error.message })
+    if (error instanceof Error) return res.status(403).send({ error: error.message })
     return res.status(403).send('Unknown error')
   }
 })
 
+router.get(`/export/:columnfilters/:sorting`, async (req, res) => {
+  let validatedValues
+  try {
+    const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({
+      columnFilters: req.params.columnfilters,
+      sorting: req.params.sorting,
+    })
+    validatedValues = values
+    if (validationErrors.length > 0) {
+      return res.status(403).send(validationErrors)
+    }
+  } catch (error) {
+    if (error instanceof Error) return res.status(403).send({ error: error.message })
+    return res.status(403).send('Unknown error')
+  }
+
+  let data
+  try {
+    data = await getCrossSearchRawSql(
+      req.user,
+      undefined,
+      undefined,
+      validatedValues.validatedColumnFilters,
+      validatedValues.validatedSorting
+    )
+  } catch (error) {
+    if (error instanceof Error) return res.status(403).send({ error: error.message })
+    return res.status(403).send('Unknown error')
+  }
+
+  res.attachment(`cross_search${currentDateAsString()}.csv`) // filename will get overwritten in frontend when fetching data from this route
+  res.on('finish', () => {
+    logger.info('Cross search export sent.')
+  })
+
+  // quoteColumns is needed to make sure linebreaks do not mess up the data
+  const stream = format({ headers: true, quoteColumns: true })
+  pipeline(stream, res, err => {
+    if (err) {
+      logger.error(`Error in crosssearch/export pipeline: ${err.message}`)
+    }
+  })
+
+  for (const row of data) {
+    stream.write(row)
+  }
+  return stream.end()
+})
+
 export default router

backend/src/services/crossSearch.ts

@@ -1,6 +1,14 @@
-import { User, CrossSearch, Role, CrossSearchRouteParameters } from '../../../frontend/src/shared/types'
+import {
+  User,
+  CrossSearch,
+  Role,
+  CrossSearchRouteParameters,
+  ParsedCrossSearchRouteParameters,
+  SortingState,
+  ColumnFilter,
+} from '../../../frontend/src/shared/types'
 import { getCrossSearchFields, getFieldsOfTables, nowDb } from '../utils/db'
-import { ColumnFilter, SortingState, generateCrossSearchSql } from './queries/crossSearchQuery'
+import { generateCrossSearchSql } from './queries/crossSearchQuery'
 import { ValidationObject } from '../../../frontend/src/shared/validators/validator'
 import { validateCrossSearchRouteParams } from '../../../frontend/src/shared/validators/crossSearch'
 
@@ -118,13 +126,43 @@ export const getCrossSearchRawSql = async (
   return result
 }
 
-export const validateCrossSearchRouteParameters = (parameters: CrossSearchRouteParameters) => {
+export const parseAndValidateCrossSearchRouteParameters = (parameters: CrossSearchRouteParameters) => {
-  const keys = Object.keys(parameters)
+  const { limit, offset, columnFilters, sorting } = parameters
-  const errors: ValidationObject[] = []
+  let parsedLimit
+  let parsedOffset
+  if (limit) {
+    parsedLimit = parseInt(limit)
+    if (isNaN(parsedLimit)) throw new Error('Limit is not a number.')
+  }
+  if (offset) {
+    parsedOffset = parseInt(offset)
+    if (isNaN(parsedOffset)) throw new Error('Offset is not a number.')
+  }
+  const parsedColumnFilters = JSON.parse(columnFilters) as unknown
+  if (!Array.isArray(parsedColumnFilters)) throw new Error('ColumnFilters is not an array.')
+  const parsedSorting = JSON.parse(sorting) as unknown
+  if (!Array.isArray(parsedSorting)) throw new Error('Sorting is not an array.')
 
+  const parsedParameters = {
+    limit: parsedLimit,
+    offset: parsedOffset,
+    columnFilters: parsedColumnFilters,
+    sorting: parsedSorting,
+  }
+  const keys = Object.keys(parsedParameters)
+  const errors: ValidationObject[] = []
   for (const key of keys) {
-    const error = validateCrossSearchRouteParams(parameters, key as keyof CrossSearchRouteParameters)
+    const error = validateCrossSearchRouteParams(parsedParameters, key as keyof ParsedCrossSearchRouteParameters)
     if (error.error) errors.push(error)
   }
-  return errors
+  const validatedColumnFilters = parsedColumnFilters as ColumnFilter[]
+  const validatedSorting = parsedSorting as SortingState[]
+
+  return {
+    validatedLimit: parsedLimit,
+    validatedOffset: parsedOffset,
+    validatedColumnFilters,
+    validatedSorting,
+    validationErrors: errors,
+  }
 }

backend/src/services/queries/crossSearchQuery.ts

@@ -1,7 +1,5 @@
 import { Prisma } from '../../../prisma/generated/now_test_client'
-
+import { ColumnFilter } from '../../../../frontend/src/shared/types'
-export type ColumnFilter = { id: string; value: string }
-export type SortingState = { desc: boolean; id: string }
 
 const generateWhereClause = (showAll: boolean, allowedLocalities: Array<number>, columnFilters: ColumnFilter[]) => {
   let userCheck: Prisma.Sql

cypress/e2e/locality.cy.js

@@ -60,7 +60,7 @@ describe('Creating a new locality', () => {
     cy.get('[id=dms_lat-textfield]').should('have.value', '48 40 48 N')
     cy.contains('This field is required')
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('4 Invalid fields')
+    cy.contains('4 invalid fields')
   })
 
   it("and filling, then erasing needed data doesn't work", () => {
@@ -95,12 +95,12 @@ describe('Creating a new locality', () => {
     cy.get('[id=min_age-textfield]').type('{backspace}{backspace}{backspace}{backspace}{backspace}')
     cy.contains('This field is required')
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('1 Invalid fields')
+    cy.contains('1 invalid field')
 
     // make sure errors in other tabs disable the write button
     cy.get('[role=tablist]').contains('Locality').click()
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('1 Invalid fields')
+    cy.contains('1 invalid field')
   })
 
   it('write button is disabled if unvisited tab has validation errors', () => {
@@ -116,12 +116,12 @@ describe('Creating a new locality', () => {
 
     // Age tab has no errors, but Locality tab has unfilled mandatory fields
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('6 Invalid fields')
+    cy.contains('6 invalid fields')
     cy.get('[id=min_age-textfield-helper-text]').should('not.exist')
     cy.get('[id=max_age-textfield-helper-text]').should('not.exist')
     cy.get('[role=tablist]').contains('Locality').click()
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('6 Invalid fields')
+    cy.contains('6 invalid fields')
   })
 
   it('composite dating method works', () => {
@@ -233,7 +233,7 @@ describe('Editing a locality', () => {
     cy.get('[id=max_age-textfield]').type('{backspace}{backspace}{backspace}{backspace}{backspace}9')
     cy.contains('Min value cannot be higher than max')
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('4 Invalid fields')
+    cy.contains('4 invalid fields')
   })
 
   it('changing fractions and basis for age updates the age correctly', () => {

cypress/e2e/reference.cy.js

@@ -0,0 +1,175 @@
+before('Reset database', () => {
+  cy.request(Cypress.env('databaseResetUrl'))
+})
+
+describe('Creating a journal', () => {
+  beforeEach('Login as admin', () => {
+    cy.login('testSu')
+  })
+
+  it('with valid data works', () => {
+    cy.visit('/reference/new')
+    cy.contains('Journal')
+    cy.contains('7 invalid fields')
+
+    cy.get('[id=title_primary-textfield]').type('New test reference')
+
+    cy.get('[data-cy=Select\\ author-button]').click()
+    cy.get('[data-cy=detailview-button-1]').click()
+    cy.contains('Close').click()
+
+    cy.get('[id=date_primary-textfield]').type('2025')
+
+    cy.get('[data-cy=Select\\ Journal-button]').click()
+    cy.get('[data-cy=detailview-button-100]').click()
+    cy.contains('Close').click()
+
+    cy.get('[id=write-button]').click()
+    cy.contains('Saved reference successfully.')
+  })
+
+  it('with missing required fields does not work', () => {
+    cy.visit('/reference/new')
+    cy.contains('Journal')
+    cy.contains('7 invalid fields')
+
+    cy.get('[id=title_primary-textfield]').type('New test reference 2')
+    cy.contains('3 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[data-cy=Select\\ author-button]').click()
+    cy.get('[data-cy=detailview-button-1]').click()
+    cy.contains('Close').click()
+    cy.contains('2 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[id=date_primary-textfield]').type('2025')
+    cy.contains('1 invalid field')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[data-cy=Select\\ Journal-button]').click()
+    cy.get('[data-cy=detailview-button-100]').click()
+    cy.contains('Close').click()
+    cy.get('[id=write-button]').should('not.be.disabled')
+
+    cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected author
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.get('[data-cy=Select\\ author-button]').click()
+    cy.get('[data-cy=detailview-button-1]').click()
+    cy.contains('Close').click()
+    cy.get('[id=write-button]').should('not.be.disabled')
+  })
+
+  it('combined check with different types of titles and gen notes works', () => {
+    cy.visit('/reference/new')
+    cy.contains('Journal')
+    cy.contains('7 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[id=title_primary-textfield]').type('Title Primary')
+    cy.contains('3 invalid fields')
+    cy.get('[id=title_primary-textfield]').clear()
+    cy.contains('7 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[id=title_secondary-textfield]').type('Title Secondary')
+    cy.contains('3 invalid fields')
+    cy.get('[id=title_secondary-textfield]').clear()
+    cy.contains('7 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[id=title_series-textfield]').type('Title Series')
+    cy.contains('3 invalid fields')
+    cy.get('[id=title_series-textfield]').clear()
+    cy.contains('7 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+
+    cy.get('[id=gen_notes-textfield]').type('Gen Notes')
+    cy.contains('3 invalid fields')
+    cy.get('[id=gen_notes-textfield]').clear()
+    cy.contains('7 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+  })
+})
+
+// TODO: add more tests for different types of references
+
+describe('Editing a journal', () => {
+  beforeEach('Login as admin', () => {
+    cy.login('testSu')
+  })
+
+  it('with valid data works', () => {
+    cy.visit('reference/10039')
+    cy.contains('A new geomagnetic polarity time scale for the Late Cretaceous and Cenozoic.')
+    cy.get('[id=edit-button]').click()
+    cy.get('[id=title_primary-textfield]').clear()
+    cy.get('[id=title_primary-textfield]').type('New primary title.')
+
+    cy.get('[data-cy=Select\\ author-button]').click()
+    cy.get('[data-cy=detailview-button-2]').click()
+    cy.contains('Close').click()
+
+    cy.get('[id=write-button]').click()
+    cy.contains('Saved reference successfully.')
+    cy.contains('New primary title.')
+    cy.contains('Cande') // Author 1
+    cy.contains('Ciochon') // Author 2
+  })
+
+  it('with missing required fields does not work', () => {
+    cy.visit('reference/10039')
+    cy.contains('New primary title.')
+    cy.get('[id=edit-button]').click()
+    cy.get('[id=title_primary-textfield]').clear()
+    cy.contains('4 invalid fields')
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.get('[id=title_primary-textfield]').type('New primary title.')
+
+    cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click()
+    cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected authors
+    cy.contains('1 invalid field')
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.get('[data-cy=Select\\ author-button]').click()
+    cy.get('[data-cy=detailview-button-1]').click()
+    cy.contains('Close').click()
+    cy.get('[id=write-button]').should('not.be.disabled')
+  })
+})
+
+// TODO: add more tests for different types of references
+
+describe('Deleting a reference', () => {
+  beforeEach('Login as admin', () => {
+    cy.login('testSu')
+  })
+
+  it('that is not linked to anything works and returns user to table view', () => {
+    cy.visit('/reference/25000')
+    cy.get('[id=delete-button]').should('exist').click()
+    cy.on('window:confirm', str => {
+      expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.')
+      // this automatically clicks the OK button and returns to table view
+    })
+    cy.contains('Deleted item successfully.')
+    cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to
+    cy.contains('A test reference that is not linked to any localities or species').should('not.exist')
+    cy.visit('/reference/25000')
+    cy.contains('Error loading data')
+  })
+
+  // Skipped because references that have linked localities or species cannot be deleted currently
+  it.skip('that is linked to a locality and a species works and returns user to table view', () => {
+    cy.visit('/reference/10039')
+    cy.get('[id=delete-button]').should('exist').click()
+    cy.on('window:confirm', str => {
+      expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.')
+      // this automatically clicks the OK button and returns to table view
+    })
+    cy.contains('Deleted item successfully.')
+    cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to
+    cy.contains('A new geomagnetic polarity time scale for the Late Cretaceous and Cenozoic').should('not.exist')
+    cy.visit('/reference/10039')
+    cy.contains('Error loading data')
+  })
+})

cypress/e2e/species.cy.js

@@ -1,3 +1,7 @@
+before('Reset database', () => {
+  cy.request(Cypress.env('databaseResetUrl'))
+})
+
 describe('Creating a species', () => {
   beforeEach('Login as admin', () => {
     cy.login('testSu')
@@ -24,7 +28,78 @@ describe('Creating a species', () => {
     cy.visit('/species/new')
     cy.contains('This field is required')
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('4 Invalid fields')
+    cy.contains('4 invalid fields')
+  })
+})
+
+describe('Editing a species', () => {
+  beforeEach('Login as admin', () => {
+    cy.login('testSu')
+  })
+
+  it('with valid data works', () => {
+    cy.visit('/species/21052/')
+    cy.contains('21052 Simplomys simplicidens')
+    cy.get('[id=edit-button]').click()
+
+    cy.get('[id=order_name-textfield]').clear()
+    cy.get('[id=order_name-textfield]').type('modifiedOrder')
+    cy.get('[id=family_name-textfield]').clear()
+    cy.get('[id=family_name-textfield]').type('modifiedFamily')
+    cy.get('[id=genus_name-textfield]').clear()
+    cy.get('[id=genus_name-textfield]').type('modifiedGenus')
+    cy.get('[id=species_name-textfield]').clear()
+    cy.get('[id=species_name-textfield]').type('modifiedSpecies')
+    cy.get('[id=subfamily_name-textfield]').type('heimo')
+
+    cy.get('[id=write-button]').should('not.be.disabled')
+    cy.get('[id=write-button]').click()
+    cy.contains('button', 'Add existing reference').click()
+    cy.get('button[data-cy^="detailview-button"]').first().click()
+    cy.contains('button', 'Close').click()
+    cy.get('[id=write-button]').should('not.be.disabled')
+    cy.get('[id=write-button]').click()
+    cy.contains('Saved species entry successfully.')
+    cy.contains('modifiedOrder')
+    cy.contains('Subfamily or Tribe heimo')
+  })
+
+  it('with missing required fields does not work', () => {
+    cy.visit('/species/21052/')
+    cy.contains('21052')
+    cy.get('[id=edit-button]').click()
+
+    cy.get('[id=order_name-textfield]').clear()
+    cy.contains('Order: This field is required')
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.contains('1 invalid field')
+
+    cy.get('[id=family_name-textfield]').clear()
+    cy.contains('Family: This field is required')
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.contains('2 invalid fields')
+
+    cy.get('[id=order_name-textfield]').type('newOrder')
+    cy.contains('Order: This field is required').should('not.exist')
+    cy.contains('Family: This field is required')
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.contains('1 invalid field')
+
+    cy.get('[id=family_name-textfield]').type('newFamily')
+    cy.contains('Family: This field is required').should('not.exist')
+    cy.get('[id=write-button]').click()
+
+    cy.contains('button', 'Add existing reference').click()
+    cy.get('button[data-cy^="detailview-button"]').first().click()
+    cy.contains('button', 'Close').click()
+    cy.get('[id=write-button]').should('not.be.disabled')
+    cy.get('[id=write-button]').click()
+
+    cy.contains('Saved species entry successfully.')
+    cy.contains('newOrder')
+    cy.contains('newFamily')
+    cy.contains('modifiedGenus')
+    cy.contains('Subfamily or Tribe heimo')
   })
 })
 

cypress/e2e/timeBound.cy.js

@@ -1,10 +1,35 @@
-describe('Editing time bound', () => {
+before('Reset database', () => {
+  cy.request(Cypress.env('databaseResetUrl'))
+})
+
+describe('Creating time bound', () => {
   beforeEach('Login as admin', () => {
     cy.login('testSu')
   })
 
-  beforeEach('Reset database', () => {
+  it('with correct values works', () => {
-    cy.request(Cypress.env('databaseResetUrl'))
+    cy.visit(`/time-bound/new/`)
+    cy.contains('Creating new time-bound')
+    cy.contains('Name: This field is required')
+    cy.contains('Age (Ma): This field is required')
+    cy.get('[id=b_name-textfield]').first().type('test bound')
+    cy.contains('Name: This field is required').should('not.exist')
+    cy.get('[id=write-button]').should('be.disabled')
+    cy.get('[id=age-textfield]').first().type('20')
+    cy.contains('Age (Ma): This field is required').should('not.exist')
+    cy.get('[id=write-button]').click()
+    cy.contains('button', 'Add existing reference').click()
+    cy.get('button[data-cy^="detailview-button"]').first().click()
+    cy.contains('button', 'Close').click()
+    cy.get('[id=write-button]').should('not.be.disabled')
+    cy.get('[id=write-button]').click()
+    cy.contains('Edited item successfully.')
+  })
+})
+
+describe('Editing time bound', () => {
+  beforeEach('Login as admin', () => {
+    cy.login('testSu')
   })
 
   it('with correct values works', () => {
@@ -20,6 +45,7 @@ describe('Editing time bound', () => {
     cy.contains('button', 'Close').click()
     cy.get('[id=write-button]').should('not.be.disabled')
     cy.get('[id=write-button]').click()
+    cy.contains('Edited item successfully.')
     cy.contains('1.5')
   })
 
@@ -30,7 +56,7 @@ describe('Editing time bound', () => {
     cy.get('[id=age-textfield]').first().type('{backspace}{backspace}{backspace}{backspace}{backspace}')
     cy.contains('This field is required')
     cy.get('[id=write-button]').should('be.disabled')
-    cy.contains('1 Invalid fields')
+    cy.contains('1 invalid field')
   })
 })
 

documentation/backend/cross_search_export.md

@@ -0,0 +1,3 @@
+### Cross Search Export
+
+Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same parameter parsing and validation function as the `/all` route, and it also uses the same SQL query, but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information.

documentation/backend/server_side_pagination.md

@@ -6,7 +6,7 @@ The server side pagination for cross search is implemented in the following way:
 
 1. The backend route is given values for limit, offset, column filters, and sorting.
    - `limit` and `offset` should be numbers, and both `columnFilters` and `sorting` should be an array of objects containing the keys `id` and `value`. `sorting` is currently an array because Material React Table supports multiple sorting values. The backend only uses the first value in the array for the SQL query.
-2. The values are validated. This is done in two places: the [validation file](../../frontend/src/shared/validators/crossSearch.ts) and the [getCrossSearchRawSql function](../../backend/src/services/crossSearch.ts).
+2. The values are validated. This is done in two places: the [parsing and validation function](../../backend/src/services/crossSearch.ts), and the [getCrossSearchRawSql function](../../backend/src/services/crossSearch.ts).
    - The checks inside getCrossSearchRawSql are very important since they check if the values given to the `WHERE` and `ORDER BY` clauses are valid column ID's. Without these the query is vulnerable to SQL injection (see below)
 3. Since the SQL query uses aliases, some values used in the query need to be converted first. This is done with the [convertIdToFieldName function](../../backend/src/services/crossSearch.ts).
 4. The values are given to [generateCrossSearchSql function](backend/src/services/queries/crossSearchQuery.ts), which generates the SQL query. The values are used in the `LIMIT`, `OFFSET`, `WHERE` and `ORDER BY` clauses, respectively. To allow TableView to show and change pages correctly, we also add a `full_count` column to every row which contains the amount of rows in the entire cross-search table.

documentation/frontend/components/TableView.md

@@ -12,7 +12,7 @@ TableView has the following required props:
 
 The table will handle paging and sorting data automatically. If your backend query for the table is using [server side pagination](../../backend/server_side_pagination.md), remember to set the `serverSidePagination` prop to True. Check the Table View section in the guide below for more information about optional props.
 
-The table itself is constructed with the useMaterialReactTable-hook. The toolbar at the top is defined in the [helpers](../../../frontend/src/components/TableView/helpers.tsx) file and rendered using the `renderToolbarInternalActions` option, and it contains buttons for showing/hiding columns and showing the table in fullscreen mode, as well as exporting the content of the table. If the toolbar needs to be changed in the future (e.g. adding more rows to it) checkout the documentation for rendering a custom toolbar [here](https://www.material-react-table.com/docs/guides/toolbar-customization#override-with-custom-toolbar-components) since the current method doesn't really allow any major changes to its layout.
+The table itself is constructed with the useMaterialReactTable-hook. The toolbar at the top is defined in the [helpers](../../../frontend/src/components/TableView/helpers.tsx) file and rendered using the `renderToolbarInternalActions` option, and it contains buttons for showing/hiding columns and showing the table in fullscreen mode, as well as exporting the content of the table. To know what components you can render here if necessary, see the [source code](https://github.com/KevinVandy/material-react-table/blob/85b98f9aaa038df48aa1dd35123560abce78ee58/packages/material-react-table/src/components/toolbar/MRT_ToolbarInternalButtons.tsx#L45). If the toolbar needs to be changed in the future (e.g. adding more rows to it) checkout the documentation for rendering a custom toolbar [here](https://www.material-react-table.com/docs/guides/toolbar-customization#override-with-custom-toolbar-components) since the current method doesn't really allow any major changes to its layout.
 
 ### Setting up a new table view
 
@@ -38,5 +38,12 @@ Following things are needed to create a new page with a table (examples are all
   - to enable a filter mode selection menu for the columns of the table, set parameter `enableColumnFilterModes` to `true`.
   - you can pass an array of filter functions to be shown in the menu to the parameter `columnFilterModeOptions`.
   - it's a good idea to disable the filter mode menu for columns for which it doesn't make sense by setting the parameter `enableColumnFilterModes` to `false` for that column. See [LocalityTable](../../../frontend/src/components/Locality/LocalityTable.tsx) for an example. For more information, see [MRT docs filter mode section](https://www.material-react-table.com/docs/guides/column-filtering#filter-modes).
-  - If the table you're creating is using server side pagination, remember to set the `serverSidePagination` prop to True so the TableView will set its pagination/sorting modes correctly. Forgetting to do this might make the table load indefinitely or crash the app.
+  - If the table you're creating is using [server side pagination](../../backend/server_side_pagination.md), remember to set the `serverSidePagination` prop to True so the TableView will set its pagination/sorting modes correctly. Forgetting to do this might make the table load indefinitely or crash the app.
+  - `isCrossSearchTable` is only set inside [CrossSearchTable.tsx](../../../frontend/src/components/CrossSearch/CrossSearchTable.tsx). See the "Exporting" section below.
   - More info can be found from official [MRT docs](https://www.material-react-table.com/).
+
+### Exporting
+
+Most tables simply call the [exportRows function](../../../frontend/src/components/TableView/helpers.tsx) when the user clicks the "Export Table" button. This function converts the entire table into a csv file and downloads it. The [export function for cross-search](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) is different since it uses server side pagination and the cross-search table does not contain all needed data at once. The `isCrossSearchTable` prop is used to render the proper toolbar and "Export Table" button for the cross-search table. See [server side pagination](../../backend/server_side_pagination.md) and [cross search export](../../backend/cross_search_export.md) docs for more information.
+
+Currently the export function for cross-search uses blob(), which waits for the entire file to be streamed before continuing. If you want to start downloading the file as soon as the backend starts streaming it, [this MDN document](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#reading_the_stream) might help. This might not speed up the download, but it shows the "download started" popup in the user's browser faster, which can make the wait a little less annoying for them.

frontend/package.json

@@ -18,7 +18,6 @@
     "@reduxjs/toolkit": "^2.2.3",
     "export-to-csv": "^1.3.0",
     "leaflet": "^1.9.4",
-    "react-leaflet": "^4.2.1",
     "lodash-es": "^4.17.21",
     "material-react-table": "^3.0.1",
     "react": "^18.2.0",

frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx

@@ -0,0 +1,59 @@
+import { useState } from 'react'
+import { useNotify } from '@/hooks/notification'
+import { BACKEND_URL } from '@/util/config'
+import { MenuItem } from '@mui/material'
+import { useUser } from '@/hooks/user'
+import { usePageContext } from '../Page'
+import { currentDateAsString } from '@/shared/currentDateAsString'
+
+export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => void }) => {
+  const [loading, setLoading] = useState(false)
+  const { sqlColumnFilters, sqlOrderBy } = usePageContext()
+  const notify = useNotify()
+
+  const token = useUser().token
+  const fetchOptions = token ? { headers: { Authorization: `Bearer ${token}` } } : {}
+
+  const filename = `cross_search${currentDateAsString()}.csv`
+
+  const fetchCSVFile = () => {
+    setLoading(true)
+    notify('Downloading CSV file, please wait...', 'info', null)
+    const URIColumnFilters = encodeURIComponent(JSON.stringify(sqlColumnFilters))
+    const URIOrderBy = encodeURIComponent(JSON.stringify(sqlOrderBy))
+
+    fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions)
+      .then(response => {
+        if (!response.ok) {
+          throw new Error('Server response was not OK.')
+        }
+        return response.blob()
+      })
+      .then(blob => {
+        const blobUrl = window.URL.createObjectURL(blob)
+        const downloadLink = document.createElement('a')
+        downloadLink.href = blobUrl
+        downloadLink.download = filename
+        document.body.appendChild(downloadLink)
+        downloadLink.click()
+        downloadLink.remove()
+        window.URL.revokeObjectURL(blobUrl)
+        notify('Download finished.')
+      })
+      .catch(_ => {
+        notify('Downloading cross-search export failed.', 'error')
+      })
+      .finally(() => setLoading(false))
+  }
+  return (
+    <MenuItem
+      onClick={() => {
+        fetchCSVFile()
+        handleClose()
+      }}
+      disabled={loading}
+    >
+      Export table
+    </MenuItem>
+  )
+}

frontend/src/components/CrossSearch/CrossSearchTable.tsx

@@ -812,6 +812,7 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros
       url="crosssearch"
       enableColumnFilterModes={true}
       serverSidePagination={true}
+      isCrossSearchTable={true}
     />
   )
 }

frontend/src/components/DetailView/common/SelectingTable.tsx

@@ -67,7 +67,7 @@ export const SelectingTable = <T extends MRT_RowData, ParentType extends object>
   if (isError) return <Box>Error fetching data for the selecting table.</Box>
   if (!data) return <CircularProgress />
   return (
-    <EditingModal buttonText={buttonText}>
+    <EditingModal dataCy={`${buttonText}-button`} buttonText={buttonText}>
       {data ? (
         <TableView<T>
           data={filteredData}

frontend/src/components/DetailView/components.tsx

@@ -113,6 +113,8 @@ export const ErrorBox = <T,>() => {
   const { fieldsWithErrors } = useDetailContext<T>()
   const fields = Object.keys(fieldsWithErrors)
 
+  const title = fields.length > 1 ? `${fields.length} invalid fields` : '1 invalid field'
+
   return (
     <Box
       sx={{
@@ -126,7 +128,7 @@ export const ErrorBox = <T,>() => {
       }}
     >
       <Typography color="text.secondary" gutterBottom>
-        {`${fields.length} Invalid fields`}
+        {title}
       </Typography>
       <List sx={{ maxHeight: '5em', maxWidth: '30em', padding: '0em 0.8em 0em 0.8em', overflow: 'auto' }}>
         {fields.map(field => (

frontend/src/components/Map/LocalitiesMap.tsx

@@ -19,19 +19,19 @@ interface Props {
   localitiesQueryIsFetching: boolean
 }
 
+type CustomMarkerOptions = L.MarkerOptions & { localityId: number }
+type CustomMarker = L.Marker & { options: CustomMarkerOptions }
+
 //  TODO: Don't do this
 const markerIcon =
   'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABkAAAApCAYAAADAk4LOAAAFgUlEQVR4Aa1XA5BjWRTN2oW17d3YaZtr2962HUzbDNpjszW24mRt28p47v7zq/bXZtrp/lWnXr337j3nPCe85NcypgSFdugCpW5YoDAMRaIMqRi6aKq5E3YqDQO3qAwjVWrD8Ncq/RBpykd8oZUb/kaJutow8r1aP9II0WmLKLIsJyv1w/kqw9Ch2MYdB++12Onxee/QMwvf4/Dk/Lfp/i4nxTXtOoQ4pW5Aj7wpici1A9erdAN2OH64x8OSP9j3Ft3b7aWkTg/Fm91siTra0f9on5sQr9INejH6CUUUpavjFNq1B+Oadhxmnfa8RfEmN8VNAsQhPqF55xHkMzz3jSmChWU6f7/XZKNH+9+hBLOHYozuKQPxyMPUKkrX/K0uWnfFaJGS1QPRtZsOPtr3NsW0uyh6NNCOkU3Yz+bXbT3I8G3xE5EXLXtCXbbqwCO9zPQYPRTZ5vIDXD7U+w7rFDEoUUf7ibHIR4y6bLVPXrz8JVZEql13trxwue/uDivd3fkWRbS6/IA2bID4uk0UpF1N8qLlbBlXs4Ee7HLTfV1j54APvODnSfOWBqtKVvjgLKzF5YdEk5ewRkGlK0i33Eofffc7HT56jD7/6U+qH3Cx7SBLNntH5YIPvODnyfIXZYRVDPqgHtLs5ABHD3YzLuespb7t79FY34DjMwrVrcTuwlT55YMPvOBnRrJ4VXTdNnYug5ucHLBjEpt30701A3Ts+HEa73u6dT3FNWwflY86eMHPk+Yu+i6pzUpRrW7SNDg5JHR4KapmM5Wv2E8Tfcb1HoqqHMHU+uWDD7zg54mz5/2BSnizi9T1Dg4QQXLToGNCkb6tb1NU+QAlGr1++eADrzhn/u8Q2YZhQVlZ5+CAOtqfbhmaUCS1ezNFVm2imDbPmPng5wmz+gwh+oHDce0eUtQ6OGDIyR0uUhUsoO3vfDmmgOezH0mZN59x7MBi++WDL1g/eEiU3avlidO671bkLfwbw5XV2P8Pzo0ydy4t2/0eu33xYSOMOD8hTf4CrBtGMSoXfPLchX+J0ruSePw3LZeK0juPJbYzrhkH0io7B3k164hiGvawhOKMLkrQLyVpZg8rHFW7E2uHOL888IBPlNZ1FPzstSJM694fWr6RwpvcJK60+0HCILTBzZLFNdtAzJaohze60T8qBzyh5ZuOg5e7uwQppofEmf2++DYvmySqGBuKaicF1blQjhuHdvCIMvp8whTTfZzI7RldpwtSzL+F1+wkdZ2TBOW2gIF88PBTzD/gpeREAMEbxnJcaJHNHrpzji0gQCS6hdkEeYt9DF/2qPcEC8RM28Hwmr3sdNyht00byAut2k3gufWNtgtOEOFGUwcXWNDbdNbpgBGxEvKkOQsxivJx33iow0Vw5S6SVTrpVq11ysA2Rp7gTfPfktc6zhtXBBC+adRLshf6sG2RfHPZ5EAc4sVZ83yCN00Fk/4kggu40ZTvIEm5g24qtU4KjBrx/BTTH8ifVASAG7gKrnWxJDcU7x8X6Ecczhm3o6YicvsLXWfh3Ch1W0k8x0nXF+0fFxgt4phz8QvypiwCCFKMqXCnqXExjq10beH+UUA7+nG6mdG/Pu0f3LgFcGrl2s0kNNjpmoJ9o4B29CMO8dMT4Q5ox8uitF6fqsrJOr8qnwNbRzv6hSnG5wP+64C7h9lp30hKNtKdWjtdkbuPA19nJ7Tz3zR/ibgARbhb4AlhavcBebmTHcFl2fvYEnW0ox9xMxKBS8btJ+KiEbq9zA4RthQXDhPa0T9TEe69gWupwc6uBUphquXgf+/FrIjweHQS4/pduMe5ERUMHUd9xv8ZR98CxkS4F2n3EUrUZ10EYNw7BWm9x1GiPssi3GgiGRDKWRYZfXlON+dfNbM+GgIwYdwAAAAASUVORK5CYII='
 
-type CustomMarkerOptions = L.MarkerOptions & { localityId: number }
-type CustomMarker = L.Marker & { options: CustomMarkerOptions }
-
 export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching }: Props) => {
   const [selectedLocality, setSelectedLocality] = useState<string | null>(null)
-  // Map instance tracked in state for attaching event listeners
-  const [mapInstance, setMapInstance] = useState<L.Map | null>(null)
   const leafletMapRef = useRef<L.Map | null>(null)
   const mapRef = useRef<HTMLDivElement | null>(null)
+  const markersRef = useRef<L.Layer | null>(null)
+  const [map, setMap] = useState<L.Map | null>(null)
   const [isOpen, setIsOpen] = useState(false)
   const columnFilters = usePageContext()
   const {
@@ -41,58 +41,20 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching }
   } = useGetLocalityDetailsQuery(selectedLocality ?? skipToken) // use skipToken to skip query when state is null
 
   useEffect(() => {
-    if (!mapRef.current || leafletMapRef.current) return
+    if (!mapRef.current) return
+
+    const mapInstance = L.map(mapRef.current, { maxZoom: 16 })
+    setMap(mapInstance)
 
-    const map = L.map(mapRef.current, { maxZoom: 16 })
-    setMapInstance(map)
     const coords: LatLngExpression = [15, 13]
-
+    mapInstance.setView(coords, 2)
-    // To prevent eslint from complaining about that 'markers' variable below:
-    /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-    /* eslint-disable @typescript-eslint/no-unsafe-call */
-
-    /* eslint-disable @typescript-eslint/no-unsafe-member-access */
-
-    // @ts-expect-error The marker cluster library is a plain javascript library
-    // with no module exports that extends 'L' when imported
-    const markers: Layer = L.markerClusterGroup()
-
-    const localityIds = columnFilters.idList as unknown as number[]
-    const filterApplied = localityIds.length > 0 && localityIds.every(id => id === null)
-
-    const validIds = localityIds.filter(id => typeof id === 'number')
-
-    const filteredLocalities = !filterApplied
-      ? localitiesQueryData?.filter(locality => validIds.includes(locality.lid))
-      : localitiesQueryData
-
-    filteredLocalities?.forEach(locality => {
-      const options: CustomMarkerOptions = {
-        icon: new Icon({ iconUrl: markerIcon, iconSize: [25, 41], iconAnchor: [12, 41] }),
-        title: locality.loc_name,
-        localityId: locality.lid,
-      }
-
-      const marker = L.marker([locality.dec_lat, locality.dec_long], options) as CustomMarker
-
-      marker.on('click', () => {
-        setSelectedLocality(marker.options.localityId.toString())
-      })
-
-      markers.addLayer(marker)
-    })
-
-    // remote add to map
-    markers.addTo(map)
-
-    map.setView(coords, 2)
 
     // BASE MAPS
     //// OpenTopoMap
     const topomap = L.tileLayer('https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png', {
       attribution: 'Map data: © <a href="https://opentopomap.org">OpenTopoMap</a> (CC-BY-SA)',
       noWrap: true,
-    }).addTo(map)
+    }).addTo(mapInstance)
 
     leafletMapRef.current = map
     //// OpenStreetMap
@@ -104,7 +66,7 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching }
     // LAYERS ON TOP OF BASE MAPS
     //// Add borders to the map
     borders.forEach(poly => {
-      L.polygon(poly as LatLngExpression[], { color: 'gray', weight: 1 }).addTo(map)
+      L.polygon(poly as LatLngExpression[], { color: 'gray', weight: 1 }).addTo(mapInstance)
     })
 
     // create a polygon layer for the country borders that is in layer control panel
@@ -125,37 +87,74 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching }
     }
 
     // Add a scale bar to the map
-    L.control.scale({ position: 'bottomright' }).addTo(map)
+    L.control.scale({ position: 'bottomright' }).addTo(mapInstance)
     // Add a layer control to the map
-    L.control.layers(baseMaps, {}, { position: 'topright' }).addTo(map)
+    L.control.layers(baseMaps, {}, { position: 'topright' }).addTo(mapInstance)
 
     return () => {
-      map.remove()
       leafletMapRef.current = null
-      setMapInstance(null)
+      mapInstance.remove()
     }
-  }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters])
+  }, [])
 
   useEffect(() => {
-    if (!mapInstance) return
+    if (!map || localitiesQueryIsFetching) return
-    const map = mapInstance
+
+    if (markersRef.current) {
+      map.removeLayer(markersRef.current)
+      markersRef.current = null
+    }
+    // To prevent eslint from complaining about that 'markers' variable below:
+    /* eslint-disable @typescript-eslint/no-unsafe-assignment */
+    /* eslint-disable @typescript-eslint/no-unsafe-call */
+
+    /* eslint-disable @typescript-eslint/no-unsafe-member-access */
+
+    // @ts-expect-error The marker cluster library is a plain javascript library
+    // with no module exports that extends 'L' when imported
+    const newMarkers: Layer = L.markerClusterGroup()
+
+    const localityIds = columnFilters.idList as unknown as number[]
+    const filterApplied = localityIds.length > 0 && localityIds.every(id => id === null)
+
+    const validIds = localityIds.filter(id => typeof id === 'number')
+
+    const filteredLocalities = !filterApplied
+      ? localitiesQueryData?.filter(locality => validIds.includes(locality.lid))
+      : localitiesQueryData
+
+    filteredLocalities?.forEach(locality => {
+      const options: CustomMarkerOptions = {
+        icon: new Icon({ iconUrl: markerIcon, iconSize: [25, 41], iconAnchor: [12, 41] }),
+        title: locality.loc_name,
+        localityId: locality.lid,
+      }
+
+      const marker = L.marker([locality.dec_lat, locality.dec_long], options) as CustomMarker
+      marker.on('click', () => {
+        setSelectedLocality(marker.options.localityId.toString())
+      })
+
+      newMarkers.addLayer(marker)
+    })
+
+    newMarkers.addTo(map)
+    markersRef.current = newMarkers
+  }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters, map])
+
+  useEffect(() => {
+    if (!map) return
+    const mapInstance = map
 
     const handleMapClick = () => {
       setSelectedLocality(null)
     }
-    map.on('click', handleMapClick)
+    mapInstance.on('click', handleMapClick)
 
     return () => {
-      map.off('click', handleMapClick)
+      mapInstance.off('click', handleMapClick)
     }
-  }, [mapInstance])
+  }, [map])
-
-  // useEffect(() => {
-  //   if (localityDetailsQueryData) {
-  //     console.log('Fetched locality details:', localityDetailsQueryData)
-  //   }
-  // })
-  // }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters])
 
   document.title = 'Map'
 

frontend/src/components/TableView/TableToolBar.tsx

@@ -7,12 +7,14 @@ import '../../styles/TableToolBar.css'
 import { useState } from 'react'
 import { Link } from 'react-router-dom'
 import AddCircleIcon from '@mui/icons-material/AddCircle'
+import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportMenuItem'
 
 export const TableToolBar = <T extends MRT_RowData>({
   table,
   combinedExport,
   kmlExport,
   exportIsLoading,
+  isCrossSearchTable,
   selectorFn,
   showNewButton,
 }: {
@@ -20,6 +22,7 @@ export const TableToolBar = <T extends MRT_RowData>({
   combinedExport?: (lids: number[]) => Promise<void>
   kmlExport?: (table: MRT_TableInstance<T>) => void
   exportIsLoading?: boolean
+  isCrossSearchTable?: boolean
   selectorFn?: (id: T) => void
   showNewButton?: boolean
 }) => {
@@ -67,14 +70,19 @@ export const TableToolBar = <T extends MRT_RowData>({
             'aria-labelledby': 'export-button',
           }}
         >
-          <MenuItem
+          {isCrossSearchTable ? (
-            onClick={() => {
+            <CrossSearchExportMenuItem handleClose={handleClose} />
-              exportRows(table)
+          ) : (
-              handleClose()
+            <MenuItem
-            }}
+              onClick={() => {
-          >
+                exportRows(table)
-            Export table
+                handleClose()
-          </MenuItem>
+              }}
+            >
+              Export table
+            </MenuItem>
+          )}
+
           {kmlExport && (
             <MenuItem
               onClick={() => {

frontend/src/components/TableView/TableView.tsx

@@ -27,6 +27,7 @@ type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination'
   
   selectorFn should only be defined if using this as a selecting table
 */
+
 export const TableView = <T extends MRT_RowData>({
   data,
   columns,
@@ -39,6 +40,7 @@ export const TableView = <T extends MRT_RowData>({
   combinedExport,
   kmlExport,
   exportIsLoading,
+  isCrossSearchTable,
   enableColumnFilterModes,
   serverSidePagination,
   isFetching,
@@ -54,6 +56,7 @@ export const TableView = <T extends MRT_RowData>({
   combinedExport?: (lids: number[]) => Promise<void>
   kmlExport?: (table: MRT_TableInstance<T>) => void
   exportIsLoading?: boolean
+  isCrossSearchTable?: boolean
   enableColumnFilterModes?: boolean
   serverSidePagination?: boolean
   isFetching: boolean
@@ -190,9 +193,7 @@ export const TableView = <T extends MRT_RowData>({
       <>
         <CircularProgress />
         <br></br>
-        {title === 'Locality-Species-Cross-Search'
+        {'Loading data...'}
-          ? 'Loading data, this might take a few seconds...'
-          : 'Loading data...'}
       </>
     )
   }
@@ -207,6 +208,7 @@ export const TableView = <T extends MRT_RowData>({
             combinedExport={combinedExport}
             kmlExport={kmlExport}
             exportIsLoading={exportIsLoading}
+            isCrossSearchTable={isCrossSearchTable}
             showNewButton={editRights.new && title != 'Locality-Species-Cross-Search'}
           />
         </div>

frontend/src/components/TableView/helpers.tsx

@@ -1,10 +1,4 @@
-import { Box } from '@mui/material'
+import { MRT_RowData, MRT_TableInstance } from 'material-react-table'
-import {
-  MRT_RowData,
-  MRT_ShowHideColumnsButton,
-  MRT_TableInstance,
-  MRT_ToggleFullScreenButton,
-} from 'material-react-table'
 import { mkConfig, generateCsv, download } from 'export-to-csv'
 import { AcceptedData } from 'node_modules/export-to-csv/output/lib/types'
 
@@ -34,16 +28,3 @@ export const exportRows = <T extends MRT_RowData>(table: MRT_TableInstance<T>) =
   const csv = generateCsv(csvConfig)(rowData)
   download(csvConfig)(csv)
 }
-
-export const renderCustomToolbar = <T extends MRT_RowData>({ table }: { table: MRT_TableInstance<T> }) => (
-  <Box>
-    <MRT_ToggleFullScreenButton table={table} />
-    <MRT_ShowHideColumnsButton table={table} />
-  </Box>
-)
-
-export const renderCustomToolbarModalVersion = <T extends MRT_RowData>({ table }: { table: MRT_TableInstance<T> }) => (
-  <Box>
-    <MRT_ShowHideColumnsButton table={table} />
-  </Box>
-)

frontend/src/shared/currentDateAsString.ts

@@ -0,0 +1,11 @@
+export const currentDateAsString = () => {
+  // returns current datetime in YYYY-MM-DD-HH-mm format, currently used for filenames
+  const today = new Date()
+  const year = today.getFullYear()
+  const month = String(today.getMonth() + 1).padStart(2, '0')
+  const day = String(today.getDate()).padStart(2, '0')
+  const hours = String(today.getHours()).padStart(2, '0')
+  const minutes = String(today.getMinutes()).padStart(2, '0')
+
+  return `${year}-${month}-${day}-${hours}-${minutes}`
+}

frontend/src/shared/types/misc.ts

@@ -27,9 +27,26 @@ type ValidationErrorItem = {
   error: string
 }
 
+export type ColumnFilter = { id: string; value: string }
+export type SortingState = { desc: boolean; id: string }
+
 export type CrossSearchRouteParameters = {
-  limit: number
+  limit?: string
-  offset: number
+  offset?: string
-  columnFilters: unknown
+  columnFilters: string
-  sorting: unknown
+  sorting: string
+}
+
+export type ParsedCrossSearchRouteParameters = {
+  limit?: number
+  offset?: number
+  columnFilters: unknown[]
+  sorting: unknown[]
+}
+
+export type ValidatedCrossSearchRouteParameters = {
+  limit?: number
+  offset?: number
+  columnFilters: ColumnFilter[]
+  sorting: SortingState[]
 }

frontend/src/shared/validators/crossSearch.ts

@@ -1,4 +1,4 @@
-import { CrossSearchRouteParameters } from '../types'
+import { ParsedCrossSearchRouteParameters } from '../types'
 import { Validators, validator, ValidationError } from './validator'
 
 const columnFilterCheck = (columnFilters: unknown) => {
@@ -31,17 +31,27 @@ const sortingCheck = (sorting: unknown) => {
 }
 
 export const validateCrossSearchRouteParams = (
-  parameters: CrossSearchRouteParameters,
+  parameters: ParsedCrossSearchRouteParameters,
-  fieldName: keyof CrossSearchRouteParameters
+  fieldName: keyof ParsedCrossSearchRouteParameters
 ) => {
-  const validators: Validators<Partial<CrossSearchRouteParameters>> = {
+  const validators: Validators<Partial<ParsedCrossSearchRouteParameters>> = {
     limit: {
       name: 'Limit',
-      required: true,
+      required: () => {
+        // TODO: change this and offset to not use the parameters object, might not be possible without changing the way validators work
+        if (typeof parameters.limit !== 'undefined' && typeof parameters.limit !== 'number')
+          return 'Limit must be undefined or a number'
+
+        return
+      },
     },
     offset: {
       name: 'Offset',
-      required: true,
+      required: () => {
+        if (typeof parameters.offset !== 'undefined' && typeof parameters.offset !== 'number')
+          return 'Offset must be undefined or a number'
+        return
+      },
     },
     columnFilters: {
       name: 'Column Filters',
@@ -53,5 +63,5 @@ export const validateCrossSearchRouteParams = (
     },
   }
 
-  return validator<CrossSearchRouteParameters>(validators, parameters, fieldName)
+  return validator<ParsedCrossSearchRouteParameters>(validators, parameters, fieldName)
 }

test_data/sqlfiles/now_test.sql

@@ -1709,7 +1709,8 @@ INSERT INTO `ref_authors` VALUES
 (21368,2,1,'Ciochon','Russell'),
 (24151,2,1,'Ogg','J.G.'),
 (24187,2,1,'Kaakinen','A.'),
-(24188,2,1,'Wang','X.');
+(24188,2,1,'Wang','X.'),
+(25000,2,1,'Doe','D.');
 /*!40000 ALTER TABLE `ref_authors` ENABLE KEYS */;
 UNLOCK TABLES;
 
@@ -2198,7 +2199,8 @@ INSERT INTO `ref_ref` VALUES
 (21368,1,100,'Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam',1996,'93',NULL,3016,3020,NULL,NULL,NULL,NULL,NULL,NULL,'abstract','http://www.pnas.org/cgi/reprint/93/7/3016',NULL,NULL,NULL,'English',NULL,NULL,1,NULL),
 (24151,2,NULL,'A Concise Geologic Time Scale: 2016',2016,NULL,NULL,NULL,240,'Elsevier',NULL,NULL,NULL,NULL,'9780444594686',NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,0),
 (24187,13,NULL,'Helsinki Asian time update',2020,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'2020-01-29',NULL,1,NULL),
-(24188,2,NULL,'Fossil Mammals of Asia',2013,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL);
+(24188,2,NULL,'Fossil Mammals of Asia',2013,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL),
+(25000,1,100,'A test reference that is not linked to any localities or species',2025,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
 /*!40000 ALTER TABLE `ref_ref` ENABLE KEYS */;
 UNLOCK TABLES;