diff --git a/backend/package-lock.json b/backend/package-lock.json index d98b1316..a64e0e0d 100755 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -24,6 +24,7 @@ "express": "^4.18.2", "express-async-errors": "^3.1.1", "express-rate-limit": "^7.5.0", + "fast-csv": "^5.0.2", "jsonwebtoken": "^9.0.2", "mariadb": "^3.3.0", "md5": "^2.3.0", @@ -2456,6 +2457,33 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@fast-csv/format": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-5.0.2.tgz", + "integrity": "sha512-fRYcWvI8vs0Zxa/8fXd/QlmQYWWkJqKZPAXM+vksnplb3owQFKTPPh9JqOtD0L3flQw/AZjjXdPkD7Kp/uHm8g==", + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/parse": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.2.tgz", + "integrity": "sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==", + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -6360,6 +6388,19 @@ "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, + "node_modules/fast-csv": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-5.0.2.tgz", + "integrity": "sha512-CnB2zYAzzeh5Ta0UhSf32NexLy2SsEsSMY+fMWPV40k1OgaLEbm9Hf5dms3z/9fASZHBjB6i834079gVeksEqQ==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "5.0.2", + "@fast-csv/parse": "5.0.2" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -9468,6 +9509,18 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -9478,11 +9531,30 @@ "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -9498,6 +9570,12 @@ "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -9510,6 +9588,12 @@ "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", diff --git a/backend/package.json b/backend/package.json index 915362fc..544290fb 100755 --- a/backend/package.json +++ b/backend/package.json @@ -53,6 +53,7 @@ "express": "^4.18.2", "express-async-errors": "^3.1.1", "express-rate-limit": "^7.5.0", + "fast-csv": "^5.0.2", "jsonwebtoken": "^9.0.2", "mariadb": "^3.3.0", "md5": "^2.3.0", diff --git a/backend/src/api-tests/crossSearch/export.test.ts b/backend/src/api-tests/crossSearch/export.test.ts new file mode 100755 index 00000000..c5a60d4c --- /dev/null +++ b/backend/src/api-tests/crossSearch/export.test.ts @@ -0,0 +1,67 @@ +import { beforeAll, afterAll, describe, it, expect } from '@jest/globals' +import { resetDatabase, send, resetDatabaseTimeout } from '../utils' +import { pool } from '../../utils/db' + +describe('Getting cross-search export', () => { + beforeAll(async () => { + await resetDatabase() + }, resetDatabaseTimeout) + afterAll(async () => { + await pool.end() + }) + + it('with invalid column filters type does not work', async () => { + const response = await send(`crosssearch/export/"not an array"/[]`, 'GET') + expect(response.status).toEqual(403) + expect(response.body).toEqual({ error: 'ColumnFilters is not an array.' }) + }) + + it('with invalid column filters content does not work', async () => { + const { body: responseBody1, status: responseStatus1 } = await send( + `crosssearch/export/[{"id": "not_a_column", "value": "Dmanisi"}]/[]`, + 'GET' + ) + expect(responseStatus1).toEqual(403) + expect(responseBody1).toEqual({ error: 'columnFilters has an invalid column id.' }) + + const { body: responseBody2, status: responseStatus2 } = await send( + `crosssearch/export/[{"value": "Dmanisi"}]/[]`, + 'GET' + ) + expect(responseStatus2).toEqual(403) + expect(responseBody2).toEqual([{ error: 'Invalid or missing id field in filter', name: 'Column Filters' }]) + + const { body: responseBody3, status: responseStatus3 } = await send( + `crosssearch/export/[{"id": "lid_now_loc' OR 1=1 --", "value": "Dmanisi"}]/[]`, + 'GET' + ) + expect(responseStatus3).toEqual(403) + expect(responseBody3).toEqual({ error: 'columnFilters has an invalid column id.' }) + }) + + it('with invalid sorting type does not work', async () => { + const response = await send(`crosssearch/export/[]/20`, 'GET') + expect(response.status).toEqual(403) + expect(response.body).toEqual({ error: 'Sorting is not an array.' }) + }) + + it('with invalid sorting content does not work', async () => { + const { body: responseBody1, status: responseStatus1 } = await send( + `crosssearch/export/[]/[{"id": "not_a_column", "desc": true}]`, + 'GET' + ) + expect(responseStatus1).toEqual(403) + expect(responseBody1).toEqual({ error: 'orderBy was not a valid column id.' }) + + const { body: responseBody2, status: responseStatus2 } = await send(`crosssearch/export/[]/[{"desc": true}]`, 'GET') + expect(responseStatus2).toEqual(403) + expect(responseBody2).toEqual([{ error: 'Invalid or missing id field in sort object', name: 'Sorting' }]) + + const { body: responseBody3, status: responseStatus3 } = await send( + `crosssearch/export/[]/[{"id": "lid_now_loc' OR 1=1 --", "desc": true}]`, + 'GET' + ) + expect(responseStatus3).toEqual(403) + expect(responseBody3).toEqual({ error: 'orderBy was not a valid column id.' }) + }) +}) diff --git a/backend/src/api-tests/region/get.test.ts b/backend/src/api-tests/region/get.test.ts index 0ad236e3..78994239 100755 --- a/backend/src/api-tests/region/get.test.ts +++ b/backend/src/api-tests/region/get.test.ts @@ -2,7 +2,7 @@ import { beforeAll, afterAll, describe, it, expect, beforeEach } from '@jest/glo import { login, logout, resetDatabase, send, resetDatabaseTimeout } from '../utils' import { pool } from '../../utils/db' -describe('Deleting a region works', () => { +describe('Getting region data works', () => { beforeAll(async () => { await resetDatabase() }, resetDatabaseTimeout) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index ecd7247a..381af6c7 100755 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -1,40 +1,93 @@ import { Router } from 'express' -import { getCrossSearchRawSql, validateCrossSearchRouteParameters } from '../services/crossSearch' +import { getCrossSearchRawSql, parseAndValidateCrossSearchRouteParameters } from '../services/crossSearch' import { fixBigInt } from '../utils/common' -import { ColumnFilter, SortingState } from '../services/queries/crossSearchQuery' +import { format } from '@fast-csv/format' +import { pipeline } from 'stream' +import { logger } from '../utils/logger' +import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' const router = Router() router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { - let limit - let offset - let columnFilters - let sorting + let validatedValues try { - limit = parseInt(req.params.limit) - offset = parseInt(req.params.offset) - columnFilters = JSON.parse(req.params.columnfilters) as unknown - sorting = JSON.parse(req.params.sorting) as unknown + const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({ + limit: req.params.limit, + offset: req.params.offset, + columnFilters: req.params.columnfilters, + sorting: req.params.sorting, + }) + validatedValues = values + if (validationErrors.length > 0) { + return res.status(403).send(validationErrors) + } } catch (error) { - return res.status(403).send({ error: 'Parsing URL parameters failed' }) - } - const validationErrors = validateCrossSearchRouteParameters({ limit, offset, columnFilters, sorting }) - if (validationErrors.length > 0) { - return res.status(403).send(validationErrors) + if (error instanceof Error) return res.status(403).send({ error: error.message }) + return res.status(403).send('Unknown error') } + try { const result = await getCrossSearchRawSql( req.user, - limit, - offset, - columnFilters as ColumnFilter[], - sorting as SortingState[] + validatedValues.validatedLimit, + validatedValues.validatedOffset, + validatedValues.validatedColumnFilters, + validatedValues.validatedSorting ) return res.status(200).send(fixBigInt(result)) } catch (error) { - if (error && typeof error === 'object' && 'message' in error) return res.status(403).send({ error: error.message }) + if (error instanceof Error) return res.status(403).send({ error: error.message }) return res.status(403).send('Unknown error') } }) +router.get(`/export/:columnfilters/:sorting`, async (req, res) => { + let validatedValues + try { + const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({ + columnFilters: req.params.columnfilters, + sorting: req.params.sorting, + }) + validatedValues = values + if (validationErrors.length > 0) { + return res.status(403).send(validationErrors) + } + } catch (error) { + if (error instanceof Error) return res.status(403).send({ error: error.message }) + return res.status(403).send('Unknown error') + } + + let data + try { + data = await getCrossSearchRawSql( + req.user, + undefined, + undefined, + validatedValues.validatedColumnFilters, + validatedValues.validatedSorting + ) + } catch (error) { + if (error instanceof Error) return res.status(403).send({ error: error.message }) + return res.status(403).send('Unknown error') + } + + res.attachment(`cross_search${currentDateAsString()}.csv`) // filename will get overwritten in frontend when fetching data from this route + res.on('finish', () => { + logger.info('Cross search export sent.') + }) + + // quoteColumns is needed to make sure linebreaks do not mess up the data + const stream = format({ headers: true, quoteColumns: true }) + pipeline(stream, res, err => { + if (err) { + logger.error(`Error in crosssearch/export pipeline: ${err.message}`) + } + }) + + for (const row of data) { + stream.write(row) + } + return stream.end() +}) + export default router diff --git a/backend/src/services/crossSearch.ts b/backend/src/services/crossSearch.ts index 5d792ef9..2a3ad948 100755 --- a/backend/src/services/crossSearch.ts +++ b/backend/src/services/crossSearch.ts @@ -1,6 +1,14 @@ -import { User, CrossSearch, Role, CrossSearchRouteParameters } from '../../../frontend/src/shared/types' +import { + User, + CrossSearch, + Role, + CrossSearchRouteParameters, + ParsedCrossSearchRouteParameters, + SortingState, + ColumnFilter, +} from '../../../frontend/src/shared/types' import { getCrossSearchFields, getFieldsOfTables, nowDb } from '../utils/db' -import { ColumnFilter, SortingState, generateCrossSearchSql } from './queries/crossSearchQuery' +import { generateCrossSearchSql } from './queries/crossSearchQuery' import { ValidationObject } from '../../../frontend/src/shared/validators/validator' import { validateCrossSearchRouteParams } from '../../../frontend/src/shared/validators/crossSearch' @@ -118,13 +126,43 @@ export const getCrossSearchRawSql = async ( return result } -export const validateCrossSearchRouteParameters = (parameters: CrossSearchRouteParameters) => { - const keys = Object.keys(parameters) - const errors: ValidationObject[] = [] +export const parseAndValidateCrossSearchRouteParameters = (parameters: CrossSearchRouteParameters) => { + const { limit, offset, columnFilters, sorting } = parameters + let parsedLimit + let parsedOffset + if (limit) { + parsedLimit = parseInt(limit) + if (isNaN(parsedLimit)) throw new Error('Limit is not a number.') + } + if (offset) { + parsedOffset = parseInt(offset) + if (isNaN(parsedOffset)) throw new Error('Offset is not a number.') + } + const parsedColumnFilters = JSON.parse(columnFilters) as unknown + if (!Array.isArray(parsedColumnFilters)) throw new Error('ColumnFilters is not an array.') + const parsedSorting = JSON.parse(sorting) as unknown + if (!Array.isArray(parsedSorting)) throw new Error('Sorting is not an array.') + const parsedParameters = { + limit: parsedLimit, + offset: parsedOffset, + columnFilters: parsedColumnFilters, + sorting: parsedSorting, + } + const keys = Object.keys(parsedParameters) + const errors: ValidationObject[] = [] for (const key of keys) { - const error = validateCrossSearchRouteParams(parameters, key as keyof CrossSearchRouteParameters) + const error = validateCrossSearchRouteParams(parsedParameters, key as keyof ParsedCrossSearchRouteParameters) if (error.error) errors.push(error) } - return errors + const validatedColumnFilters = parsedColumnFilters as ColumnFilter[] + const validatedSorting = parsedSorting as SortingState[] + + return { + validatedLimit: parsedLimit, + validatedOffset: parsedOffset, + validatedColumnFilters, + validatedSorting, + validationErrors: errors, + } } diff --git a/backend/src/services/queries/crossSearchQuery.ts b/backend/src/services/queries/crossSearchQuery.ts index 1ceb76af..4436d619 100755 --- a/backend/src/services/queries/crossSearchQuery.ts +++ b/backend/src/services/queries/crossSearchQuery.ts @@ -1,7 +1,5 @@ import { Prisma } from '../../../prisma/generated/now_test_client' - -export type ColumnFilter = { id: string; value: string } -export type SortingState = { desc: boolean; id: string } +import { ColumnFilter } from '../../../../frontend/src/shared/types' const generateWhereClause = (showAll: boolean, allowedLocalities: Array, columnFilters: ColumnFilter[]) => { let userCheck: Prisma.Sql diff --git a/cypress/e2e/locality.cy.js b/cypress/e2e/locality.cy.js index 1ded9816..186ff104 100755 --- a/cypress/e2e/locality.cy.js +++ b/cypress/e2e/locality.cy.js @@ -60,7 +60,7 @@ describe('Creating a new locality', () => { cy.get('[id=dms_lat-textfield]').should('have.value', '48 40 48 N') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') }) it("and filling, then erasing needed data doesn't work", () => { @@ -95,12 +95,12 @@ describe('Creating a new locality', () => { cy.get('[id=min_age-textfield]').type('{backspace}{backspace}{backspace}{backspace}{backspace}') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') // make sure errors in other tabs disable the write button cy.get('[role=tablist]').contains('Locality').click() cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') }) it('write button is disabled if unvisited tab has validation errors', () => { @@ -116,12 +116,12 @@ describe('Creating a new locality', () => { // Age tab has no errors, but Locality tab has unfilled mandatory fields cy.get('[id=write-button]').should('be.disabled') - cy.contains('6 Invalid fields') + cy.contains('6 invalid fields') cy.get('[id=min_age-textfield-helper-text]').should('not.exist') cy.get('[id=max_age-textfield-helper-text]').should('not.exist') cy.get('[role=tablist]').contains('Locality').click() cy.get('[id=write-button]').should('be.disabled') - cy.contains('6 Invalid fields') + cy.contains('6 invalid fields') }) it('composite dating method works', () => { @@ -233,7 +233,7 @@ describe('Editing a locality', () => { cy.get('[id=max_age-textfield]').type('{backspace}{backspace}{backspace}{backspace}{backspace}9') cy.contains('Min value cannot be higher than max') cy.get('[id=write-button]').should('be.disabled') - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') }) it('changing fractions and basis for age updates the age correctly', () => { diff --git a/cypress/e2e/reference.cy.js b/cypress/e2e/reference.cy.js new file mode 100755 index 00000000..0a0817a5 --- /dev/null +++ b/cypress/e2e/reference.cy.js @@ -0,0 +1,175 @@ +before('Reset database', () => { + cy.request(Cypress.env('databaseResetUrl')) +}) + +describe('Creating a journal', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it('with valid data works', () => { + cy.visit('/reference/new') + cy.contains('Journal') + cy.contains('7 invalid fields') + + cy.get('[id=title_primary-textfield]').type('New test reference') + + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + + cy.get('[id=date_primary-textfield]').type('2025') + + cy.get('[data-cy=Select\\ Journal-button]').click() + cy.get('[data-cy=detailview-button-100]').click() + cy.contains('Close').click() + + cy.get('[id=write-button]').click() + cy.contains('Saved reference successfully.') + }) + + it('with missing required fields does not work', () => { + cy.visit('/reference/new') + cy.contains('Journal') + cy.contains('7 invalid fields') + + cy.get('[id=title_primary-textfield]').type('New test reference 2') + cy.contains('3 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + cy.contains('2 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[id=date_primary-textfield]').type('2025') + cy.contains('1 invalid field') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[data-cy=Select\\ Journal-button]').click() + cy.get('[data-cy=detailview-button-100]').click() + cy.contains('Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + + cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected author + cy.get('[id=write-button]').should('be.disabled') + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + }) + + it('combined check with different types of titles and gen notes works', () => { + cy.visit('/reference/new') + cy.contains('Journal') + cy.contains('7 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[id=title_primary-textfield]').type('Title Primary') + cy.contains('3 invalid fields') + cy.get('[id=title_primary-textfield]').clear() + cy.contains('7 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[id=title_secondary-textfield]').type('Title Secondary') + cy.contains('3 invalid fields') + cy.get('[id=title_secondary-textfield]').clear() + cy.contains('7 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[id=title_series-textfield]').type('Title Series') + cy.contains('3 invalid fields') + cy.get('[id=title_series-textfield]').clear() + cy.contains('7 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[id=gen_notes-textfield]').type('Gen Notes') + cy.contains('3 invalid fields') + cy.get('[id=gen_notes-textfield]').clear() + cy.contains('7 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + }) +}) + +// TODO: add more tests for different types of references + +describe('Editing a journal', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it('with valid data works', () => { + cy.visit('reference/10039') + cy.contains('A new geomagnetic polarity time scale for the Late Cretaceous and Cenozoic.') + cy.get('[id=edit-button]').click() + cy.get('[id=title_primary-textfield]').clear() + cy.get('[id=title_primary-textfield]').type('New primary title.') + + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-2]').click() + cy.contains('Close').click() + + cy.get('[id=write-button]').click() + cy.contains('Saved reference successfully.') + cy.contains('New primary title.') + cy.contains('Cande') // Author 1 + cy.contains('Ciochon') // Author 2 + }) + + it('with missing required fields does not work', () => { + cy.visit('reference/10039') + cy.contains('New primary title.') + cy.get('[id=edit-button]').click() + cy.get('[id=title_primary-textfield]').clear() + cy.contains('4 invalid fields') + cy.get('[id=write-button]').should('be.disabled') + cy.get('[id=title_primary-textfield]').type('New primary title.') + + cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() + cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected authors + cy.contains('1 invalid field') + cy.get('[id=write-button]').should('be.disabled') + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + }) +}) + +// TODO: add more tests for different types of references + +describe('Deleting a reference', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it('that is not linked to anything works and returns user to table view', () => { + cy.visit('/reference/25000') + cy.get('[id=delete-button]').should('exist').click() + cy.on('window:confirm', str => { + expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.') + // this automatically clicks the OK button and returns to table view + }) + cy.contains('Deleted item successfully.') + cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to + cy.contains('A test reference that is not linked to any localities or species').should('not.exist') + cy.visit('/reference/25000') + cy.contains('Error loading data') + }) + + // Skipped because references that have linked localities or species cannot be deleted currently + it.skip('that is linked to a locality and a species works and returns user to table view', () => { + cy.visit('/reference/10039') + cy.get('[id=delete-button]').should('exist').click() + cy.on('window:confirm', str => { + expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.') + // this automatically clicks the OK button and returns to table view + }) + cy.contains('Deleted item successfully.') + cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to + cy.contains('A new geomagnetic polarity time scale for the Late Cretaceous and Cenozoic').should('not.exist') + cy.visit('/reference/10039') + cy.contains('Error loading data') + }) +}) diff --git a/cypress/e2e/species.cy.js b/cypress/e2e/species.cy.js index 5bd82509..cdb62933 100755 --- a/cypress/e2e/species.cy.js +++ b/cypress/e2e/species.cy.js @@ -1,3 +1,7 @@ +before('Reset database', () => { + cy.request(Cypress.env('databaseResetUrl')) +}) + describe('Creating a species', () => { beforeEach('Login as admin', () => { cy.login('testSu') @@ -24,7 +28,78 @@ describe('Creating a species', () => { cy.visit('/species/new') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') + }) +}) + +describe('Editing a species', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it('with valid data works', () => { + cy.visit('/species/21052/') + cy.contains('21052 Simplomys simplicidens') + cy.get('[id=edit-button]').click() + + cy.get('[id=order_name-textfield]').clear() + cy.get('[id=order_name-textfield]').type('modifiedOrder') + cy.get('[id=family_name-textfield]').clear() + cy.get('[id=family_name-textfield]').type('modifiedFamily') + cy.get('[id=genus_name-textfield]').clear() + cy.get('[id=genus_name-textfield]').type('modifiedGenus') + cy.get('[id=species_name-textfield]').clear() + cy.get('[id=species_name-textfield]').type('modifiedSpecies') + cy.get('[id=subfamily_name-textfield]').type('heimo') + + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + cy.contains('button', 'Add existing reference').click() + cy.get('button[data-cy^="detailview-button"]').first().click() + cy.contains('button', 'Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + cy.contains('Saved species entry successfully.') + cy.contains('modifiedOrder') + cy.contains('Subfamily or Tribe heimo') + }) + + it('with missing required fields does not work', () => { + cy.visit('/species/21052/') + cy.contains('21052') + cy.get('[id=edit-button]').click() + + cy.get('[id=order_name-textfield]').clear() + cy.contains('Order: This field is required') + cy.get('[id=write-button]').should('be.disabled') + cy.contains('1 invalid field') + + cy.get('[id=family_name-textfield]').clear() + cy.contains('Family: This field is required') + cy.get('[id=write-button]').should('be.disabled') + cy.contains('2 invalid fields') + + cy.get('[id=order_name-textfield]').type('newOrder') + cy.contains('Order: This field is required').should('not.exist') + cy.contains('Family: This field is required') + cy.get('[id=write-button]').should('be.disabled') + cy.contains('1 invalid field') + + cy.get('[id=family_name-textfield]').type('newFamily') + cy.contains('Family: This field is required').should('not.exist') + cy.get('[id=write-button]').click() + + cy.contains('button', 'Add existing reference').click() + cy.get('button[data-cy^="detailview-button"]').first().click() + cy.contains('button', 'Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + + cy.contains('Saved species entry successfully.') + cy.contains('newOrder') + cy.contains('newFamily') + cy.contains('modifiedGenus') + cy.contains('Subfamily or Tribe heimo') }) }) diff --git a/cypress/e2e/timeBound.cy.js b/cypress/e2e/timeBound.cy.js index df1a34bd..4099711f 100755 --- a/cypress/e2e/timeBound.cy.js +++ b/cypress/e2e/timeBound.cy.js @@ -1,10 +1,35 @@ -describe('Editing time bound', () => { +before('Reset database', () => { + cy.request(Cypress.env('databaseResetUrl')) +}) + +describe('Creating time bound', () => { beforeEach('Login as admin', () => { cy.login('testSu') }) - beforeEach('Reset database', () => { - cy.request(Cypress.env('databaseResetUrl')) + it('with correct values works', () => { + cy.visit(`/time-bound/new/`) + cy.contains('Creating new time-bound') + cy.contains('Name: This field is required') + cy.contains('Age (Ma): This field is required') + cy.get('[id=b_name-textfield]').first().type('test bound') + cy.contains('Name: This field is required').should('not.exist') + cy.get('[id=write-button]').should('be.disabled') + cy.get('[id=age-textfield]').first().type('20') + cy.contains('Age (Ma): This field is required').should('not.exist') + cy.get('[id=write-button]').click() + cy.contains('button', 'Add existing reference').click() + cy.get('button[data-cy^="detailview-button"]').first().click() + cy.contains('button', 'Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + cy.contains('Edited item successfully.') + }) +}) + +describe('Editing time bound', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') }) it('with correct values works', () => { @@ -20,6 +45,7 @@ describe('Editing time bound', () => { cy.contains('button', 'Close').click() cy.get('[id=write-button]').should('not.be.disabled') cy.get('[id=write-button]').click() + cy.contains('Edited item successfully.') cy.contains('1.5') }) @@ -30,7 +56,7 @@ describe('Editing time bound', () => { cy.get('[id=age-textfield]').first().type('{backspace}{backspace}{backspace}{backspace}{backspace}') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') }) }) diff --git a/documentation/backend/cross_search_export.md b/documentation/backend/cross_search_export.md new file mode 100755 index 00000000..d5d5d6be --- /dev/null +++ b/documentation/backend/cross_search_export.md @@ -0,0 +1,3 @@ +### Cross Search Export + +Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same parameter parsing and validation function as the `/all` route, and it also uses the same SQL query, but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file diff --git a/documentation/backend/server_side_pagination.md b/documentation/backend/server_side_pagination.md index 9a15a1e6..9d3c10a3 100755 --- a/documentation/backend/server_side_pagination.md +++ b/documentation/backend/server_side_pagination.md @@ -6,7 +6,7 @@ The server side pagination for cross search is implemented in the following way: 1. The backend route is given values for limit, offset, column filters, and sorting. - `limit` and `offset` should be numbers, and both `columnFilters` and `sorting` should be an array of objects containing the keys `id` and `value`. `sorting` is currently an array because Material React Table supports multiple sorting values. The backend only uses the first value in the array for the SQL query. -2. The values are validated. This is done in two places: the [validation file](../../frontend/src/shared/validators/crossSearch.ts) and the [getCrossSearchRawSql function](../../backend/src/services/crossSearch.ts). +2. The values are validated. This is done in two places: the [parsing and validation function](../../backend/src/services/crossSearch.ts), and the [getCrossSearchRawSql function](../../backend/src/services/crossSearch.ts). - The checks inside getCrossSearchRawSql are very important since they check if the values given to the `WHERE` and `ORDER BY` clauses are valid column ID's. Without these the query is vulnerable to SQL injection (see below) 3. Since the SQL query uses aliases, some values used in the query need to be converted first. This is done with the [convertIdToFieldName function](../../backend/src/services/crossSearch.ts). 4. The values are given to [generateCrossSearchSql function](backend/src/services/queries/crossSearchQuery.ts), which generates the SQL query. The values are used in the `LIMIT`, `OFFSET`, `WHERE` and `ORDER BY` clauses, respectively. To allow TableView to show and change pages correctly, we also add a `full_count` column to every row which contains the amount of rows in the entire cross-search table. diff --git a/documentation/frontend/components/TableView.md b/documentation/frontend/components/TableView.md index 57a0a83c..8a4e2619 100755 --- a/documentation/frontend/components/TableView.md +++ b/documentation/frontend/components/TableView.md @@ -12,7 +12,7 @@ TableView has the following required props: The table will handle paging and sorting data automatically. If your backend query for the table is using [server side pagination](../../backend/server_side_pagination.md), remember to set the `serverSidePagination` prop to True. Check the Table View section in the guide below for more information about optional props. -The table itself is constructed with the useMaterialReactTable-hook. The toolbar at the top is defined in the [helpers](../../../frontend/src/components/TableView/helpers.tsx) file and rendered using the `renderToolbarInternalActions` option, and it contains buttons for showing/hiding columns and showing the table in fullscreen mode, as well as exporting the content of the table. If the toolbar needs to be changed in the future (e.g. adding more rows to it) checkout the documentation for rendering a custom toolbar [here](https://www.material-react-table.com/docs/guides/toolbar-customization#override-with-custom-toolbar-components) since the current method doesn't really allow any major changes to its layout. +The table itself is constructed with the useMaterialReactTable-hook. The toolbar at the top is defined in the [helpers](../../../frontend/src/components/TableView/helpers.tsx) file and rendered using the `renderToolbarInternalActions` option, and it contains buttons for showing/hiding columns and showing the table in fullscreen mode, as well as exporting the content of the table. To know what components you can render here if necessary, see the [source code](https://github.com/KevinVandy/material-react-table/blob/85b98f9aaa038df48aa1dd35123560abce78ee58/packages/material-react-table/src/components/toolbar/MRT_ToolbarInternalButtons.tsx#L45). If the toolbar needs to be changed in the future (e.g. adding more rows to it) checkout the documentation for rendering a custom toolbar [here](https://www.material-react-table.com/docs/guides/toolbar-customization#override-with-custom-toolbar-components) since the current method doesn't really allow any major changes to its layout. ### Setting up a new table view @@ -38,5 +38,12 @@ Following things are needed to create a new page with a table (examples are all - to enable a filter mode selection menu for the columns of the table, set parameter `enableColumnFilterModes` to `true`. - you can pass an array of filter functions to be shown in the menu to the parameter `columnFilterModeOptions`. - it's a good idea to disable the filter mode menu for columns for which it doesn't make sense by setting the parameter `enableColumnFilterModes` to `false` for that column. See [LocalityTable](../../../frontend/src/components/Locality/LocalityTable.tsx) for an example. For more information, see [MRT docs filter mode section](https://www.material-react-table.com/docs/guides/column-filtering#filter-modes). - - If the table you're creating is using server side pagination, remember to set the `serverSidePagination` prop to True so the TableView will set its pagination/sorting modes correctly. Forgetting to do this might make the table load indefinitely or crash the app. + - If the table you're creating is using [server side pagination](../../backend/server_side_pagination.md), remember to set the `serverSidePagination` prop to True so the TableView will set its pagination/sorting modes correctly. Forgetting to do this might make the table load indefinitely or crash the app. + - `isCrossSearchTable` is only set inside [CrossSearchTable.tsx](../../../frontend/src/components/CrossSearch/CrossSearchTable.tsx). See the "Exporting" section below. - More info can be found from official [MRT docs](https://www.material-react-table.com/). + +### Exporting + +Most tables simply call the [exportRows function](../../../frontend/src/components/TableView/helpers.tsx) when the user clicks the "Export Table" button. This function converts the entire table into a csv file and downloads it. The [export function for cross-search](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) is different since it uses server side pagination and the cross-search table does not contain all needed data at once. The `isCrossSearchTable` prop is used to render the proper toolbar and "Export Table" button for the cross-search table. See [server side pagination](../../backend/server_side_pagination.md) and [cross search export](../../backend/cross_search_export.md) docs for more information. + +Currently the export function for cross-search uses blob(), which waits for the entire file to be streamed before continuing. If you want to start downloading the file as soon as the backend starts streaming it, [this MDN document](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#reading_the_stream) might help. This might not speed up the download, but it shows the "download started" popup in the user's browser faster, which can make the wait a little less annoying for them. diff --git a/frontend/package.json b/frontend/package.json index 58b74bed..b2fdf882 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,7 +18,6 @@ "@reduxjs/toolkit": "^2.2.3", "export-to-csv": "^1.3.0", "leaflet": "^1.9.4", - "react-leaflet": "^4.2.1", "lodash-es": "^4.17.21", "material-react-table": "^3.0.1", "react": "^18.2.0", @@ -61,4 +60,4 @@ "vite": "^5.2.5", "vite-plugin-istanbul": "^6.0.2" } -} +} \ No newline at end of file diff --git a/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx b/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx new file mode 100755 index 00000000..65146f3b --- /dev/null +++ b/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx @@ -0,0 +1,59 @@ +import { useState } from 'react' +import { useNotify } from '@/hooks/notification' +import { BACKEND_URL } from '@/util/config' +import { MenuItem } from '@mui/material' +import { useUser } from '@/hooks/user' +import { usePageContext } from '../Page' +import { currentDateAsString } from '@/shared/currentDateAsString' + +export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { + const [loading, setLoading] = useState(false) + const { sqlColumnFilters, sqlOrderBy } = usePageContext() + const notify = useNotify() + + const token = useUser().token + const fetchOptions = token ? { headers: { Authorization: `Bearer ${token}` } } : {} + + const filename = `cross_search${currentDateAsString()}.csv` + + const fetchCSVFile = () => { + setLoading(true) + notify('Downloading CSV file, please wait...', 'info', null) + const URIColumnFilters = encodeURIComponent(JSON.stringify(sqlColumnFilters)) + const URIOrderBy = encodeURIComponent(JSON.stringify(sqlOrderBy)) + + fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions) + .then(response => { + if (!response.ok) { + throw new Error('Server response was not OK.') + } + return response.blob() + }) + .then(blob => { + const blobUrl = window.URL.createObjectURL(blob) + const downloadLink = document.createElement('a') + downloadLink.href = blobUrl + downloadLink.download = filename + document.body.appendChild(downloadLink) + downloadLink.click() + downloadLink.remove() + window.URL.revokeObjectURL(blobUrl) + notify('Download finished.') + }) + .catch(_ => { + notify('Downloading cross-search export failed.', 'error') + }) + .finally(() => setLoading(false)) + } + return ( + { + fetchCSVFile() + handleClose() + }} + disabled={loading} + > + Export table + + ) +} diff --git a/frontend/src/components/CrossSearch/CrossSearchTable.tsx b/frontend/src/components/CrossSearch/CrossSearchTable.tsx index b8195a79..28baa44b 100755 --- a/frontend/src/components/CrossSearch/CrossSearchTable.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchTable.tsx @@ -812,6 +812,7 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros url="crosssearch" enableColumnFilterModes={true} serverSidePagination={true} + isCrossSearchTable={true} /> ) } diff --git a/frontend/src/components/DetailView/common/SelectingTable.tsx b/frontend/src/components/DetailView/common/SelectingTable.tsx index 46427135..2b43bae4 100755 --- a/frontend/src/components/DetailView/common/SelectingTable.tsx +++ b/frontend/src/components/DetailView/common/SelectingTable.tsx @@ -67,7 +67,7 @@ export const SelectingTable = if (isError) return Error fetching data for the selecting table. if (!data) return return ( - + {data ? ( data={filteredData} diff --git a/frontend/src/components/DetailView/components.tsx b/frontend/src/components/DetailView/components.tsx index f530089d..cc8b5681 100755 --- a/frontend/src/components/DetailView/components.tsx +++ b/frontend/src/components/DetailView/components.tsx @@ -113,6 +113,8 @@ export const ErrorBox = () => { const { fieldsWithErrors } = useDetailContext() const fields = Object.keys(fieldsWithErrors) + const title = fields.length > 1 ? `${fields.length} invalid fields` : '1 invalid field' + return ( () => { }} > - {`${fields.length} Invalid fields`} + {title} {fields.map(field => ( diff --git a/frontend/src/components/Map/LocalitiesMap.tsx b/frontend/src/components/Map/LocalitiesMap.tsx index 76e03885..6b830cd3 100755 --- a/frontend/src/components/Map/LocalitiesMap.tsx +++ b/frontend/src/components/Map/LocalitiesMap.tsx @@ -19,19 +19,19 @@ interface Props { localitiesQueryIsFetching: boolean } +type CustomMarkerOptions = L.MarkerOptions & { localityId: number } +type CustomMarker = L.Marker & { options: CustomMarkerOptions } + // TODO: Don't do this const markerIcon = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABkAAAApCAYAAADAk4LOAAAFgUlEQVR4Aa1XA5BjWRTN2oW17d3YaZtr2962HUzbDNpjszW24mRt28p47v7zq/bXZtrp/lWnXr337j3nPCe85NcypgSFdugCpW5YoDAMRaIMqRi6aKq5E3YqDQO3qAwjVWrD8Ncq/RBpykd8oZUb/kaJutow8r1aP9II0WmLKLIsJyv1w/kqw9Ch2MYdB++12Onxee/QMwvf4/Dk/Lfp/i4nxTXtOoQ4pW5Aj7wpici1A9erdAN2OH64x8OSP9j3Ft3b7aWkTg/Fm91siTra0f9on5sQr9INejH6CUUUpavjFNq1B+Oadhxmnfa8RfEmN8VNAsQhPqF55xHkMzz3jSmChWU6f7/XZKNH+9+hBLOHYozuKQPxyMPUKkrX/K0uWnfFaJGS1QPRtZsOPtr3NsW0uyh6NNCOkU3Yz+bXbT3I8G3xE5EXLXtCXbbqwCO9zPQYPRTZ5vIDXD7U+w7rFDEoUUf7ibHIR4y6bLVPXrz8JVZEql13trxwue/uDivd3fkWRbS6/IA2bID4uk0UpF1N8qLlbBlXs4Ee7HLTfV1j54APvODnSfOWBqtKVvjgLKzF5YdEk5ewRkGlK0i33Eofffc7HT56jD7/6U+qH3Cx7SBLNntH5YIPvODnyfIXZYRVDPqgHtLs5ABHD3YzLuespb7t79FY34DjMwrVrcTuwlT55YMPvOBnRrJ4VXTdNnYug5ucHLBjEpt30701A3Ts+HEa73u6dT3FNWwflY86eMHPk+Yu+i6pzUpRrW7SNDg5JHR4KapmM5Wv2E8Tfcb1HoqqHMHU+uWDD7zg54mz5/2BSnizi9T1Dg4QQXLToGNCkb6tb1NU+QAlGr1++eADrzhn/u8Q2YZhQVlZ5+CAOtqfbhmaUCS1ezNFVm2imDbPmPng5wmz+gwh+oHDce0eUtQ6OGDIyR0uUhUsoO3vfDmmgOezH0mZN59x7MBi++WDL1g/eEiU3avlidO671bkLfwbw5XV2P8Pzo0ydy4t2/0eu33xYSOMOD8hTf4CrBtGMSoXfPLchX+J0ruSePw3LZeK0juPJbYzrhkH0io7B3k164hiGvawhOKMLkrQLyVpZg8rHFW7E2uHOL888IBPlNZ1FPzstSJM694fWr6RwpvcJK60+0HCILTBzZLFNdtAzJaohze60T8qBzyh5ZuOg5e7uwQppofEmf2++DYvmySqGBuKaicF1blQjhuHdvCIMvp8whTTfZzI7RldpwtSzL+F1+wkdZ2TBOW2gIF88PBTzD/gpeREAMEbxnJcaJHNHrpzji0gQCS6hdkEeYt9DF/2qPcEC8RM28Hwmr3sdNyht00byAut2k3gufWNtgtOEOFGUwcXWNDbdNbpgBGxEvKkOQsxivJx33iow0Vw5S6SVTrpVq11ysA2Rp7gTfPfktc6zhtXBBC+adRLshf6sG2RfHPZ5EAc4sVZ83yCN00Fk/4kggu40ZTvIEm5g24qtU4KjBrx/BTTH8ifVASAG7gKrnWxJDcU7x8X6Ecczhm3o6YicvsLXWfh3Ch1W0k8x0nXF+0fFxgt4phz8QvypiwCCFKMqXCnqXExjq10beH+UUA7+nG6mdG/Pu0f3LgFcGrl2s0kNNjpmoJ9o4B29CMO8dMT4Q5ox8uitF6fqsrJOr8qnwNbRzv6hSnG5wP+64C7h9lp30hKNtKdWjtdkbuPA19nJ7Tz3zR/ibgARbhb4AlhavcBebmTHcFl2fvYEnW0ox9xMxKBS8btJ+KiEbq9zA4RthQXDhPa0T9TEe69gWupwc6uBUphquXgf+/FrIjweHQS4/pduMe5ERUMHUd9xv8ZR98CxkS4F2n3EUrUZ10EYNw7BWm9x1GiPssi3GgiGRDKWRYZfXlON+dfNbM+GgIwYdwAAAAASUVORK5CYII=' -type CustomMarkerOptions = L.MarkerOptions & { localityId: number } -type CustomMarker = L.Marker & { options: CustomMarkerOptions } - export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching }: Props) => { const [selectedLocality, setSelectedLocality] = useState(null) - // Map instance tracked in state for attaching event listeners - const [mapInstance, setMapInstance] = useState(null) const leafletMapRef = useRef(null) const mapRef = useRef(null) + const markersRef = useRef(null) + const [map, setMap] = useState(null) const [isOpen, setIsOpen] = useState(false) const columnFilters = usePageContext() const { @@ -41,58 +41,20 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching } } = useGetLocalityDetailsQuery(selectedLocality ?? skipToken) // use skipToken to skip query when state is null useEffect(() => { - if (!mapRef.current || leafletMapRef.current) return + if (!mapRef.current) return + + const mapInstance = L.map(mapRef.current, { maxZoom: 16 }) + setMap(mapInstance) - const map = L.map(mapRef.current, { maxZoom: 16 }) - setMapInstance(map) const coords: LatLngExpression = [15, 13] - - // To prevent eslint from complaining about that 'markers' variable below: - /* eslint-disable @typescript-eslint/no-unsafe-assignment */ - /* eslint-disable @typescript-eslint/no-unsafe-call */ - - /* eslint-disable @typescript-eslint/no-unsafe-member-access */ - - // @ts-expect-error The marker cluster library is a plain javascript library - // with no module exports that extends 'L' when imported - const markers: Layer = L.markerClusterGroup() - - const localityIds = columnFilters.idList as unknown as number[] - const filterApplied = localityIds.length > 0 && localityIds.every(id => id === null) - - const validIds = localityIds.filter(id => typeof id === 'number') - - const filteredLocalities = !filterApplied - ? localitiesQueryData?.filter(locality => validIds.includes(locality.lid)) - : localitiesQueryData - - filteredLocalities?.forEach(locality => { - const options: CustomMarkerOptions = { - icon: new Icon({ iconUrl: markerIcon, iconSize: [25, 41], iconAnchor: [12, 41] }), - title: locality.loc_name, - localityId: locality.lid, - } - - const marker = L.marker([locality.dec_lat, locality.dec_long], options) as CustomMarker - - marker.on('click', () => { - setSelectedLocality(marker.options.localityId.toString()) - }) - - markers.addLayer(marker) - }) - - // remote add to map - markers.addTo(map) - - map.setView(coords, 2) + mapInstance.setView(coords, 2) // BASE MAPS //// OpenTopoMap const topomap = L.tileLayer('https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png', { attribution: 'Map data: © OpenTopoMap (CC-BY-SA)', noWrap: true, - }).addTo(map) + }).addTo(mapInstance) leafletMapRef.current = map //// OpenStreetMap @@ -104,7 +66,7 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching } // LAYERS ON TOP OF BASE MAPS //// Add borders to the map borders.forEach(poly => { - L.polygon(poly as LatLngExpression[], { color: 'gray', weight: 1 }).addTo(map) + L.polygon(poly as LatLngExpression[], { color: 'gray', weight: 1 }).addTo(mapInstance) }) // create a polygon layer for the country borders that is in layer control panel @@ -125,37 +87,74 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching } } // Add a scale bar to the map - L.control.scale({ position: 'bottomright' }).addTo(map) + L.control.scale({ position: 'bottomright' }).addTo(mapInstance) // Add a layer control to the map - L.control.layers(baseMaps, {}, { position: 'topright' }).addTo(map) + L.control.layers(baseMaps, {}, { position: 'topright' }).addTo(mapInstance) return () => { - map.remove() leafletMapRef.current = null - setMapInstance(null) + mapInstance.remove() } - }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters]) + }, []) useEffect(() => { - if (!mapInstance) return - const map = mapInstance + if (!map || localitiesQueryIsFetching) return + + if (markersRef.current) { + map.removeLayer(markersRef.current) + markersRef.current = null + } + // To prevent eslint from complaining about that 'markers' variable below: + /* eslint-disable @typescript-eslint/no-unsafe-assignment */ + /* eslint-disable @typescript-eslint/no-unsafe-call */ + + /* eslint-disable @typescript-eslint/no-unsafe-member-access */ + + // @ts-expect-error The marker cluster library is a plain javascript library + // with no module exports that extends 'L' when imported + const newMarkers: Layer = L.markerClusterGroup() + + const localityIds = columnFilters.idList as unknown as number[] + const filterApplied = localityIds.length > 0 && localityIds.every(id => id === null) + + const validIds = localityIds.filter(id => typeof id === 'number') + + const filteredLocalities = !filterApplied + ? localitiesQueryData?.filter(locality => validIds.includes(locality.lid)) + : localitiesQueryData + + filteredLocalities?.forEach(locality => { + const options: CustomMarkerOptions = { + icon: new Icon({ iconUrl: markerIcon, iconSize: [25, 41], iconAnchor: [12, 41] }), + title: locality.loc_name, + localityId: locality.lid, + } + + const marker = L.marker([locality.dec_lat, locality.dec_long], options) as CustomMarker + marker.on('click', () => { + setSelectedLocality(marker.options.localityId.toString()) + }) + + newMarkers.addLayer(marker) + }) + + newMarkers.addTo(map) + markersRef.current = newMarkers + }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters, map]) + + useEffect(() => { + if (!map) return + const mapInstance = map const handleMapClick = () => { setSelectedLocality(null) } - map.on('click', handleMapClick) + mapInstance.on('click', handleMapClick) return () => { - map.off('click', handleMapClick) + mapInstance.off('click', handleMapClick) } - }, [mapInstance]) - - // useEffect(() => { - // if (localityDetailsQueryData) { - // console.log('Fetched locality details:', localityDetailsQueryData) - // } - // }) - // }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters]) + }, [map]) document.title = 'Map' diff --git a/frontend/src/components/TableView/TableToolBar.tsx b/frontend/src/components/TableView/TableToolBar.tsx index 5aff9614..22ed6c1d 100755 --- a/frontend/src/components/TableView/TableToolBar.tsx +++ b/frontend/src/components/TableView/TableToolBar.tsx @@ -7,12 +7,14 @@ import '../../styles/TableToolBar.css' import { useState } from 'react' import { Link } from 'react-router-dom' import AddCircleIcon from '@mui/icons-material/AddCircle' +import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportMenuItem' export const TableToolBar = ({ table, combinedExport, kmlExport, exportIsLoading, + isCrossSearchTable, selectorFn, showNewButton, }: { @@ -20,6 +22,7 @@ export const TableToolBar = ({ combinedExport?: (lids: number[]) => Promise kmlExport?: (table: MRT_TableInstance) => void exportIsLoading?: boolean + isCrossSearchTable?: boolean selectorFn?: (id: T) => void showNewButton?: boolean }) => { @@ -67,14 +70,19 @@ export const TableToolBar = ({ 'aria-labelledby': 'export-button', }} > - { - exportRows(table) - handleClose() - }} - > - Export table - + {isCrossSearchTable ? ( + + ) : ( + { + exportRows(table) + handleClose() + }} + > + Export table + + )} + {kmlExport && ( { diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index 713a4efa..604910de 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -27,6 +27,7 @@ type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' selectorFn should only be defined if using this as a selecting table */ + export const TableView = ({ data, columns, @@ -39,6 +40,7 @@ export const TableView = ({ combinedExport, kmlExport, exportIsLoading, + isCrossSearchTable, enableColumnFilterModes, serverSidePagination, isFetching, @@ -54,6 +56,7 @@ export const TableView = ({ combinedExport?: (lids: number[]) => Promise kmlExport?: (table: MRT_TableInstance) => void exportIsLoading?: boolean + isCrossSearchTable?: boolean enableColumnFilterModes?: boolean serverSidePagination?: boolean isFetching: boolean @@ -190,9 +193,7 @@ export const TableView = ({ <>

- {title === 'Locality-Species-Cross-Search' - ? 'Loading data, this might take a few seconds...' - : 'Loading data...'} + {'Loading data...'} ) } @@ -207,6 +208,7 @@ export const TableView = ({ combinedExport={combinedExport} kmlExport={kmlExport} exportIsLoading={exportIsLoading} + isCrossSearchTable={isCrossSearchTable} showNewButton={editRights.new && title != 'Locality-Species-Cross-Search'} /> diff --git a/frontend/src/components/TableView/helpers.tsx b/frontend/src/components/TableView/helpers.tsx index a59cacd4..ce305ccf 100755 --- a/frontend/src/components/TableView/helpers.tsx +++ b/frontend/src/components/TableView/helpers.tsx @@ -1,10 +1,4 @@ -import { Box } from '@mui/material' -import { - MRT_RowData, - MRT_ShowHideColumnsButton, - MRT_TableInstance, - MRT_ToggleFullScreenButton, -} from 'material-react-table' +import { MRT_RowData, MRT_TableInstance } from 'material-react-table' import { mkConfig, generateCsv, download } from 'export-to-csv' import { AcceptedData } from 'node_modules/export-to-csv/output/lib/types' @@ -34,16 +28,3 @@ export const exportRows = (table: MRT_TableInstance) = const csv = generateCsv(csvConfig)(rowData) download(csvConfig)(csv) } - -export const renderCustomToolbar = ({ table }: { table: MRT_TableInstance }) => ( - - - - -) - -export const renderCustomToolbarModalVersion = ({ table }: { table: MRT_TableInstance }) => ( - - - -) diff --git a/frontend/src/shared/currentDateAsString.ts b/frontend/src/shared/currentDateAsString.ts new file mode 100755 index 00000000..7e825cc9 --- /dev/null +++ b/frontend/src/shared/currentDateAsString.ts @@ -0,0 +1,11 @@ +export const currentDateAsString = () => { + // returns current datetime in YYYY-MM-DD-HH-mm format, currently used for filenames + const today = new Date() + const year = today.getFullYear() + const month = String(today.getMonth() + 1).padStart(2, '0') + const day = String(today.getDate()).padStart(2, '0') + const hours = String(today.getHours()).padStart(2, '0') + const minutes = String(today.getMinutes()).padStart(2, '0') + + return `${year}-${month}-${day}-${hours}-${minutes}` +} diff --git a/frontend/src/shared/types/misc.ts b/frontend/src/shared/types/misc.ts index cefe9003..c3e5904f 100755 --- a/frontend/src/shared/types/misc.ts +++ b/frontend/src/shared/types/misc.ts @@ -27,9 +27,26 @@ type ValidationErrorItem = { error: string } +export type ColumnFilter = { id: string; value: string } +export type SortingState = { desc: boolean; id: string } + export type CrossSearchRouteParameters = { - limit: number - offset: number - columnFilters: unknown - sorting: unknown + limit?: string + offset?: string + columnFilters: string + sorting: string +} + +export type ParsedCrossSearchRouteParameters = { + limit?: number + offset?: number + columnFilters: unknown[] + sorting: unknown[] +} + +export type ValidatedCrossSearchRouteParameters = { + limit?: number + offset?: number + columnFilters: ColumnFilter[] + sorting: SortingState[] } diff --git a/frontend/src/shared/validators/crossSearch.ts b/frontend/src/shared/validators/crossSearch.ts index 344d58e9..1e0ab7c2 100755 --- a/frontend/src/shared/validators/crossSearch.ts +++ b/frontend/src/shared/validators/crossSearch.ts @@ -1,4 +1,4 @@ -import { CrossSearchRouteParameters } from '../types' +import { ParsedCrossSearchRouteParameters } from '../types' import { Validators, validator, ValidationError } from './validator' const columnFilterCheck = (columnFilters: unknown) => { @@ -31,17 +31,27 @@ const sortingCheck = (sorting: unknown) => { } export const validateCrossSearchRouteParams = ( - parameters: CrossSearchRouteParameters, - fieldName: keyof CrossSearchRouteParameters + parameters: ParsedCrossSearchRouteParameters, + fieldName: keyof ParsedCrossSearchRouteParameters ) => { - const validators: Validators> = { + const validators: Validators> = { limit: { name: 'Limit', - required: true, + required: () => { + // TODO: change this and offset to not use the parameters object, might not be possible without changing the way validators work + if (typeof parameters.limit !== 'undefined' && typeof parameters.limit !== 'number') + return 'Limit must be undefined or a number' + + return + }, }, offset: { name: 'Offset', - required: true, + required: () => { + if (typeof parameters.offset !== 'undefined' && typeof parameters.offset !== 'number') + return 'Offset must be undefined or a number' + return + }, }, columnFilters: { name: 'Column Filters', @@ -53,5 +63,5 @@ export const validateCrossSearchRouteParams = ( }, } - return validator(validators, parameters, fieldName) + return validator(validators, parameters, fieldName) } diff --git a/test_data/sqlfiles/now_test.sql b/test_data/sqlfiles/now_test.sql index 18596cd4..f4833a7e 100755 --- a/test_data/sqlfiles/now_test.sql +++ b/test_data/sqlfiles/now_test.sql @@ -1709,7 +1709,8 @@ INSERT INTO `ref_authors` VALUES (21368,2,1,'Ciochon','Russell'), (24151,2,1,'Ogg','J.G.'), (24187,2,1,'Kaakinen','A.'), -(24188,2,1,'Wang','X.'); +(24188,2,1,'Wang','X.'), +(25000,2,1,'Doe','D.'); /*!40000 ALTER TABLE `ref_authors` ENABLE KEYS */; UNLOCK TABLES; @@ -2198,7 +2199,8 @@ INSERT INTO `ref_ref` VALUES (21368,1,100,'Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam',1996,'93',NULL,3016,3020,NULL,NULL,NULL,NULL,NULL,NULL,'abstract','http://www.pnas.org/cgi/reprint/93/7/3016',NULL,NULL,NULL,'English',NULL,NULL,1,NULL), (24151,2,NULL,'A Concise Geologic Time Scale: 2016',2016,NULL,NULL,NULL,240,'Elsevier',NULL,NULL,NULL,NULL,'9780444594686',NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,0), (24187,13,NULL,'Helsinki Asian time update',2020,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'2020-01-29',NULL,1,NULL), -(24188,2,NULL,'Fossil Mammals of Asia',2013,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL); +(24188,2,NULL,'Fossil Mammals of Asia',2013,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL), +(25000,1,100,'A test reference that is not linked to any localities or species',2025,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL); /*!40000 ALTER TABLE `ref_ref` ENABLE KEYS */; UNLOCK TABLES;