From ff32b55a5d343f4602b22ce9375d636cacadd774 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 15:16:34 +0300 Subject: [PATCH 01/30] fixed region get test title --- backend/src/api-tests/region/get.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api-tests/region/get.test.ts b/backend/src/api-tests/region/get.test.ts index 0ad236e3..78994239 100644 --- a/backend/src/api-tests/region/get.test.ts +++ b/backend/src/api-tests/region/get.test.ts @@ -2,7 +2,7 @@ import { beforeAll, afterAll, describe, it, expect, beforeEach } from '@jest/glo import { login, logout, resetDatabase, send, resetDatabaseTimeout } from '../utils' import { pool } from '../../utils/db' -describe('Deleting a region works', () => { +describe('Getting region data works', () => { beforeAll(async () => { await resetDatabase() }, resetDatabaseTimeout) From 747070563b5bc7de91b848e1827a22c76cb2f008 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 15:27:30 +0300 Subject: [PATCH 02/30] add test for time bound creation --- cypress/e2e/timeBound.cy.js | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/cypress/e2e/timeBound.cy.js b/cypress/e2e/timeBound.cy.js index df1a34bd..2392cc4e 100644 --- a/cypress/e2e/timeBound.cy.js +++ b/cypress/e2e/timeBound.cy.js @@ -1,10 +1,35 @@ -describe('Editing time bound', () => { +before('Reset database', () => { + cy.request(Cypress.env('databaseResetUrl')) +}) + +describe('Creating time bound', () => { beforeEach('Login as admin', () => { cy.login('testSu') }) - beforeEach('Reset database', () => { - cy.request(Cypress.env('databaseResetUrl')) + it('with correct values works', () => { + cy.visit(`/time-bound/new/`) + cy.contains('Creating new time-bound') + cy.contains('Name: This field is required') + cy.contains('Age (Ma): This field is required') + cy.get('[id=b_name-textfield]').first().type('test bound') + cy.contains('Name: This field is required').should('not.exist') + cy.get('[id=write-button]').should('be.disabled') + cy.get('[id=age-textfield]').first().type('20') + cy.contains('Age (Ma): This field is required').should('not.exist') + cy.get('[id=write-button]').click() + cy.contains('button', 'Add existing reference').click() + cy.get('button[data-cy^="detailview-button"]').first().click() + cy.contains('button', 'Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + cy.contains('Edited item successfully.') + }) +}) + +describe('Editing time bound', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') }) it('with correct values works', () => { @@ -20,6 +45,7 @@ describe('Editing time bound', () => { cy.contains('button', 'Close').click() cy.get('[id=write-button]').should('not.be.disabled') cy.get('[id=write-button]').click() + cy.contains('Edited item successfully.') cy.contains('1.5') }) From 5388c23dd9af2245088b0ae8ae6e055e8b52a52f Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 15:39:59 +0300 Subject: [PATCH 03/30] add tests for species editing --- cypress/e2e/species.cy.js | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/cypress/e2e/species.cy.js b/cypress/e2e/species.cy.js index 5bd82509..9eae87ca 100644 --- a/cypress/e2e/species.cy.js +++ b/cypress/e2e/species.cy.js @@ -1,3 +1,7 @@ +before('Reset database', () => { + cy.request(Cypress.env('databaseResetUrl')) +}) + describe('Creating a species', () => { beforeEach('Login as admin', () => { cy.login('testSu') @@ -28,6 +32,77 @@ describe('Creating a species', () => { }) }) +describe('Editing a species', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it('with valid data works', () => { + cy.visit('/species/21052/') + cy.contains('21052 Simplomys simplicidens') + cy.get('[id=edit-button]').click() + + cy.get('[id=order_name-textfield]').clear() + cy.get('[id=order_name-textfield]').type('modifiedOrder') + cy.get('[id=family_name-textfield]').clear() + cy.get('[id=family_name-textfield]').type('modifiedFamily') + cy.get('[id=genus_name-textfield]').clear() + cy.get('[id=genus_name-textfield]').type('modifiedGenus') + cy.get('[id=species_name-textfield]').clear() + cy.get('[id=species_name-textfield]').type('modifiedSpecies') + cy.get('[id=subfamily_name-textfield]').type('heimo') + + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + cy.contains('button', 'Add existing reference').click() + cy.get('button[data-cy^="detailview-button"]').first().click() + cy.contains('button', 'Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + cy.contains('Saved species entry successfully.') + cy.contains('modifiedOrder') + cy.contains('Subfamily or Tribe heimo') + }) + + it('with missing required fields does not work', () => { + cy.visit('/species/21052/') + cy.contains('21052') + cy.get('[id=edit-button]').click() + + cy.get('[id=order_name-textfield]').clear() + cy.contains('Order: This field is required') + cy.get('[id=write-button]').should('be.disabled') + cy.contains('1 Invalid fields') + + cy.get('[id=family_name-textfield]').clear() + cy.contains('Family: This field is required') + cy.get('[id=write-button]').should('be.disabled') + cy.contains('2 Invalid fields') + + cy.get('[id=order_name-textfield]').type('newOrder') + cy.contains('Order: This field is required').should('not.exist') + cy.contains('Family: This field is required') + cy.get('[id=write-button]').should('be.disabled') + cy.contains('1 Invalid fields') + + cy.get('[id=family_name-textfield]').type('newFamily') + cy.contains('Family: This field is required').should('not.exist') + cy.get('[id=write-button]').click() + + cy.contains('button', 'Add existing reference').click() + cy.get('button[data-cy^="detailview-button"]').first().click() + cy.contains('button', 'Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + cy.get('[id=write-button]').click() + + cy.contains('Saved species entry successfully.') + cy.contains('newOrder') + cy.contains('newFamily') + cy.contains('modifiedGenus') + cy.contains('Subfamily or Tribe heimo') + }) +}) + describe('Deleting a species', () => { beforeEach('Login as admin', () => { cy.login('testSu') From 9b0c74f56eef1673766ff00a8c38a9944268ced6 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 15:49:32 +0300 Subject: [PATCH 04/30] add skeleton for e2e reference tests --- cypress/e2e/reference.cy.js | 63 +++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 cypress/e2e/reference.cy.js diff --git a/cypress/e2e/reference.cy.js b/cypress/e2e/reference.cy.js new file mode 100644 index 00000000..88ec8e9e --- /dev/null +++ b/cypress/e2e/reference.cy.js @@ -0,0 +1,63 @@ +before('Reset database', () => { + cy.request(Cypress.env('databaseResetUrl')) +}) + +describe('Creating a reference', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it.skip('with valid data works', () => {}) + + it.skip('with missing required fields does no work', () => {}) + + // TODO: add more tests for different types of references etc. +}) + +describe('Editing a reference', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it.skip('with valid data works', () => {}) + + it.skip('with missing required fields does no work', () => {}) + + // TODO: add more tests for different types of references etc. +}) + +describe('Deleting a reference', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it.skip('that is not linked to anything works and returns user to table view', () => { + // TODO: Create a new reference in anon database that is not linked to any localities or species + cy.visit('/reference/') + cy.get('[id=delete-button]').should('exist').click() + cy.on('window:confirm', str => { + expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.') + // this automatically clicks the OK button and returns to table view + }) + cy.contains('Deleted item successfully.') + cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to + cy.contains('').should('not.exist') + cy.visit('/reference/') + cy.contains('Error loading data') + }) + + // Skipped because referecnes that have linked localities or species cannot be deleted currently. + it.skip('that is linked to a locality and a species works and returns user to table view', () => { + cy.visit('/reference/10039') + cy.get('[id=delete-button]').should('exist').click() + cy.on('window:confirm', str => { + expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.') + // this automatically clicks the OK button and returns to table view + }) + cy.contains('Deleted item successfully.') + cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to + cy.contains('A new geomagnetic polarity time scale for the Late Cretaceous and Cenozoic').should('not.exist') + cy.visit('/reference/10039') + cy.contains('Error loading data') + }) +}) From 4965138cd04e624d2cb424e7ad842057ba53c682 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Fri, 9 May 2025 14:56:36 +0300 Subject: [PATCH 05/30] added new test reference to anon database and made a deletion test --- cypress/e2e/reference.cy.js | 15 +++++++-------- test_data/sqlfiles/now_test.sql | 6 ++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cypress/e2e/reference.cy.js b/cypress/e2e/reference.cy.js index 88ec8e9e..188cfea8 100644 --- a/cypress/e2e/reference.cy.js +++ b/cypress/e2e/reference.cy.js @@ -9,7 +9,7 @@ describe('Creating a reference', () => { it.skip('with valid data works', () => {}) - it.skip('with missing required fields does no work', () => {}) + it.skip('with missing required fields does not work', () => {}) // TODO: add more tests for different types of references etc. }) @@ -21,7 +21,7 @@ describe('Editing a reference', () => { it.skip('with valid data works', () => {}) - it.skip('with missing required fields does no work', () => {}) + it.skip('with missing required fields does not work', () => {}) // TODO: add more tests for different types of references etc. }) @@ -31,9 +31,8 @@ describe('Deleting a reference', () => { cy.login('testSu') }) - it.skip('that is not linked to anything works and returns user to table view', () => { - // TODO: Create a new reference in anon database that is not linked to any localities or species - cy.visit('/reference/') + it('that is not linked to anything works and returns user to table view', () => { + cy.visit('/reference/25000') cy.get('[id=delete-button]').should('exist').click() cy.on('window:confirm', str => { expect(str).to.eq('Are you sure you want to delete this item? This operation cannot be undone.') @@ -41,12 +40,12 @@ describe('Deleting a reference', () => { }) cy.contains('Deleted item successfully.') cy.contains('Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam') // check that table view has been navigated to - cy.contains('').should('not.exist') - cy.visit('/reference/') + cy.contains('A test reference that is not linked to any localities or species').should('not.exist') + cy.visit('/reference/25000') cy.contains('Error loading data') }) - // Skipped because referecnes that have linked localities or species cannot be deleted currently. + // Skipped because references that have linked localities or species cannot be deleted currently it.skip('that is linked to a locality and a species works and returns user to table view', () => { cy.visit('/reference/10039') cy.get('[id=delete-button]').should('exist').click() diff --git a/test_data/sqlfiles/now_test.sql b/test_data/sqlfiles/now_test.sql index 18596cd4..f4833a7e 100755 --- a/test_data/sqlfiles/now_test.sql +++ b/test_data/sqlfiles/now_test.sql @@ -1709,7 +1709,8 @@ INSERT INTO `ref_authors` VALUES (21368,2,1,'Ciochon','Russell'), (24151,2,1,'Ogg','J.G.'), (24187,2,1,'Kaakinen','A.'), -(24188,2,1,'Wang','X.'); +(24188,2,1,'Wang','X.'), +(25000,2,1,'Doe','D.'); /*!40000 ALTER TABLE `ref_authors` ENABLE KEYS */; UNLOCK TABLES; @@ -2198,7 +2199,8 @@ INSERT INTO `ref_ref` VALUES (21368,1,100,'Dated co-occurrence of Homo erectus and Gigantopithecus from Tham Khuyen Cave, Vietnam',1996,'93',NULL,3016,3020,NULL,NULL,NULL,NULL,NULL,NULL,'abstract','http://www.pnas.org/cgi/reprint/93/7/3016',NULL,NULL,NULL,'English',NULL,NULL,1,NULL), (24151,2,NULL,'A Concise Geologic Time Scale: 2016',2016,NULL,NULL,NULL,240,'Elsevier',NULL,NULL,NULL,NULL,'9780444594686',NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,1,0), (24187,13,NULL,'Helsinki Asian time update',2020,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'2020-01-29',NULL,1,NULL), -(24188,2,NULL,'Fossil Mammals of Asia',2013,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL); +(24188,2,NULL,'Fossil Mammals of Asia',2013,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,NULL), +(25000,1,100,'A test reference that is not linked to any localities or species',2025,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL); /*!40000 ALTER TABLE `ref_ref` ENABLE KEYS */; UNLOCK TABLES; From c1d77497a2068203e3a1511014b4c11971a0f8ec Mon Sep 17 00:00:00 2001 From: juhanikat Date: Fri, 9 May 2025 15:28:05 +0300 Subject: [PATCH 06/30] changed SelectingTable to give an id to EditingModal, and added creation tests for references --- cypress/e2e/reference.cy.js | 106 +++++++++++++++--- .../DetailView/common/SelectingTable.tsx | 2 +- 2 files changed, 92 insertions(+), 16 deletions(-) diff --git a/cypress/e2e/reference.cy.js b/cypress/e2e/reference.cy.js index 188cfea8..9dc79d3b 100644 --- a/cypress/e2e/reference.cy.js +++ b/cypress/e2e/reference.cy.js @@ -2,7 +2,94 @@ before('Reset database', () => { cy.request(Cypress.env('databaseResetUrl')) }) -describe('Creating a reference', () => { +describe('Creating a journal', () => { + beforeEach('Login as admin', () => { + cy.login('testSu') + }) + + it('with valid data works', () => { + cy.visit('/reference/new') + cy.contains('Journal') + cy.contains('7 Invalid fields') + + cy.get('[id=title_primary-textfield]').type('New test reference') + + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + + cy.get('[id=date_primary-textfield]').type('2025') + + cy.get('[data-cy=Select\\ Journal-button]').click() + cy.get('[data-cy=detailview-button-100]').click() + cy.contains('Close').click() + + cy.get('[id=write-button]').click() + cy.contains('Saved reference successfully.') + }) + + it('with missing required fields does not work', () => { + cy.visit('/reference/new') + cy.contains('Journal') + cy.contains('7 Invalid fields') + + cy.get('[id=title_primary-textfield]').type('New test reference 2') + cy.contains('3 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + cy.contains('2 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[id=date_primary-textfield]').type('2025') + cy.contains('1 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') + + cy.get('[data-cy=Select\\ Journal-button]').click() + cy.get('[data-cy=detailview-button-100]').click() + cy.contains('Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + + cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected author + cy.get('[id=write-button]').should('be.disabled') + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + }) + + it('combined check with different types of titles and gen notes works', () => { + cy.visit('/reference/new') + cy.contains('Journal') + cy.contains('7 Invalid fields') + + cy.get('[id=title_primary-textfield]').type('Title Primary') + cy.contains('3 Invalid fields') + cy.get('[id=title_primary-textfield]').clear() + cy.contains('7 Invalid fields') + + cy.get('[id=title_secondary-textfield]').type('Title Secondary') + cy.contains('3 Invalid fields') + cy.get('[id=title_secondary-textfield]').clear() + cy.contains('7 Invalid fields') + + cy.get('[id=title_series-textfield]').type('Title Series') + cy.contains('3 Invalid fields') + cy.get('[id=title_series-textfield]').clear() + cy.contains('7 Invalid fields') + + cy.get('[id=gen_notes-textfield]').type('Gen Notes') + cy.contains('3 Invalid fields') + cy.get('[id=gen_notes-textfield]').clear() + cy.contains('7 Invalid fields') + }) +}) + +// TODO: add more tests for different types of references + +describe('Editing a journal', () => { beforeEach('Login as admin', () => { cy.login('testSu') }) @@ -10,28 +97,17 @@ describe('Creating a reference', () => { it.skip('with valid data works', () => {}) it.skip('with missing required fields does not work', () => {}) - - // TODO: add more tests for different types of references etc. }) -describe('Editing a reference', () => { - beforeEach('Login as admin', () => { - cy.login('testSu') - }) - - it.skip('with valid data works', () => {}) - - it.skip('with missing required fields does not work', () => {}) - - // TODO: add more tests for different types of references etc. -}) +// TODO: add more tests for different types of references describe('Deleting a reference', () => { beforeEach('Login as admin', () => { cy.login('testSu') }) - it('that is not linked to anything works and returns user to table view', () => { + // REMOVE SKIP + it.skip('that is not linked to anything works and returns user to table view', () => { cy.visit('/reference/25000') cy.get('[id=delete-button]').should('exist').click() cy.on('window:confirm', str => { diff --git a/frontend/src/components/DetailView/common/SelectingTable.tsx b/frontend/src/components/DetailView/common/SelectingTable.tsx index 46427135..2b43bae4 100755 --- a/frontend/src/components/DetailView/common/SelectingTable.tsx +++ b/frontend/src/components/DetailView/common/SelectingTable.tsx @@ -67,7 +67,7 @@ export const SelectingTable = if (isError) return Error fetching data for the selecting table. if (!data) return return ( - + {data ? ( data={filteredData} From 121eb29d0ae921cce8876cb8d99e595fddf8fb66 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Fri, 9 May 2025 15:41:10 +0300 Subject: [PATCH 07/30] added editing tests for references --- cypress/e2e/reference.cy.js | 45 +++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/cypress/e2e/reference.cy.js b/cypress/e2e/reference.cy.js index 9dc79d3b..7034dab9 100644 --- a/cypress/e2e/reference.cy.js +++ b/cypress/e2e/reference.cy.js @@ -64,26 +64,31 @@ describe('Creating a journal', () => { cy.visit('/reference/new') cy.contains('Journal') cy.contains('7 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_primary-textfield]').type('Title Primary') cy.contains('3 Invalid fields') cy.get('[id=title_primary-textfield]').clear() cy.contains('7 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_secondary-textfield]').type('Title Secondary') cy.contains('3 Invalid fields') cy.get('[id=title_secondary-textfield]').clear() cy.contains('7 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_series-textfield]').type('Title Series') cy.contains('3 Invalid fields') cy.get('[id=title_series-textfield]').clear() cy.contains('7 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') cy.get('[id=gen_notes-textfield]').type('Gen Notes') cy.contains('3 Invalid fields') cy.get('[id=gen_notes-textfield]').clear() cy.contains('7 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') }) }) @@ -94,9 +99,42 @@ describe('Editing a journal', () => { cy.login('testSu') }) - it.skip('with valid data works', () => {}) + it('with valid data works', () => { + cy.visit('reference/10039') + cy.contains('A new geomagnetic polarity time scale for the Late Cretaceous and Cenozoic.') + cy.get('[id=edit-button]').click() + cy.get('[id=title_primary-textfield]').clear() + cy.get('[id=title_primary-textfield]').type('New primary title.') - it.skip('with missing required fields does not work', () => {}) + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-2]').click() + cy.contains('Close').click() + + cy.get('[id=write-button]').click() + cy.contains('Saved reference successfully.') + cy.contains('New primary title.') + cy.contains('Cande') // Author 1 + cy.contains('Ciochon') // Author 2 + }) + + it('with missing required fields does not work', () => { + cy.visit('reference/10039') + cy.contains('New primary title.') + cy.get('[id=edit-button]').click() + cy.get('[id=title_primary-textfield]').clear() + cy.contains('4 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') + cy.get('[id=title_primary-textfield]').type('New primary title.') + + cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() + cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected authors + cy.contains('1 Invalid fields') + cy.get('[id=write-button]').should('be.disabled') + cy.get('[data-cy=Select\\ author-button]').click() + cy.get('[data-cy=detailview-button-1]').click() + cy.contains('Close').click() + cy.get('[id=write-button]').should('not.be.disabled') + }) }) // TODO: add more tests for different types of references @@ -106,8 +144,7 @@ describe('Deleting a reference', () => { cy.login('testSu') }) - // REMOVE SKIP - it.skip('that is not linked to anything works and returns user to table view', () => { + it('that is not linked to anything works and returns user to table view', () => { cy.visit('/reference/25000') cy.get('[id=delete-button]').should('exist').click() cy.on('window:confirm', str => { From 71f1394b9ec374b8883d4438d12bd48c96b6279e Mon Sep 17 00:00:00 2001 From: juhanikat Date: Fri, 9 May 2025 15:44:53 +0300 Subject: [PATCH 08/30] chnaged errorbox title slightly --- cypress/e2e/locality.cy.js | 12 +++---- cypress/e2e/reference.cy.js | 32 +++++++++---------- cypress/e2e/species.cy.js | 8 ++--- cypress/e2e/timeBound.cy.js | 2 +- .../src/components/DetailView/components.tsx | 4 ++- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/cypress/e2e/locality.cy.js b/cypress/e2e/locality.cy.js index 1ded9816..186ff104 100644 --- a/cypress/e2e/locality.cy.js +++ b/cypress/e2e/locality.cy.js @@ -60,7 +60,7 @@ describe('Creating a new locality', () => { cy.get('[id=dms_lat-textfield]').should('have.value', '48 40 48 N') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') }) it("and filling, then erasing needed data doesn't work", () => { @@ -95,12 +95,12 @@ describe('Creating a new locality', () => { cy.get('[id=min_age-textfield]').type('{backspace}{backspace}{backspace}{backspace}{backspace}') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') // make sure errors in other tabs disable the write button cy.get('[role=tablist]').contains('Locality').click() cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') }) it('write button is disabled if unvisited tab has validation errors', () => { @@ -116,12 +116,12 @@ describe('Creating a new locality', () => { // Age tab has no errors, but Locality tab has unfilled mandatory fields cy.get('[id=write-button]').should('be.disabled') - cy.contains('6 Invalid fields') + cy.contains('6 invalid fields') cy.get('[id=min_age-textfield-helper-text]').should('not.exist') cy.get('[id=max_age-textfield-helper-text]').should('not.exist') cy.get('[role=tablist]').contains('Locality').click() cy.get('[id=write-button]').should('be.disabled') - cy.contains('6 Invalid fields') + cy.contains('6 invalid fields') }) it('composite dating method works', () => { @@ -233,7 +233,7 @@ describe('Editing a locality', () => { cy.get('[id=max_age-textfield]').type('{backspace}{backspace}{backspace}{backspace}{backspace}9') cy.contains('Min value cannot be higher than max') cy.get('[id=write-button]').should('be.disabled') - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') }) it('changing fractions and basis for age updates the age correctly', () => { diff --git a/cypress/e2e/reference.cy.js b/cypress/e2e/reference.cy.js index 7034dab9..0a0817a5 100644 --- a/cypress/e2e/reference.cy.js +++ b/cypress/e2e/reference.cy.js @@ -10,7 +10,7 @@ describe('Creating a journal', () => { it('with valid data works', () => { cy.visit('/reference/new') cy.contains('Journal') - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=title_primary-textfield]').type('New test reference') @@ -31,20 +31,20 @@ describe('Creating a journal', () => { it('with missing required fields does not work', () => { cy.visit('/reference/new') cy.contains('Journal') - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=title_primary-textfield]').type('New test reference 2') - cy.contains('3 Invalid fields') + cy.contains('3 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[data-cy=Select\\ author-button]').click() cy.get('[data-cy=detailview-button-1]').click() cy.contains('Close').click() - cy.contains('2 Invalid fields') + cy.contains('2 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[id=date_primary-textfield]').type('2025') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') cy.get('[id=write-button]').should('be.disabled') cy.get('[data-cy=Select\\ Journal-button]').click() @@ -63,31 +63,31 @@ describe('Creating a journal', () => { it('combined check with different types of titles and gen notes works', () => { cy.visit('/reference/new') cy.contains('Journal') - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_primary-textfield]').type('Title Primary') - cy.contains('3 Invalid fields') + cy.contains('3 invalid fields') cy.get('[id=title_primary-textfield]').clear() - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_secondary-textfield]').type('Title Secondary') - cy.contains('3 Invalid fields') + cy.contains('3 invalid fields') cy.get('[id=title_secondary-textfield]').clear() - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_series-textfield]').type('Title Series') - cy.contains('3 Invalid fields') + cy.contains('3 invalid fields') cy.get('[id=title_series-textfield]').clear() - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[id=gen_notes-textfield]').type('Gen Notes') - cy.contains('3 Invalid fields') + cy.contains('3 invalid fields') cy.get('[id=gen_notes-textfield]').clear() - cy.contains('7 Invalid fields') + cy.contains('7 invalid fields') cy.get('[id=write-button]').should('be.disabled') }) }) @@ -122,13 +122,13 @@ describe('Editing a journal', () => { cy.contains('New primary title.') cy.get('[id=edit-button]').click() cy.get('[id=title_primary-textfield]').clear() - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') cy.get('[id=write-button]').should('be.disabled') cy.get('[id=title_primary-textfield]').type('New primary title.') cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() cy.get('[data-testid=RemoveCircleOutlineIcon]').first().click() // removes the selected authors - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') cy.get('[id=write-button]').should('be.disabled') cy.get('[data-cy=Select\\ author-button]').click() cy.get('[data-cy=detailview-button-1]').click() diff --git a/cypress/e2e/species.cy.js b/cypress/e2e/species.cy.js index 9eae87ca..cdb62933 100644 --- a/cypress/e2e/species.cy.js +++ b/cypress/e2e/species.cy.js @@ -28,7 +28,7 @@ describe('Creating a species', () => { cy.visit('/species/new') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('4 Invalid fields') + cy.contains('4 invalid fields') }) }) @@ -72,18 +72,18 @@ describe('Editing a species', () => { cy.get('[id=order_name-textfield]').clear() cy.contains('Order: This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') cy.get('[id=family_name-textfield]').clear() cy.contains('Family: This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('2 Invalid fields') + cy.contains('2 invalid fields') cy.get('[id=order_name-textfield]').type('newOrder') cy.contains('Order: This field is required').should('not.exist') cy.contains('Family: This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') cy.get('[id=family_name-textfield]').type('newFamily') cy.contains('Family: This field is required').should('not.exist') diff --git a/cypress/e2e/timeBound.cy.js b/cypress/e2e/timeBound.cy.js index 2392cc4e..4099711f 100644 --- a/cypress/e2e/timeBound.cy.js +++ b/cypress/e2e/timeBound.cy.js @@ -56,7 +56,7 @@ describe('Editing time bound', () => { cy.get('[id=age-textfield]').first().type('{backspace}{backspace}{backspace}{backspace}{backspace}') cy.contains('This field is required') cy.get('[id=write-button]').should('be.disabled') - cy.contains('1 Invalid fields') + cy.contains('1 invalid field') }) }) diff --git a/frontend/src/components/DetailView/components.tsx b/frontend/src/components/DetailView/components.tsx index f530089d..cc8b5681 100755 --- a/frontend/src/components/DetailView/components.tsx +++ b/frontend/src/components/DetailView/components.tsx @@ -113,6 +113,8 @@ export const ErrorBox = () => { const { fieldsWithErrors } = useDetailContext() const fields = Object.keys(fieldsWithErrors) + const title = fields.length > 1 ? `${fields.length} invalid fields` : '1 invalid field' + return ( () => { }} > - {`${fields.length} Invalid fields`} + {title} {fields.map(field => ( From dcbeb85d531525059ceb371e59864f7d2fa353ec Mon Sep 17 00:00:00 2001 From: juhanikat Date: Sat, 12 Apr 2025 19:58:23 +0300 Subject: [PATCH 09/30] add testing route for cross search export --- backend/package-lock.json | 84 +++++++++++++++++++++++++++++++ backend/package.json | 1 + backend/src/routes/crossSearch.ts | 20 ++++++++ 3 files changed, 105 insertions(+) diff --git a/backend/package-lock.json b/backend/package-lock.json index d98b1316..a64e0e0d 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -24,6 +24,7 @@ "express": "^4.18.2", "express-async-errors": "^3.1.1", "express-rate-limit": "^7.5.0", + "fast-csv": "^5.0.2", "jsonwebtoken": "^9.0.2", "mariadb": "^3.3.0", "md5": "^2.3.0", @@ -2456,6 +2457,33 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@fast-csv/format": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-5.0.2.tgz", + "integrity": "sha512-fRYcWvI8vs0Zxa/8fXd/QlmQYWWkJqKZPAXM+vksnplb3owQFKTPPh9JqOtD0L3flQw/AZjjXdPkD7Kp/uHm8g==", + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/parse": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.2.tgz", + "integrity": "sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==", + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -6360,6 +6388,19 @@ "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, + "node_modules/fast-csv": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-5.0.2.tgz", + "integrity": "sha512-CnB2zYAzzeh5Ta0UhSf32NexLy2SsEsSMY+fMWPV40k1OgaLEbm9Hf5dms3z/9fASZHBjB6i834079gVeksEqQ==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "5.0.2", + "@fast-csv/parse": "5.0.2" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -9468,6 +9509,18 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -9478,11 +9531,30 @@ "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -9498,6 +9570,12 @@ "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -9510,6 +9588,12 @@ "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", diff --git a/backend/package.json b/backend/package.json index 915362fc..544290fb 100644 --- a/backend/package.json +++ b/backend/package.json @@ -53,6 +53,7 @@ "express": "^4.18.2", "express-async-errors": "^3.1.1", "express-rate-limit": "^7.5.0", + "fast-csv": "^5.0.2", "jsonwebtoken": "^9.0.2", "mariadb": "^3.3.0", "md5": "^2.3.0", diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index ecd7247a..9d61c520 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -2,6 +2,7 @@ import { Router } from 'express' import { getCrossSearchRawSql, validateCrossSearchRouteParameters } from '../services/crossSearch' import { fixBigInt } from '../utils/common' import { ColumnFilter, SortingState } from '../services/queries/crossSearchQuery' +import { format } from '@fast-csv/format' const router = Router() @@ -37,4 +38,23 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { } }) +router.get(`/export`, async (req, res) => { + const stream = format() + stream.pipe(res) + res.attachment('test.csv') + + const result = await getCrossSearchRawSql(req.user) + + const headers = [] + for (const key of Object.keys(result[0])) { + headers.push(key) + } + stream.write(headers) + + for (const row of result) { + stream.write(row) + } + stream.end() +}) + export default router From ba248d309b93db77a402859c1a7685af2aeb77f7 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 14 Apr 2025 14:41:07 +0300 Subject: [PATCH 10/30] improve export route --- backend/src/routes/crossSearch.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index 9d61c520..5bf508e8 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -3,6 +3,8 @@ import { getCrossSearchRawSql, validateCrossSearchRouteParameters } from '../ser import { fixBigInt } from '../utils/common' import { ColumnFilter, SortingState } from '../services/queries/crossSearchQuery' import { format } from '@fast-csv/format' +import { pipeline } from 'stream' +import { logger } from '../utils/logger' const router = Router() @@ -39,16 +41,21 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { }) router.get(`/export`, async (req, res) => { + res.attachment('cross_search_export.csv') + res.on('finish', () => { + logger.info('Cross search export sent.') + }) + const stream = format() - stream.pipe(res) - res.attachment('test.csv') + pipeline(stream, res, err => { + if (err) { + logger.error(`Error in pipeline: ${err.message}`) + } + }) const result = await getCrossSearchRawSql(req.user) - const headers = [] - for (const key of Object.keys(result[0])) { - headers.push(key) - } + const headers = Object.keys(result[0]) stream.write(headers) for (const row of result) { From 777035bd50a6a2efdd34b172372d491bee63143e Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 14 Apr 2025 15:31:33 +0300 Subject: [PATCH 11/30] new button for exporting cross search table --- .../CrossSearch/CrossSearchTable.tsx | 1 + .../src/components/TableView/TableView.tsx | 32 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/CrossSearch/CrossSearchTable.tsx b/frontend/src/components/CrossSearch/CrossSearchTable.tsx index b8195a79..28baa44b 100755 --- a/frontend/src/components/CrossSearch/CrossSearchTable.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchTable.tsx @@ -812,6 +812,7 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros url="crosssearch" enableColumnFilterModes={true} serverSidePagination={true} + isCrossSearchTable={true} /> ) } diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index 713a4efa..0b05cc2f 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -18,6 +18,7 @@ import { useUser } from '@/hooks/user' import { defaultPagination, defaultPaginationSmall } from '@/common' import '../../styles/TableView.css' import { TableToolBar } from './TableToolBar' +import { BACKEND_URL } from '@/util/config' type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' @@ -27,6 +28,31 @@ type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' selectorFn should only be defined if using this as a selecting table */ + +const CrossSearchExportButton = () => { + // loading is set to false after a timeout because actually knowing + // once the file is downloaded is impossible when implemented this way + const [loading, setLoading] = useState(false) + + return ( + + + {loading && } + + ) +} export const TableView = ({ data, columns, @@ -41,6 +67,7 @@ export const TableView = ({ exportIsLoading, enableColumnFilterModes, serverSidePagination, + isCrossSearchTable, isFetching, }: { title?: string @@ -56,6 +83,7 @@ export const TableView = ({ exportIsLoading?: boolean enableColumnFilterModes?: boolean serverSidePagination?: boolean + isCrossSearchTable?: boolean isFetching: boolean }) => { const location = useLocation() @@ -190,9 +218,7 @@ export const TableView = ({ <>

- {title === 'Locality-Species-Cross-Search' - ? 'Loading data, this might take a few seconds...' - : 'Loading data...'} + {'Loading data...'} ) } From dcf30bc0d8db2a47c1fd52cf8df32201d8a95c85 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 13:42:08 +0300 Subject: [PATCH 12/30] improved export button --- .../CrossSearch/CrossSearchExportButton.tsx | 44 +++++++++++++++++++ .../src/components/TableView/TableView.tsx | 33 ++++---------- frontend/src/components/TableView/helpers.tsx | 30 +++++++++++++ 3 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 frontend/src/components/CrossSearch/CrossSearchExportButton.tsx diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx new file mode 100644 index 00000000..e3b27adc --- /dev/null +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -0,0 +1,44 @@ +import { useState } from 'react' +import { useNotify } from '@/hooks/notification' +import { BACKEND_URL } from '@/util/config' +import { Box, Button, CircularProgress } from '@mui/material' + +export const CrossSearchExportButton = () => { + const [loading, setLoading] = useState(false) + const notify = useNotify() + + const fetchCSVFile = () => { + setLoading(true) + notify('Download started, please wait...') + + fetch(`${BACKEND_URL}/crosssearch/export`) + .then(response => { + if (!response.ok) { + throw new Error('Server response was not OK.') + } + return response.blob() + }) + .then(blob => { + const url = window.URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = 'cross_search_export.csv' + document.body.appendChild(a) + a.click() + a.remove() + window.URL.revokeObjectURL(url) + }) + .catch(_ => { + notify('Downloading cross-search export failed.', 'error') + }) + .finally(() => setLoading(false)) + } + return ( + + + {loading && } + + ) +} diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index 0b05cc2f..bd16626a 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -29,30 +29,6 @@ type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' selectorFn should only be defined if using this as a selecting table */ -const CrossSearchExportButton = () => { - // loading is set to false after a timeout because actually knowing - // once the file is downloaded is impossible when implemented this way - const [loading, setLoading] = useState(false) - - return ( - - - {loading && } - - ) -} export const TableView = ({ data, columns, @@ -135,6 +111,15 @@ export const TableView = ({ else rowCount = data.length } + let toolbar + if (isCrossSearchTable) { + toolbar = renderCustomToolbarCrossSearchVersion + } else if (selectorFn) { + toolbar = renderCustomToolbarModalVersion + } else { + toolbar = renderCustomToolbar + } + const table = useMaterialReactTable({ columns: columns, data: data || [], diff --git a/frontend/src/components/TableView/helpers.tsx b/frontend/src/components/TableView/helpers.tsx index a59cacd4..7b359006 100755 --- a/frontend/src/components/TableView/helpers.tsx +++ b/frontend/src/components/TableView/helpers.tsx @@ -7,6 +7,7 @@ import { } from 'material-react-table' import { mkConfig, generateCsv, download } from 'export-to-csv' import { AcceptedData } from 'node_modules/export-to-csv/output/lib/types' +import { CrossSearchExportButton } from '../CrossSearch/CrossSearchExportButton' export type ExportFn = (data: T) => { [k: string]: AcceptedData } @@ -47,3 +48,32 @@ export const renderCustomToolbarModalVersion = ({ table } ) +export const renderCustomToolbarCrossSearchVersion = ({ + table, +}: { + table: MRT_TableInstance +}) => ( + <> + + + + + + + + +) + +export const renderCustomToolbar = ({ table }: { table: MRT_TableInstance }) => ( + <> + + + + + + + + +) From 4ae8013c6d9876d00d764e77b4e6c6cb5296b433 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 14:07:06 +0300 Subject: [PATCH 13/30] added and updated documentation --- documentation/backend/cross_search_export.md | 3 +++ documentation/frontend/components/TableView.md | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 documentation/backend/cross_search_export.md diff --git a/documentation/backend/cross_search_export.md b/documentation/backend/cross_search_export.md new file mode 100644 index 00000000..a65fcb0a --- /dev/null +++ b/documentation/backend/cross_search_export.md @@ -0,0 +1,3 @@ +### Cross Search Export + +Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same SQL query as the normal cross-search route but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file diff --git a/documentation/frontend/components/TableView.md b/documentation/frontend/components/TableView.md index 57a0a83c..8a4e2619 100644 --- a/documentation/frontend/components/TableView.md +++ b/documentation/frontend/components/TableView.md @@ -12,7 +12,7 @@ TableView has the following required props: The table will handle paging and sorting data automatically. If your backend query for the table is using [server side pagination](../../backend/server_side_pagination.md), remember to set the `serverSidePagination` prop to True. Check the Table View section in the guide below for more information about optional props. -The table itself is constructed with the useMaterialReactTable-hook. The toolbar at the top is defined in the [helpers](../../../frontend/src/components/TableView/helpers.tsx) file and rendered using the `renderToolbarInternalActions` option, and it contains buttons for showing/hiding columns and showing the table in fullscreen mode, as well as exporting the content of the table. If the toolbar needs to be changed in the future (e.g. adding more rows to it) checkout the documentation for rendering a custom toolbar [here](https://www.material-react-table.com/docs/guides/toolbar-customization#override-with-custom-toolbar-components) since the current method doesn't really allow any major changes to its layout. +The table itself is constructed with the useMaterialReactTable-hook. The toolbar at the top is defined in the [helpers](../../../frontend/src/components/TableView/helpers.tsx) file and rendered using the `renderToolbarInternalActions` option, and it contains buttons for showing/hiding columns and showing the table in fullscreen mode, as well as exporting the content of the table. To know what components you can render here if necessary, see the [source code](https://github.com/KevinVandy/material-react-table/blob/85b98f9aaa038df48aa1dd35123560abce78ee58/packages/material-react-table/src/components/toolbar/MRT_ToolbarInternalButtons.tsx#L45). If the toolbar needs to be changed in the future (e.g. adding more rows to it) checkout the documentation for rendering a custom toolbar [here](https://www.material-react-table.com/docs/guides/toolbar-customization#override-with-custom-toolbar-components) since the current method doesn't really allow any major changes to its layout. ### Setting up a new table view @@ -38,5 +38,12 @@ Following things are needed to create a new page with a table (examples are all - to enable a filter mode selection menu for the columns of the table, set parameter `enableColumnFilterModes` to `true`. - you can pass an array of filter functions to be shown in the menu to the parameter `columnFilterModeOptions`. - it's a good idea to disable the filter mode menu for columns for which it doesn't make sense by setting the parameter `enableColumnFilterModes` to `false` for that column. See [LocalityTable](../../../frontend/src/components/Locality/LocalityTable.tsx) for an example. For more information, see [MRT docs filter mode section](https://www.material-react-table.com/docs/guides/column-filtering#filter-modes). - - If the table you're creating is using server side pagination, remember to set the `serverSidePagination` prop to True so the TableView will set its pagination/sorting modes correctly. Forgetting to do this might make the table load indefinitely or crash the app. + - If the table you're creating is using [server side pagination](../../backend/server_side_pagination.md), remember to set the `serverSidePagination` prop to True so the TableView will set its pagination/sorting modes correctly. Forgetting to do this might make the table load indefinitely or crash the app. + - `isCrossSearchTable` is only set inside [CrossSearchTable.tsx](../../../frontend/src/components/CrossSearch/CrossSearchTable.tsx). See the "Exporting" section below. - More info can be found from official [MRT docs](https://www.material-react-table.com/). + +### Exporting + +Most tables simply call the [exportRows function](../../../frontend/src/components/TableView/helpers.tsx) when the user clicks the "Export Table" button. This function converts the entire table into a csv file and downloads it. The [export function for cross-search](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) is different since it uses server side pagination and the cross-search table does not contain all needed data at once. The `isCrossSearchTable` prop is used to render the proper toolbar and "Export Table" button for the cross-search table. See [server side pagination](../../backend/server_side_pagination.md) and [cross search export](../../backend/cross_search_export.md) docs for more information. + +Currently the export function for cross-search uses blob(), which waits for the entire file to be streamed before continuing. If you want to start downloading the file as soon as the backend starts streaming it, [this MDN document](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#reading_the_stream) might help. This might not speed up the download, but it shows the "download started" popup in the user's browser faster, which can make the wait a little less annoying for them. From f9c4bd59b90665dd108a8339062927739a5a92c1 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Wed, 16 Apr 2025 15:58:58 +0300 Subject: [PATCH 14/30] add user token to request headers --- backend/src/routes/crossSearch.ts | 2 +- .../components/CrossSearch/CrossSearchExportButton.tsx | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index 5bf508e8..ae1ca8f6 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -49,7 +49,7 @@ router.get(`/export`, async (req, res) => { const stream = format() pipeline(stream, res, err => { if (err) { - logger.error(`Error in pipeline: ${err.message}`) + logger.error(`Error in crosssearch/export pipeline: ${err.message}`) } }) diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index e3b27adc..5914d554 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -2,16 +2,20 @@ import { useState } from 'react' import { useNotify } from '@/hooks/notification' import { BACKEND_URL } from '@/util/config' import { Box, Button, CircularProgress } from '@mui/material' +import { useUser } from '@/hooks/user' export const CrossSearchExportButton = () => { const [loading, setLoading] = useState(false) const notify = useNotify() + const token = useUser().token + const headers = token ? { Authorization: `Bearer ${token}` } : {} + const fetchCSVFile = () => { setLoading(true) notify('Download started, please wait...') - fetch(`${BACKEND_URL}/crosssearch/export`) + fetch(`${BACKEND_URL}/crosssearch/export`, { headers: headers }) .then(response => { if (!response.ok) { throw new Error('Server response was not OK.') @@ -36,9 +40,8 @@ export const CrossSearchExportButton = () => { return ( - {loading && } ) } From b1c61023f060f013db68cc536173f7f8a2c2e11c Mon Sep 17 00:00:00 2001 From: juhanikat Date: Wed, 16 Apr 2025 16:54:59 +0300 Subject: [PATCH 15/30] added column filtering and sorting to export, and fixed bug with crosssearch/all route --- backend/src/routes/crossSearch.ts | 42 ++++++++++++++++--- .../CrossSearch/CrossSearchExportButton.tsx | 6 ++- frontend/src/shared/types/misc.ts | 4 +- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index ae1ca8f6..d04f1f71 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -15,7 +15,9 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { let sorting try { limit = parseInt(req.params.limit) + if (isNaN(limit)) throw new Error('Limit is not a number') offset = parseInt(req.params.offset) + if (isNaN(offset)) throw new Error('Offset is not a number') columnFilters = JSON.parse(req.params.columnfilters) as unknown sorting = JSON.parse(req.params.sorting) as unknown } catch (error) { @@ -40,7 +42,37 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { } }) -router.get(`/export`, async (req, res) => { +router.get(`/export/:columnfilters/:sorting`, async (req, res) => { + let columnFilters + let sorting + let data + try { + columnFilters = JSON.parse(req.params.columnfilters) as unknown + sorting = JSON.parse(req.params.sorting) as unknown + } catch (error) { + return res.status(403).send({ error: 'Parsing URL parameters failed' }) + } + const validationErrors = validateCrossSearchRouteParameters({ + columnFilters, + sorting, + }) + if (validationErrors.length > 0) { + return res.status(403).send(validationErrors) + } + + try { + data = await getCrossSearchRawSql( + req.user, + undefined, + undefined, + columnFilters as ColumnFilter[], + sorting as SortingState[] + ) + } catch (error) { + if (error && typeof error === 'object' && 'message' in error) return res.status(403).send({ error: error.message }) + return res.status(403).send('Unknown error') + } + res.attachment('cross_search_export.csv') res.on('finish', () => { logger.info('Cross search export sent.') @@ -53,15 +85,13 @@ router.get(`/export`, async (req, res) => { } }) - const result = await getCrossSearchRawSql(req.user) - - const headers = Object.keys(result[0]) + const headers = Object.keys(data[0]) stream.write(headers) - for (const row of result) { + for (const row of data) { stream.write(row) } - stream.end() + return stream.end() }) export default router diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index 5914d554..09a9a78f 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -3,9 +3,11 @@ import { useNotify } from '@/hooks/notification' import { BACKEND_URL } from '@/util/config' import { Box, Button, CircularProgress } from '@mui/material' import { useUser } from '@/hooks/user' +import { usePageContext } from '../Page' export const CrossSearchExportButton = () => { const [loading, setLoading] = useState(false) + const { sqlColumnFilters, sqlOrderBy } = usePageContext() const notify = useNotify() const token = useUser().token @@ -15,7 +17,9 @@ export const CrossSearchExportButton = () => { setLoading(true) notify('Download started, please wait...') - fetch(`${BACKEND_URL}/crosssearch/export`, { headers: headers }) + fetch(`${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, { + headers: headers, + }) .then(response => { if (!response.ok) { throw new Error('Server response was not OK.') diff --git a/frontend/src/shared/types/misc.ts b/frontend/src/shared/types/misc.ts index cefe9003..6bdd4e67 100755 --- a/frontend/src/shared/types/misc.ts +++ b/frontend/src/shared/types/misc.ts @@ -28,8 +28,8 @@ type ValidationErrorItem = { } export type CrossSearchRouteParameters = { - limit: number - offset: number + limit?: number + offset?: number columnFilters: unknown sorting: unknown } From 961b961e64f8bcc8eae5e8e4fd310574c05c69b2 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Thu, 17 Apr 2025 14:47:43 +0300 Subject: [PATCH 16/30] put every field in quotes to ensure linebreaks do not mess up the data --- backend/src/routes/crossSearch.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index d04f1f71..a5844452 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -78,16 +78,14 @@ router.get(`/export/:columnfilters/:sorting`, async (req, res) => { logger.info('Cross search export sent.') }) - const stream = format() + // quoteColumns is needed to make sure linebreaks do not mess up the data + const stream = format({ headers: true, quoteColumns: true }) pipeline(stream, res, err => { if (err) { logger.error(`Error in crosssearch/export pipeline: ${err.message}`) } }) - const headers = Object.keys(data[0]) - stream.write(headers) - for (const row of data) { stream.write(row) } From 55d377f46d76f33f46bb621183163977728c4a82 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Thu, 17 Apr 2025 15:44:04 +0300 Subject: [PATCH 17/30] changed validators and updated button look --- backend/src/routes/crossSearch.ts | 2 ++ .../CrossSearch/CrossSearchExportButton.tsx | 14 ++++++++------ frontend/src/shared/types/misc.ts | 4 ++-- frontend/src/shared/validators/crossSearch.ts | 14 ++++++++++++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index a5844452..3b01fd9a 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -53,6 +53,8 @@ router.get(`/export/:columnfilters/:sorting`, async (req, res) => { return res.status(403).send({ error: 'Parsing URL parameters failed' }) } const validationErrors = validateCrossSearchRouteParameters({ + limit: undefined, + offset: undefined, columnFilters, sorting, }) diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index 09a9a78f..156e3165 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -11,15 +11,16 @@ export const CrossSearchExportButton = () => { const notify = useNotify() const token = useUser().token - const headers = token ? { Authorization: `Bearer ${token}` } : {} + const fetchOptions = token ? { headers: { Authorization: `Bearer ${token}` } } : {} const fetchCSVFile = () => { setLoading(true) - notify('Download started, please wait...') + notify('Downloading CSV file, please wait...', 'info', null) - fetch(`${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, { - headers: headers, - }) + fetch( + `${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, + fetchOptions + ) .then(response => { if (!response.ok) { throw new Error('Server response was not OK.') @@ -35,6 +36,7 @@ export const CrossSearchExportButton = () => { a.click() a.remove() window.URL.revokeObjectURL(url) + notify('Download finished.') }) .catch(_ => { notify('Downloading cross-search export failed.', 'error') @@ -44,7 +46,7 @@ export const CrossSearchExportButton = () => { return ( ) diff --git a/frontend/src/shared/types/misc.ts b/frontend/src/shared/types/misc.ts index 6bdd4e67..d13b0ac0 100755 --- a/frontend/src/shared/types/misc.ts +++ b/frontend/src/shared/types/misc.ts @@ -28,8 +28,8 @@ type ValidationErrorItem = { } export type CrossSearchRouteParameters = { - limit?: number - offset?: number + limit: number | undefined + offset: number | undefined columnFilters: unknown sorting: unknown } diff --git a/frontend/src/shared/validators/crossSearch.ts b/frontend/src/shared/validators/crossSearch.ts index 344d58e9..17d0d4d2 100755 --- a/frontend/src/shared/validators/crossSearch.ts +++ b/frontend/src/shared/validators/crossSearch.ts @@ -37,11 +37,21 @@ export const validateCrossSearchRouteParams = ( const validators: Validators> = { limit: { name: 'Limit', - required: true, + required: () => { + // TODO: change this and offset to not use the parameters object, might not be possible without changing the way validators work + if (typeof parameters.limit !== 'undefined' && typeof parameters.limit !== 'number') + return 'Limit must be undefined or a number' + + return + }, }, offset: { name: 'Offset', - required: true, + required: () => { + if (typeof parameters.offset !== 'undefined' && typeof parameters.offset !== 'number') + return 'Offset must be undefined or a number' + return + }, }, columnFilters: { name: 'Column Filters', From f0da34396a95ccc10ae385dc1b0ec63509d8ab56 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Fri, 18 Apr 2025 15:35:24 +0300 Subject: [PATCH 18/30] updated validation functions --- backend/src/routes/crossSearch.ts | 67 ++++++++----------- backend/src/services/crossSearch.ts | 54 +++++++++++++-- .../src/services/queries/crossSearchQuery.ts | 4 +- frontend/src/shared/types/misc.ts | 21 +++++- frontend/src/shared/validators/crossSearch.ts | 10 +-- 5 files changed, 102 insertions(+), 54 deletions(-) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index 3b01fd9a..00fe33f8 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -1,7 +1,6 @@ import { Router } from 'express' -import { getCrossSearchRawSql, validateCrossSearchRouteParameters } from '../services/crossSearch' +import { getCrossSearchRawSql, parseAndValidateCrossSearchRouteParameters } from '../services/crossSearch' import { fixBigInt } from '../utils/common' -import { ColumnFilter, SortingState } from '../services/queries/crossSearchQuery' import { format } from '@fast-csv/format' import { pipeline } from 'stream' import { logger } from '../utils/logger' @@ -9,31 +8,28 @@ import { logger } from '../utils/logger' const router = Router() router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { - let limit - let offset - let columnFilters - let sorting + let validatedValues try { - limit = parseInt(req.params.limit) - if (isNaN(limit)) throw new Error('Limit is not a number') - offset = parseInt(req.params.offset) - if (isNaN(offset)) throw new Error('Offset is not a number') - columnFilters = JSON.parse(req.params.columnfilters) as unknown - sorting = JSON.parse(req.params.sorting) as unknown + const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({ + limit: req.params.limit, + offset: req.params.offset, + columnFilters: req.params.columnfilters, + sorting: req.params.sorting, + }) + validatedValues = values + if (validationErrors.length > 0) { + return res.status(403).send(validationErrors) + } } catch (error) { - return res.status(403).send({ error: 'Parsing URL parameters failed' }) - } - const validationErrors = validateCrossSearchRouteParameters({ limit, offset, columnFilters, sorting }) - if (validationErrors.length > 0) { - return res.status(403).send(validationErrors) + return res.status(403).send({ error: error }) } try { const result = await getCrossSearchRawSql( req.user, - limit, - offset, - columnFilters as ColumnFilter[], - sorting as SortingState[] + validatedValues.validatedLimit, + validatedValues.validatedOffset, + validatedValues.validatedColumnFilters, + validatedValues.validatedSorting ) return res.status(200).send(fixBigInt(result)) } catch (error) { @@ -43,32 +39,27 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { }) router.get(`/export/:columnfilters/:sorting`, async (req, res) => { - let columnFilters - let sorting + let validatedValues let data try { - columnFilters = JSON.parse(req.params.columnfilters) as unknown - sorting = JSON.parse(req.params.sorting) as unknown + const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({ + columnFilters: req.params.columnfilters, + sorting: req.params.sorting, + }) + validatedValues = values + if (validationErrors.length > 0) { + return res.status(403).send(validationErrors) + } } catch (error) { - return res.status(403).send({ error: 'Parsing URL parameters failed' }) + return res.status(403).send({ error: error }) } - const validationErrors = validateCrossSearchRouteParameters({ - limit: undefined, - offset: undefined, - columnFilters, - sorting, - }) - if (validationErrors.length > 0) { - return res.status(403).send(validationErrors) - } - try { data = await getCrossSearchRawSql( req.user, undefined, undefined, - columnFilters as ColumnFilter[], - sorting as SortingState[] + validatedValues.validatedColumnFilters, + validatedValues.validatedSorting ) } catch (error) { if (error && typeof error === 'object' && 'message' in error) return res.status(403).send({ error: error.message }) diff --git a/backend/src/services/crossSearch.ts b/backend/src/services/crossSearch.ts index 5d792ef9..de8ada42 100644 --- a/backend/src/services/crossSearch.ts +++ b/backend/src/services/crossSearch.ts @@ -1,6 +1,14 @@ -import { User, CrossSearch, Role, CrossSearchRouteParameters } from '../../../frontend/src/shared/types' +import { + User, + CrossSearch, + Role, + CrossSearchRouteParameters, + ParsedCrossSearchRouteParameters, + SortingState, + ColumnFilter, +} from '../../../frontend/src/shared/types' import { getCrossSearchFields, getFieldsOfTables, nowDb } from '../utils/db' -import { ColumnFilter, SortingState, generateCrossSearchSql } from './queries/crossSearchQuery' +import { generateCrossSearchSql } from './queries/crossSearchQuery' import { ValidationObject } from '../../../frontend/src/shared/validators/validator' import { validateCrossSearchRouteParams } from '../../../frontend/src/shared/validators/crossSearch' @@ -118,13 +126,47 @@ export const getCrossSearchRawSql = async ( return result } -export const validateCrossSearchRouteParameters = (parameters: CrossSearchRouteParameters) => { - const keys = Object.keys(parameters) +export const parseAndValidateCrossSearchRouteParameters = (parameters: CrossSearchRouteParameters) => { + const { limit, offset, columnFilters, sorting } = parameters + let parsedLimit + let parsedOffset + let parsedColumnFilters + let parsedSorting + try { + if (limit) { + parsedLimit = parseInt(limit) + if (isNaN(parsedLimit)) throw new Error('Limit is not a number') + } + if (offset) { + parsedOffset = parseInt(offset) + if (isNaN(parsedOffset)) throw new Error('Offset is not a number') + } + parsedColumnFilters = JSON.parse(columnFilters) as unknown + parsedSorting = JSON.parse(sorting) as unknown + } catch (error) { + throw new Error('Parsing URL parameters failed') + } + const parsedParameters = { + limit: parsedLimit, + offset: parsedOffset, + columnFilters: parsedColumnFilters, + sorting: parsedSorting, + } + const keys = Object.keys(parsedParameters) const errors: ValidationObject[] = [] for (const key of keys) { - const error = validateCrossSearchRouteParams(parameters, key as keyof CrossSearchRouteParameters) + const error = validateCrossSearchRouteParams(parsedParameters, key as keyof ParsedCrossSearchRouteParameters) if (error.error) errors.push(error) } - return errors + const validatedColumnFilters = parsedColumnFilters as ColumnFilter[] + const validatedSorting = parsedSorting as SortingState[] + + return { + validatedLimit: parsedLimit, + validatedOffset: parsedOffset, + validatedColumnFilters, + validatedSorting, + validationErrors: errors, + } } diff --git a/backend/src/services/queries/crossSearchQuery.ts b/backend/src/services/queries/crossSearchQuery.ts index 1ceb76af..4436d619 100644 --- a/backend/src/services/queries/crossSearchQuery.ts +++ b/backend/src/services/queries/crossSearchQuery.ts @@ -1,7 +1,5 @@ import { Prisma } from '../../../prisma/generated/now_test_client' - -export type ColumnFilter = { id: string; value: string } -export type SortingState = { desc: boolean; id: string } +import { ColumnFilter } from '../../../../frontend/src/shared/types' const generateWhereClause = (showAll: boolean, allowedLocalities: Array, columnFilters: ColumnFilter[]) => { let userCheck: Prisma.Sql diff --git a/frontend/src/shared/types/misc.ts b/frontend/src/shared/types/misc.ts index d13b0ac0..530ad624 100755 --- a/frontend/src/shared/types/misc.ts +++ b/frontend/src/shared/types/misc.ts @@ -27,9 +27,26 @@ type ValidationErrorItem = { error: string } +export type ColumnFilter = { id: string; value: string } +export type SortingState = { desc: boolean; id: string } + export type CrossSearchRouteParameters = { - limit: number | undefined - offset: number | undefined + limit?: string + offset?: string + columnFilters: string + sorting: string +} + +export type ParsedCrossSearchRouteParameters = { + limit?: number + offset?: number columnFilters: unknown sorting: unknown } + +export type ValidatedCrossSearchRouteParameters = { + limit?: number + offset?: number + columnFilters: ColumnFilter[] + sorting: SortingState[] +} diff --git a/frontend/src/shared/validators/crossSearch.ts b/frontend/src/shared/validators/crossSearch.ts index 17d0d4d2..1e0ab7c2 100755 --- a/frontend/src/shared/validators/crossSearch.ts +++ b/frontend/src/shared/validators/crossSearch.ts @@ -1,4 +1,4 @@ -import { CrossSearchRouteParameters } from '../types' +import { ParsedCrossSearchRouteParameters } from '../types' import { Validators, validator, ValidationError } from './validator' const columnFilterCheck = (columnFilters: unknown) => { @@ -31,10 +31,10 @@ const sortingCheck = (sorting: unknown) => { } export const validateCrossSearchRouteParams = ( - parameters: CrossSearchRouteParameters, - fieldName: keyof CrossSearchRouteParameters + parameters: ParsedCrossSearchRouteParameters, + fieldName: keyof ParsedCrossSearchRouteParameters ) => { - const validators: Validators> = { + const validators: Validators> = { limit: { name: 'Limit', required: () => { @@ -63,5 +63,5 @@ export const validateCrossSearchRouteParams = ( }, } - return validator(validators, parameters, fieldName) + return validator(validators, parameters, fieldName) } From 2aeda642cf1c452292a59f1b753a92e4c2e7d2df Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 21 Apr 2025 15:36:23 +0300 Subject: [PATCH 19/30] made tests for cross search export and improved error handling --- .../src/api-tests/crossSearch/export.test.ts | 67 +++++++++++++++++++ backend/src/routes/crossSearch.ts | 14 ++-- backend/src/services/crossSearch.ts | 28 ++++---- frontend/src/shared/types/misc.ts | 4 +- 4 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 backend/src/api-tests/crossSearch/export.test.ts diff --git a/backend/src/api-tests/crossSearch/export.test.ts b/backend/src/api-tests/crossSearch/export.test.ts new file mode 100644 index 00000000..c5a60d4c --- /dev/null +++ b/backend/src/api-tests/crossSearch/export.test.ts @@ -0,0 +1,67 @@ +import { beforeAll, afterAll, describe, it, expect } from '@jest/globals' +import { resetDatabase, send, resetDatabaseTimeout } from '../utils' +import { pool } from '../../utils/db' + +describe('Getting cross-search export', () => { + beforeAll(async () => { + await resetDatabase() + }, resetDatabaseTimeout) + afterAll(async () => { + await pool.end() + }) + + it('with invalid column filters type does not work', async () => { + const response = await send(`crosssearch/export/"not an array"/[]`, 'GET') + expect(response.status).toEqual(403) + expect(response.body).toEqual({ error: 'ColumnFilters is not an array.' }) + }) + + it('with invalid column filters content does not work', async () => { + const { body: responseBody1, status: responseStatus1 } = await send( + `crosssearch/export/[{"id": "not_a_column", "value": "Dmanisi"}]/[]`, + 'GET' + ) + expect(responseStatus1).toEqual(403) + expect(responseBody1).toEqual({ error: 'columnFilters has an invalid column id.' }) + + const { body: responseBody2, status: responseStatus2 } = await send( + `crosssearch/export/[{"value": "Dmanisi"}]/[]`, + 'GET' + ) + expect(responseStatus2).toEqual(403) + expect(responseBody2).toEqual([{ error: 'Invalid or missing id field in filter', name: 'Column Filters' }]) + + const { body: responseBody3, status: responseStatus3 } = await send( + `crosssearch/export/[{"id": "lid_now_loc' OR 1=1 --", "value": "Dmanisi"}]/[]`, + 'GET' + ) + expect(responseStatus3).toEqual(403) + expect(responseBody3).toEqual({ error: 'columnFilters has an invalid column id.' }) + }) + + it('with invalid sorting type does not work', async () => { + const response = await send(`crosssearch/export/[]/20`, 'GET') + expect(response.status).toEqual(403) + expect(response.body).toEqual({ error: 'Sorting is not an array.' }) + }) + + it('with invalid sorting content does not work', async () => { + const { body: responseBody1, status: responseStatus1 } = await send( + `crosssearch/export/[]/[{"id": "not_a_column", "desc": true}]`, + 'GET' + ) + expect(responseStatus1).toEqual(403) + expect(responseBody1).toEqual({ error: 'orderBy was not a valid column id.' }) + + const { body: responseBody2, status: responseStatus2 } = await send(`crosssearch/export/[]/[{"desc": true}]`, 'GET') + expect(responseStatus2).toEqual(403) + expect(responseBody2).toEqual([{ error: 'Invalid or missing id field in sort object', name: 'Sorting' }]) + + const { body: responseBody3, status: responseStatus3 } = await send( + `crosssearch/export/[]/[{"id": "lid_now_loc' OR 1=1 --", "desc": true}]`, + 'GET' + ) + expect(responseStatus3).toEqual(403) + expect(responseBody3).toEqual({ error: 'orderBy was not a valid column id.' }) + }) +}) diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index 00fe33f8..b10be798 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -21,8 +21,10 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { return res.status(403).send(validationErrors) } } catch (error) { - return res.status(403).send({ error: error }) + if (error instanceof Error) return res.status(403).send({ error: error.message }) + return res.status(403).send('Unknown error') } + try { const result = await getCrossSearchRawSql( req.user, @@ -33,14 +35,13 @@ router.get(`/all/:limit/:offset/:columnfilters/:sorting`, async (req, res) => { ) return res.status(200).send(fixBigInt(result)) } catch (error) { - if (error && typeof error === 'object' && 'message' in error) return res.status(403).send({ error: error.message }) + if (error instanceof Error) return res.status(403).send({ error: error.message }) return res.status(403).send('Unknown error') } }) router.get(`/export/:columnfilters/:sorting`, async (req, res) => { let validatedValues - let data try { const { validationErrors, ...values } = parseAndValidateCrossSearchRouteParameters({ columnFilters: req.params.columnfilters, @@ -51,8 +52,11 @@ router.get(`/export/:columnfilters/:sorting`, async (req, res) => { return res.status(403).send(validationErrors) } } catch (error) { - return res.status(403).send({ error: error }) + if (error instanceof Error) return res.status(403).send({ error: error.message }) + return res.status(403).send('Unknown error') } + + let data try { data = await getCrossSearchRawSql( req.user, @@ -62,7 +66,7 @@ router.get(`/export/:columnfilters/:sorting`, async (req, res) => { validatedValues.validatedSorting ) } catch (error) { - if (error && typeof error === 'object' && 'message' in error) return res.status(403).send({ error: error.message }) + if (error instanceof Error) return res.status(403).send({ error: error.message }) return res.status(403).send('Unknown error') } diff --git a/backend/src/services/crossSearch.ts b/backend/src/services/crossSearch.ts index de8ada42..2a3ad948 100644 --- a/backend/src/services/crossSearch.ts +++ b/backend/src/services/crossSearch.ts @@ -130,22 +130,19 @@ export const parseAndValidateCrossSearchRouteParameters = (parameters: CrossSear const { limit, offset, columnFilters, sorting } = parameters let parsedLimit let parsedOffset - let parsedColumnFilters - let parsedSorting - try { - if (limit) { - parsedLimit = parseInt(limit) - if (isNaN(parsedLimit)) throw new Error('Limit is not a number') - } - if (offset) { - parsedOffset = parseInt(offset) - if (isNaN(parsedOffset)) throw new Error('Offset is not a number') - } - parsedColumnFilters = JSON.parse(columnFilters) as unknown - parsedSorting = JSON.parse(sorting) as unknown - } catch (error) { - throw new Error('Parsing URL parameters failed') + if (limit) { + parsedLimit = parseInt(limit) + if (isNaN(parsedLimit)) throw new Error('Limit is not a number.') } + if (offset) { + parsedOffset = parseInt(offset) + if (isNaN(parsedOffset)) throw new Error('Offset is not a number.') + } + const parsedColumnFilters = JSON.parse(columnFilters) as unknown + if (!Array.isArray(parsedColumnFilters)) throw new Error('ColumnFilters is not an array.') + const parsedSorting = JSON.parse(sorting) as unknown + if (!Array.isArray(parsedSorting)) throw new Error('Sorting is not an array.') + const parsedParameters = { limit: parsedLimit, offset: parsedOffset, @@ -154,7 +151,6 @@ export const parseAndValidateCrossSearchRouteParameters = (parameters: CrossSear } const keys = Object.keys(parsedParameters) const errors: ValidationObject[] = [] - for (const key of keys) { const error = validateCrossSearchRouteParams(parsedParameters, key as keyof ParsedCrossSearchRouteParameters) if (error.error) errors.push(error) diff --git a/frontend/src/shared/types/misc.ts b/frontend/src/shared/types/misc.ts index 530ad624..c3e5904f 100755 --- a/frontend/src/shared/types/misc.ts +++ b/frontend/src/shared/types/misc.ts @@ -40,8 +40,8 @@ export type CrossSearchRouteParameters = { export type ParsedCrossSearchRouteParameters = { limit?: number offset?: number - columnFilters: unknown - sorting: unknown + columnFilters: unknown[] + sorting: unknown[] } export type ValidatedCrossSearchRouteParameters = { From d7ec215cd32132712ebbe0efe109e3553b8ef1e3 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 21 Apr 2025 16:02:30 +0300 Subject: [PATCH 20/30] updated docs --- documentation/backend/cross_search_export.md | 2 +- documentation/backend/server_side_pagination.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/backend/cross_search_export.md b/documentation/backend/cross_search_export.md index a65fcb0a..d5d5d6be 100644 --- a/documentation/backend/cross_search_export.md +++ b/documentation/backend/cross_search_export.md @@ -1,3 +1,3 @@ ### Cross Search Export -Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same SQL query as the normal cross-search route but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file +Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same parameter parsing and validation function as the `/all` route, and it also uses the same SQL query, but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file diff --git a/documentation/backend/server_side_pagination.md b/documentation/backend/server_side_pagination.md index 9a15a1e6..9d3c10a3 100644 --- a/documentation/backend/server_side_pagination.md +++ b/documentation/backend/server_side_pagination.md @@ -6,7 +6,7 @@ The server side pagination for cross search is implemented in the following way: 1. The backend route is given values for limit, offset, column filters, and sorting. - `limit` and `offset` should be numbers, and both `columnFilters` and `sorting` should be an array of objects containing the keys `id` and `value`. `sorting` is currently an array because Material React Table supports multiple sorting values. The backend only uses the first value in the array for the SQL query. -2. The values are validated. This is done in two places: the [validation file](../../frontend/src/shared/validators/crossSearch.ts) and the [getCrossSearchRawSql function](../../backend/src/services/crossSearch.ts). +2. The values are validated. This is done in two places: the [parsing and validation function](../../backend/src/services/crossSearch.ts), and the [getCrossSearchRawSql function](../../backend/src/services/crossSearch.ts). - The checks inside getCrossSearchRawSql are very important since they check if the values given to the `WHERE` and `ORDER BY` clauses are valid column ID's. Without these the query is vulnerable to SQL injection (see below) 3. Since the SQL query uses aliases, some values used in the query need to be converted first. This is done with the [convertIdToFieldName function](../../backend/src/services/crossSearch.ts). 4. The values are given to [generateCrossSearchSql function](backend/src/services/queries/crossSearchQuery.ts), which generates the SQL query. The values are used in the `LIMIT`, `OFFSET`, `WHERE` and `ORDER BY` clauses, respectively. To allow TableView to show and change pages correctly, we also add a `full_count` column to every row which contains the amount of rows in the entire cross-search table. From 7f855dc8e6b8ec873b62549a405ac46d6e200765 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 21 Apr 2025 16:38:12 +0300 Subject: [PATCH 21/30] changed filename of downloaded CSV file --- backend/src/routes/crossSearch.ts | 3 ++- .../CrossSearch/CrossSearchExportButton.tsx | 19 +++++++++++-------- frontend/src/shared/currentDateAsString.ts | 11 +++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 frontend/src/shared/currentDateAsString.ts diff --git a/backend/src/routes/crossSearch.ts b/backend/src/routes/crossSearch.ts index b10be798..381af6c7 100644 --- a/backend/src/routes/crossSearch.ts +++ b/backend/src/routes/crossSearch.ts @@ -4,6 +4,7 @@ import { fixBigInt } from '../utils/common' import { format } from '@fast-csv/format' import { pipeline } from 'stream' import { logger } from '../utils/logger' +import { currentDateAsString } from '../../../frontend/src/shared/currentDateAsString' const router = Router() @@ -70,7 +71,7 @@ router.get(`/export/:columnfilters/:sorting`, async (req, res) => { return res.status(403).send('Unknown error') } - res.attachment('cross_search_export.csv') + res.attachment(`cross_search${currentDateAsString()}.csv`) // filename will get overwritten in frontend when fetching data from this route res.on('finish', () => { logger.info('Cross search export sent.') }) diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index 156e3165..e5d7ff22 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -4,6 +4,7 @@ import { BACKEND_URL } from '@/util/config' import { Box, Button, CircularProgress } from '@mui/material' import { useUser } from '@/hooks/user' import { usePageContext } from '../Page' +import { currentDateAsString } from '@/shared/currentDateAsString' export const CrossSearchExportButton = () => { const [loading, setLoading] = useState(false) @@ -13,6 +14,8 @@ export const CrossSearchExportButton = () => { const token = useUser().token const fetchOptions = token ? { headers: { Authorization: `Bearer ${token}` } } : {} + const filename = `cross_search${currentDateAsString()}.csv` + const fetchCSVFile = () => { setLoading(true) notify('Downloading CSV file, please wait...', 'info', null) @@ -28,14 +31,14 @@ export const CrossSearchExportButton = () => { return response.blob() }) .then(blob => { - const url = window.URL.createObjectURL(blob) - const a = document.createElement('a') - a.href = url - a.download = 'cross_search_export.csv' - document.body.appendChild(a) - a.click() - a.remove() - window.URL.revokeObjectURL(url) + const blobUrl = window.URL.createObjectURL(blob) + const downloadLink = document.createElement('a') + downloadLink.href = blobUrl + downloadLink.download = filename + document.body.appendChild(downloadLink) + downloadLink.click() + downloadLink.remove() + window.URL.revokeObjectURL(blobUrl) notify('Download finished.') }) .catch(_ => { diff --git a/frontend/src/shared/currentDateAsString.ts b/frontend/src/shared/currentDateAsString.ts new file mode 100644 index 00000000..4f69a8fd --- /dev/null +++ b/frontend/src/shared/currentDateAsString.ts @@ -0,0 +1,11 @@ +export const currentDateAsString = () => { + // returns current datetime in YYYYMMDDHHSS format, currently used for filenames + const today = new Date() + const year = today.getFullYear() + const month = String(today.getMonth() + 1).padStart(2, '0') + const day = String(today.getDate()).padStart(2, '0') + const hours = String(today.getHours()).padStart(2, '0') + const minutes = String(today.getMinutes()).padStart(2, '0') + + return `${year}-${month}-${day}-${hours}-${minutes}` +} From d9b26b0fe274de80222a899e4bf5305c16c63fd1 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 14 Apr 2025 15:31:33 +0300 Subject: [PATCH 22/30] new button for exporting cross search table --- .../src/components/TableView/TableView.tsx | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index bd16626a..da9c38ea 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -29,6 +29,30 @@ type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' selectorFn should only be defined if using this as a selecting table */ +const CrossSearchExportButton = () => { + // loading is set to false after a timeout because actually knowing + // once the file is downloaded is impossible when implemented this way + const [loading, setLoading] = useState(false) + + return ( + + + {loading && } + + ) +} export const TableView = ({ data, columns, @@ -44,6 +68,7 @@ export const TableView = ({ enableColumnFilterModes, serverSidePagination, isCrossSearchTable, + isCrossSearchTable, isFetching, }: { title?: string @@ -60,6 +85,7 @@ export const TableView = ({ enableColumnFilterModes?: boolean serverSidePagination?: boolean isCrossSearchTable?: boolean + isCrossSearchTable?: boolean isFetching: boolean }) => { const location = useLocation() From f63118aa0d6433bf4ed5ccf9cc87b89869ee80aa Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 13:42:08 +0300 Subject: [PATCH 23/30] improved export button --- .../src/components/TableView/TableView.tsx | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index da9c38ea..c5ef1932 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -29,30 +29,6 @@ type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' selectorFn should only be defined if using this as a selecting table */ -const CrossSearchExportButton = () => { - // loading is set to false after a timeout because actually knowing - // once the file is downloaded is impossible when implemented this way - const [loading, setLoading] = useState(false) - - return ( - - - {loading && } - - ) -} export const TableView = ({ data, columns, From eacc6e07e3418b8d8eb86c00e2a2588fc981179c Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 15 Apr 2025 14:07:06 +0300 Subject: [PATCH 24/30] added and updated documentation --- documentation/backend/cross_search_export.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/backend/cross_search_export.md b/documentation/backend/cross_search_export.md index d5d5d6be..a65fcb0a 100644 --- a/documentation/backend/cross_search_export.md +++ b/documentation/backend/cross_search_export.md @@ -1,3 +1,3 @@ ### Cross Search Export -Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same parameter parsing and validation function as the `/all` route, and it also uses the same SQL query, but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file +Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same SQL query as the normal cross-search route but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file From 2b4e86b3aa5a1b4cad4824d9bc44d8af2bc4dab0 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Wed, 16 Apr 2025 16:54:59 +0300 Subject: [PATCH 25/30] added column filtering and sorting to export, and fixed bug with crosssearch/all route --- .../components/CrossSearch/CrossSearchExportButton.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index e5d7ff22..bf6752fa 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -4,7 +4,10 @@ import { BACKEND_URL } from '@/util/config' import { Box, Button, CircularProgress } from '@mui/material' import { useUser } from '@/hooks/user' import { usePageContext } from '../Page' +<<<<<<< HEAD import { currentDateAsString } from '@/shared/currentDateAsString' +======= +>>>>>>> 2dfd88b (added column filtering and sorting to export, and fixed bug with crosssearch/all route) export const CrossSearchExportButton = () => { const [loading, setLoading] = useState(false) @@ -20,10 +23,16 @@ export const CrossSearchExportButton = () => { setLoading(true) notify('Downloading CSV file, please wait...', 'info', null) +<<<<<<< HEAD fetch( `${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, fetchOptions ) +======= + fetch(`${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, { + headers: headers, + }) +>>>>>>> 2dfd88b (added column filtering and sorting to export, and fixed bug with crosssearch/all route) .then(response => { if (!response.ok) { throw new Error('Server response was not OK.') From 82094e22708dbdea28ae5b51164328d51d1b4f3a Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 21 Apr 2025 16:02:30 +0300 Subject: [PATCH 26/30] updated docs --- documentation/backend/cross_search_export.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/backend/cross_search_export.md b/documentation/backend/cross_search_export.md index a65fcb0a..d5d5d6be 100644 --- a/documentation/backend/cross_search_export.md +++ b/documentation/backend/cross_search_export.md @@ -1,3 +1,3 @@ ### Cross Search Export -Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same SQL query as the normal cross-search route but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file +Cross search has a [`/export` route](../../backend/src/routes/crossSearch.ts) which is used by the ["Export Table"](../../../frontend/src/components/CrossSearch/CrossSearchExportButton.tsx) button in a cross-search table view. This route uses the same parameter parsing and validation function as the `/all` route, and it also uses the same SQL query, but with LIMIT set to undefined. The resulting data is [streamed](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) to the user, where the browser will download it as a csv file. See the "Exporting" section in [TableView docs](../frontend/components/TableView.md) for more information. \ No newline at end of file From dc78cff078cb73371f57cf7503bc9fc3f5f94752 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 27 May 2025 16:39:23 +0300 Subject: [PATCH 27/30] fixed lint issues and removed code that is no longer needed --- .../CrossSearch/CrossSearchExportButton.tsx | 9 ---- .../CrossSearch/CrossSearchTable.tsx | 1 - .../src/components/TableView/TableView.tsx | 14 ----- frontend/src/components/TableView/helpers.tsx | 51 +------------------ 4 files changed, 1 insertion(+), 74 deletions(-) diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index bf6752fa..e5d7ff22 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -4,10 +4,7 @@ import { BACKEND_URL } from '@/util/config' import { Box, Button, CircularProgress } from '@mui/material' import { useUser } from '@/hooks/user' import { usePageContext } from '../Page' -<<<<<<< HEAD import { currentDateAsString } from '@/shared/currentDateAsString' -======= ->>>>>>> 2dfd88b (added column filtering and sorting to export, and fixed bug with crosssearch/all route) export const CrossSearchExportButton = () => { const [loading, setLoading] = useState(false) @@ -23,16 +20,10 @@ export const CrossSearchExportButton = () => { setLoading(true) notify('Downloading CSV file, please wait...', 'info', null) -<<<<<<< HEAD fetch( `${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, fetchOptions ) -======= - fetch(`${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, { - headers: headers, - }) ->>>>>>> 2dfd88b (added column filtering and sorting to export, and fixed bug with crosssearch/all route) .then(response => { if (!response.ok) { throw new Error('Server response was not OK.') diff --git a/frontend/src/components/CrossSearch/CrossSearchTable.tsx b/frontend/src/components/CrossSearch/CrossSearchTable.tsx index 28baa44b..b8195a79 100755 --- a/frontend/src/components/CrossSearch/CrossSearchTable.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchTable.tsx @@ -812,7 +812,6 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros url="crosssearch" enableColumnFilterModes={true} serverSidePagination={true} - isCrossSearchTable={true} /> ) } diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index c5ef1932..a7aa85ec 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -18,7 +18,6 @@ import { useUser } from '@/hooks/user' import { defaultPagination, defaultPaginationSmall } from '@/common' import '../../styles/TableView.css' import { TableToolBar } from './TableToolBar' -import { BACKEND_URL } from '@/util/config' type TableStateInUrl = 'sorting' | 'columnfilters' | 'pagination' @@ -43,8 +42,6 @@ export const TableView = ({ exportIsLoading, enableColumnFilterModes, serverSidePagination, - isCrossSearchTable, - isCrossSearchTable, isFetching, }: { title?: string @@ -60,8 +57,6 @@ export const TableView = ({ exportIsLoading?: boolean enableColumnFilterModes?: boolean serverSidePagination?: boolean - isCrossSearchTable?: boolean - isCrossSearchTable?: boolean isFetching: boolean }) => { const location = useLocation() @@ -113,15 +108,6 @@ export const TableView = ({ else rowCount = data.length } - let toolbar - if (isCrossSearchTable) { - toolbar = renderCustomToolbarCrossSearchVersion - } else if (selectorFn) { - toolbar = renderCustomToolbarModalVersion - } else { - toolbar = renderCustomToolbar - } - const table = useMaterialReactTable({ columns: columns, data: data || [], diff --git a/frontend/src/components/TableView/helpers.tsx b/frontend/src/components/TableView/helpers.tsx index 7b359006..ce305ccf 100755 --- a/frontend/src/components/TableView/helpers.tsx +++ b/frontend/src/components/TableView/helpers.tsx @@ -1,13 +1,6 @@ -import { Box } from '@mui/material' -import { - MRT_RowData, - MRT_ShowHideColumnsButton, - MRT_TableInstance, - MRT_ToggleFullScreenButton, -} from 'material-react-table' +import { MRT_RowData, MRT_TableInstance } from 'material-react-table' import { mkConfig, generateCsv, download } from 'export-to-csv' import { AcceptedData } from 'node_modules/export-to-csv/output/lib/types' -import { CrossSearchExportButton } from '../CrossSearch/CrossSearchExportButton' export type ExportFn = (data: T) => { [k: string]: AcceptedData } @@ -35,45 +28,3 @@ export const exportRows = (table: MRT_TableInstance) = const csv = generateCsv(csvConfig)(rowData) download(csvConfig)(csv) } - -export const renderCustomToolbar = ({ table }: { table: MRT_TableInstance }) => ( - - - - -) - -export const renderCustomToolbarModalVersion = ({ table }: { table: MRT_TableInstance }) => ( - - - -) -export const renderCustomToolbarCrossSearchVersion = ({ - table, -}: { - table: MRT_TableInstance -}) => ( - <> - - - - - - - - -) - -export const renderCustomToolbar = ({ table }: { table: MRT_TableInstance }) => ( - <> - - - - - - - - -) From de5c6cc88b0bb4d27171b5b70dc7f6eec8787bb4 Mon Sep 17 00:00:00 2001 From: juhanikat Date: Tue, 27 May 2025 17:00:20 +0300 Subject: [PATCH 28/30] added cross search export back to table view --- frontend/package.json | 3 +-- .../CrossSearch/CrossSearchExportButton.tsx | 18 ++++++++------ .../CrossSearch/CrossSearchTable.tsx | 1 + .../src/components/TableView/TableToolBar.tsx | 24 ++++++++++++------- .../src/components/TableView/TableView.tsx | 3 +++ 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 58b74bed..b2fdf882 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,7 +18,6 @@ "@reduxjs/toolkit": "^2.2.3", "export-to-csv": "^1.3.0", "leaflet": "^1.9.4", - "react-leaflet": "^4.2.1", "lodash-es": "^4.17.21", "material-react-table": "^3.0.1", "react": "^18.2.0", @@ -61,4 +60,4 @@ "vite": "^5.2.5", "vite-plugin-istanbul": "^6.0.2" } -} +} \ No newline at end of file diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx index e5d7ff22..173799e2 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx @@ -1,12 +1,12 @@ import { useState } from 'react' import { useNotify } from '@/hooks/notification' import { BACKEND_URL } from '@/util/config' -import { Box, Button, CircularProgress } from '@mui/material' +import { MenuItem } from '@mui/material' import { useUser } from '@/hooks/user' import { usePageContext } from '../Page' import { currentDateAsString } from '@/shared/currentDateAsString' -export const CrossSearchExportButton = () => { +export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => void }) => { const [loading, setLoading] = useState(false) const { sqlColumnFilters, sqlOrderBy } = usePageContext() const notify = useNotify() @@ -47,10 +47,14 @@ export const CrossSearchExportButton = () => { .finally(() => setLoading(false)) } return ( - - - + { + fetchCSVFile() + handleClose() + }} + disabled={loading} + > + Export table + ) } diff --git a/frontend/src/components/CrossSearch/CrossSearchTable.tsx b/frontend/src/components/CrossSearch/CrossSearchTable.tsx index b8195a79..28baa44b 100755 --- a/frontend/src/components/CrossSearch/CrossSearchTable.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchTable.tsx @@ -812,6 +812,7 @@ export const CrossSearchTable = ({ selectorFn }: { selectorFn?: (newObject: Cros url="crosssearch" enableColumnFilterModes={true} serverSidePagination={true} + isCrossSearchTable={true} /> ) } diff --git a/frontend/src/components/TableView/TableToolBar.tsx b/frontend/src/components/TableView/TableToolBar.tsx index 5aff9614..e3884a98 100644 --- a/frontend/src/components/TableView/TableToolBar.tsx +++ b/frontend/src/components/TableView/TableToolBar.tsx @@ -7,12 +7,14 @@ import '../../styles/TableToolBar.css' import { useState } from 'react' import { Link } from 'react-router-dom' import AddCircleIcon from '@mui/icons-material/AddCircle' +import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportButton' export const TableToolBar = ({ table, combinedExport, kmlExport, exportIsLoading, + isCrossSearchTable, selectorFn, showNewButton, }: { @@ -20,6 +22,7 @@ export const TableToolBar = ({ combinedExport?: (lids: number[]) => Promise kmlExport?: (table: MRT_TableInstance) => void exportIsLoading?: boolean + isCrossSearchTable?: boolean selectorFn?: (id: T) => void showNewButton?: boolean }) => { @@ -67,14 +70,19 @@ export const TableToolBar = ({ 'aria-labelledby': 'export-button', }} > - { - exportRows(table) - handleClose() - }} - > - Export table - + {isCrossSearchTable ? ( + + ) : ( + { + exportRows(table) + handleClose() + }} + > + Export table + + )} + {kmlExport && ( { diff --git a/frontend/src/components/TableView/TableView.tsx b/frontend/src/components/TableView/TableView.tsx index a7aa85ec..604910de 100755 --- a/frontend/src/components/TableView/TableView.tsx +++ b/frontend/src/components/TableView/TableView.tsx @@ -40,6 +40,7 @@ export const TableView = ({ combinedExport, kmlExport, exportIsLoading, + isCrossSearchTable, enableColumnFilterModes, serverSidePagination, isFetching, @@ -55,6 +56,7 @@ export const TableView = ({ combinedExport?: (lids: number[]) => Promise kmlExport?: (table: MRT_TableInstance) => void exportIsLoading?: boolean + isCrossSearchTable?: boolean enableColumnFilterModes?: boolean serverSidePagination?: boolean isFetching: boolean @@ -206,6 +208,7 @@ export const TableView = ({ combinedExport={combinedExport} kmlExport={kmlExport} exportIsLoading={exportIsLoading} + isCrossSearchTable={isCrossSearchTable} showNewButton={editRights.new && title != 'Locality-Species-Cross-Search'} /> From eba769bd9030eae9d9dff78894f29f35e97ec567 Mon Sep 17 00:00:00 2001 From: arttu Date: Mon, 2 Jun 2025 10:47:31 +0300 Subject: [PATCH 29/30] fixed a bug where filtering the map reset the map view --- frontend/src/components/Map/LocalitiesMap.tsx | 90 +++++++++++-------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/frontend/src/components/Map/LocalitiesMap.tsx b/frontend/src/components/Map/LocalitiesMap.tsx index 5f48d08e..b145847f 100755 --- a/frontend/src/components/Map/LocalitiesMap.tsx +++ b/frontend/src/components/Map/LocalitiesMap.tsx @@ -25,52 +25,26 @@ const markerIcon = export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching }: Props) => { const mapRef = useRef(null) + const markersRef = useRef(null) + const [map, setMap] = useState(null) const [isOpen, setIsOpen] = useState(false) const columnFilters = usePageContext() useEffect(() => { - if (!mapRef.current || localitiesQueryIsFetching) return + if (!mapRef.current) return + + const mapInstance = L.map(mapRef.current, { maxZoom: 16 }) + setMap(mapInstance) - const map = L.map(mapRef.current, { maxZoom: 16 }) const coords: LatLngExpression = [15, 13] - - // To prevent eslint from complaining about that 'markers' variable below: - /* eslint-disable @typescript-eslint/no-unsafe-assignment */ - /* eslint-disable @typescript-eslint/no-unsafe-call */ - /* eslint-disable @typescript-eslint/no-unsafe-return */ - /* eslint-disable @typescript-eslint/no-unsafe-member-access */ - - // @ts-expect-error The marker cluster library is a plain javascript library - // with no module exports that extends 'L' when imported - const markers: Layer = L.markerClusterGroup() - - const localityIds = columnFilters.idList as unknown as number[] - const filterApplied = localityIds.length > 0 && localityIds.every(id => id === null) - - const validIds = localityIds.filter(id => typeof id === 'number') - - const filteredLocalities = !filterApplied - ? localitiesQueryData?.filter(locality => validIds.includes(locality.lid)) - : localitiesQueryData - - filteredLocalities?.forEach(locality => - markers.addLayer( - L.marker([locality.dec_lat, locality.dec_long], { - icon: new Icon({ iconUrl: markerIcon, iconSize: [25, 41], iconAnchor: [12, 41] }), - }) - ) - ) - - markers.addTo(map) - - map.setView(coords, 2) + mapInstance.setView(coords, 2) // BASE MAPS //// OpenTopoMap const topomap = L.tileLayer('https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png', { attribution: 'Map data: © OpenTopoMap (CC-BY-SA)', noWrap: true, - }).addTo(map) + }).addTo(mapInstance) //// OpenStreetMap const osm = L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { @@ -81,7 +55,7 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching } // LAYERS ON TOP OF BASE MAPS //// Add borders to the map borders.forEach(poly => { - L.polygon(poly as LatLngExpression[], { color: 'gray', weight: 1 }).addTo(map) + L.polygon(poly as LatLngExpression[], { color: 'gray', weight: 1 }).addTo(mapInstance) }) // create a polygon layer for the country borders that is in layer control panel @@ -102,14 +76,52 @@ export const LocalitiesMap = ({ localitiesQueryData, localitiesQueryIsFetching } } // Add a scale bar to the map - L.control.scale({ position: 'bottomright' }).addTo(map) + L.control.scale({ position: 'bottomright' }).addTo(mapInstance) // Add a layer control to the map - L.control.layers(baseMaps, {}, { position: 'topright' }).addTo(map) + L.control.layers(baseMaps, {}, { position: 'topright' }).addTo(mapInstance) return () => { - map.remove() + mapInstance.remove() } - }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters]) + }, []) + + useEffect(() => { + if (!map || localitiesQueryIsFetching) return + + if (markersRef.current) { + map.removeLayer(markersRef.current) + markersRef.current = null + } + // To prevent eslint from complaining about that 'markers' variable below: + /* eslint-disable @typescript-eslint/no-unsafe-assignment */ + /* eslint-disable @typescript-eslint/no-unsafe-call */ + /* eslint-disable @typescript-eslint/no-unsafe-return */ + /* eslint-disable @typescript-eslint/no-unsafe-member-access */ + + // @ts-expect-error The marker cluster library is a plain javascript library + // with no module exports that extends 'L' when imported + const newMarkers: Layer = L.markerClusterGroup() + + const localityIds = columnFilters.idList as unknown as number[] + const filterApplied = localityIds.length > 0 && localityIds.every(id => id === null) + + const validIds = localityIds.filter(id => typeof id === 'number') + + const filteredLocalities = !filterApplied + ? localitiesQueryData?.filter(locality => validIds.includes(locality.lid)) + : localitiesQueryData + + filteredLocalities?.forEach(locality => + newMarkers.addLayer( + L.marker([locality.dec_lat, locality.dec_long], { + icon: new Icon({ iconUrl: markerIcon, iconSize: [25, 41], iconAnchor: [12, 41] }), + }) + ) + ) + + newMarkers.addTo(map) + markersRef.current = newMarkers + }, [localitiesQueryData, localitiesQueryIsFetching, columnFilters, map]) document.title = 'Map' From 90f4a0197d00d2c6549f1bb522909fc8af6a2e4c Mon Sep 17 00:00:00 2001 From: juhanikat Date: Mon, 2 Jun 2025 11:55:24 +0300 Subject: [PATCH 30/30] changed CrossSearchExportButton filename and added encodeURIcomponent to fetch() --- ...earchExportButton.tsx => CrossSearchExportMenuItem.tsx} | 7 +++---- frontend/src/components/TableView/TableToolBar.tsx | 2 +- frontend/src/shared/currentDateAsString.ts | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) rename frontend/src/components/CrossSearch/{CrossSearchExportButton.tsx => CrossSearchExportMenuItem.tsx} (87%) diff --git a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx b/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx similarity index 87% rename from frontend/src/components/CrossSearch/CrossSearchExportButton.tsx rename to frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx index 173799e2..65146f3b 100644 --- a/frontend/src/components/CrossSearch/CrossSearchExportButton.tsx +++ b/frontend/src/components/CrossSearch/CrossSearchExportMenuItem.tsx @@ -19,11 +19,10 @@ export const CrossSearchExportMenuItem = ({ handleClose }: { handleClose: () => const fetchCSVFile = () => { setLoading(true) notify('Downloading CSV file, please wait...', 'info', null) + const URIColumnFilters = encodeURIComponent(JSON.stringify(sqlColumnFilters)) + const URIOrderBy = encodeURIComponent(JSON.stringify(sqlOrderBy)) - fetch( - `${BACKEND_URL}/crosssearch/export/${JSON.stringify(sqlColumnFilters)}/${JSON.stringify(sqlOrderBy)}`, - fetchOptions - ) + fetch(`${BACKEND_URL}/crosssearch/export/${URIColumnFilters}/${URIOrderBy}`, fetchOptions) .then(response => { if (!response.ok) { throw new Error('Server response was not OK.') diff --git a/frontend/src/components/TableView/TableToolBar.tsx b/frontend/src/components/TableView/TableToolBar.tsx index e3884a98..22ed6c1d 100644 --- a/frontend/src/components/TableView/TableToolBar.tsx +++ b/frontend/src/components/TableView/TableToolBar.tsx @@ -7,7 +7,7 @@ import '../../styles/TableToolBar.css' import { useState } from 'react' import { Link } from 'react-router-dom' import AddCircleIcon from '@mui/icons-material/AddCircle' -import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportButton' +import { CrossSearchExportMenuItem } from '../CrossSearch/CrossSearchExportMenuItem' export const TableToolBar = ({ table, diff --git a/frontend/src/shared/currentDateAsString.ts b/frontend/src/shared/currentDateAsString.ts index 4f69a8fd..7e825cc9 100644 --- a/frontend/src/shared/currentDateAsString.ts +++ b/frontend/src/shared/currentDateAsString.ts @@ -1,5 +1,5 @@ export const currentDateAsString = () => { - // returns current datetime in YYYYMMDDHHSS format, currently used for filenames + // returns current datetime in YYYY-MM-DD-HH-mm format, currently used for filenames const today = new Date() const year = today.getFullYear() const month = String(today.getMonth() + 1).padStart(2, '0')