diff --git a/backend/src/routes/localitySpecies.ts b/backend/src/routes/localitySpecies.ts index dd991813..8c6d3a82 100644 --- a/backend/src/routes/localitySpecies.ts +++ b/backend/src/routes/localitySpecies.ts @@ -1,5 +1,5 @@ import { Request, Router } from 'express' -import { getLocalitySpeciesList } from '../services/locality' +import { getLocalitySpeciesList } from '../services/localitySpeciesExport' const router = Router() @@ -7,8 +7,8 @@ router.post('/', async (req: Request, res) = const user = req.user const lids = req.body.lids if (!lids) return res.status(400).send({ message: 'Missing lids' }) - const localitySpeciesList = await getLocalitySpeciesList(lids, user) - return res.status(200).json(localitySpeciesList) + const exportList = await getLocalitySpeciesList(lids, user, res) + return res.status(200).json(exportList) }) export default router diff --git a/backend/src/services/locality.ts b/backend/src/services/locality.ts index 79ef1cba..6fa3d194 100644 --- a/backend/src/services/locality.ts +++ b/backend/src/services/locality.ts @@ -1,68 +1,11 @@ -import { logDb, nowDb, pool } from '../utils/db' import { EditDataType, LocalityDetailsType, User } from '../../../frontend/src/backendTypes' -import Prisma from '../../prisma/generated/now_test_client' +import { Role } from '../../../frontend/src/types' import { validateLocality } from '../../../frontend/src/validators/locality' import { ValidationObject } from '../../../frontend/src/validators/validator' -import { fixBigInt } from '../utils/common' -import { Role } from '../../../frontend/src/types' +import Prisma from '../../prisma/generated/now_test_client' import { AccessError } from '../middlewares/authorizer' -import { NOW_DB_NAME } from '../utils/config' -import { PoolConnection } from 'mariadb' - -const CHUNK_SIZE = 20000 - -const getChunk = async ( - conn: PoolConnection, - limit: number, - offset: number, - lids: number[], - includeDrafts: boolean -) => { - const excludeDraftsString = includeDrafts - ? '' - : `AND (loc_status = 0 AND lid NOT IN (SELECT DISTINCT ${NOW_DB_NAME}.now_plr.lid FROM ${NOW_DB_NAME}.now_plr JOIN ${NOW_DB_NAME}.now_proj ON ${NOW_DB_NAME}.now_plr.pid = ${NOW_DB_NAME}.now_proj.pid WHERE ${NOW_DB_NAME}.now_proj.proj_records = 1))` - const result: { [index: string]: string | number | null | bigint | boolean }[] = await conn.query( - ` - SELECT * FROM ${NOW_DB_NAME}.now_v_export_locsp WHERE lid IN (${lids.map(() => '?').join(', ')}) ${excludeDraftsString} ORDER BY loc_name LIMIT ${limit} OFFSET ${offset} - `, - [...lids] - ) - return result -} - -const formatValue = (val: unknown) => { - if (typeof val === 'bigint') return `"${Number(val)}"` - if (val === null) return `""` - return `"${val as string}"` -} - -const getExportList = async (conn: PoolConnection, lids: number[], includeDrafts: boolean) => { - const chunks = [] - - let columnHeaders: null | string[] = null - - for (let i = 0; ; i += 1) { - const chunk = await getChunk(conn, CHUNK_SIZE, i * CHUNK_SIZE, lids, includeDrafts) - if (chunk.length === 0) break - - // Get column headers - if (!columnHeaders) columnHeaders = Object.keys(chunk[0]) - - chunks.push(...chunk.map(c => Object.values(c).map(val => formatValue(val)))) - } - return [columnHeaders, ...chunks] -} - -export const getLocalitySpeciesList = async (lids: number[], user: User | undefined) => { - const conn = await pool.getConnection() - const exportList = await getExportList( - conn, - lids, - (user && [Role.Admin, Role.EditUnrestricted].includes(user.role)) || false - ) - await conn.end() - return exportList -} +import { fixBigInt } from '../utils/common' +import { logDb, nowDb } from '../utils/db' const getIdsOfUsersProjects = async (user: User) => { const usersProjects = await nowDb.now_proj_people.findMany({ diff --git a/backend/src/services/localitySpeciesExport.ts b/backend/src/services/localitySpeciesExport.ts new file mode 100644 index 00000000..615a5eb8 --- /dev/null +++ b/backend/src/services/localitySpeciesExport.ts @@ -0,0 +1,51 @@ +import { PoolConnection } from 'mariadb' +import { User } from '../../../frontend/src/backendTypes' +import { NOW_DB_NAME } from '../utils/config' +import { Role } from '../../../frontend/src/types' +import { Response } from 'express' +import { pool } from '../utils/db' + +const getExportList = async ( + conn: PoolConnection, + lids: number[], + includeDrafts: boolean, + rowHandler: (row: unknown) => void +) => { + return new Promise(resolve => { + const excludeDraftsString = includeDrafts + ? '' + : `AND (loc_status = 0 AND lid NOT IN (SELECT DISTINCT ${NOW_DB_NAME}.now_plr.lid FROM ${NOW_DB_NAME}.now_plr JOIN ${NOW_DB_NAME}.now_proj ON ${NOW_DB_NAME}.now_plr.pid = ${NOW_DB_NAME}.now_proj.pid WHERE ${NOW_DB_NAME}.now_proj.proj_records = 1))` + resolve( + conn + .queryStream( + ` + SELECT * FROM ${NOW_DB_NAME}.now_v_export_locsp WHERE lid IN (${lids.map(() => '?').join(', ')}) ${excludeDraftsString} ORDER BY loc_name + `, + [...lids] + ) + .on('data', rowHandler) + ) + }) +} + +const formatValue = (val: unknown) => { + if (typeof val === 'bigint') return `"${Number(val)}"` + if (val === null) return `""` + return `"${val as string}"` +} + +export const getLocalitySpeciesList = async (lids: number[], user: User | undefined, res: Response) => { + const exportList: unknown[] = [] + + const transformRow = (row: unknown) => { + const values = Object.values(row as object) + return values.map(value => formatValue(value)) + } + + const conn = await pool.getConnection() + await getExportList(conn, lids, (user && [Role.Admin, Role.EditUnrestricted].includes(user.role)) || false, row => { + exportList.push(transformRow(row)) + }) + await conn.end() + return exportList +}