iritmaximus/nowdatabase
1
0
Fork 0
mirror of https://github.com/nowcommunity/nowdatabase.git synced 2026-10-12 19:13:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

Stream the export from db to backend

Browse Source
This commit is contained in:
arttu
2024-09-03 18:02:10 +03:00
parent f33456a89c
commit cf564dbd10
3 changed files with 58 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/localitySpecies.ts
+3 -3
View File
@@ -1,5 +1,5 @@
import { Request, Router } from 'express'
import { getLocalitySpeciesList } from '../services/locality'
import { getLocalitySpeciesList } from '../services/localitySpeciesExport'
const router = Router()
@@ -7,8 +7,8 @@ router.post('/', async (req: Request<object, object, { lids: number[] }>, res) =
const user = req.user
const lids = req.body.lids
if (!lids) return res.status(400).send({ message: 'Missing lids' })
const localitySpeciesList = await getLocalitySpeciesList(lids, user)
return res.status(200).json(localitySpeciesList)
const exportList = await getLocalitySpeciesList(lids, user, res)
return res.status(200).json(exportList)
})
export default router
backend/src/services/locality.ts
+4 -61
View File
@@ -1,68 +1,11 @@
import { logDb, nowDb, pool } from '../utils/db'
import { EditDataType, LocalityDetailsType, User } from '../../../frontend/src/backendTypes'
import Prisma from '../../prisma/generated/now_test_client'
import { Role } from '../../../frontend/src/types'
import { validateLocality } from '../../../frontend/src/validators/locality'
import { ValidationObject } from '../../../frontend/src/validators/validator'
import { fixBigInt } from '../utils/common'
import { Role } from '../../../frontend/src/types'
import Prisma from '../../prisma/generated/now_test_client'
import { AccessError } from '../middlewares/authorizer'
import { NOW_DB_NAME } from '../utils/config'
import { PoolConnection } from 'mariadb'
const CHUNK_SIZE = 20000
const getChunk = async (
conn: PoolConnection,
limit: number,
offset: number,
lids: number[],
includeDrafts: boolean
) => {
const excludeDraftsString = includeDrafts
? ''
: `AND (loc_status = 0 AND lid NOT IN (SELECT DISTINCT ${NOW_DB_NAME}.now_plr.lid FROM ${NOW_DB_NAME}.now_plr JOIN ${NOW_DB_NAME}.now_proj ON ${NOW_DB_NAME}.now_plr.pid = ${NOW_DB_NAME}.now_proj.pid WHERE ${NOW_DB_NAME}.now_proj.proj_records = 1))`
const result: { [index: string]: string | number | null | bigint | boolean }[] = await conn.query(
`
SELECT * FROM ${NOW_DB_NAME}.now_v_export_locsp WHERE lid IN (${lids.map(() => '?').join(', ')}) ${excludeDraftsString} ORDER BY loc_name LIMIT ${limit} OFFSET ${offset}
`,
[...lids]
)
return result
}
const formatValue = (val: unknown) => {
if (typeof val === 'bigint') return `"${Number(val)}"`
if (val === null) return `""`
return `"${val as string}"`
}
const getExportList = async (conn: PoolConnection, lids: number[], includeDrafts: boolean) => {
const chunks = []
let columnHeaders: null | string[] = null
for (let i = 0; ; i += 1) {
const chunk = await getChunk(conn, CHUNK_SIZE, i * CHUNK_SIZE, lids, includeDrafts)
if (chunk.length === 0) break
// Get column headers
if (!columnHeaders) columnHeaders = Object.keys(chunk[0])
chunks.push(...chunk.map(c => Object.values(c).map(val => formatValue(val))))
}
return [columnHeaders, ...chunks]
}
export const getLocalitySpeciesList = async (lids: number[], user: User | undefined) => {
const conn = await pool.getConnection()
const exportList = await getExportList(
conn,
lids,
(user && [Role.Admin, Role.EditUnrestricted].includes(user.role)) || false
)
await conn.end()
return exportList
}
import { fixBigInt } from '../utils/common'
import { logDb, nowDb } from '../utils/db'
const getIdsOfUsersProjects = async (user: User) => {
const usersProjects = await nowDb.now_proj_people.findMany({
backend/src/services/localitySpeciesExport.ts
+51
View File
@@ -0,0 +1,51 @@
import { PoolConnection } from 'mariadb'
import { User } from '../../../frontend/src/backendTypes'
import { NOW_DB_NAME } from '../utils/config'
import { Role } from '../../../frontend/src/types'
import { Response } from 'express'
import { pool } from '../utils/db'
const getExportList = async (
conn: PoolConnection,
lids: number[],
includeDrafts: boolean,
rowHandler: (row: unknown) => void
) => {
return new Promise(resolve => {
const excludeDraftsString = includeDrafts
? ''
: `AND (loc_status = 0 AND lid NOT IN (SELECT DISTINCT ${NOW_DB_NAME}.now_plr.lid FROM ${NOW_DB_NAME}.now_plr JOIN ${NOW_DB_NAME}.now_proj ON ${NOW_DB_NAME}.now_plr.pid = ${NOW_DB_NAME}.now_proj.pid WHERE ${NOW_DB_NAME}.now_proj.proj_records = 1))`
resolve(
conn
.queryStream(
`
SELECT * FROM ${NOW_DB_NAME}.now_v_export_locsp WHERE lid IN (${lids.map(() => '?').join(', ')}) ${excludeDraftsString} ORDER BY loc_name
`,
[...lids]
)
.on('data', rowHandler)
)
})
}
const formatValue = (val: unknown) => {
if (typeof val === 'bigint') return `"${Number(val)}"`
if (val === null) return `""`
return `"${val as string}"`
}
export const getLocalitySpeciesList = async (lids: number[], user: User | undefined, res: Response) => {
const exportList: unknown[] = []
const transformRow = (row: unknown) => {
const values = Object.values(row as object)
return values.map(value => formatValue(value))
}
const conn = await pool.getConnection()
await getExportList(conn, lids, (user && [Role.Admin, Role.EditUnrestricted].includes(user.role)) || false, row => {
exportList.push(transformRow(row))
})
await conn.end()
return exportList
}