60 lines
1.8 KiB
TypeScript
60 lines
1.8 KiB
TypeScript
import axios from 'axios';
|
|
|
|
const pgp = require('pg-promise')({
|
|
capSQL: true
|
|
});
|
|
|
|
export async function bulkBatchInsert(pool, tablename, columns, inserts, batchSize, schema) {
|
|
// Creates bulk insert statements from an array of inserts - avoids performance cost of insert roundtrips to the server
|
|
// Batches the inserts to stop the individual statements getting too large
|
|
// All inserts are done in a transaction - so if one fails they all will
|
|
|
|
const table = new pgp.helpers.TableName({table: tablename, schema: schema})
|
|
|
|
if (inserts.length === 0) {
|
|
return
|
|
} else if (batchSize < 1) {
|
|
throw 'batchSize must be at least 1'
|
|
}
|
|
|
|
let client = await pool.connect()
|
|
const cs = new pgp.helpers.ColumnSet(columns, {table: table});
|
|
try {
|
|
await client.query('BEGIN')
|
|
|
|
for (let i = 0, j = inserts.length; i < j; i += batchSize) {
|
|
let insertsBatch = inserts.slice(i, i + batchSize);
|
|
let insertsSql = pgp.helpers.insert(insertsBatch, cs);
|
|
await client.query(insertsSql)
|
|
|
|
console.log('inserted ' + (i + insertsBatch.length) + ' of ' + inserts.length)
|
|
}
|
|
|
|
await client.query('COMMIT')
|
|
|
|
} catch (e) {
|
|
await client.query('ROLLBACK')
|
|
throw e
|
|
} finally {
|
|
client.release()
|
|
}
|
|
}
|
|
|
|
export function notify(content) {
|
|
if (process.env.WEBHOOK_URL) {
|
|
axios.post(process.env.WEBHOOK_URL, {content});
|
|
}
|
|
}
|
|
|
|
export function getLatestObjPerCombination(arr, combinationFields) {
|
|
// Utility function - iterates over arr and return the element with the highest index per set of combinationFields
|
|
|
|
let latestCombinations = {};
|
|
for (let values of arr) {
|
|
let combination = combinationFields.map((e) => values[e]);
|
|
latestCombinations[combination] = values;
|
|
}
|
|
|
|
return Object.values(latestCombinations);
|
|
}
|