44 lines
1.3 KiB
TypeScript
44 lines
1.3 KiB
TypeScript
import axios from 'axios';
|
|
|
|
const pgp = require('pg-promise')({
|
|
capSQL: true
|
|
});
|
|
|
|
export async function bulkBatchInsert(pool, table, columns, inserts, batchSize) {
|
|
// Creates bulk insert statements from an array of inserts - avoids performance cost of insert roundtrips to the server
|
|
// Batches the inserts to stop the individual statements getting too large
|
|
// All inserts are done in a transaction - so if one fails they all will
|
|
|
|
|
|
if (inserts.length === 0) {
|
|
return
|
|
} else if (batchSize < 1) {
|
|
throw 'batchSize must be at least 1'
|
|
}
|
|
|
|
|
|
let client = await pool.connect()
|
|
const cs = new pgp.helpers.ColumnSet(columns, {table: table});
|
|
try {
|
|
await client.query('BEGIN')
|
|
|
|
for (let i = 0, j = inserts.length; i < j; i += batchSize) {
|
|
let insertsBatch = inserts.slice(i, i + batchSize);
|
|
let updatesSql = pgp.helpers.insert(insertsBatch, cs);
|
|
await client.query(updatesSql)
|
|
}
|
|
|
|
await client.query('COMMIT')
|
|
} catch (e) {
|
|
await client.query('ROLLBACK')
|
|
throw e
|
|
} finally {
|
|
client.release()
|
|
}
|
|
}
|
|
|
|
export function notify(content) {
|
|
if (process.env.TRANSACTIONS_SCRAPER_WEBHOOK_URL) {
|
|
axios.post(process.env.TRANSACTIONS_SCRAPER_WEBHOOK_URL, {content});
|
|
}
|
|
} |