Last active
March 8, 2021 01:10
-
-
Save sairamkrish/477d20980611202f46a2d44648f7b14b to your computer and use it in GitHub Desktop.
Javascript bulk insert into postgresql using pg-copy-streams. The code is explained here - https://medium.com/@sairamkrish/how-to-do-postgresql-bulk-insert-with-node-pg-copy-streams-f4301ec29388
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
Pseudo code - to serve as a help guide. | |
*/ | |
const copyFrom = require('pg-copy-streams').from; | |
const Readable = require('stream').Readable; | |
const { Pool,Client } = require('pg'); | |
const fs = require('fs'); | |
const path = require('path'); | |
const datasourcesConfigFilePath = path.join(__dirname,'..','..','server','datasources.json'); | |
const datasources = JSON.parse(fs.readFileSync(datasourcesConfigFilePath, 'utf8')); | |
const pool = new Pool({ | |
user: datasources.PG.user, | |
host: datasources.PG.host, | |
database: datasources.PG.database, | |
password: datasources.PG.password, | |
port: datasources.PG.port, | |
}); | |
export const bulkInsert = (employees) => { | |
pool.connect().then(client=>{ | |
let done = () => { | |
client.release(); | |
} | |
var stream = client.query(copyFrom('COPY employee (name,age,salary) FROM STDIN')); | |
var rs = new Readable; | |
let currentIndex = 0; | |
rs._read = function () { | |
if (currentIndex === employees.length) { | |
rs.push(null); | |
} else { | |
let employee = employees[currentIndex]; | |
rs.push(employee.name + '\t' + employee.age + '\t' + employee.salary + '\n'); | |
currentIndex = currentIndex+1; | |
} | |
}; | |
let onError = strErr => { | |
console.error('Something went wrong:', strErr); | |
done(); | |
}; | |
rs.on('error', onError); | |
stream.on('error', onError); | |
stream.on('end',done); | |
rs.pipe(stream); | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment