Created
June 29, 2023 15:07
-
-
Save corporatepiyush/6caf07d69db0bb0f4e7ea1a7b649c776 to your computer and use it in GitHub Desktop.
Taking a day wise backup of large SQL Table incrementally
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const pg = require('pg-promise')(); | |
const fs = require('fs'); | |
const db = pg({ | |
user: "****", | |
password: "*****", | |
host: "******", | |
port: 5432, | |
database: "*****", | |
ssl: { | |
rejectUnauthorized: true, | |
ca: fs.readFileSync("./rds.pem").toString(), | |
}, | |
}); | |
const numberOfDays = 1000; // for backup in past 3 years | |
const dayInMilliSecond = 86400000; | |
async function getRecords(from, to) { | |
return await db.query(`SELECT * FROM <table> WHERE <any timestamp indexed column> between '${from}' and '${to}'`); | |
} | |
async function fetchAndSave() { | |
let to = Date.now(); | |
let from = to - dayInMilliSecond; | |
for (let dayIndex = numberOfDays; dayIndex > 0; dayIndex--) { | |
const records = await getRecords(new Date(from).toISOString(), new Date(to).toISOString()); | |
console.log('Records fetched for the day ' + new Date(from).toISOString() + ' =', records.length); | |
const formattedRecords = records.map(record => Object.values(record).toString()).join('\n'); | |
await fs.promises.appendFile('records.csv', formattedRecords); | |
to = from; | |
from = to - dayInMilliSecond; | |
} | |
} | |
fetchAndSave(); //without CSV header |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment