Skip to content

Instantly share code, notes, and snippets.

@danielsan
Last active April 4, 2020 14:26
Show Gist options
  • Save danielsan/302c8c4ba623d76e93149e4d37807a08 to your computer and use it in GitHub Desktop.
Save danielsan/302c8c4ba623d76e93149e4d37807a08 to your computer and use it in GitHub Desktop.
map athena partition for redshift backups
var s3 = new (require('aws-sdk/clients/s3'))()
var doIt = async (m, days, y = new Date().getFullYear()) => {
for (const d of days) {
await s3.listObjectsV2({ MaxKeys: 1000, Bucket: 'em-data-analytics-bucket', Prefix:`redshift-backups/public/em_farenet2_001/year=${y}/month=${m}/day=${d}/airlineiatacode=` }).promise()
.then((res) => {
const m = res.Contents.reduce((acc, i) => acc.add(path.dirname(i.Key)), new Set()); console.log(Array.from(m.values()).map((s, i) => {
const { year, month, day, airl, jt } = s.substr(24).match(/(?<tb>\w+)\/year=(?<year>\w+)\/month=(?<month>\w+)\/day=(?<day>\w+)\/\w+=(?<airl>\w+)\/\w+=(?<jt>\w+)/).groups;
return `PARTITION (year = '${year}', month = '${month}', day='${day}', airlineiatacode='${airl}', journeytype='${jt}') ${i % 4 === 0 ? '\n' : ''}`
}).join(' '))
})
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment