retrieve_all_files.js

const AWS = require('aws-sdk')
const fs = require('fs')
const path = require('path')

const s3 = new AWS.S3()

const BUCKET = 'gil-test-bucket'
const PREFIX = 'Clarizen/salesforce/'
const OUTDIR = '/tmp/aa'

const retrieve = async key => {
  console.log(key)
  const inStream = s3.getObject({
    Bucket: BUCKET,
    Key: key,
  }).createReadStream()
  const filename = path.join(OUTDIR, key)
  await fs.promises.mkdir(path.dirname(filename), { recursive: true })
  const outStream = fs.createWriteStream(filename)
  inStream.pipe(outStream)
  return new Promise((resolve, reject) => {
    outStream.once('end', resolve)
    outStream.once('error', reject)
  })
}

const main = async () => {
  const promises = []
  const next = async (continuationToken) => {
    const response = await s3.listObjectsV2({ 
      Bucket: BUCKET, 
      Prefix: PREFIX,
      ContinuationToken: continuationToken, 
    }).promise()
    promises.push(...response.Contents.map(e => retrieve(e.Key)))
    if (response.NextContinuationToken) {
      return next(response.NextContinuationToken)
    }
    return
  }
  await next()
  await Promise.all(promises)
}

main().catch(e => {
  console.error(e.stack || e)
  process.exit(1)
})