s3-run-in-batch.js

const S3Client = require('aws-sdk/clients/s3')
const sharp = require('sharp')

const ACCESS_KEY_ID = ''
const SECRET_ACCESS_KEY = ''
const REGION = ''
const BUCKET = ''
const PREFIX = ''
const CHUNK_SIZE = 500
const FILTER = (filesName) => {
  return filesName.filter(
    (fileName) =>
      (fileName.endsWith('.jpg') || fileName.endsWith('.png')) &&
      !fileName.includes('-x2.') &&
      !filesName.includes(fileName.replace('.', '-x2.'))
  )
}
const PROCESS = async (fileName) => {
  const result = await getFile(fileName)

  const imageStream = sharp(result.Body)
  const imageMetadata = await imageStream.metadata()
  const { data } = await imageStream
    .resize({
      width: imageMetadata.width * 2,
      height: imageMetadata.height * 2,
      fit: 'inside',
    })
    .toBuffer({
      resolveWithObject: true,
    })

  await s3
    .upload({
      Key: fileName.replace('.', '-x2.'),
      Body: data,
      Bucket: BUCKET,
      ContentType: result.ContentType,
      ACL: 'public-read',
    })
    .promise()
}

const s3 = new S3Client({
  accessKeyId: ACCESS_KEY_ID,
  secretAccessKey: SECRET_ACCESS_KEY,
  region: REGION,
})

const splitEvery = (n, arr) => {
  let idx = 0
  const chunks = []
  while (idx < arr.length) {
    chunks.push(arr.slice(idx, (idx += n)))
  }
  return chunks
}

const getFile = async (fileName) => {
  return s3
    .getObject({
      Bucket: BUCKET,
      Key: fileName,
    })
    .promise()
}

const getFiles = async (files = []) => {
  const result = await s3
    .listObjectsV2({
      Bucket: BUCKET,
      Prefix: PREFIX,
      MaxKeys: 1000,
      StartAfter: files.length > 0 ? files[files.length - 1] : undefined,
    })
    .promise()

  const filesName = result.Contents.map((file) => file.Key)

  const list = [...files, ...filesName]

  return filesName.length === 1000 ? getFiles(list) : list
}

const run = async () => {
  const allFiles = await getFiles()
  console.log('allFiles', allFiles.length)

  const filteredFiles = FILTER(allFiles)
  console.log('filteredFiles', filteredFiles.length)

  const chunks = splitEvery(CHUNK_SIZE, filteredFiles)
  console.log('chunks', chunks.length)

  await Promise.all(
    chunks.map(async (chunk, index) => {
      const chunkNumber = (index + 1).toString().padStart(2, '0')
      const chunkLength = chunk.length
      let i = 1
      for (let fileName of chunk) {
        console.log(`[chunk ${chunkNumber}] ${i++}/${chunkLength}`)
        await PROCESS(fileName)
      }
    })
  )
}

run()