S3.listObjectsV2 as stream

s3_list_objects_stream.js

"use strict";
const {Readable} = require('readable-stream');
const {S3} = require('aws-sdk');

// Convert the given S3::listObjectsV2 request to a Readable stream
// req should be an un-sent S3::listObjectsV2() request
// opts are typical stream options. Object mode is required.
function s3_list_stream( req, opts ){
  opts = Object.assign({}, opts, {read, objectMode: true});
  const stream = new Readable(opts);

  return stream;

  function read(){
    if( !req ) return;

    const _req = req;
    req = null;
    _req.send(page_handler);
  }

  function page_handler( e, data ){
    if( e ) return stream.destroy(e);

    for( let obj of data.Contents ){
      obj.Bucket = data.Name;
      stream.push(obj);
    }

    req = this.hasNextPage() ? this.nextPage() : null;
    if( !req ) stream.push(null);
  }
}

function test_count_objects(){
  const list_req = new S3().listObjectsV2({
    Bucket: 'NEED',
    Prefix: 'MAYBE NEED'
  });
  const list_stream = s3_list_stream(list_req);

  let count = 0;
  list_stream.on('data', _ => count++);
  return new Promise(done => list_stream.once('end', _ => done(count)));
}