extraIfBranch.js

// Count processed rows and batches and compare it with TotlaRecords (which comes from getSize() function beforehand)
let recordsProcessed = 0;
let batchNumber = 1;
let batchRecords = [];
...
// save in batch mode:
else if (output === 's3') {
    s1.on('result', (row) => {
        ++recordsProcessed;
        batchRecords.push(row);

        if (recordsProcessed === (BATCH_SIZE * batchNumber) || recordsProcessed === totalRecords) {
            if (!dryRun) {
                connection.pause();
                pr(` batch ${batchNumber}, pushing to s3, ${batchRecords.length}, totalRecordsProcessed = ${recordsProcessed}`);

                // batch process here. saves in batch mode, split file into smaller files.
                const params = { Bucket: bucket, Key: s3key + batchNumber, Body: JSON.stringify(batchRecords) };

                // save chunk to s3
                s3.upload(params).promise()
                    .then(data => { connection.resume(); });
                pr(`saved ${batchNumber} to aws s3 cp s3://${bucket}/${s3key}${batchNumber} ./tmp/${s3key}${batchNumber}.csv`);
            }
            batchRecords = [];
            ++batchNumber;
        }
    });
    s1.on('end', () => { resolve(recordsProcessed); });
}