Proof of concept of streaming a file through zlib and into s3, without storing the entire file in memory. knox-mpu provides support for Amazon's multipart upload, which allows us to stream an arbitrary amount of data without specifying the content-length ahead of time.

testGzipKnox.js

/**
 * Proof of concept of streaming a file through zlib and into s3,
 * without storing the entire file in memory. knox-mpu provides support
 * for Amazon's multipart upload, which allows us to stream an arbitrary
 * amount of data without specifying the content-length ahead of time.
 */
var knox = require('knox'),
  fs = require('fs'),
  zlib = require('zlib'),
  MultiPartUpload = require('knox-mpu');

var knoxClient = knox.createClient({
  key: 'your s3 key',
  secret: 'your s3 secret',
  bucket: 'your s3 bucket'
});

// Source path on disk
var srcPath = './test.txt';
// Desination path on s3
var dstPath = '/destination.gz';

var inStream = fs.createReadStream(srcPath);
var gzip = zlib.createGzip();
var headers = {
  'Content-Type': 'application/x-gzip'
}
// Pipe the input file into a gzip stream, then pipe that into s3
var upload = new MultiPartUpload(
  {
    client: knoxClient,
    objectName: dstPath,
    headers: headers,
    stream: inStream.pipe(gzip)
  },
  function(err, res) {
    if (err) return handleError(err);
    console.log(res);
  }
);

var handleError = function(err) {
  console.log(err);
  process.exit(1);
}