dechunk-to-utf8.js

var http = require('http'),
  Iconv = require('iconv').Iconv,
  buffertools = require('buffertools');

function goGetAmazon() {
  var headers;
  var chunks = [];

  var req = http.request({
    method: 'GET',
    host: 'www.amazon.com',
    path: '/',
    port: 80,
//  headers: {
//    Accept-Encoding: 'gzip,deflate'
//  }
  }, function (res) {
    headers = res.headers;
    //console.log(headers);
    res.on('data', function(chunk) {
      // Add chunks to an array to avoid implicit conversion to utf8 and to
      // hold off on "coalescing" via copy into a single contiguous buffer
      // until later.
      chunks.push(chunk);
    });
    res.on('end', function() {
      var conv = new Iconv('ISO-8859-1','utf8');
      var unconv = new Iconv('utf8','ISO-8859-1');

      var buf = buffertools.concat.apply(buffertools, chunks);
      chunks = null;
//    if (headers['content-encoding'].match(/\b(gzip|deflate)\b/)) {
//      buf = zlib.inflate(buf);
//    }
      matches = buf.inspect().match(/46 75 72 6e 69 74 75 72 65 20 26 61 6d 70 3b 20 44 .+? 63 6f 72/g);
      console.log(matches);
 
      // TODO: re-deflate/-gzip the buf? OR
      // TODO: pass through 'Content-Encoding: identity' after undoing?
      cb(res.headers['content-type'], unconv.convert(conv.convert(buf)));
    } );
  });
 
  req.on('error', function(e) {
    console.log('problem with request: ' + e.message);
  });
 
  req.end();
}