Demonstrate memory use of streaming HTTP clients in ruby.

_result.md

This gist contains a set of tests to see how much memory gets used by various ruby HTTP client libraries while reading a large HTTP response.

Results:

_run

#!/bin/sh -e

echo "Fetching dependencies..."
bundle install --quiet --path .bundle

bundle exec stripmem ruby run.rb

Gemfile

source 'https://rubygems.org'

# Graph memory use.
gem 'stripmem'

gem 'typhoeus', :git => 'https://github.com/spraints/typhoeus', :ref => 'stream'
gem 'ethon',    :git => 'https://github.com/spraints/ethon',    :ref => 'streaming'

gem 'em-http-request'

Gemfile.lock

GIT
  remote: https://github.com/spraints/ethon
  revision: dedb07d81ec6bfdcc6831954f3df53267f8ed8c6
  ref: streaming
  specs:
    ethon (0.6.1)
      ffi (>= 1.3.0)
      mime-types (~> 1.18)

GIT
  remote: https://github.com/spraints/typhoeus
  revision: 3fe5d8d4c6e8dca8c31b4cad57148f459a1de10d
  ref: stream
  specs:
    typhoeus (0.6.6)
      ethon (~> 0.6.1)

GEM
  remote: https://rubygems.org/
  specs:
    addressable (2.3.5)
    cookiejar (0.3.0)
    em-http-request (1.0.3)
      addressable (>= 2.2.3)
      cookiejar
      em-socksify
      eventmachine (>= 1.0.0.beta.4)
      http_parser.rb (>= 0.5.3)
    em-socksify (0.3.0)
      eventmachine (>= 1.0.0.beta.4)
    em-websocket (0.5.0)
      eventmachine (>= 0.12.9)
      http_parser.rb (~> 0.5.3)
    eventmachine (1.0.3)
    ffi (1.9.3)
    http_parser.rb (0.5.3)
    json (1.8.1)
    mime-types (1.25.1)
    rack (1.5.2)
    rack-protection (1.5.1)
      rack
    sinatra (1.4.4)
      rack (~> 1.4)
      rack-protection (~> 1.4)
      tilt (~> 1.3, >= 1.3.4)
    stripmem (0.0.3)
      em-websocket
      json
      sinatra
    tilt (1.4.1)

PLATFORMS
  ruby

DEPENDENCIES
  em-http-request
  ethon!
  stripmem
  typhoeus!

run.rb

class App
  def initialize
    @pids = {}
    @port = '3100'
    @size = '511222333'
    @url = "http://localhost:#{@port}/#{@size}"
  end

  def main
    forward_signals

    start_server
    sleep 5

    start 'typhoeus'
    start 'nostreamtyphoeus'
    start 'em'
    start 'nethttp'
    start 'nostream'
    start_curl

    wait
  end

  private

  def start_server
    fork do
      exec 'node', 'server.js', @port
    end
  end

  def start(client)
    fork do
      exec 'ruby', "test-#{client}.rb", @url
    end
  end

  def start_curl
    fork do
      exec 'curl', '-o', '/dev/null', @url
    end
  end

  def fork(&block)
    pid = super(&block)
    @pids[pid] = pid
  end

  def forward_signals
    trap(:INT) { @pids.keys.each { |pid| Process.kill(:INT, pid) } }
  end

  def wait
    while true
      pid, status = Process.waitpid2(-1)
      @pids.delete(pid)
      if @pids.size == 1
        Process.kill(:INT, @pids.keys.first)
      end
    end
  rescue Errno::ECHILD
    # done
  end
end

App.new.main

server.js

var http = require('http');

var s = "abcdefg\n";
require('crypto').randomBytes(1000, function(ex, buf) { s = buf.toString('hex') + "\n"; });

var responder = function(request, response) {
  var length = getResponseLength(request);
  response.writeHead(200, {'Content-Length': length, 'Content-Type': 'text/plain'});
  var writer = function() {
    makeToken(length, function(token) {
      response.write(token);
      length -= token.length;
      if(length == 0) {
        response.end();
      } else {
        setImmediate(writer);
      }
    });
  };
  setImmediate(writer);
};

var makeToken = function(maxSize, callback) {
  callback(s.slice(0,maxSize));
};

var getResponseLength = function(request) {
  var m = request.url.match(/\d+/);
  if(m) { return parseInt(m[0]); }
  return 100;
};

var server = http.createServer(responder);
server.listen(process.argv[2], function() { console.log("Listening on " + server.address().port); });

test-em.rb

require 'em-http-request'

size = 0
EM.run do
  req = EM::HttpRequest.new(ARGV[0]).get
  req.stream { |chunk| size += chunk.bytesize }
  req.errback { raise 'boom' }
  req.callback { EM.stop }
end
puts "#$0: read #{size} bytes"

test-nethttp.rb

require 'net/http'
require 'uri'

size = 0
Net::HTTP.get_response(URI.parse(ARGV[0])) do |response|
  response.read_body do |chunk|
    size += chunk.bytesize
  end
end
puts "#{$0}: read #{size} bytes"

test-nostream.rb

require 'net/http'
require 'uri'

size = 0
Net::HTTP.get_response(URI.parse(ARGV[0])) do |response|
  size = response.body.bytesize
end
puts "#{$0}: read #{size} bytes"

test-nostreamtyphoeus.rb

require 'typhoeus'

hydra = Typhoeus::Hydra.new
request = Typhoeus::Request.new(ARGV[0])
size = 0
request.on_complete do |response|
  size += response.body.bytesize
end
hydra.queue request
hydra.run
puts "#$0: read #{size} bytes"

test-typhoeus.rb

require 'typhoeus'

hydra = Typhoeus::Hydra.new
request = Typhoeus::Request.new(ARGV[0])
size = 0
request.on_body do |chunk|
  size += chunk.bytesize
end
hydra.queue request
hydra.run
puts "#$0: read #{size} bytes"