Compare memory performance between File.open block and EventMachine streaming HTTP.

file.rb

require 'yajl'

def get_memory_usage
  `ps -o rss= -p #{Process.pid}`.to_i / 1024
end

before = get_memory_usage
parsed_lines = []
File.open("public/sample.json", 'r') do |f|
  f.each_line do |line|
     parsed = Yajl::Parser.parse(line)
     puts parsed
     # parsed_lines << parsed # uncomment this line to see how memory is impacted by storing collection
  end
end
after = get_memory_usage

puts "Finished"
puts "Memory Before: #{before.to_s}KB; After: #{after.to_s}KB"

stream.rb

require 'eventmachine'
require 'em-http'
require 'yajl'

def get_memory_usage
  `ps -o rss= -p #{Process.pid}`.to_i / 1024
end

before = get_memory_usage
parsed_lines = []
EventMachine.run do
  buffer = ""

  # Modify URL to point to a large JSON file. I used a 350 MB JSON file hosted locally through POW.cx
  http = EventMachine::HttpRequest.new('http://jstream.dev/sample.json').get
  http.stream do |chunk|
    buffer += chunk

    while line = buffer.slice!(/.+\n/)
      parsed = Yajl::Parser.parse(line)
      # parsed_lines << parsed # uncomment this line to see how memory is impacted by storing collection
    end
  end
  http.callback { |chunk| EventMachine.stop }
end
after = get_memory_usage

puts "Finished"
puts "Memory Before: #{before.to_s}KB; After: #{after.to_s}KB"