streaming gzip

em-http-gzip-streaming.rb

require 'rubygems'
require 'em-http-request'

# Monkey-patched Gzip Decoder to handle
# Gzip streams.
#
# This takes advantage of the fact that
# Zlib::GzipReader takes an IO object &
# reads from it as it decompresses.
#
# It also relies on Zlib only checking for
# nil as the method of determining whether
# it has reached EOF.
#
# `IO#read(len, buf)` can also denote EOF by returning a string
# shorter than `len`, but Zlib doesn't care about that.
#
module EventMachine::HttpDecoders
  class GZip < Base
    class LazyStringIO
      def initialize string=""
        @stream=string        
      end

      def << string
        @stream << string
      end

      def read length=nil,buffer=nil
        buffer||=""
        length||=0
        buffer << @stream[0..(length-1)]
        @stream = @stream[length..-1]
        buffer
      end
      
      def size
        @stream.size
      end
    end

    def self.encoding_names
      %w(gzip compressed)
    end

    def decompress(compressed)
      @buf ||= LazyStringIO.new
      @buf << compressed
      
       # Zlib::GzipReader loads input in 2048 byte chunks
      if @buf.size > 2048
        @gzip ||= Zlib::GzipReader.new @buf
        @gzip.readline # lines are bigger than compressed chunks, so this works
                       # you could also use #readpartial, but then you need to tune
                       # the max length
                       # don't use #read, because it will attempt to read the full file
                       # readline uses #gets under the covers, so you could try that too.
      end
    end

    def finalize
      @gzip.read
    end
  end
end


url = "my-streaming-url"
user = "my-user"
password = "my-password"

EventMachine.run do

  http = EventMachine::HttpRequest.new(url).get :head => {
             'Accept-Encoding' => 'gzip',
             'Authorization' => [ user, password ] }

  http.headers do |hash|
    p [:status, http.response_header.status]
    p [:headers, hash]
    if http.response_header.status > 299
      puts 'unsuccessful request'
      EM.stop
    end
  end

  http.stream do |chunk|
     print chunk
  end
  
  http.callback do
   p "done"
   EM.stop
  end

  http.errback do
   puts "there was an error"
   p http.error
   p http.response
   EM.stop
  end

end

Thanks, this helped a lot getting my Powertrack client to interface with Gnip console.

the monkey patch bit of this has been cleaned up and merged into em-http-request, which rocks!

igrigorik/em-http-request#186

Excellent!

@baroquebobcat Thanks for your work producing this. There is discussion that could use your attention at igrigorik/em-http-request#204