Inflating Multiple SPDY packets in Python and Ruby

zlib_inflate.py

import struct
import ctypes as C
from ctypes import util

_zlib = C.cdll.LoadLibrary(util.find_library('libz'))

class _z_stream(C.Structure):
    _fields_ = [
        ("next_in", C.POINTER(C.c_ubyte)),
        ("avail_in", C.c_uint),
        ("total_in", C.c_ulong),
        ("next_out", C.POINTER(C.c_ubyte)),
        ("avail_out", C.c_uint),
        ("total_out", C.c_ulong),
        ("msg", C.c_char_p),
        ("state", C.c_void_p),
        ("zalloc", C.c_void_p),
        ("zfree", C.c_void_p),
        ("opaque", C.c_void_p),
        ("data_type", C.c_int),
        ("adler", C.c_ulong),
        ("reserved", C.c_ulong),

    ]

ZLIB_VERSION = C.c_char_p("1.2.3")
Z_SYNC_FLUSH = 0x02

CHUNK = 1024 * 128

dictionary = "optionsgetheadpostputdeletetraceacceptaccept-charsetaccept-encodingaccept-languageauthorizationexpectfromhostif-modified-sinceif-matchif-none-matchif-rangeif-unmodifiedsincemax-forwardsproxy-authorizationrangerefererteuser-agent100101200201202203204205206300301302303304305306307400401402403404405406407408409410411412413414415416417500501502503504505accept-rangesageetaglocationproxy-authenticatepublicretry-afterservervarywarningwww-authenticateallowcontent-basecontent-encodingcache-controlconnectiondatetrailertransfer-encodingupgradeviawarningcontent-languagecontent-lengthcontent-locationcontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookieMondayTuesdayWednesdayThursdayFridaySaturdaySundayJanFebMarAprMayJunJulAugSepOctNovDecchunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplication/xhtmltext/plainpublicmax-agecharset=iso-8859-1utf-8gzipdeflateHTTP/1.1statusversionurl\x00"

octets_1 = [0x38,0xea,0xdf,0xa2,0x51,0xb2,0x62,0xe0,0x62,0x60,0x83,0xa4,0x17,0x06,0x7b,0xb8,0x0b,0x75,0x30,0x2c,0xd6,0xae,0x40,0x17,0xcd,0xcd,0xb1,0x2e,0xb4,0x35,0xd0,0xb3,0xd4,0xd1,0xd2,0xd7,0x02,0xb3,0x2c,0x18,0xf8,0x50,0x73,0x2c,0x83,0x9c,0x67,0xb0,0x3f,0xd4,0x3d,0x3a,0x60,0x07,0x81,0xd5,0x99,0xeb,0x40,0xd4,0x1b,0x33,0xf0,0xa3,0xe5,0x69,0x06,0x41,0x90,0x8b,0x75,0xa0,0x4e,0xd6,0x29,0x4e,0x49,0xce,0x80,0xab,0x81,0x25,0x03,0x06,0xbe,0xd4,0x3c,0xdd,0xd0,0x60,0x9d,0xd4,0x3c,0xa8,0xa5,0x2c,0xa0,0x3c,0xce,0xc0,0x0f,0x4a,0x08,0x39,0x20,0xa6,0x15,0x30,0xe3,0x19,0x18,0x30,0xb0,0xe5,0x02,0x0b,0x97,0xfc,0x14,0x06,0x66,0x77,0xd7,0x10,0x06,0xb6,0x62,0x60,0x7a,0xcc,0x4d,0x65,0x60,0xcd,0x28,0x29,0x29,0x28,0x66,0x60,0x06,0x79,0x9c,0x51,0x9f,0x81,0x0b,0x91,0x5b,0x19,0xd2,0x7d,0xf3,0xab,0x32,0x73,0x72,0x12,0xf5,0x4d,0xf5,0x0c,0x14,0x34,0x00,0x8a,0x30,0x34,0xb4,0x56,0xf0,0xc9,0xcc,0x2b,0xad,0x50,0xa8,0xb0,0x30,0x8b,0x37,0x33,0xd1,0x54,0x70,0x04,0x7a,0x3e,0x35,0x3c,0x35,0xc9,0x3b,0xb3,0x44,0xdf,0xd4,0xd8,0x44,0xcf,0x18,0xa8,0xcc,0xdb,0x23,0xc4,0xd7,0x47,0x47,0x21,0x27,0x33,0x3b,0x55,0xc1,0x3d,0x35,0x39,0x3b,0x5f,0x53,0xc1,0x39,0x03,0x58,0xec,0xa4,0xea,0x1b,0x1a,0xe9,0x01,0x7d,0x6a,0x62,0x04,0x52,0x16,0x9c,0x98,0x96,0x58,0x94,0x09,0xd5,0xc4,0xc0,0x0e,0x0d,0x7c,0x06,0x0e,0x58,0x9c,0x00,0x00,0x00,0x00,0xff,0xff]

octets_2 = [0x42,0x8a,0x02,0x66,0x60,0x60,0x0e,0xad,0x60,0xe4,0xd1,0x4f,0x4b,0x2c,0xcb,0x04,0x66,0x33,0x3d,0x20,0x31,0x58,0x42,0x14,0x00,0x00,0x00,0xff,0xff]

d1 = struct.pack('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB', *octets_1)

d2 = struct.pack('BBBBBBBBBBBBBBBBBBBBBBBBBBBBB', *octets_2)

# z_stream object for zlib
z_stream = _z_stream()

# Initiate inflate
_zlib.inflateInit2_(C.byref(z_stream), 15, ZLIB_VERSION, C.sizeof(z_stream))

# The out-buffer is just a range of memory to store the results
outbuf = C.create_string_buffer(CHUNK)

z_stream.avail_in  = len(d1)
z_stream.next_in   = C.cast(C.c_char_p(d1), C.POINTER(C.c_ubyte))
z_stream.avail_out = CHUNK
z_stream.next_out  = C.cast(outbuf, C.POINTER(C.c_ubyte))

# Try inflate, it fails because it needs a dictionary
_zlib.inflate(C.byref(z_stream), Z_SYNC_FLUSH)

# Set the dictionary
_zlib.inflateSetDictionary(
  C.byref(z_stream),
  C.cast(C.c_char_p(dictionary), C.POINTER(C.c_ubyte)),
  len(dictionary))

# Inflate for real now that the dictionary is set
_zlib.inflate(C.byref(z_stream), Z_SYNC_FLUSH)

outbuf[:CHUNK-z_stream.avail_out]

outbuf = C.create_string_buffer(CHUNK)
z_stream.avail_in  = len(d2)
z_stream.next_in   = C.cast(C.c_char_p(d2), C.POINTER(C.c_ubyte))
z_stream.avail_out = CHUNK
z_stream.next_out  = C.cast(outbuf, C.POINTER(C.c_ubyte))
_zlib.inflate(C.byref(z_stream), Z_SYNC_FLUSH)

outbuf[:CHUNK-z_stream.avail_out]

zlib_inflate.rb

require 'ffi/zlib'

DICT = \
"optionsgetheadpostputdeletetraceacceptaccept-charsetaccept-encodingaccept-" \
"languageauthorizationexpectfromhostif-modified-sinceif-matchif-none-matchi" \
"f-rangeif-unmodifiedsincemax-forwardsproxy-authorizationrangerefererteuser" \
"-agent10010120020120220320420520630030130230330430530630740040140240340440" \
"5406407408409410411412413414415416417500501502503504505accept-rangesageeta" \
"glocationproxy-authenticatepublicretry-afterservervarywarningwww-authentic" \
"ateallowcontent-basecontent-encodingcache-controlconnectiondatetrailertran" \
"sfer-encodingupgradeviawarningcontent-languagecontent-lengthcontent-locati" \
"oncontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookieMo" \
"ndayTuesdayWednesdayThursdayFridaySaturdaySundayJanFebMarAprMayJunJulAugSe" \
"pOctNovDecchunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplic" \
"ation/xhtmltext/plainpublicmax-agecharset=iso-8859-1utf-8gzipdeflateHTTP/1" \
".1statusversionurl\0"

CHUNK = 10*1024 # this is silly, but it'll do for now

octets_1 = [0x38,0xea,0xdf,0xa2,0x51,0xb2,0x62,0xe0,0x62,0x60,0x83,0xa4,0x17,0x06,0x7b,0xb8,0x0b,0x75,0x30,0x2c,0xd6,0xae,0x40,0x17,0xcd,0xcd,0xb1,0x2e,0xb4,0x35,0xd0,0xb3,0xd4,0xd1,0xd2,0xd7,0x02,0xb3,0x2c,0x18,0xf8,0x50,0x73,0x2c,0x83,0x9c,0x67,0xb0,0x3f,0xd4,0x3d,0x3a,0x60,0x07,0x81,0xd5,0x99,0xeb,0x40,0xd4,0x1b,0x33,0xf0,0xa3,0xe5,0x69,0x06,0x41,0x90,0x8b,0x75,0xa0,0x4e,0xd6,0x29,0x4e,0x49,0xce,0x80,0xab,0x81,0x25,0x03,0x06,0xbe,0xd4,0x3c,0xdd,0xd0,0x60,0x9d,0xd4,0x3c,0xa8,0xa5,0x2c,0xa0,0x3c,0xce,0xc0,0x0f,0x4a,0x08,0x39,0x20,0xa6,0x15,0x30,0xe3,0x19,0x18,0x30,0xb0,0xe5,0x02,0x0b,0x97,0xfc,0x14,0x06,0x66,0x77,0xd7,0x10,0x06,0xb6,0x62,0x60,0x7a,0xcc,0x4d,0x65,0x60,0xcd,0x28,0x29,0x29,0x28,0x66,0x60,0x06,0x79,0x9c,0x51,0x9f,0x81,0x0b,0x91,0x5b,0x19,0xd2,0x7d,0xf3,0xab,0x32,0x73,0x72,0x12,0xf5,0x4d,0xf5,0x0c,0x14,0x34,0x00,0x8a,0x30,0x34,0xb4,0x56,0xf0,0xc9,0xcc,0x2b,0xad,0x50,0xa8,0xb0,0x30,0x8b,0x37,0x33,0xd1,0x54,0x70,0x04,0x7a,0x3e,0x35,0x3c,0x35,0xc9,0x3b,0xb3,0x44,0xdf,0xd4,0xd8,0x44,0xcf,0x18,0xa8,0xcc,0xdb,0x23,0xc4,0xd7,0x47,0x47,0x21,0x27,0x33,0x3b,0x55,0xc1,0x3d,0x35,0x39,0x3b,0x5f,0x53,0xc1,0x39,0x03,0x58,0xec,0xa4,0xea,0x1b,0x1a,0xe9,0x01,0x7d,0x6a,0x62,0x04,0x52,0x16,0x9c,0x98,0x96,0x58,0x94,0x09,0xd5,0xc4,0xc0,0x0e,0x0d,0x7c,0x06,0x0e,0x58,0x9c,0x00,0x00,0x00,0x00,0xff,0xff]

octets_2 = [0x42,0x8a,0x02,0x66,0x60,0x60,0x0e,0xad,0x60,0xe4,0xd1,0x4f,0x4b,0x2c,0xcb,0x04,0x66,0x33,0x3d,0x20,0x31,0x58,0x42,0x14,0x00,0x00,0x00,0xff,0xff]

d1 = octets_1.pack("C*")
d2 = octets_2.pack("C*")

# z_stream object for zlib
zstream = FFI::Zlib::Z_stream.new

# Initiate inflate
#FFI::Zlib.inflateInit(zstream)
FFI::Zlib.inflateInit2(zstream, 15)

# The out-buffer is just a range of memory to store the results
out_buf = FFI::MemoryPointer.new(CHUNK)

# Zlib needs an in-buffer and an out-buffer
# The in-buffer is the compressed data in the first packet
in_buf  = FFI::MemoryPointer.from_string(d1)

zstream[:avail_in]  = in_buf.size
zstream[:next_in]   = in_buf
zstream[:avail_out] = CHUNK
zstream[:next_out]  = out_buf

# Try inflate, it fails because it needs a dictionary
FFI::Zlib.inflate(zstream, FFI::Zlib::Z_SYNC_FLUSH)

# Set the dictionary
FFI::Zlib.inflateSetDictionary(zstream, DICT, DICT.size)

# Inflate for real now that the dictionary is set
FFI::Zlib.inflate(zstream, FFI::Zlib::Z_SYNC_FLUSH)

# Uncompressed data is now in the output buffer
out_buf.get_bytes(0, zstream[:total_out])

out_buf = FFI::MemoryPointer.new(CHUNK)
in_buf = FFI::MemoryPointer.from_string(d2)
zstream[:avail_in]  = in_buf.size
zstream[:next_in]   = in_buf
zstream[:avail_out] = CHUNK
zstream[:next_out]  = out_buf

FFI::Zlib.inflate(zstream, FFI::Zlib::Z_SYNC_FLUSH)

The first packet works in both python and ruby, resulting in:

 => "\x00\n\x00\x06accept\x00?text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8\x00\x0Eaccept-charset\x00\x1EISO-8859-1,utf-8;q=0.7,;q=0.3\x00\x0Faccept-encoding\x00\x11gzip,deflate,sdch\x00\x0Faccept-language\x00\x0Een-US,en;q=0.8\x00\x04host\x00\x0Flocalhost:10000\x00\x06method\x00\x03GET\x00\x06scheme\x00\x05https\x00\x03url\x00\x01/\x00\nuser-agent\x00gMozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.30 Safari/534.30\x00\aversion\x00\bHTTP/1.1"

The second works in Python, producing:

'\x00\n\x00\x06accept\x00\x03/\x00\x0eaccept-charset\x00\x1eISO-8859-1,utf-8;q=0.7,;q=0.3\x00\x0faccept-encoding\x00\x11gzip,deflate,sdch\x00\x0faccept-language\x00\x0een-US,en;q=0.8\x00\x04host\x00\x0flocalhost:10000\x00\x06method\x00\x03GET\x00\x06scheme\x00\x05https\x00\x03url\x00\x0c/favicon.ico\x00\nuser-agent\x00gMozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.30 Safari/534.30\x00\x07version\x00\x08HTTP/1.1'

In Ruby, the second inflate results in:

ruby-1.9.2-p180 :065 >   FFI::Zlib.inflate(zstream, FFI::Zlib::Z_SYNC_FLUSH)

=> -3