bench-citylots.py

import io
from msgpack import fallback, packb
try:
    from msgpack import _unpacker, _packer
    has_ext = True
except ImportError:
    has_ext = False
import timeit


def profile(name, func):
    times = timeit.repeat(func, number=1, repeat=5)
    times = ', '.join(["%5f" % t for t in times])
    print("%-30s %40s" % (name, times))


def unpack_oneshot(mod, data):
    return mod.unpackb(data)


def unpack_feed(mod, data):
    up = mod.Unpacker()
    pos = 0
    while pos < len(data):
        up.feed(data[pos:pos+1000])
        pos += 1000
        for x in up:
            id(x)


def unpack_read(mod, data):
    file = io.BytesIO(data)
    up = mod.Unpacker(file, read_size=1000)
    for x in up:
        id(x)


def unpack_bench(mod, data):
    profile('unpackb', lambda: unpack_oneshot(mod, data))
    #profile('feed', lambda: unpack_feed(mod, data))
    #profile('read', lambda: unpack_read(mod, data))


def simple(name, data):
    data = packb(data, use_bin_type=True)
    if has_ext:
        print("=== cython ===")
        unpack_bench(_unpacker, data)
    print("=== pure Python ===")
    unpack_bench(fallback, data)


def main():
    import json
    import gzip
    # citylots.json is from https://github.com/zemirco/sf-city-lots-json
    with gzip.open('citylots.json.gz') as f:
        data = json.load(f)
    simple("citylots", data)


main()

bytearray.patch

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 181d7e2..3178122 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -196,7 +196,7 @@ class Unpacker(object):
             self._fb_feeding = False
 
         #: array of bytes feeded.
-        self._buffer = b""
+        self._buffer = bytearray()
         #: Which position we currently reads
         self._buff_i = 0
 
@@ -247,9 +247,7 @@ class Unpacker(object):
 
         if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size):
             raise BufferFull
-        # bytes + bytearray -> bytearray
-        # So cast before append
-        self._buffer += bytes(next_bytes)
+        self._buffer += next_bytes
 
     def _fb_consume(self):
         """ Gets rid of the used parts of the buffer. """
@@ -282,7 +280,7 @@ class Unpacker(object):
 
         # Strip buffer before checkpoint before reading file.
         if self._buf_checkpoint > 0:
-            self._buffer = self._buffer[self._buf_checkpoint:]
+            del self._buffer[:self._buf_checkpoint]
             self._buff_i -= self._buf_checkpoint
             self._buf_checkpoint = 0
 
@@ -304,7 +302,7 @@ class Unpacker(object):
         if len(self._buffer) == n:
             # checkpoint == 0
             ret = self._buffer
-            self._buffer = b""
+            self._buffer = bytearray()
             self._buff_i = 0
         else:
             ret = self._buffer[self._buff_i:self._buff_i+n]
@@ -519,11 +517,13 @@ class Unpacker(object):
         if typ == TYPE_RAW:
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
+            else:
+                obj = bytes(obj)
             return obj
         if typ == TYPE_EXT:
-            return self._ext_hook(n, obj)
+            return self._ext_hook(n, bytes(obj))
         if typ == TYPE_BIN:
-            return obj
+            return bytes(obj)
         assert typ == TYPE_IMMEDIATE
         return obj