Read/write arbitrary data to Mongo

mongo_hungry.py

from pymongo.son_manipulator import SONManipulator
from bson import Binary
import numpy as np
import pickle


NUMPY_SUBTYPE = 128


class Transform(SONManipulator):
    def transform_incoming(self, son, collection):
        for (key, value) in son.items():
            if isinstance(value, np.ndarray):
                son[key] = Binary(pickle.dumps(value, protocol=-1), 
                                  subtype=NUMPY_SUBTYPE)
            elif isinstance(value, dict): # Make sure we recurse into sub-docs
                son[key] = self.transform_incoming(value, collection)
        return son

    def transform_outgoing(self, son, collection):
        for (key, value) in son.items():
            if isinstance(value, Binary) and value.subtype == NUMPY_SUBTYPE:
                son[key] = pickle.loads(value)
            elif isinstance(value, dict):
                # Again, make sure to recurse into sub-docs
                son[key] = self.transform_outgoing(value, collection)
        return son


if __name__ == '__main__':
    
    from pymongo import Connection

    conn = Connection()
    db = conn.test_db
    db.add_son_manipulator(Transform())

    bdata = np.ones((8000, 1000), dtype=np.uint8)

    data = {"name": "Alyssa",  "yo": [None, "hey"],
            "favorite_number": 256, "fixed_big": bdata,
            'inner': dict(b=10, inner2=dict(foo=3))}

    import time
    t0 = time.time()
    test2 = db.test2
    test2.insert(data)

    data['name'] = 'Bob'
    data['favorite_number'] = 'forty'
    test2.insert(data)
    print(test2.find_one())
    print(time.time() - t0)

    for i in range(30):
        data['name'] = str(i)
        test2.insert(data)
    print(time.time() - t0)