http://stackoverflow.com/questions/35257305/mongoengine-is-very-slow-on-large-documents-comapred-to-native-pymongo-usage Example using Embeded documents

35257305_better.py

import datetime
import itertools
import random
import sys
import timeit
from collections import defaultdict

import mongoengine as db
from pycallgraph.output.graphviz import GraphvizOutput
from pycallgraph.pycallgraph import PyCallGraph

db.connect("test-dicts")


class Data(db.EmbeddedDocument):
    subf0 = db.ListField(db.IntField())
    subf1 = db.ListField(db.IntField())
    subf2 = db.ListField(db.IntField())
    subf3 = db.ListField(db.IntField())
    subf4 = db.ListField(db.IntField())


class MyDictModel(db.Document):
    date = db.DateTimeField(required=True, default=datetime.date.today)
    data_dict_1 = db.DictField()


class MyEmbedModel(db.Document):
    date = db.DateTimeField(required=True, default=datetime.date.today)
    data_dict_1 = db.EmbeddedDocumentField(Data)


MyDictModel.drop_collection()
MyEmbedModel.drop_collection()

data = ["subf{}".format(f) for f in range(5)]

m_dict = MyDictModel()
my_dict = dict([(d, list(random.sample(range(50000), 20000))) for d in data])
m_dict.data_dict_1 = my_dict
m_dict.save()

m_embed = MyEmbedModel()
my_data = Data()
for f in data:
    my_data[f] = list(random.sample(range(50000), 20000))
m_embed.data_dict_1 = my_data
m_embed.save()


def pymongo_dict_doc():
    return db.connection.get_connection()["test-dicts"]['my_dict_model'].find_one()


def pymongo_embed_doc():
    return db.connection.get_connection()["test-dicts"]['my_embed_model'].find_one()


def mongoengine_dict_doc():
    return MyDictModel.objects.first()


def mongoengine_embed_doc():
    return MyEmbedModel.objects.first()


if __name__ == '__main__':
    print("pymongo with dict took {:2.2f}s".format(timeit.timeit(pymongo_dict_doc, number=10)))
    print("pymongo with embed took {:2.2f}s".format(timeit.timeit(pymongo_embed_doc, number=10)))
    print("mongoengine with dict took", timeit.timeit(mongoengine_dict_doc, number=10))
    print("mongoengine with embed took", timeit.timeit(mongoengine_embed_doc, number=10))
    out1 = GraphvizOutput()
    out1.output_file = "viz_embed.png"
    out2 = GraphvizOutput()
    out2.output_file = "viz_dict.png"
    with PyCallGraph(output=out1):
        mongoengine_embed_doc()
    with PyCallGraph(output=out2):
        mongoengine_dict_doc()