-
-
Save mhugo/dec469223e578ea7ec94946edcd43e6f to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# coding: utf-8 | |
from __future__ import print_function | |
import resource | |
import gc | |
import time | |
import six | |
# set IJSON_BACKEND=yajl2_c or yajl2_cffi | |
import ijson | |
def memusage(): | |
"""memory usage in MB. getrusage defaults to KB on Linux.""" | |
return str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e3) | |
if six.PY2: | |
from cStringIO import StringIO as BytesIO | |
else: | |
from io import BytesIO | |
if __name__ == "__main__": | |
print("using backend", ijson.backend) | |
print("using ijson version", ijson.__version__) | |
print("starting memory usage:", memusage(), 'MB') | |
t_start = time.time() | |
for i in range(400000): | |
data = BytesIO(b"""{"blah": {}}""") | |
for item in ijson.items(data, "blah"): | |
pass | |
print("spent time: {:.2f}".format(time.time() - t_start)) | |
print("memory usage after ijson calls:", memusage(), 'MB') | |
gc.collect() | |
print("memory usage after garbage collection:", memusage(), 'MB') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment