Skip to content

Instantly share code, notes, and snippets.

@chipx86
Last active April 22, 2016 21:07
Show Gist options
  • Save chipx86/ae3cf4e7cf353b69b1ba2c96f123852a to your computer and use it in GitHub Desktop.
Save chipx86/ae3cf4e7cf353b69b1ba2c96f123852a to your computer and use it in GitHub Desktop.
#!/usr/bin/env python
from __future__ import unicode_literals
import gc
import os
import sys
import psutil
import time
from io import BytesIO
process = psutil.Process(os.getpid())
last_time = time.clock()
print 'RSS: %s bytes' % process.memory_info().rss
print
print 'Reading lines from data file'
with open('datafile', 'r') as fp:
src_lines = fp.readlines()
print '%s lines' % len(src_lines)
now = time.clock()
print 'Elapsed: %s seconds' % (now - last_time)
last_time = now
print 'RSS: %s bytes' % process.memory_info().rss
print
if sys.argv[1] == 'concat':
print 'Concatenating strings'
data = b''
for line in src_lines:
data += line + b'\n'
print 'RSS: %s bytes' % process.memory_info().rss
elif sys.argv[1] == 'listjoin':
print 'Building and joining list'
data = []
for line in src_lines:
data.append(line)
data.append('\n')
print 'RSS: %s bytes' % process.memory_info().rss
now = time.clock()
print 'Elapsed: %s seconds' % (now - last_time)
last_time = now
print 'RSS (new list, pre-join): %s bytes' % process.memory_info().rss
data = b''.join(data)
elif sys.argv[1] == 'bytesio':
print 'Building BytesIO'
data = BytesIO()
for line in src_lines:
data.write(line)
data.write(b'\n')
print 'RSS: %s bytes' % process.memory_info().rss
data = data.getvalue()
now = time.clock()
print 'Elapsed: %s seconds' % (now - last_time)
gc.collect()
print 'RSS: %s bytes' % process.memory_info().rss
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment