urlfetch mem test

ndb_urlfetch.py

import sys, os
sys.path.insert(0, os.environ.get('GOOGLE_APPENGINE_SDK_PATH'))
sys.path.insert(0, os.environ.get('GOOGLE_APPENGINE_NDB_PATH'))
sys.modules.pop('google', None)
import dev_appserver
dev_appserver.fix_sys_path()
from google.appengine.ext import testbed
from google.appengine.api import urlfetch_errors
from google.appengine.runtime import apiproxy_errors
#from guppy import hpy
#from meliae import scanner
import logging
import time
import thread
import threading
logging.basicConfig(level=5,
                    #format='%(levelname)-8s %(message)s',
                    format='%(asctime)s %(levelname)-8s %(message)s',
                    datefmt='%H:%M:%S',
                    filename='/tmp/ndb.%d.log' % int(time.time()),
                    filemode='w')
t = testbed.Testbed()
os.environ['APPLICATION_ID'] = 'foo'
t.activate()
t.init_urlfetch_stub()
#t.init_datastore_v3_stub(use_sqlite=True, datastore_file='100GzipText.sqlite')
#t.init_memcache_stub()
import ndb
import gc, inspect, weakref, pprint
from pprint import pprint as pp


def mem():
  #gc.collect()
  #time.sleep(1) # Sometimes cleanup happen asynchronously...
  t = 0
  for l in open('/proc/%d/smaps' % os.getpid()):
    if l.startswith('Private'):
      t += int(l.split()[1])
  return t


@ndb.tasklet
def doFetch():
  ctx = ndb.get_context()
  try:
   f = ctx.urlfetch(":-(", method='POST', payload='a'*1024*1024)
   f.get_result().content
  except (urlfetch_errors.InvalidURLError,):# apiproxy_errors.RequestTooLargeError):
    pass

def run(iters=1, threaded=False, pool=False, toplevel=False, samples=1):
  fun = ndb.toplevel(doFetch) if toplevel else doFetch
  if pool:
    from multiprocessing.pool import ThreadPool
    TPool = ThreadPool(1)
  for i in xrange(1, iters + 1):
    if pool:
      TPool.apply(fun)
    elif threaded:
      t = threading.Thread(target=fun)
      t.start(); t.join()
    else:
      fun()
    if samples and i % int(iters / samples + .9999) == 0:
      print 'mem %d %.3f' % (i, mem() / 1024.0)