google appengine dev_appserver commands automated with paver.

dev_appservers.yaml

default:
  args:
    url: localhost
    port: 8888
    address: 0.0.0.0
    partition: s
  flags:
    - use_sqlite
    - high_replication
    - allow_skipped_files
    - disable_task_running
    - skip_sdk_update_check
    - disable_static_caching

pavement.py

import os
import yaml
import time
import shutil
import ConfigParser
from paver.easy import *
from jinja2 import Environment
from jinja2.loaders import DictLoader


project = Bunch(
    basedir=path('.'),
    srcdir=path('./src'),
    gae_datadir=path('./.data'),
    gae_blobstoredir=path('./.data/blobstore'),
    gae_datastoredir=path('./.data/datastore'),
    gae_datastorepath=path('./.data/datastore/dev_appserver.sqlite'),
)


@task
@needs('clean_datastore')
@needs('clean_blobstore')
def bootstrap():
  """Creates the data directories for the local App Engine SDK server."""
  project.gae_datadir.mkdir()
  project.gae_datastoredir.mkdir()
  project.gae_blobstoredir.mkdir()
  print('---> bootstrap success\n')


@task
def clean_datastore():
  """Cleans the local App Engine SDK server datastore."""
  # todo: this does not work!
  sh('rm -rf {}'.format(project.gae_datastoredir), cwd=project.basedir)
  project.gae_datastoredir.mkdir()
  # an error is raised if the file doesn't exist, so create an empty placeholder..
  sh('touch dev_appserver.sqlite', cwd=project.gae_datastoredir)


@task
def clean_blobstore():
  """Cleans the local App Engine SDK server blobstore."""
  sh('rm -rf {}'.format(project.gae_blobstoredir), cwd=project.basedir)
  project.gae_blobstoredir.mkdir()


def get_dev_appserver_config(id):
  f = open('dev_appservers.yaml', 'r')
  config = yaml.safe_load(f).get(id)
  f.close()
  config['args']['blobstore_path'] = project.gae_blobstoredir
  config['args']['datastore_path'] = project.gae_datastorepath
  return config


@task
@cmdopts([
  ('config_id=', 'c', '''Name of the configuration profile, defined in
dev_appserver.yaml, to run the server with.''')
])
def serve(options):
  """Starts a Google App Engine server for local development."""
  config_id = 'default'
  if hasattr(options, 'config_id'):
    config_id = options.config_id
  config = get_dev_appserver_config(config_id)
  command = build_dev_appserver_cmd(config)
  try:
    # capture the process id, so we can manage by writing
    # the process id to ".pid"..
    sh('''
nohup {} >/var/tmp/dev_appserver.log 2>.dev_appserver &
PID=$!
echo $PID > .pid
    '''.format(command), cwd=project.basedir)
    # this command can be run to clear the terminal..
    # /usr/bin/open -a Terminal
    # /usr/bin/osascript -e 'tell application "System Events" to tell process "Terminal" to keystroke "k" using command down'
  except:
    print '''An error occurred and the server failed to start.'''


@task
def open_admin(options):
  """Opens the Google App Engine SDK admin console."""
  config_id = 'default'
  if hasattr(options, 'config_id'):
    config_id = options.config_id
  config = get_dev_appserver_config(config_id)
  sh('open http://{url}:{port}/_ah/admin'.format(**config.get('args')),
    cwd=project.basedir)


@task
def stop_server():
  """Stops the local Google App Engine SDK server."""
  f = project.basedir / '.pid'
  try:
    pid = f.text().strip()
    if pid and len(pid) > 0:
      result = sh('kill -9 {} && rm -f .pid'.format(pid),
        cwd=project.basedir, ignore_error=True, capture=True)
      if 'No such process' in result:
        raise ValueError('No such process')
  except (ValueError, IOError):
    pid = sh('rm -f .pid && ps -ef | grep "dev_appserver.py" | awk "{print $2}"',
      capture=True).strip().split(' ')[1]
    sh('kill -9 {}'.format(pid),
      cwd=project.basedir, ignore_error=True, capture=True)


@task
@cmdopts([
  ('config_id=', 'c', '''Name of the configuration profile, defined in
dev_appserver.yaml, to run the server with.''')
])
def restart_server(options):
  """Restarts the local Google App Engine SDK server."""
  stop_server()
  serve(options)


@task
def tail():
  """View the dev_appserver logs by running the unix native "tail" command.

    :example:
      tail -f ./.dev_appserver
  """
  sh('tail -f {}'.format('.dev_appserver'), cwd=project.basedir)


watch_ignore = [
  '.git',
  '.data',
  'lib',
  'public',
  'generators',
  'node_modules',
  'reference_data',
]

def get_watched(root):
  result = []
  for root, dirs, files in os.walk(root, followlinks=False):
    for wi in watch_ignore:
      if wi in dirs:
        dirs.remove(wi)
    result.extend([
      os.path.join(root, f) for f in files
        if f.endswith('.py') or
           f.endswith('.cfg') or
           f.endswith('.yaml') or
           f.endswith('.coffee')
    ])
  return result


@task
@cmdopts([
  ('config_id=', 'c', '''Name of the configuration profile, defined in
dev_appserver.yaml, to run the server with.''')
])
def watch(options):
  """Watches file system for changes and restarts the server."""
  # todo: this currently only supports changes to files, need to add in
  # support for addition & removal of dirs/files.
  stop_server()
  clean()
  build()
  serve(options)
  root = os.getcwd()
  print "Watching for changes in", root, "..."
  watched = [{"file": w} for w in get_watched(root)]
  for w in watched:
    w["output"] = os.path.splitext(w["file"])[0] + ".out"
    w["modtime"] = os.path.getmtime(w["file"])
    # print "  watching:", w["file"]
  while True:
    # sleep for a while
    time.sleep(0.5)
    # check if anything changed
    for w in watched:
      # http://timgolden.me.uk/python/win32_how_do_i/watch_directory_for_changes.html
      # http://stackoverflow.com/questions/182197/how-do-i-watch-a-file-for-changes-using-python
      # is mod time of file is newer than the one registered?
      # os.stat(filename).st_mtime
      if os.path.getmtime(w["file"]) > w["modtime"]:
        # store new time and...
        w["modtime"] = os.path.getmtime(w["file"])
        print w["file"], "has changed, restarting server..."
        # response to events here..
        if w["file"].endswith('.py'):
          # restart the sdk server for python changes..
          restart_server()
        else:
          # rerun the watcher for building the ui..
          return watch(options)


@task
@cmdopts([
  ('email=', 'e', 'Email address of the Google App Engine account.')
])
def deploy(options):
  """Deploys the app to Google App Engine production servers."""
  command = 'appcfg.py update --oauth --email={} .'.format(options.email)
  sh(command, cwd=project.basedir)