TheBeege/twisted_http_server.py

## twisted_http_server.py
#!/usr/bin/env python2.7

# README:
# So this puppy won't run as is.
# I've commented out the LogAdapter lines that assign
# the logger var. This is because I strongly, strongly
# recommend you have good logging, take the time to
# setup your logger properly, and customize your
# own logging format. Your operations folks will
# thank you (assuming you log useful stuff).

# You should actually start reading this towards the
# bottom AFTER all the objects have been defined.
# It'll make more sense.
# HINT: Search for OH HAI


# Native Python imports
import cgi
import logging
import logging.handlers
import sys
import simplejson
import signal
import ConfigParser
from zope.interface import Interface, Attribute, implements
from datetime import datetime

# Twisted imports
from twisted.internet import epollreactor
epollreactor.install()

from twisted.application import internet, service
from twisted.cred import error as credError
from twisted.cred.credentials import IUsernameHashedPassword, IUsernamePassword
from twisted.cred.checkers import ICredentialsChecker
from twisted.cred.portal import Portal, IRealm
from twisted.enterprise import adbapi
from twisted.internet import ssl, reactor
from twisted.internet.defer import Deferred
from twisted.python.components import registerAdapter
from twisted.python.log import PythonLoggingObserver
from twisted.web.http import HTTPChannel
from twisted.web.guard import HTTPAuthSessionWrapper
from twisted.web.guard import DigestCredentialFactory
from twisted.web.guard import BasicCredentialFactory
from twisted.web.server import Site, Session
from twisted.web.static import Data
from twisted.web.resource import Resource, IResource

# Here's the Logger Adapter we'll be using:
class MBLogAdapter(logging.LoggerAdapter):

  # Super simple. It just adds the provided params at instantion to the
  # log message. This will make more sense when you see it in action.
  def process(self, msg, kwargs):
    return '%s - %s - %s - %s' % (self.extra['file'], self.extra['object'], self.extra['method'], msg), kwargs


# If you don't get what this does, do some reading on HTTP and sessions.
# I'd link a tutorial myself, but there are plenty out there of various styles.
# If you're not familiar with Python, the class name in parentheses a class
# we want this new class to inherit from.
class MySession(Session):
  sessionTimeout = 2592000 # 30 days

# This is a custom interface that will allow us to store a user object in
# a given session.
class IUser(Interface):
  user_id = Attribute("The id from the database representing the current user.")

# This is the aforementioned user object. We can place arbitrary values into this
# object that we can retrieve anytime a user is authenticated. In our case,
# we're storing a user object that has an id, which is pulled from our back-end DB.
class User(object):
  # If you're not familiar with zope, zope permits interfaces. Python doesn't
  # natively support traditional interfaces. Zope fixes that, and this is how.
  implements(IUser)

  # If you're not familiar with Python, the __init__ method is an object's constructor.
  # You'll find that every function takes in "self" as its first parameter. This is
  # because Python can be written in an object oriented or scripted fashion. The
  # self parameter is passing an instance of the object to the function.
  # TL;DR: always include "self" as the first parameter for an instance method
  def __init__(self, session):
    self.baseLogger = logging.getLogger('MyApp.User')
    # Say hello to our custom Log Adapter. I recommend reading up on Log Adapters in Python's documentation.
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'User', 'method': '__init__'})
    logger.info('Creating new User')

    logger.debug('session="%s"', str(session))
    # You can access instance variables on the "self" object. If "self" didn't make
    # sense before, hopefully this helps.
    self.user_id = 0

# This line injects our user object to our custom session object so that we
# can actually retrieve it.
registerAdapter(User, MySession, IUser)

# While I'm cutting out our proprietary stuff, I'm leaving some of the
# more generic things in here for you to use as a reference and for
# you to give us feedback if we're doing something dumb and don't
# realize it.

# This is our Registration class. In Twisted, Resources are essentially
# HTTP endpoints (read as, web pages). Our Registration class inherits
# from Twisted's Resource class and will hold the logic executed when
# someone accesses the Registration resource. We'll cover more on how
# accessing resources works later.
class Registration(Resource):

  baseLogger = None

  def __init__(self):
    self.baseLogger = logging.getLogger('MB-Server.Registration')
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'Registration', 'method': '__init__'})
    logger.info('Creating new Registration')

    # This is how you call the parent class's constructor IF the parent class is an old-style object.
    # Curious about old-style? It's a super annoying Python quirk. Just call parent class constructors
    # using this syntax when using Twisted classes. They're all old-style.
    Resource.__init__(self)

  # This is where Twisted makes things easy. If you're not familiar with HTTP methods, you should
  # read up on that before continuing.
  # Twisted uses these render_* methods on resources to respond to different HTTP methods. For example,
  # you can define render_GET, render_POST, render_PUT, and others, and Twisted will just pick up these
  # methods and execute them in response to the appropriate HTTP method.
  # Long story short, when someone hits the Registration resource using the "POST" HTTP method, this
  # method will be executed.
  def render_POST(self, request):
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'Registration', 'method': 'render_POST'})

    logger.debug('request="%s"', str(request))

    # simplejson (sometimes just json, depending on Python version) is a super useful module
    # that will "load" JSON into Python dictionaries and "dump" Python dictionaries to JSON
    # text. I recommend it.
    # request is the Request object as delivered by Twisted. You can use
    # request.content.getvalue() to access the body of the HTTP request.
    # Now, using JSON as a POST method's body is NOT standard HTTP. We decided we wanted to
    # use JSON instead of the typical URL encoding used in standard POST. Rage all you want;
    # it's easier to pass complex data.
    jsonContents = simplejson.loads(request.content.getvalue())

    username = jsonContents['username']
    password = jsonContents['password']
    email = jsonContents['email']

    # Don't worry, in our final, we're removing password from this logging statement.
    # We have a big, bold, highlighted task on the backlog to do this.
    logger.debug('username="%s" password="%s" email="%s"', username, password, email)

    # TODO: Your custom registration logic/method call/whatever here!

    # This is the inverse of the other JSON object above. This time, Python dictionary
    # to JSON text. It's that easy.
    response_text = simplejson.dumps({ 'status': output })

    # You can set your response's header key-value pairs by using
    # setHeader on the request. Slightly confusing logically, but it's easy.
    request.setHeader("content-type", "application/json")
    # You can also manually set the response code like so:
    request.setResponseCode(200)
    logger.debug('returning response="%s"', response_text)
    # Whatever text you return will be the HTTP response's body (read as, web page)
    return response_text

# This is a Credentials Checker we ripped from the core examples that utilizes
# credentials stored in a database to validate.
# I strongly recommend reading through and understanding it. We actually did a
# little of our own customization.
# It's also forcibly broken, so you'll need to go through it anyway :-P
# http://twistedmatrix.com/documents/current/core/examples/dbcred.py
class DBCredentialsChecker(object):
  implements(ICredentialsChecker)

  baseLogger = None

  def __init__(self, runQuery,
      query="""
      CALL checkLogin(%s);
      """, customCheckFunc=None, caseSensitivePasswords=True):
    """@param runQuery: This will be called to get the info from the db.
       Generally, you'd want to create a L{twisted.enterprise.abdapi.ConnectionPool}
       and pass its runQuery method here. Otherwise pass a function with the same prototype.
       @type runQuery: C{callable}

       @param query: query used to authenticate user.
       @type query: C{str}

       @param customCheckFunc: Use this if the passwords in the db are stored as hashed. We'll just call this, so you can do the checking yourself. It takes the following params: (username, suppliedPass, dbPass) and must return a boolean.
       @type customCheckFunc: C{callable}

       @param caseSensitivePasswords: If true requires that every letter in C{credentials.password} is exactly the same case as its counterpart character in the database. This is only relevant if C{customCheckFunc} is not used.
       @type caseSensitivePasswords: C{bool}
    """

    #set_trace()
    self.baseLogger = logging.getLogger('MB-Server.DBCredentialsChecker')
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'DBCredentialsChecker', 'method': '__init__'})
    logger.debug('Starting method...')
    logger.debug('runQuery="%s" caseSensitivePasswords="%s" customCheckFunc="%s" query="%s"', str(runQuery), str(caseSensitivePasswords), str(customCheckFunc), str(query))

    self.runQuery = runQuery
    self.caseSensitivePasswords = caseSensitivePasswords
    self.customCheckFunc = customCheckFunc
    # We can't support hashed password credentials if we only have a hash in the DB
    if customCheckFunc:
      self.credentialInterfaces = (IUsernamePassword,)
    else:
      self.credentialInterfaces = (IUsernamePassword, IUsernameHashedPassword,)

    self.sql = query

  def requestAvatarId(self, credentials):
    """
    Authenticates the kiosk against the database.
    """
    #set_trace()
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'DBCredentialsChecker', 'method': 'requestAvatarId'})
    logger.debug('Starting method...')
    logger.debug('credentials="%s"', str(credentials))
    # Check that the credentials instance implements at least one of our interfaces
    for interface in self.credentialInterfaces:
      if interface.providedBy(credentials):
        break
      else:
        raise credError.UnhandledCredentials()

    # Ask the database for the username and password
    logger.debug('sql="%s" credentials.username="%s"', str(self.sql), str(credentials.username))

    import ConfigParser
    config = ConfigParser.RawConfigParser()
    config.read('server.conf')

    saltVal = config.get('server','salt')
    # The hashing algorithm we use is slightly obscure, and we would
    # prefer to keep it that way.
    import REDACTED
    passwordHash = REDACTED.crypt(credentials.password, saltVal, 50)
    credentials.password = passwordHash

    dbDeferred = self.runQuery(self.sql, (credentials.username, ))
    # Setup our deferred result
    deferred = Deferred()
    dbDeferred.addCallbacks(self._cbAuthenticate, self._ebAuthenticate,
      callbackArgs=(credentials, deferred),
      errbackArgs=(credentials, deferred))
    return deferred

  def _cbAuthenticate(self, result, credentials, deferred):
    """
    Checks to see if the authentication was good. Called once the info has been retrieved from the DB.
    """

    #set_trace()
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'DBCredentialsChecker', 'method': '_cbAuthenticate'})
    logger.debug('Starting method...')
    logger.debug('result="%s" credentials="%s" deferred="%s"', str(result), str(credentials), str(deferred))
    logger.debug('credentials.username="%s" credentials.password="%s"', credentials.username, credentials.password)
    if len(result) == 0:
      # Username not found in db
      deferred.errback(credError.UnauthorizedLogin('Username unknown'))
    else:
      username, password = result[0]
      if self.customCheckFunc:
        # Let the owner do the checking
        if self.customCheckFunc(
            username, credentials.password, password):
          deferred.callback(credentials.username)
        else:
          deferred.errback(credError.UnauthorizedLogin('Password mismatch'))
      else:
        # It's up to us or the credentials object to do the checking now
        if IUsernameHashedPassword.providedBy(credentials):
          logger.debug('IUsernameHashedPassword.providedBy(%s) returned true', str(credentials))
          # Let the hashed password checker do the checking
          if credentials.checkPassword(password):
            deferred.callback(credentials.username)
          else:
            deferred.errback(credError.UnauthorizedLogin('Password mismatch'))
        elif IUsernamePassword.providedBy(credentials):
          logger.debug('IUsernamePassword.providedBy(%s) returned true', str(credentials))
          # Compare the passwords, deciding whether or not to use case sensitivity
          if self.caseSensitivePasswords:
            passOk = (password.lower() == credentials.password.lower())
          else:
            passOk = password == credentials.password

          # See if they match
          if passOk:
            deferred.callback(credentials.username)
          else:
            deferred.errback(credError.UnauthorizedLogin('Password mismatch'))
        else:
          # OK, we don't know how to check this
          deferred.errback(credError.UnhandledCredentials())

  def _ebAuthenticate(self, message, credentials, deferred):
    """
    The database lookup failed for some reason.
    """
    #set_trace()
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'DBCredentialsChecker', 'method': '_ebAuthenticate'})
    logger.debug('Starting method...')
    logger.debug('message="%s" credentials="%s" deferred="%s"', str(message), str(credentials), str(deferred))
    deferred.errback(credError.LoginFailed(message))

# This realm is for resources that should be password-protected.
class HttpPasswordRealm(object):
  # Implementing IRealm is what makes most of the above-mentioned magic happen
  implements(IRealm)

  baseLogger = None

  # What's authWall? A resource we created that will hide other resources that require
  # authentication to get access to.
  def __init__(self, authWall):
    self.baseLogger = logging.getLogger('MB-Server.HttpPasswordRealm')
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'HttpPasswordRealm', 'method': '__init__'})
    logger.info('Creating new HttpPasswordRealm')
    # We hold onto the authWall so that we can just re-pass it
    # when we need to authenticate a user. More below.
    self.authWall = authWall

  # This method is responsible for initiating the request for credentials.
  # I'm not exactly sure how...?
  def requestAvatar(self, user, mind, *interfaces):
    logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'HttpPasswordRealm', 'method': 'requestAvatar'})
    logger.debug('requestAvatar(%s, %s, %s) called', str(user), str(mind), str(interfaces))

    if IResource in interfaces:
      logger.debug('IResource="%s"', str(IResource))

      # authWall is passed regardless of user
      # This puppy goes, "Oh, you're in this realm? Well, here's the Resource you should get"
      # ...I think
      return (IResource, self.authWall, lambda: None)
    raise NotImplementedError()

class AuthWall(Resource):

  baseLogger = None

  def __init__(self):
    self.baseLogger = logging.getLogger('MB-Server.AuthWall')
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'AuthWall', 'method': '__init__'})
    logger.info('Creating new AuthWall')

    Resource.__init__(self)

  # Remember how we mentioned authWall hides the other resources? You can create child resources (read as,
  # sub-pages) using the getChild method. In authWall's case, it will just grab a session and pass the
  # requestor through to the child resources, assuming the user has an authenticated session.
  def getChild(self, path, request):
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'AuthWall', 'method': 'getChild'})
    session = request.getSession()
    logger.debug('session="%s"', str(session))

    Resource.getChild(self, path, request)

# This Login Resource will service as our Login endpoint. Now, we're
# doing weird stuff here. All this guy does is add the user object
# to the user's session and return. We use this as an opportunity
# to record the user's ID in their session (removed from here).
# HTTP Basic Authentication actually handles the passing of
# credentials.
class Login(Resource):

  baseLogger = None

  def __init__(self):
    self.baseLogger = logging.getLogger('MB-Server.Login')
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'Login', 'method': '__init__'})
    logger.info('Creating new Login')

    Resource.__init__(self)

  def render_GET(self, request):
    #logger = MBLogAdapter(self.baseLogger, {'file': 'main.tac', 'object': 'Login', 'method': 'render_GET'})

    # Here, we grab the user session corresponding to the request
    # hitting this page.
    session = request.getSession()
    # We create an instance of a new user.
    # I know this -looks- like we're instantiating an
    # interface, but remember that registerAdapter call
    # we made towards the top of the file? It does some
    # weird stuff under the covers, creating a normal
    # User object and binding it to the provided session.
    user = IUser(session)
    logging.debug('user="%s"', str(user))

    # Since we're using HTTP Basic Auth, the username is
    # actually accessible in the request. Twisted provides
    # this easy method to extract the username from the
    # auth header.
    username = request.getUser()
    # I don't know why we do this twice...? I thought
    # I had a comment here at some point describing it.
    user = IUser(session)

    # TODO: Your code here for logging in logic!

    # You've seen this stuff before
    request.setHeader("content-type", "application/json")
    request.setResponseCode(200)
    logger.debug('returning response="%s"', response_text)
    return response_text


# OH HAI! Now that all the objects are over, let's do stuff that
# should execute when we initially try to start the server.

# CONFIGS
# ConfigParser is super useful for using configuration files.
# One caveat: if a file doesn't exist, it will fail when you try to
# access a configuration value, not when you try to read the file.
# Why? You can write out configs with ConfigParser, too.
config = ConfigParser.RawConfigParser()
config.read('server.conf')

# For this example, the config file may look like this:
# [server]
# port = 80
# hostname = localhost
PORT = int(config.get('server','port'))

# LOGGING
# In Python, you have five steps to setup logging:
# 1) Get a logger for your application
# 2) Create a formatter(s)
# 3) Create a handler(s)
# 4) Set the formatter for your handler(s)
# 5) Set the handler(s) for your logger
# Loggers follow a hierarchy according to dot structure.
# So I could have logging.getLogger('MyApp'), and then
# have logging.getLogger('MyApp.MyChild'). The
# MyApp.MyChild logger would inherit any handlers
# (and therefore, formatters) that MyApp would have.
# For a brief on what a formatter can contain and
# what handlers are available, check out the
# Python doc site.
log = logging.getLogger('MB-Server')
# We're gonna write the ascii time, the logging level (DEBUG,
# INFO, WARN, ERROR, etc), the thread id, and whatever
# message is being passed to the logger to be written.
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(thread)d - %(message)s')

# The StreamHandler writes output to the console by default.
consoleHandler = logging.StreamHandler()
# Set the StreamHandler to use the formatter we defined
# above. You can have multiple handlers for a logger, but
# you can only have one formatter per handler.
consoleHandler.setFormatter(formatter)
# Set the level (it will inherit this from the logger
# if not defined).
consoleHandler.setLevel(logging.DEBUG)
# Add the handler to the logger
log.addHandler(consoleHandler)

# We're adding another handler, this time for writing to a
# rotating file.
fileHandler = logging.handlers.RotatingFileHandler('log/server.log', maxBytes=52428800) # 50 MB files
fileHandler.setFormatter(formatter)
fileHandler.setLevel(logging.DEBUG)
log.addHandler(fileHandler)

log.setLevel(logging.DEBUG)
# A log adapter allows you to set specific, customizable
# additional options for a logger. You can use this to
# reuse the same logger (handlers, formatters) while only
# changing a couple of parameters.
#log = MBLogAdapter(log, {'file': 'main.tac', 'object': 'raw', 'method': 'init'})

# This will cause Twisted's built-in logging to utilize
# the native Python logging. The constructor for
# PythonLoggingObserver works much like getLogger.
# It will inherit the settings we've made above.
twisted_logger = PythonLoggingObserver('MB-Server.Twisted')
twisted_logger.logger.setLevel(logging.DEBUG)
# One difference is that we need to explicitly start
# the Twisted logger.
twisted_logger.start()

# When logging, you can do substitution using %s,
# then pass your variables to sub in as additional
# parameters. The substition order is positional.
log.info('Server starting on port="%s"...', PORT)

# SERVER SETUP
# A resource in Twisted is an HTTP endpoint - a location
# that an entity can send a request to.
# In this case, we're not actually publishing any
# content at the site root, so we're going to just
# supply a blank Resource
root = Resource()

# The AuthWall and Login classes are resources that we define above.
# The AuthWall is a Resource that requires child resources
# to have an authenticated session open on the server,
# read as "logged in."
auth_wall = AuthWall()
# The Login is a Resource that allows a user to login (surprise!)
login = Login()

# Now, we did something funky. You may not want to do it this
# way. We're using Basic HTTP authentication:
# See http://www.ietf.org/rfc/rfc2617.txt at line 242
# Or even better: http://www.httpwatch.com/httpgallery/authentication/
# All the Login page does is return success if you can hit
# it. The HTTP protocol handles the actual passing of
# credentials from client to server through Basic Auth.
# This specific line places the Login Resources behind
# the auth wall resource, so the URL would look like
# this: "chainsawrabbitgames.com/<path_to_authWall>/login"
auth_wall.putChild('login', login)

# Instead of doing database connections directly, we utilize
# Twisted's connection pooling feature. Twisted will create
# and manage a pool of database connections efficiently
# for the various threads it will open to support requests.
pool = adbapi.ConnectionPool('MySQLdb',config.get('database','host'),config.get('database','user'),config.get('database','password'),config.get('database','database'))

# This instantiates a DB-based credentials checker. It takes
# in the DB connection pool instance and an option for
# what query to use to grab the credentials to be matched.
checker = DBCredentialsChecker(pool.runQuery, query="SELECT `name`, password FROM user WHERE `name` = (%s)")

# An HTTP Realm is a "space" where sessions and other server-side stuff can live segregated.
# For example, you could have a store and a game account on the same server and require
# different credentials for each using two different realms.
# This defines a realm where the auth wall and all child resources
# will live.
realm = HttpPasswordRealm(auth_wall)
# Portals are how you tell Twisted, "Hey! User auth is required here!"
p = Portal(realm)
# We register our DB Credentials Checker with the portal so that
# the portal actually knows how to authenticate its users.
p.registerChecker(checker)
log.debug('p="%s"', str(p))

# I'm not sure as to the utilit of this, but it's required.
credentialFactory = BasicCredentialFactory("Chainsaw Rabbit Games")
log.debug('credentialFactory="%s"', str(credentialFactory))
# To tie all the auth stuff together, we use an auth session wrapper
# to bind the portal to the credential factory.
protected_resource = HTTPAuthSessionWrapper(p, [credentialFactory])
log.debug('protected_resource="%s"', str(protected_resource))

# Now that the auth wall is all configured, we can finally
# add resources to our root path. This will create two new
# "pages":
# chainsawrabbitgames.com/register
# chainsawrabbitgames.com/auth_wall
# Since we already defined login as a child of auth_wall,
# we'll also have:
# chainsawrabbitgames.com/auth_wall/login
root.putChild('register', Registration())
root.putChild('auth_wall', protected_resource)

# I'm not sure why the docs called this factory.
# Here, we actually create the site using our
# root Resource as the... well, the root.
factory = Site(root)
# We're using HTTP...
factory.protocol = HTTPChannel
# We defined our own custom session object so
# that we can fine-tune things line session
# expiration. See above.
factory.sessionFactory = MBSession

log.debug('listCredentialsInterfaces="%s"', str(p.listCredentialsInterfaces()))

# If we define a globally-scoped variable named
# "application", we can utilize the twistd daemon.
# This will allow us to run the server in the
# background and not tied to a user. A must-have
# for user-facing systems. We just gotta give a name.
application = service.Application('MonsterBrawl')

# If you want to make it an SSL server, use the
# commented out lines, where the server.key and
# server.crt are a private key and server
# certificate, respectively, that the server
# can use for establishing SSL (HTTPS) comms.
#game_service = internet.SSLServer(PORT, factory,
#                  ssl.DefaultOpenSSLContextFactory(
#                    'ssl/server.key', 'ssl/server.crt'))
# If you don't want SSL, just do this.
# To tie things in, "factory" is our Site, which has
# the root Resource and all the children.
game_service = internet.TCPServer(PORT, factory)

log.debug('game_service="%s"', game_service)

# Lastly, setting our game_service as a child
# of the application will cause twistd to
# run our server when we invoke it.
game_service.setServiceParent(application)

# Once you've got all your customizations made
# and things fixed, you can run the server with
# twistd -y filename.tac
# Questions? Hit us up via any of the
# following channels:
# https://www.facebook.com/ChainsawRabbitGames
# @ChainsawRabbitG
# http://chainsawrabbitgames.com