symbol-server-proxy.py

#!/usr/bin/env python
from BaseHTTPServer import BaseHTTPRequestHandler
from SocketServer import ThreadingTCPServer
from spotify.util import logging_support
import distutils.spawn
import logging
import os
import requests
import shutil
import signal
import subprocess
import tempfile
import time
import urllib

from boto.s3.connection import S3Connection
from boto.s3.key import Key

logging_support.setup(progname='SocorroSymbolProxy', daemon=True)
logging.basicConfig(level=logging.INFO)
logging.getLogger('boto').setLevel(logging.INFO)
log = logging.getLogger(__name__)

profile_name = 'socorro-symbols'
symbol_bucket = 'com.spotify.socorro2.symbols'
bucket_path = '/v1'

MS_URL = 'http://msdl.microsoft.com/download/symbols/'
MS_USER_AGENT = 'Microsoft-Symbol-Server/6.3.0.0'
MS_DUMP_SYMS_CMD = ['wine64', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'dump_syms.exe')]
MS_CABEXTRACT_CMD = ['cabextract', '-d']

def sigterm_handler(signum, frame):
    # serve_forever will only quit cleanly on SIGINT
    # shutdown() can't be called because of threading issues
    os.kill(os.getpid(), signal.SIGINT)

def ms_fetch_symbol(debug_id, debug_file):
    '''
    Attempt to fetch a PDB file from Microsoft's symbol server.
    '''
    url = MS_URL + "/".join([debug_file, debug_id, debug_file[:-1] + '_'])
    r = requests.get(url, headers={'User-Agent': MS_USER_AGENT})
    if r.status_code == 200:
        return r.content
    return None

def ms_fetch_and_dump_symbols(debug_id, debug_file):
    pdb_bytes = ms_fetch_symbol(debug_id, debug_file)
    if not pdb_bytes or not pdb_bytes.startswith(b'MSCF'):
        return False
    try:
        tmpdir = tempfile.mkdtemp(prefix='symbols')
        pdb_path = os.path.join(tmpdir, debug_file[:-1] + '_')
        with open(pdb_path, 'wb') as f:
            f.write(pdb_bytes)
        # Decompress it
        subprocess.check_call(MS_CABEXTRACT_CMD + [tmpdir, pdb_path])
        pdb_path = os.path.join(tmpdir, debug_file)
        # Dump it
        return subprocess.check_output(MS_DUMP_SYMS_CMD + [pdb_path])
    except subprocess.CalledProcessError:
        return None
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True)

class Handler(BaseHTTPRequestHandler):

    def log_request(self, code='-', size='-'):
        log.info("%s - - %s",
                          self.address_string(),
                          '"%s" %s %s' % (self.requestline, str(code), str(size)))

    def log_error(self, format, *args):
        log.error("%s - - %s",
                          self.address_string(),
                          format % args)

    def redirect_key(self, key):
        if key:
            self.send_response(301)
            self.send_header('Location', key.generate_url(expires_in=60, force_http=True))
            self.end_headers()
        else:
            self.send_response(404)
            self.end_headers()
            self.wfile.write("Not found\n")

    # Find a file on S3 and redirect to a signed URL for it
    def s3_redirect(self, path):
        conn = S3Connection(profile_name=profile_name)
        bucket = conn.get_bucket(symbol_bucket, validate=False)
        k = bucket.lookup(path)
        self.redirect_key(k)

    # Find a symbol on Microsoft Debug server, run dump_syms, uploads to S3 and return a signed URL for it
    def ms_redirect(self, path):
        debug_file, debug_id, _ = path.split('/', 2)
        symbols = ms_fetch_and_dump_symbols(debug_id, debug_file)
        if not symbols:
            self.redirect_key(None)
            return
        conn = S3Connection(profile_name=profile_name)
        bucket = conn.get_bucket(symbol_bucket, validate=False)
        key = Key(bucket, '%s/%s/%s/%s.sym' % (bucket_path, debug_file, debug_id, os.path.splitext(debug_file)[0]))
        key.set_contents_from_string(symbols)
        self.redirect_key(key)

    def do_GET(self):
        if self.path.startswith('/s3/'):
            self.s3_redirect(urllib.unquote(self.path[4:]))
        elif self.path.startswith('/ms/') and self.path.find('.pdb/'):
            self.ms_redirect(urllib.unquote(self.path[4:]))
        else:
            self.redirect_key(None)

class TCPServer(ThreadingTCPServer):
    allow_reuse_address = True

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, sigterm_handler)

    subprocess.check_call(['wine64', 'regsvr32', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'msdia120.dll')])

    server = TCPServer(('localhost', 8080), Handler)
    log.info('Starting server, use <Ctrl-C> to stop or send SIGTERM')
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass

    server.socket.close()

    log.info('Closing')