Example generic HTTPS-capable caching proxy script for mitmproxy

mitmproxy_cacheanything.py

import time
import sys
import os
import re
from static.apps import Cling
from libmproxy.flow import AppRegistry, version
from netlib import wsgi
import requests


def catchall(environ, start_response):
    """Simplest possible application object"""
    url = '%(wsgi.url_scheme)s://%(HTTP_HOST)s%(PATH_INFO)s' % environ

    download_path = 'cache/%(HTTP_HOST)s%(PATH_INFO)s' % environ

    if url.endswith('/'):
        download_path += 'index.html'
    elif re.search('pypi.python\.org/simple/\w+$', url):
        download_path += '/index.html'
        url += '/'

    if not os.path.exists(download_path):
        download_dir = u'/'.join(download_path.split('/')[:-1])

        if not os.path.exists(download_dir):
            os.makedirs(download_dir)

        headers = {}

        if 'HTTP_AUTHORIZATION' in environ:
            headers['Authorization'] = environ['HTTP_AUTHORIZATION']

        r = requests.get(url, stream=True, headers=headers)

        if r.status_code >= 400:
            os.symlink(str(r.status_code), download_path)
        else:
            with open(download_path, 'wb+') as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk: # filter out keep-alive new chunks
                        f.write(chunk)
                        f.flush()

    if os.path.islink(download_path):
        start_response(os.path.readlink(download_path))
        return []
    else:
        return Cling('cache/%(HTTP_HOST)s' % environ)(environ, start_response)


class MyAppRegistry(AppRegistry):
    def get(self, request):
        app = AppRegistry.get(self, request)

        if app:
            return app

        host = request.headers.get('Host')[0]
        return wsgi.WSGIAdaptor(catchall, host, request.port,
            version.NAMEVERSION)


def start(context, argv):
    context.log("start")
    context._master.apps = MyAppRegistry()