Proxy to get around authentication requirements for web crawlers

config.yaml

confluence:
  loginUrl: https://confluence.atlassian.com/dologin.action
  loginFields:
    - os_username: ArnoldFacepalmer
    - os_password: password123
    - os_cookie: true
    - login: Log in
    - os_destination: /index.action
  cookies:
    - JSESSIONID

prox_your_sox_off.py

#!/usr/bin/env python
from libmproxy import controller, proxy
import os
import requests
import yaml

configStream = open('config.yaml', 'r')
config = yaml.load(configStream)

class AuthProxy(controller.Master):
    def __init__(self, server, config):
        controller.Master.__init__(self, server)
        self.config = config
        self.cookies = None

    def run(self):
        try:
            self.perform_login()
            return controller.Master.run(self)
        except KeyboardInterrupt:
            self.shutdown()

    def handle_request(self, msg):
        if self.cookies is not None:
            msg.headers['Cookie'] = [self.cookies]

        msg.reply()

    # Initialize a session
    def perform_login(self):
        data = dict()
        for field in self.config['loginFields']:
            for name, value in field.iteritems():
                data[name] = value

        session = requests.Session()
        session.post(self.config['loginUrl'], data)

        # Save our auth cookies
        cookies = list()
        for cookie in self.config['cookies']:
            cookies.append('%s=%s' % (cookie, session.cookies[cookie]))
        self.cookies = ';'.join(cookies) + ';'

confluence = config['confluence']
proxyConfig = proxy.ProxyConfig(
    cacert=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem"),

)
server = proxy.ProxyServer(proxyConfig, 10001)

master = AuthProxy(server, confluence)
master.run()