OONI Test for Wikipedia Page Filtering

gistfile1.py

# -*- encoding: utf-8 -*-
#
# :authors: Collin Anderson
# :licence: see LICENSE

import random
from twisted.internet import defer
from twisted.python import usage
from urllib import quote, unquote
from ooni.utils import log
from ooni.utils.net import userAgents
from ooni.templates import httpt
from ooni.errors import failureToString, handleAllFailures

class UsageOptions(usage.Options):
    optParameters = [
                     ['remote_host', 'r', None, 'Hostname for Response "onlyreturnstrue.com"'],
                     ['control_body', 'b', 'true', 'Expected Responses'],
                     ['experiment_hostname', 'n', None, 'Remote Server for Response'],
                    ]


class WikipediaTest(httpt.HTTPTest):

    """
        There are two methods of performing this test: 
            1.) Comparing responses between a direct request and one made over a more trusted channel (in http_requests, Tor).
            2.) Comparing responses to an expected answer. This would ideally occur if the resource in question gave predictable answers, however, a server that responses in a fixed way to any hostname and request should also suffice (e.g. onlyreturnstrue.com). This however would only detect hostname and request filtering, missing if returned content triggers a filtering response.
        
        
        
    """
    name = "Wikipedia Requests Test"
    author = "Collin Anderson"
    version = "0.0"

    usageOptions = UsageOptions
    inputFile = ['file', 'f', None, 'Wikimedia provided article titles list to perform requests to.']    
    
    def setUp(self):

        if self.input:
            self.url = self.input
        else:
            raise Exception("No input specified")
    
        if self.localOptions['experiment_hostname']:
            self.experiment_hostname = self.localOptions['experiment_hostname']
        else:
            log.msg("No hostname specified, assuming English Wikipedia.")
            self.experiment_hostname = 'en.wikipedia.org'
        
        if self.localOptions['remote_host']:
            self.remote_host = self.localOptions['remote_host']
        elif self.localOptions['remote_host'] is None and self.experiment_hostname:
            self.remote_host = self.experiment_hostname
        else:
            raise Exception("No Wikipedia hostname or test server specificed.")

        self.control_body = self.localOptions['control_body']

    def test_wikipedia(self):
        def callback(experiment):
            experiment_succeeded, experiment_result = experiment.pop()

            self.report['experiment_succeeded'] = experiment_succeeded
            self.report['correct_content'] = True if experiment_result.body == self.control_body else False

        headers = {
                'User-Agent': [random.choice(userAgents)],
                'Host': [self.experiment_hostname]
            }
        
        experimental_url = "http://%s/%s/%s" % (self.remote_host, 'wiki', quote(self.url))
        log.msg("Performing GET request to %s" % experimental_url)
 
        experiment_request = self.doRequest(experimental_url, method = "GET", headers = headers)

        l = []
        l.append(experiment_request)
        deferredlist = defer.DeferredList(l, consumeErrors=True)
        deferredlist.addCallback(callback)
        
        return deferredlist