reportclean.py

# This is an example of how to parse ooniprobe reports

import yaml
import sys
import os
import shutil
from glob import glob
from tempfile import mkstemp

import pygeoip

asnum = pygeoip.GeoIP('/Users/x/code/networking/bgp/bgpfoo/data/GeoIPASNum.dat')
ipd = pygeoip.GeoIP('/Users/x/code/networking/bgp/bgpfoo/data/GeoIP.dat')
ip_address = '203.81.70.29'

def rewrite_header(report_filename):
    with open(report_filename) as f:

        yamloo = yaml.load_all(f)

        header = yamloo.next()

        def fill_in_missing_data():
            subargs = []
            if 'subargs' in header:
                for x in header['subargs']:
                    if x.startswith('/'):
                        x = hashlib.sha256(x).hexdigest()[:10] + '.lst'
                    subargs.append(x)
            header['subargs'] = subargs

            header['probe_ip'] = ip_address

            header['probe_asn'] = asnum.org_by_addr(ip_address).split(' ')[0]
            header['probe_cc'] = ipd.country_code_by_addr(ip_address)

            return header

        header = fill_in_missing_data()

        tmp_fd, tmp_file_path = mkstemp()
        tmp_f = os.fdopen(tmp_fd, 'w+')
        tmp_f.write(yaml.safe_dump(header))

        for entry in yamloo:
            d = yaml.dump(entry)
            tmp_f.write(d)

        tmp_f.close()

    shutil.move(tmp_file_path, report_filename)

for report_filename in glob("*.yamloo"):
    print "Opening %s" % report_filename
    rewrite_header(report_filename)

# f = open(sys.argv[1])
# yamloo = yaml.safe_load_all(f)
#
# report_header = yamloo.next()
# print "ASN: %s" % report_header['probe_asn']
# print "CC: %s" % report_header['probe_cc']
# print "IP: %s" % report_header['probe_ip']
# print "Start Time: %s" % report_header['start_time']
# print "Test name: %s" % report_header['test_name']
# print "Test version: %s" % report_header['test_version']
#
# for report_entry in yamloo:
#     print "Test: %s" % report_entry['test_name']
#     print "Input: %s" % report_entry['input']
#     print "Report: %s" % report_entry['report']
#
# f.close()