MITM Search

MITM Search.ipynb

MITM Search.py

# coding: utf-8

# In[1]:

from moztelemetry import get_pings_properties, get_one_ping_per_client
from moztelemetry.dataset import Dataset


# In[2]:

pings = Dataset.from_source("telemetry")     .where(submissionDate=lambda xx: xx.startswith("201612"))     .where(docType="OTHER")     .records(sc, sample=1)


# In[3]:

experiment_data = pings.filter(lambda xx: xx['meta']['docType'] == "mitm-prevalence-beta51")


# In[4]:

example = experiment_data.take(5)


# In[5]:

experiment_data


# In[6]:

example


# In[30]:

expected = ["197feaf3faa0f0ad637a89c97cb91336bfc114b6b3018203cbd9c3d10c7fa86c",
            "154c433c491929c5ef686e838e323664a00e6a0d822ccc958fb4dab03e49a08f",
            "4348a0e9444c78cb265e058d5e8944b4d84f9662bd26db257f8934a443c70161"]
def isChainAsExpected(xx):
    if 'payload' not in xx:
        return
    if 'chain' not in xx['payload']:
        return
    if len(xx['payload']['chain']) != 3:
        return xx['payload']['chain']
    for idx, val in enumerate(expected):
        chainEntry = xx['payload']['chain'][idx]['sha256Fingerprint']
        if chainEntry != val:
            #return "Mismatch on {} - {} != {}".format(idx, chainEntry, val)
            return xx['payload']['chain']
    return
test_data = map(isChainAsExpected, example)


# In[34]:

filter(lambda xx: isChainAsExpected(xx) != None, example)


# In[35]:

abnormal_data = experiment_data.filter(lambda xx: isChainAsExpected(xx) != None)


# In[36]:

abnormal_data


# In[37]:

abnormal_result = abnormal_data.first()


# In[38]:

abnormal_data.count()


# In[39]:

abnormal_result


# In[48]:

def isChainBuiltInOrError(xx):
    if 'payload' not in xx:
        return
    if 'chain' not in xx['payload']:
        return
    if xx['payload']['errorCode'] == 0:
        return
    for chainEntry in xx['payload']['chain']:
        if chainEntry['isBuiltInRoot'] == True:
            return xx
    return
#map(isChainBuiltInOrError, example)


# In[49]:

mitm_data = abnormal_data.filter(lambda xx: isChainBuiltInOrError(xx) != None)


# In[50]:

mitm_result = mitm_data.take(10)


# In[90]:

from collections import defaultdict, Counter

def countRoots(accum, xx):
    if 'isAccum' not in accum:
        # Happens on the first execution
        accum = countRoots({'isAccum':True, 
                            'total_roots':Counter(), 
                            'total_errors':Counter(), 
                            'rooterrors':Counter()
                           }, accum)
    if 'isAccum' in xx:
        # Happens on the final executions, merging intermediate states
        for k,v in xx['total_errors'].iteritems():
            accum['total_errors'][k] += v
        for k,v in xx['total_roots'].iteritems():
            accum['total_roots'][k] += v
        for k,v in xx['rooterrors'].iteritems():
            accum['rooterrors'][k] += v
        return accum

    # Primary analysis
    if 'payload' not in xx:
        return accum
    if 'errorCode' not in xx['payload']:
        return accum
    if 'chain' not in xx['payload']:
        return accum

    code=xx['payload']['errorCode']
    if code != 0:
        accum['total_errors'][code] += 1
    for chainEntry in xx['payload']['chain']:
        if chainEntry['isBuiltInRoot'] == True:
            fp = chainEntry['sha256Fingerprint']
            accum['total_roots'][fp] += 1
    
            fpcode = "{} {}".format(fp,code)
            accum['rooterrors'][fpcode] += 1
    return accum

x=reduce(countRoots, mitm_result)
countRoots(x,x)


# In[86]:

mitm_count_roots = mitm_data.reduce(countRoots)


# In[87]:

mitm_count_roots


# In[91]:

experiment_data.reduce(countRoots)


# In[ ]: