Untitled

non-experiment-slow-script-analysis.ipynb

non-experiment-slow-script-analysis.py

# coding: utf-8

# In[2]:

import ujson as json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import plotly.plotly as py
import IPython

from __future__ import division
from moztelemetry.spark import get_pings, get_one_ping_per_client, get_pings_properties
from montecarlino import grouped_permutation_test

get_ipython().magic(u'pylab inline')
IPython.core.pylabtools.figsize(16, 7)


# In[3]:

sc.defaultParallelism


# In[46]:

def chi2_distance(xs, ys, eps = 1e-10, normalize = True):
    histA = xs.sum(axis=0)
    histB = ys.sum(axis=0)
    
    if normalize:
        histA = histA/histA.sum()
        histB = histB/histB.sum()
    
    d = 0.5 * np.sum([((a - b) ** 2) / (a + b + eps)
        for (a, b) in zip(histA, histB)])

    return d

def median_diff(xs, ys):
    return np.median(xs) - np.median(ys)

def normalize_uptime_hour(frame):
    frame = frame[frame["payload/simpleMeasurements/uptime"] > 0]
    frame = 60 * frame.apply(lambda x: x/frame["payload/simpleMeasurements/uptime"]) # Metric per hour
    frame.drop('payload/simpleMeasurements/uptime', axis=1, inplace=True)
    return frame
    
def compare_count_histograms(pings, *histograms_names):
    values = get_pings_properties(pings, [
        "payload/histograms/SLOW_SCRIPT_NOTICE_COUNT",
        "payload/histograms/SLOW_SCRIPT_PAGE_COUNT",
        "payload/simpleMeasurements/uptime",
        "environment/settings/e10sEnabled",
    ])
    frame = pd.DataFrame(values.collect())

    e10s = frame[frame["environment/settings/e10sEnabled"] == True]
    e10s = normalize_uptime_hour(e10s)
    
    none10s = frame[frame["environment/settings/e10sEnabled"] == False]
    none10s = normalize_uptime_hour(none10s)
    
    for histogram in e10s.columns:
        if histogram == "environment/settings/e10sEnabled" or histogram.endswith("_parent") or histogram.endswith("_children"):
            continue
            
        compare_scalars(histogram + " per hour", e10s[histogram].dropna(), none10s[histogram].dropna())

def compare_scalars(metric, *groups):
    print "Median difference in {} is {:.2f}, ({:.2f}, {:.2f}).".format(metric,
                                                                        median_diff(*groups), 
                                                                        np.median(groups[0]),
                                                                        np.median(groups[1]))
    print "The probability of this effect being purely by chance is {:.2f}.".         format(grouped_permutation_test(median_diff, groups, num_samples=10000))


# In[20]:

pings = get_pings(sc, app="Firefox", channel="nightly", submission_date=("20160405", "20160405"), fraction=1)


# In[43]:

pings.count()


# In[47]:

compare_count_histograms(pings, "")


# In[49]:

pings = get_pings(sc, app="Firefox", channel="aurora", submission_date=("20160405", "20160405"), fraction=1)


# In[50]:

pings.count()


# In[51]:

compare_count_histograms(pings, "")


# In[61]:

pings = get_pings(sc, app="Firefox", channel="nightly", build_id=("20160402000000", "20160405999999"), fraction=1)


# In[62]:

pings.count()


# In[63]:

compare_count_histograms(pings, "")


# In[58]:

pings = get_pings(sc, app="Firefox", channel="aurora", build_id=("20160402000000", "20160405999999"), fraction=1)


# In[59]:

pings.count()


# In[60]:

compare_count_histograms(pings, "")


# In[ ]: