bug 1436914 - VSIZE 0s

bug 1436914 - VSIZE 0s.ipynb

bug 1436914 - VSIZE 0s.py

# coding: utf-8
---
title: "bug 1436914 - MEMORY_VSIZE is reporting 0s all of a sudden"
authors:
- chutten
tags:
- memory
- vsize
- telemetry
- mystery
created_at: 2018-02-09
updated_at: 2018-02-09
tldr: What can we learn about clients sending us 0s in MEMORY_VSIZE all of a sudden
---
# ### MEMORY_VSIZE 0s

# In[1]:

import ujson as json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import plotly.plotly as py

from plotly.graph_objs import *
from moztelemetry import get_pings_properties, get_one_ping_per_client
from moztelemetry.dataset import Dataset

get_ipython().magic(u'matplotlib inline')


# In[2]:

sc.defaultParallelism


# In[3]:

Dataset.from_source("telemetry").schema


# In[50]:

pings = Dataset.from_source("telemetry")     .select(
        'clientId',
        'environment',
        vsize='payload.histograms.MEMORY_VSIZE',
    ).where(
        docType='main',
        appName='Firefox',
        appUpdateChannel='nightly',
        appBuildId=lambda x: x > '20180202',
    ).records(sc, sample=1)


# In[51]:

pings.count()


# In[52]:

has_zeros = pings.filter(lambda p: p['vsize'] and p['vsize'].get('values', {}).get('0', None))


# In[53]:

has_zeros.count()


# In[55]:

not_zeros = pings.filter(lambda p: not(p['vsize'] and p['vsize'].get('values', {}).get('0', None)))


# Do clients that have sent 0s also send non-0s?

# In[56]:

not_zero_clients = not_zeros.map(lambda p: p['clientId'])
zero_clients = has_zeros.map(lambda p: p['clientId'])

both = zero_clients.intersection(not_zero_clients)


# In[57]:

both.count()


# In[58]:

not_zero_clients.distinct().count()


# In[59]:

zero_clients.distinct().count()


# When a client sends 0s, does the histogram contain only 0s?

# In[60]:

has_zeros.map(lambda p: p['vsize']['values']).take(4)


# In[61]:

def has_nonzero_samples(p):
    values = p['vsize']['values']
    for k,v in values.iteritems():
        if k != '0' and v != 0:
            return True
    return False


# In[62]:

has_zeros.filter(has_nonzero_samples).count()


# Yup. If you are recording any 0s, you're recording _only_ 0s.

# #### Environment

# In[63]:

environment_counts = has_zeros.map(lambda p: ((p['environment']['build']['architecture'], str(p['environment']['settings']['sandbox']), str(p['environment']['system']['os']['version'])), 1)).countByKey()


# In[64]:

environment_counts


# In[ ]: