bug 1298758 - corrupt places DB analysis

bug 1298758 - corrupt places DB analysis.ipynb

bug 1298758 - corrupt places DB analysis.py

# coding: utf-8

# In[1]:

from moztelemetry import get_pings, get_pings_properties

# XXX - we should:
# * Work out how to exclude records before bug 1288445 landed (the same user is represented with
#   2 different IDs if pings before and after that patch are considered)
# * Work out some sane way of excluding "problem pings" that are probably yet to
#   have places auto-fix their DB.
pings = get_pings_properties(get_pings(sc, doc_type='sync', fraction=1.0),
                             ["payload/uid", "payload/when", "payload/engines"])


# In[2]:

# Filter and group by userid.

# Only pings with a bookmark Sync.
def enginePings(p):
    return p.get("payload/engines") and len(filter(lambda e: e["name"] == "bookmarks", p["payload/engines"])) > 0

# group them by the UID.
grouped = pings.filter(enginePings).map(lambda p: (p["payload/uid"], p)).groupByKey()


# In[3]:

# grouped is now a "list" of (uid, all_bookmark_pings_for_user)

# Use a map function to simplify the structure and flag where the errors are seen.
# XXX - note that I'm also seeing this error for the "tabs" engine in some cases.
# I predict history will see it too - but for now, just consider bookmarks.
def mapErrors((uid, pings)):
    sawError = fixed = False
    for ping in sorted(pings, key=lambda p: p["payload/when"]):
        engines = ping["payload/engines"]
        for e in engines:
            if e["name"] == "bookmarks":
                if "failureReason" not in e:
                    # It worked! If this after we saw the corruption error it must have been "fixed".
                    if sawError:
                        fixed = True
                        break # there's no need to inspect any more...
                elif e["failureReason"]["name"] == "unexpectederror" and                     e["failureReason"]["error"] == "Error: Error(s) encountered during statement execution: database disk image is malformed":
                    # It's one of ours
                    sawError = True
                # else it's some other error...
    return sawError, fixed

print "Number of users with a bookmark sync:", grouped.count()

withErrors = grouped.map(mapErrors).filter(lambda (sawError, fixed): sawError)
print "Number with errors:", withErrors.count()
print "Number where errors went away:", withErrors.filter(lambda (sawError, fixed): fixed).count()


# In[ ]: