process.py

from __future__ import print_function

import json
import os.path
import glob
import math
import statistics
from shutil import copyfile
from operator import itemgetter

os.chdir("browsertime-results-copy");
files = glob.glob("*")

metrics = ["pageLoadTime","SpeedIndex","PerceptualSpeedIndex", "ContentfulSpeedIndex", "FirstVisualChange","VisualComplete85"]

def filter_outliers(loads, threshold):
  length = len(loads)
  median = statistics.median(loads)
  mean   = statistics.mean(loads)
  stdev  = statistics.stdev(loads)

  ratio = stdev/mean
  while ratio > threshold:
    maxDiff = 0
    maxIndex = 0
    for i,val in enumerate(loads):
      if abs(val-median) > maxDiff:
        maxDiff = abs(val-median)
        maxIndex = i

    #print("Removing index " + str(maxIndex) + " with value " + str(loads[maxIndex]))
    del loads[maxIndex]
    if len(loads) <= 5:
      return

    mean = statistics.mean(loads)
    median = statistics.median(loads)
    ratio = statistics.stdev(loads)/mean
    #print("----> ratio = " + str(ratio))


sortedResults = []
for url in files:
  os.chdir(url)
  print("\nProcessing " + url)

  report = []
  sessions = sorted(glob.glob("*"))
  skipURL=0
  for k,session in enumerate(sessions):
    print("    Processing {}".format(session));
    with open(session + "/browsertime.json") as f:
      data = json.load(f)


      pageLoadTime = []
      SpeedIndex = []
      PerceptualSpeedIndex = []
      ContentfulSpeedIndex = []
      FirstVisualChange = []
      VisualComplete85 = []

      print("pageLoadTime  length = " + str(len(data[0]['browserScripts'])));
      print("visualMetrics length = " + str(len(data[0]['visualMetrics'])));

      #if (len(data[0]['browserScripts']) != len(data[0]['visualMetrics'])):
      #  skipURL=1
      #  break

      iterations=len(data[0]['browserScripts'])
      for i in range(0,iterations):
        pageLoadTime.append( data[0]['browserScripts'][i]['timings']['pageTimings']['pageLoadTime'] )

      iterations=len(data[0]['visualMetrics'])
      for i in range(0,iterations):
        SpeedIndex.append( data[0]['visualMetrics'][i]['SpeedIndex'] )
        PerceptualSpeedIndex.append( data[0]['visualMetrics'][i]['PerceptualSpeedIndex'] )
        ContentfulSpeedIndex.append( data[0]['visualMetrics'][i]['ContentfulSpeedIndex'] )
        FirstVisualChange.append( data[0]['visualMetrics'][i]['FirstVisualChange'] )
        VisualComplete85.append( data[0]['visualMetrics'][i]['VisualComplete85'] )

      #filter_outliers(pageLoadTime, 0.1)
      #filter_outliers(SpeedIndex, 0.1)
      #filter_outliers(VisualComplete85, 0.1)
      #filter_outliers(FirstVisualChange, 0.1)

      print(pageLoadTime)
      instance = {}
      instance['pageLoadTimeMean']        = statistics.mean(pageLoadTime);
      instance['pageLoadTimeStdev']       = statistics.stdev(pageLoadTime);
      instance['SpeedIndexMean']          = statistics.mean(SpeedIndex);
      instance['SpeedIndexStdev']         = statistics.stdev(SpeedIndex);
      instance['PerceptualSpeedIndexMean']  = statistics.mean(PerceptualSpeedIndex);
      instance['PerceptualSpeedIndexStdev'] = statistics.stdev(PerceptualSpeedIndex);
      instance['ContentfulSpeedIndexMean']  = statistics.mean(ContentfulSpeedIndex);
      instance['ContentfulSpeedIndexStdev'] = statistics.stdev(ContentfulSpeedIndex);
      instance['FirstVisualChangeMean']   = statistics.mean(FirstVisualChange);
      instance['FirstVisualChangeStdev']  = statistics.stdev(FirstVisualChange);
      instance['VisualComplete85Mean']    = statistics.mean(VisualComplete85);
      instance['VisualComplete85Stdev']   = statistics.stdev(VisualComplete85);
      instance['value']                   = session
      instance['timestamp']               = data[0]['info']['timestamp']
      instance['url']                     = url
      report.append(instance)

  sortedResults.append( sorted(report, key=itemgetter('timestamp')) )
  os.chdir("..")

# Print data
for metric in metrics:
  print(metric + ":")
  print("-"*106)
  for j,l in enumerate(sortedResults):
    meanIndex = metric + "Mean"
    stdevIndex = metric + "Stdev"

    print("%-50s"% sortedResults[j][0]["url"], end="")
    print("| ", end="")
    for i,instance in enumerate(sortedResults[j]):
      print("%9.2f"%  instance[meanIndex]   + " ", end="")
      print("(+-%4.0f"% instance[stdevIndex] + ") ", end="")

      if i == 0:
        baseValue = instance[meanIndex]
        print(" "*5, end="")
        print("|", end="")
      else:
        speedup = (baseValue-instance[meanIndex])/baseValue*100
        print("%7.2f"% speedup + "%", end="")
        print(" |", end="")
    print("")
  print("\n")