test_time_trends.py

#!/usr/bin/env python3
import boto3
import os
import bz2
import json
import subprocess
from datetime import datetime


def get_git_commit_history(path, branch="master"):
    rc = subprocess.check_output(['git', '-C', path, 'log', '--pretty=format:%H %ct' ,branch]).decode("latin-1")
    return [(x[0], datetime.fromtimestamp(int(x[1]))) for x in [line.split(" ") for line in rc.split("\n")]]


def get_ossci_json(bucket, sha, config='pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test2'):
   objs = list(bucket.objects.filter(Prefix=f"test_time/{sha}/{config}"))
   if len(objs) == 0:
       return {}
   return json.loads(bz2.decompress(objs[0].get()['Body'].read()))



def search_for_commit(bucket, commits, suite_name, test_name, start_sha=None, delta = 12*3600):
    last_sha = None
    prev_time = datetime.now()
    for sha, time in commits:
        if start_sha != None:
            if start_sha != sha:
                continue
            start_sha = None
        if (prev_time - time).total_seconds() < delta:
            continue
        data = get_ossci_json(bucket, sha)
        suites = data['suites'] if 'suites' in data else {}
        if suite_name not in suites:
            data = get_ossci_json(bucket, sha, 'pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test')
            suites = data['suites'] if 'suites' in data else {}
            if suite_name not in suites:
                print(f"{time} {sha}: Can't find {suite_name} in {suites.keys()}")
                continue
        testcase_times = {case['name']:case['seconds'] for case in suites[suite_name]['cases']}
        if test_name not in testcase_times:
            break
        last_sha = sha
        prev_time = time
        print(f"{time} {sha} {testcase_times[test_name]}")
    return last_sha

if __name__ == "__main__":
    pytorch_git_path = os.path.expanduser("~/git/pytorch/pytorch")
    commits = get_git_commit_history(pytorch_git_path)

    s3 = boto3.resource("s3")
    bucket = s3.Bucket('ossci-metrics')

    last_sha = search_for_commit(bucket, commits, 'TestAutogradDeviceTypeCUDA', 'test_qr_square_many_batched_complex_cuda', start_sha = None, delta = 12*3600)
    last_sha = search_for_commit(bucket, commits, 'TestAutogradDeviceTypeCUDA', 'test_qr_square_many_batched_complex_cuda', start_sha = last_sha, delta = 3600)