script to fetch pod startup latency test results for k8s storage performance tests

fetch_podstartup_times.py

from lxml import html
import requests
import json

def find_build_ids(test_case='max-emptydir-vol-per-pod'):
	page = requests.get('https://gcsweb.k8s.io/gcs/kubernetes-jenkins/logs/ci-kubernetes-storage-scalability-{}/'.format(test_case))
	tree = html.fromstring(page.content)

	raw_build_ids = tree.xpath('//li[@class="pure-g grid-row"]/div[@class="pure-u-2-5"]/a/text()')

	build_ids = [''.join(c for c in build_id if c.isdigit()) for build_id in raw_build_ids]
	build_ids = list(filter(lambda bi: bi != '', build_ids))

	return build_ids

def find_podstartup_times(test_case='max-emptydir-vol-per-pod', build_id='1154857195515416577'):
	page = requests.get('https://gcsweb.k8s.io/gcs/kubernetes-jenkins/logs/ci-kubernetes-storage-scalability-{test_case}/{build_id}/artifacts/'.format(test_case=test_case, build_id=build_id))
	tree = html.fromstring(page.content)

	raw_pod_startup_artifact_name = tree.xpath('//a[text()[contains(., "PodStartupLatency_PodWithMultiVolumeStartupLatency")]]/text()')
	if not raw_pod_startup_artifact_name:
		return None

	pod_startup_artifact_name = raw_pod_startup_artifact_name[0][1:]

	pod_startup_artifact_json_resp = requests.get('https://storage.googleapis.com/kubernetes-jenkins/logs/ci-kubernetes-storage-scalability-{test_case}/{bid}/artifacts/{artifact}'.
		format(test_case=test_case, bid=build_id, artifact=pod_startup_artifact_name))

	# to extract only pod startup time
	# pod_startup_times = [data_item for data_item in json.loads(pod_startup_artifact_json_resp.content)['dataItems'] if data_item['labels']['Metric'] == 'pod_startup'] 

	pod_startup_times = json.loads(pod_startup_artifact_json_resp.content)
	pod_startup_times.update({'date': pod_startup_artifact_name.split('_')[3][:-5]})
	return pod_startup_times

test_cases = [
'max-emptydir-vol-per-pod', 'max-configmap-vol-per-pod', 'max-downwardapi-vol-per-pod', 'max-secret-vol-per-pod',
 'max-emptydir-vol-per-node', 'max-configmap-vol-per-node', 'max-downwardapi-vol-per-node', 'max-secret-vol-per-node'
]

test_case_test_results = {}
for test_case in test_cases:
	print('Find build ids for test case {}'.format(test_case))
	build_ids = find_build_ids(test_case)
	print('There are {} build ids for test case {}'.format(len(build_ids), test_case))
	build_id_test_results = {}
	for build_id in build_ids:
		print('Find podstartup times for build id {} and test case {}'.format(build_id, test_case))
		pod_startup_times = find_podstartup_times(test_case, build_id)
		build_id_test_results.update({build_id: pod_startup_times})
	test_case_test_results.update({test_case: build_id_test_results})

print(test_case_test_results)
print('Write the results to file...')
json_f = open('volume_podstartup_time_results.json', 'w')
json_f.write(json.dumps(test_case_test_results))
json_f.close()
print('All Done!')