20000 and 15000 samples are not reliable for with tags because there is some OOM kills that restart the process
-
-
Save bunelr/7d0255a827e84dc28125 to your computer and use it in GitHub Desktop.
Dogstatsd Benchmark
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import subprocess | |
| from collections import defaultdict | |
| import json | |
| import time | |
| import random | |
| import threading | |
| from statsd import statsd | |
| count = 0 | |
| try: | |
| with open("bench.json","r") as previous_bench: | |
| results= json.load(previous_bench) | |
| print "Loaded previously started benchmark" | |
| print results | |
| except Exception as e: | |
| print e | |
| results=defaultdict(dict) | |
| if not results: | |
| results = defaultdict(dict) | |
| tags = [] | |
| #for i in range(1,100): | |
| # tags.append("tag:"+str(i)) | |
| nb_of_samples_per_second = [100,500,1000,1500,2000,5000,7500,10000,15000,20000,40000] | |
| nb_of_different_metric = [100,500,1000,1500,2000,5000,7500,10000,15000,20000,40000,50000,75000,100000] | |
| def get_process_pid(user, process): | |
| reply = subprocess.check_output(["ps -u {0} -f | grep {1}".format(user,process)],shell=True) | |
| return reply.split()[1] | |
| def get_memory_usage(pid): | |
| ps_reply = subprocess.check_output(["ps -v -p{0}".format(pid)],shell=True) | |
| ps_lines = ps_reply.split("\n") | |
| columns_line = ps_lines[0] | |
| value_line = ps_lines[1] | |
| mem_index = columns_line.split().index("RSS") | |
| mem_value = value_line.split()[mem_index] | |
| return mem_value | |
| def get_dogstatsd_memory_usage(): | |
| pid = get_process_pid("dd-agent","dogstatsd") | |
| return get_memory_usage(pid) | |
| def start_agent(): | |
| subprocess.call(["sudo", "service", "datadog-agent", "start"]) | |
| time.sleep(5) | |
| def stop_agent(): | |
| subprocess.call(["sudo", "service", "datadog-agent", "stop"]) | |
| time.sleep(5) | |
| def feed_data(nb_hist,nb_of_calls_per_second): | |
| global count | |
| second_done = threading.Event() | |
| end_second = threading.Timer(1, lambda: second_done.set()) | |
| end_second.start() | |
| for call_nb in range(nb_of_calls_per_second): | |
| value = random.random() | |
| statsd.gauge("benchmark_perf.metric"+ str(count % nb_hist), value, tags = random.sample(tags, 0)) | |
| count +=1 | |
| second_done.wait() | |
| for nb_of_samples in nb_of_samples_per_second: | |
| for nb_of_hist in nb_of_different_metric: | |
| try: | |
| results[unicode(nb_of_samples)][unicode(nb_of_hist)] | |
| except KeyError: | |
| print "New data-point - looking into it" | |
| pass | |
| else: | |
| print "Already exist - skipping" | |
| continue | |
| should_feed_data = threading.Event() | |
| feeding_timer = threading.Timer(300, lambda: should_feed_data.clear()) | |
| stop_agent() | |
| start_agent() | |
| should_feed_data.set() | |
| feeding_timer.start() | |
| while should_feed_data.is_set(): | |
| feed_data(nb_of_hist,nb_of_samples) | |
| memory_usage = get_dogstatsd_memory_usage() | |
| results[unicode(nb_of_samples)][unicode(nb_of_hist)] = memory_usage | |
| with open("bench.json","w") as output_file: | |
| json.dump(dict(results),output_file,indent =4) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import matplotlib.pyplot as plt | |
| import json | |
| files_to_open = ["bench.json.withtags", "bench.json.notags.gauge", "bench.json.notags"] | |
| for file in files_to_open: | |
| plt.figure(figsize=(15,5)) | |
| with open(file, "r") as bench_results: | |
| results = json.load(bench_results) | |
| plt.xlabel("nb of different metrics") | |
| plt.ylabel("memory usage in kb") | |
| for nb_of_samples in sorted(results.keys(),key=int,reverse=True): | |
| abs = [] | |
| ord = [] | |
| for hist_nb, mem_value in results[nb_of_samples].items(): | |
| abs.append(int(hist_nb)) | |
| ord.append(int(mem_value)) | |
| ord = sorted(ord,key=lambda x:abs[ord.index(x)]) | |
| abs.sort() | |
| plt.plot(abs, ord, label="{0} samples per second".format(int(nb_of_samples))) | |
| lgd = plt.legend(bbox_to_anchor=(1, 1), loc="upper left") | |
| plt.savefig(file+".png", bbox_extra_artists=(lgd,), bbox_inches='tight') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment