# -*- coding: utf-8 -*- | |
import requests | |
import time | |
import json | |
from StringIO import StringIO | |
import re | |
import sys | |
import os | |
import pandas as pd | |
import numpy as np | |
import matplotlib.pyplot as plt | |
import logging | |
logging.basicConfig() | |
logger = logging.getLogger(__name__) | |
logger.setLevel(level=logging.INFO) | |
MACKEREL_APIKEY = os.environ.get("MACKEREL_APIKEY") | |
BASEURL="https://mackerel.io" | |
epoch_time = int(time.time()) | |
#duration = 60 * 60 * 24 | |
duration = 60 * 60 * 24 * 7 | |
THRESHOLD = 1 | |
def fetch_hosts(): | |
headers = {'X-Api-Key': MACKEREL_APIKEY} | |
payload = {} | |
r = requests.get(BASEURL+"/api/v0/hosts", headers=headers, params=payload) | |
time.sleep(2.0) # wait | |
hosts = json.load(StringIO(r.content)) | |
return hosts['hosts'] | |
def fetch_metrics(hostid, name, time_from, time_to): | |
payload = {'name': name, 'from': time_from, 'to': time_to} | |
headers = {'X-Api-Key': MACKEREL_APIKEY} | |
r = requests.get(BASEURL+"/api/v0/hosts/"+hostid+"/metrics", params=payload, headers=headers) | |
time.sleep(2.0) # wait | |
metrics = json.load(StringIO(r.content)) | |
if metrics.has_key('metrics'): | |
return metrics['metrics'] | |
else: | |
return [] | |
def import_df(metrics, size): | |
# {u'metrics': [{u'value': 6.85, u'time': 1450575540}, {u'value': 7.03, u'time': 1450575600}]} | |
times = [] | |
values = [] | |
for metric in metrics: | |
times.append(metric['time'] - (epoch_time - duration)) | |
values.append(metric['value']) | |
df = pd.DataFrame({'time': times, 'value': values}) | |
#plt.plot(times, values, 'bo') | |
model = pd.ols(y=df['value'], x=df['time'], intercept=True) | |
#print model | |
#plt.plot(model.x['x'], model.y_fitted, 'g-') | |
#plt.hlines([size, size * 0.8], times[0], times[-1], linestyles="dashed") | |
#plt.show() | |
return model | |
def prepare_targets(): | |
dat = {} | |
hosts = fetch_hosts() | |
for host in hosts: | |
if not dat.has_key(host['name']): | |
dat[host['name']] = {'id': host['id']} | |
meta = host['meta'] | |
if meta.has_key('memory'): | |
dat[host['name']]['memory'] = meta['memory'].keys() | |
return dat | |
def human_readable_size(s): | |
s = float(s) | |
if s > 10**12: | |
return "%.2f TB" % (s / 10**12) | |
if s > 10**9: | |
return "%.2f GB" % (s / 10**9) | |
if s > 10**6: | |
return "%.2f MB" % (s / 10**6) | |
if s > 10**3: | |
return "%.2f MB" % (s / 10**3) | |
return "%d B" % s | |
def human_readable_time(s): | |
r = "" | |
unit = [("d", 60*60*24),("h", 60*60),("m", 60)] | |
for u in unit: | |
if s >= u[1]: | |
i = int(s / u[1]) | |
r = r + "%s%s" % (i, u[0]) | |
s = s - i * u[1] | |
return "%s%fs" % (r, s) | |
def process_targets(dat): | |
res = [] | |
num = 0 | |
for hostname in dat.keys(): | |
num = num + 1 | |
logger.info('fetching %s (%d/%d)', hostname, num, len(dat)) | |
record = {'hostname': hostname, 'mem': [], 'nearest': sys.maxint} | |
if not dat[hostname].has_key('memory'): | |
continue | |
mem = dat[hostname]['memory'] | |
metric_name = "memory.used" | |
metrics_size = fetch_metrics(dat[hostname]['id'], "memory.total", epoch_time, epoch_time - 60 * 10) | |
if len(metrics_size) == 0: | |
logger.info("skip the mem: %s" % (metric_name)) | |
continue | |
metrics = fetch_metrics(dat[hostname]['id'], metric_name, epoch_time, epoch_time - duration) | |
size = 0 | |
for m in metrics_size: | |
size = max(size, m['value']) | |
latest = 0 | |
for m in metrics[::-1]: | |
if m['value'] > 0: | |
latest = m['value'] | |
break | |
model = import_df(metrics, size) | |
x = model.beta.x | |
i = model.beta.intercept | |
logger.debug("metric:%s, x: %f, intercept: %f, size: %s, current: %s" % (metric_name, x, i, size, latest)) | |
if x > 0: | |
target = size * THRESHOLD | |
e = (target-i)/x | |
record['mem'].append({'name': metric_name, 'latest': latest, 'size': size, 'x': x, 'i': i, 'fullin': e}) | |
record['nearest'] = min(record['nearest'], e) | |
logger.debug("mem: %s, size: %s/%s, estimated %s" % ( | |
mem, | |
human_readable_size(latest), | |
human_readable_size(size), | |
human_readable_time(e))) | |
else: | |
record['mem'].append({'name': metric_name, 'latest': latest, 'size': size, 'x': x, 'i': i}) | |
logger.debug("mem: %s, used size is reducing." % (mem)) | |
res.append(record) | |
return res | |
def output_results(res): | |
res = sorted(res, key=lambda x: x['nearest']) | |
for host in res: | |
for mem in sorted(host['mem'], key=lambda x: x['fullin'] if x.has_key('fullin') else sys.maxint): | |
if mem.has_key('fullin'): | |
print "%s:%s, size: %s/%s, full in %s" % ( | |
host['hostname'], mem['name'], | |
human_readable_size(mem['latest']), | |
human_readable_size(mem['size']), | |
human_readable_time(mem['fullin'])) | |
else: | |
print "%s:%s, size: %s/%s, never full" % ( | |
host['hostname'], mem['name'], | |
human_readable_size(mem['latest']), | |
human_readable_size(mem['size'])) | |
if __name__ == "__main__": | |
dat = prepare_targets() | |
res = process_targets(dat) | |
output_results(res) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment