mackerel_estimate_memory_lifetime.py

# -*- coding: utf-8 -*-

import requests
import time
import json
from StringIO import StringIO
import re
import sys
import os

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import logging
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(level=logging.INFO)

MACKEREL_APIKEY = os.environ.get("MACKEREL_APIKEY")
BASEURL="https://mackerel.io"

epoch_time = int(time.time())
#duration = 60 * 60 * 24
duration = 60 * 60 * 24 * 7
THRESHOLD = 1

def fetch_hosts():
    headers = {'X-Api-Key': MACKEREL_APIKEY}
    payload = {}
    r = requests.get(BASEURL+"/api/v0/hosts", headers=headers, params=payload)

    time.sleep(2.0) # wait

    hosts = json.load(StringIO(r.content))
    return hosts['hosts']

def fetch_metrics(hostid, name, time_from, time_to):
    payload = {'name': name, 'from': time_from, 'to': time_to}
    headers = {'X-Api-Key': MACKEREL_APIKEY}
    r = requests.get(BASEURL+"/api/v0/hosts/"+hostid+"/metrics", params=payload, headers=headers)

    time.sleep(2.0) # wait

    metrics = json.load(StringIO(r.content))
    if metrics.has_key('metrics'):
        return metrics['metrics']
    else:
        return []

def import_df(metrics, size):
    # {u'metrics': [{u'value': 6.85, u'time': 1450575540}, {u'value': 7.03, u'time': 1450575600}]}
    times = []
    values = []
    for metric in metrics:
        times.append(metric['time'] - (epoch_time - duration))
        values.append(metric['value'])
    df = pd.DataFrame({'time': times, 'value': values})
    #plt.plot(times, values, 'bo')

    model = pd.ols(y=df['value'], x=df['time'], intercept=True)
    #print model
    #plt.plot(model.x['x'], model.y_fitted, 'g-')
    #plt.hlines([size, size * 0.8], times[0], times[-1], linestyles="dashed")
    #plt.show()
    return model

def prepare_targets():
    dat = {}
    hosts = fetch_hosts()
    for host in hosts:
        if not dat.has_key(host['name']):
            dat[host['name']] = {'id': host['id']}
        meta = host['meta']
        if meta.has_key('memory'):
            dat[host['name']]['memory'] = meta['memory'].keys()

    return dat

def human_readable_size(s):
    s = float(s)
    if s > 10**12:
        return "%.2f TB" % (s / 10**12)
    if s > 10**9:
        return "%.2f GB" % (s / 10**9)
    if s > 10**6:
        return "%.2f MB" % (s / 10**6)
    if s > 10**3:
        return "%.2f MB" % (s / 10**3)
    return "%d B" % s

def human_readable_time(s):
    r = ""
    unit = [("d", 60*60*24),("h", 60*60),("m", 60)]
    for u in unit:
        if s >= u[1]:
            i = int(s / u[1])
            r = r + "%s%s" % (i, u[0])
            s = s - i * u[1]
    return "%s%fs" % (r, s)

def process_targets(dat):
    res = []
    num = 0
    for hostname in dat.keys():
        num = num + 1
        logger.info('fetching %s (%d/%d)', hostname, num, len(dat))
        record = {'hostname': hostname, 'mem': [], 'nearest': sys.maxint}
        if not dat[hostname].has_key('memory'):
            continue
        mem = dat[hostname]['memory']
        metric_name = "memory.used"
        metrics_size = fetch_metrics(dat[hostname]['id'], "memory.total", epoch_time, epoch_time - 60 * 10)
        if len(metrics_size) == 0:
            logger.info("skip the mem: %s" % (metric_name))
            continue

        metrics = fetch_metrics(dat[hostname]['id'], metric_name, epoch_time, epoch_time - duration)

        size = 0
        for m in metrics_size:
            size = max(size, m['value'])

        latest = 0
        for m in metrics[::-1]:
            if m['value'] > 0:
                latest = m['value']
                break

        model = import_df(metrics, size)
        x =  model.beta.x
        i =  model.beta.intercept
        logger.debug("metric:%s, x: %f, intercept: %f, size: %s, current: %s" % (metric_name, x, i, size, latest))
        if x > 0:
            target = size * THRESHOLD
            e = (target-i)/x
            record['mem'].append({'name': metric_name, 'latest': latest, 'size': size, 'x': x, 'i': i, 'fullin': e})
            record['nearest'] = min(record['nearest'], e)
            logger.debug("mem: %s, size: %s/%s, estimated %s" % (
                mem,
                human_readable_size(latest),
                human_readable_size(size),
                human_readable_time(e)))
        else:
            record['mem'].append({'name': metric_name, 'latest': latest, 'size': size, 'x': x, 'i': i})
            logger.debug("mem: %s, used size is reducing." % (mem))
        res.append(record)
    return res

def output_results(res):
    res = sorted(res, key=lambda x: x['nearest'])
    for host in res:
        for mem in sorted(host['mem'], key=lambda x: x['fullin'] if x.has_key('fullin') else sys.maxint):
            if mem.has_key('fullin'):
                print "%s:%s, size: %s/%s, full in %s" % (
                    host['hostname'], mem['name'],
                    human_readable_size(mem['latest']),
                    human_readable_size(mem['size']),
                    human_readable_time(mem['fullin']))
            else:
                print "%s:%s, size: %s/%s, never full" % (
                    host['hostname'], mem['name'],
                    human_readable_size(mem['latest']),
                    human_readable_size(mem['size']))

if __name__ == "__main__":
    dat = prepare_targets()
    res = process_targets(dat)
    output_results(res)