trending.py

from IPython.core.display import display, HTML, Javascript
from string import Template
import pandas as pd
import numpy as np
import requests
import fnmatch
import datetime
import pytz

defaultRestURL = "https://lsst-camera-dev.slac.stanford.edu/CCSWebTrending/rest/"
defaultSite = "ir2"

class Channel:
    def __init__(self, restURL, path, node):
        self.restURL = restURL
        self.path = path
        self.text = node['text']
        self.id = node['id']
        self.data = int(node.get('data')) if 'data' in node else None
        self.hasChildren = node['children']
        self.children = {}

    def full_path(self):
        return self.path + '/' + self.text if self.path else self.text 
        
    def __repr__(self):
        return "Channel (%s (%s) %s)" % (self.full_path(), self.hasChildren, self.data)
    
    def find(self, name):
        if self.hasChildren and not self.children:
            self.__load_children()
        tokens = name.split('/',1)
        child = self.children[tokens[0]]
        if len(tokens) == 1:
            return child
        else:
            return child.find(tokens[1])
        
    def find_all(self, path):
        if self.hasChildren and not self.children:
            self.__load_children()
        tokens = path.split('/',1)
        result = []
        for key,child in self.children.items():
            if fnmatch.fnmatchcase(key,tokens[0]):
                if len(tokens) == 1:
                    result.append(child)
                else:
                    result += child.find_all(tokens[1])
        return result
    
    def ls(self, recursive=False):
        if self.hasChildren and not self.children:
            self.__load_children()
        if recursive:
            l = [];
            for key,value in self.children.items():
                l.append(key)
                if value.hasChildren:
                    l.append(value.ls(True))
            return l
        else:
            return self.children.keys()
    
    def __load_children(self):
        if self.hasChildren and not self.children:
            url = self.restURL
            if id != 0:
                url += "?id=%d" % self.id
            r = requests.get(url)
            r.raise_for_status()
            json = r.json()
            full_path = self.full_path()
            for node in json:
               c = Channel(self.restURL,full_path,node)
               self.children[c.text] = c        
        
class ChannelMap:
    def __init__(self, site=defaultSite, restURL=defaultRestURL):
        self.root = Channel(restURL+site+"/channels/", "",  {'text': '', 'id': 0, 'children': True})
    
    def find(self, name):
        return self.root.find(name)
    
    def find_all(self, path):
        return self.root.find_all(path)
    
    def __repr__(self):
        return self.root.__repr__()
    
    def ls(self, recursive=False):
        return self.root.ls(recursive)
    
class ChannelDataReader:

    def __init__(self, site=defaultSite, restURL=defaultRestURL):
        self.restURL = restURL+site
        
    def read_data(self, ids, names, timePeriod, nBins=100,):
        
        url = "%s?n=%d" % (self.restURL, nBins)
        url += "&t1=%d&t2=%d" % timePeriod.as_millis()
        for id in ids:
            url += "&key=%d" % id
        r = requests.get(url)
        r.raise_for_status()
        json = r.json()
        data = np.array(json['data'])
        df = pd.DataFrame(data=data[:,1:],index=data[:,0],columns=names)
        df.index = pd.to_datetime(df.index,unit='ms')
        return df
    

class TimePeriod:
    epoch = pytz.utc.localize(datetime.datetime.utcfromtimestamp(0))
    def to_millis(dt):
        return int ((dt-TimePeriod.epoch).total_seconds()*1000)
    
    def as_millis(self):
        pass
    
    def as_ccs_string(self):
        pass
    
    def for_range(range):
        if isinstance(range, datetime.timedelta):
            return DeltaTimePeriod(range)
        elif isinstance(range, tuple):
            return StartEndTimePeriod(range[0], range[1])
        else:
            raise RuntimeError("Unsupported argument type for for_range "+range)
        
    
class StartEndTimePeriod(TimePeriod):
    def __init__(self,start,end):
        self.start = start
        self.end = end
        
    def as_millis(self):
        return (TimePeriod.to_millis(self.start), TimePeriod.to_millis(self.end))
    
    def as_ccs_string(self):
        return "{start: new Date(%d), end: new Date(%d)}" % ( TimePeriod.to_millis(self.start), TimePeriod.to_millis(self.end) )
    
class DeltaTimePeriod(TimePeriod):
    def __init__(self,delta):
        self.delta = delta

    def as_millis(self):
        now = pytz.utc.localize(datetime.datetime.utcnow())
        return (TimePeriod.to_millis(now-self.delta), TimePeriod.to_millis(now))
    
    def as_ccs_string(self):
        return "%d" % self.delta.total_seconds()
    
class CCSTrending:
    """A simple jupyter interface to CCS trending"""
    
    def __init__(self, title="Trending Plot", data=None, range=datetime.timedelta(days=1), site=defaultSite, restURL=defaultRestURL):
        self.plots = []
        self.title = title
        self.range = range
        self.restURL = restURL+site
        self.cm = ChannelMap(site=site, restURL=restURL)
        self.dr = ChannelDataReader(site=site, restURL=restURL)

        if data:
            if isinstance(data,str):
                self.add_all(self.cm.find_all(data))
            elif isinstance(data, dict):
                for key, value in data.items():
                    self.add_channel(key,value)
            else:
                raise RuntimeError("Unsupported argument type for data "+data)
        
    @property
    def range(self):
        return self.__range
    
    @range.setter
    def range(self, range):
        self.__range = TimePeriod.for_range(range)
    
    def add_channel(self, id, key=None):
        if isinstance(id, int):
            self.plots.append([id, key])
        elif isinstance(id, Channel):
            self.plots.append([id.data, key if key else id.full_path()])
        elif isinstance(id, str):
            channel = self.cm.find(id);
            self.plots.append([channel.data, key if key else channel.text])
        else:
            raise RuntimeError('Unsuppored argument type for add_channel '+ id)
            
    def add_all(self, channels):
        for channel in channels:
            self.add_channel(channel)
        
    def __repr__(self):
        return str(self.plots)
    
    def as_dataframe(self):
        ids = []
        keys = []
        for id, key in self.plots:
            ids.append(id)
            keys.append(key)
        return self.dr.read_data(ids, keys, self.__range)
    
    def plot(self):
        code = Template('''
            $$('<style>.dygraph-legend {left: 70px !important;}</style>').appendTo(element);
            var div = $$('<div style="width:100%;height:400px"></div>').appendTo(element)
            setTimeout(function(){
                var g = new CCSTrendingPlot(
                        div[0], {
                    title: '${title}',
                    restURL: '${restURL}', 
                    range: ${range},
                    data: { ${data} }
                });
                var zoomDict = {'hour': 3600, '3 hour': 3*3600, '6 hour': 6*3600, '12 hour': 12*3600,
                   'day': 86400, 'week': 604800, 'month': 30 * 86400 };
                var errorBarDict = { 'none': 'NONE', 'minmax': 'MINMAX', 'rms': 'RMS'}
                function zoom(event) { g.zoom(event.data); }
                function setErrorBars(event) { g.setErrorBars(event.data); }

                $$('<b>Zoom:&nbsp;</b>').appendTo(element);
                for (var key in zoomDict) {
                  var a = $$('<a href="#">'+key+'</a>').appendTo(element);
                  a.click(zoomDict[key],zoom);
                  element.append('&nbsp;');
                }
                $$('<b>Error Bars:&nbsp;</b>').appendTo(element);
                for (var key in errorBarDict) {
                  var a = $$('<a href="#">'+key+'</a>').appendTo(element);
                  a.click(errorBarDict[key],setErrorBars);
                  element.append('&nbsp;');
                }
              },0);
           ''')
        def output(plots):
            result = ""
            for id, key in plots:
                result += "%d: {title: '%s'}," % (id, key)
            return result
        
        js = Javascript(code.substitute(title=self.title,data=output(self.plots),range=self.range.as_ccs_string(),restURL=self.restURL), 
                           lib = ["//cdnjs.cloudflare.com/ajax/libs/dygraph/1.1.1/dygraph-combined.js",
                                  "https://lsst-camera-dev.slac.stanford.edu/CCSWebTrending/CCSTrending.js"]);
        display(js);