Make CSCDOC HV summary table

make_hv_summary.py

import os
import sys
import commands
import datetime
import time

"""
1. Visit this exact URL in Google Chrome
https://cmswbm.cern.ch/cmsdb/servlet/ConditionBrowser?DISPLAY=1&SUBMIT=1D&OPTION=vs_time&STEP=3&BEGIN=2017.10.29_09:50:16&END=2017.10.31_09:50:16&PRESCALER=1&cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..M13_C03..10=1&cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..M13_C03..11=1&

2. After authenticating you should see a plot. Press cmd+alt+i on Mac to open developer tools and go to Network tab

3. Refresh the page. You should see an entry starting with "ConditionBrowser?DISPLAY" at the top of the stuff that shows up. Right click on this entry  > Copy > Copy as cURL

4. Paste the curl command into the variable below


Create a text file `channels.txt` with content copied from the "Show as plain text" output in 
HV Channel Management of DCS. Example lines:
ME-1/3/01 ch. 18 | 3550V
ME-1/3/03 ch. 11 | 3400V
ME-1/3/16 ch. 10 | 3500V
ME-1/3/16 ch. 16 | 3400V

Executing this script will download images of current vs time for the past 2 days for the 
specified channels and a reference channel, as well as a .csv with the same information, into
the `outdir` folder. The csv will be analyzed and if the channel is ok to have the voltage raised,
the new voltage is calculated. A summary table will be printed out at the end. Remember that you,
of course, need to actually raise the voltages in the DCS HV Channel Management panel.
"""

template_curl_cmd = "curl 'https://cmswbm.cern.ch/cmsdb/servlet/ConditionBrowser?DISPLAY=1&SUBMIT=1D&OPTION=vs_time&STEP=3&BEGIN=2017.10.29_09:50:16&END=2017.10.31_09:50:16&PRESCALER=1&cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..M13_C03..10=1&cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..M13_C03..11=1&' -H 'Pragma: no-cache' -H 'DNT: 1' -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: no-cache' -H 'Referer: https://cmswbm.cern.ch/cmsdb/servlet/ConditionBrowser' -H 'Cookie: JSESSIONID=47B7C8F3129488ACE9174CC71AD9096C; __utma=203412483.2092000603.1481656321.1482954929.1483208678.11; AI_SESSION=28BFF16A104A5B797EE26A6222D8A3541A6464ED5DC3E0624FE7734780023331686955199896CC01B060E3DEF06B3AB58D7A769167DD64465E7CC3236B461627C2935A9F775EEF56573F6757F47796B3CA4F5FE64777A64CAA906C01DE86DC83569B2303; _shibstate_1509350998_6e98=https%3A%2F%2Fcmswbm.cern.ch%2Fimages%2FWBM_4C_100_yellow.jpg; _shibstate_1509445509_3b0b=https%3A%2F%2Fcmswbm.cern.ch%2Fcmsdb%2Fservlet%2FConditionBrowser%3FDISPLAY%3D1%26SUBMIT%3D1D%26OPTION%3Dvs_time%26STEP%3D3%26BEGIN%3D2017.10.29_09%3A50%3A16%26END%3D2017.10.31_09%3A50%3A16%26PRESCALER%3D1%26cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..M13_C03..10%3D1%26cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..M13_C03..11%3D1%26; _shibsession_636d7377626d5f6d6f6e69746f7268747470733a2f2f636d7377626d2e6365726e2e63682f53686962626f6c6574682e73736f2f41444653=_7772b99fc67a25e1b75b47dc391bcb39; _saml_idp=aHR0cHM6Ly9jZXJuLmNoL2xvZ2lu' -H 'Connection: keep-alive' --compressed --insecure"
outdir = "outputs/"

# print commands.getstatusoutput(curl_cmd)
def modify_curl_url(curl_cmd, new_url):
    new_curl_cmd = curl_cmd[:]
    new_curl_cmd = curl_cmd[:].split()
    new_curl_cmd[1] = "'{}'".format(new_url)
    new_curl_cmd = " ".join(new_curl_cmd)
    return new_curl_cmd

def add_curl_output(curl_cmd, output_fname):
    return curl_cmd + " -o {}".format(output_fname)

def is_channel_bad(fname):
    """
    Given a csv file name, return (True, max_current, max_current_ref) if the
    channel is still being noisy/spiking, along with the max current for
    selected channel and reference channel
    """
    data = []
    max_current = -1
    max_current_ref = -1
    with open(fname,"r") as fhin:
        found_first = False
        is_reference = False
        for line in fhin:
            # We want to break before reading the reference. We don't care at the moment
            if "cms_omds_lb" in line:
                if not found_first: found_first = True
                else: is_reference = True
            if "WEIGHT" in line: continue
            if ",,," in line: continue
            _, _, t1, current = line.strip().split(",")
            t1 = int(t1)
            current = float(current)
            if is_reference:
                if current > max_current_ref: max_current_ref = current
            else:
                if current > max_current: max_current = current
    is_bad = (max_current > 350.) or ((max_current >= 2.0 * max_current_ref) and (max_current_ref > 0))
    return is_bad, max_current, max_current_ref

def get_raised_voltage(station,ring,voltage):
    """
    If a channel is deemed good by the other function, what do we raise the
    voltage to?  Implemented logic following slide 12 of
    https://indico.cern.ch/event/672452/contributions/2751023/attachments/1539303/2413088/CSCDOCReport20171011.pdf
    """
    if station == 1 and ring == 1:
        if voltage <= 2800.: return voltage+20
        elif 2800. < voltage < 2850.: return voltage+10
        elif voltage >= 2850.: return 2900
    else:
        if voltage <= 3500.: return voltage+50
        elif 3500. < voltage < 3550.: return voltage+10
        elif voltage >= 3550.: return 3600

dry_run = False

template_url = """https://cmswbm.cern.ch/cmsdb/servlet/ConditionBrowser?DISPLAY=1&SUBMIT=1D&OPTION=vs_time&STEP=3&BEGIN={beginstr}&END={endstr}&PRESCALER=1&{chamberstr1}&{chamberstr2}&"""
datestr_begin = (datetime.datetime.now()-datetime.timedelta(days=2)).strftime("%Y.%m.%d_%H:%M:%S")
datestr_end = (datetime.datetime.now()+datetime.timedelta(hours=9)).strftime("%Y.%m.%d_%H:%M:%S")

commands.getstatusoutput("mkdir -p {}".format(outdir))

infos = []
with open("channels.txt","r") as fhin:
    for line in fhin:
        if "ME" not in line: continue
        parts = line.split()
        chamberstr = parts[0]
        channel = int(parts[2])
        voltage = int(parts[-1].replace("V",""))
        station, ring, chamber = chamberstr.replace("ME","").split("/")
        chamber = int(chamber)
        ring = int(ring)
        station = int(station)
        endcap = 2 if station < 0 else 1
        station = abs(station)
        infos.append( [endcap, station, ring, chamber, channel, voltage] )

# LOOP OVER CHAMBERCHANNELS
# for endcap, station, ring, chamber, channel, voltage in [(2,1,3,3,10,3400)]:
to_print = []
for endcap, station, ring, chamber, channel, voltage in infos:
    # Reference channel is an adjacent channel with this directionality so that I don't have
    # to deal with edge cases
    channel_ref = channel+1 if channel < 10 else channel-1

    chamber_channel_tag = "ME{endcap}{station}/{ring}/{chamber} ch. {channel}".format(endcap="-" if endcap == 2 else "+", station=station, ring=ring, chamber=str(chamber).zfill(2), channel=channel)
    print ">>> Working on {tag} ({voltage}V) with reference channel {channelref}".format(tag=chamber_channel_tag, channelref=channel_ref, voltage=voltage)

    chamber_str1 = "cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..{endcap}{station}{ring}_C{chamber}..{channel}=1".format(endcap="M" if endcap == 2 else "P", station=station, ring=ring, chamber=str(chamber).zfill(2), channel=channel) # first is red
    chamber_str2 = "cms_omds_lb.CMS_CSC_PVSS_COND.CSC_HV.CURRENT..{endcap}{station}{ring}_C{chamber}..{channel}=1".format(endcap="M" if endcap == 2 else "P", station=station, ring=ring, chamber=str(chamber).zfill(2), channel=channel_ref) # second is blue

    url = template_url.format( beginstr=datestr_begin, endstr=datestr_end, chamberstr1=chamber_str1, chamberstr2=chamber_str2 )
    curl_cmd = modify_curl_url(template_curl_cmd, url)

    output_fname_base = "ME{endcap}{station}_{ring}_{chamber}_chan{channel}_ref{channelref}".format(endcap="m" if endcap == 2 else "p", station=station, ring=ring, chamber=str(chamber).zfill(2), channel=channel, channelref=channel_ref)
    csv_fname = "{}/{}.csv".format(outdir,output_fname_base)
    img_fname = "{}/{}.png".format(outdir,output_fname_base)

    do_fetch = not dry_run

    # If file already exists, don't fetch it again.
    # This means you need to update the output directory or delete old
    # files if you want to fetch a fresh set of data
    if os.path.exists(csv_fname):
        do_fetch = False

    if do_fetch:
        out = commands.getoutput(curl_cmd)
        time.sleep(1)
    else:
        out = open("out.html","r").read()

    csv_link, img_link = None, None
    for line in out.splitlines():
        if "../../cmsdb/data/ConditionBrowser" not in line: continue
        parts = line.split("|")
        for part in parts:
            if "csv" in part:
                csv_link = "https://cmswbm.cern.ch/cmsdb/servlet/"+part.split("HREF=")[-1].split(">")[0]
            if "png" in part:
                img_link = "https://cmswbm.cern.ch/cmsdb/servlet/"+part.split("SRC=")[-1].split(">")[0]

    if not csv_link or not img_link:
        print "ERROR with this channel!"
        continue

    csv_curl_cmd = add_curl_output(modify_curl_url(curl_cmd, csv_link), csv_fname)
    img_curl_cmd = add_curl_output(modify_curl_url(curl_cmd, img_link), img_fname)

    if do_fetch:
        commands.getstatusoutput(csv_curl_cmd)
        time.sleep(1)
        commands.getstatusoutput(img_curl_cmd)
        time.sleep(1)

    is_bad, current, current_ref = is_channel_bad(csv_fname)
    vchange = "no change"
    notes = "-"
    if not is_bad: 
        new_voltage = get_raised_voltage(station, ring, voltage)
        vchange = "+{}V".format(new_voltage - voltage)
        notes = "normal, raising"
    to_print.append( "| {:16s} |  {:4d}  | {:9} | {:19} |".format( chamber_channel_tag, voltage, vchange, notes ) )



print "|-------------------------------------------------------------|"
print "|    CHANNEL       | HV (V) |  Change   |       Notes         |"
print "|-------------------------------------------------------------|"
print "\n".join(to_print)
print "|-------------------------------------------------------------|"