habitat_field_names_transition.py

import logging
import couchdbkit
db = couchdbkit.Server("http://habitat.habhub.org/")['habitat']
#db = couchdbkit.Server("http://localhost:5984/")['test_fieldnames']

mapping = {
    'Sats': ('satellites', None),
    'temp_int': ('temperature_internal', None),
    'main_battery_volts': ('battery', None),
    'pressure_hpa': ('pressure', 100.0),
    'temp_iext': ('temperature_external', None),
    'temp_radio': ('temperature_radio', None),
    'battery_volts': ('battery', None),
    'cycle_count': ('sentence_id', None),
    'INTTEMP': ('temperature_internal', None),
    'external_temperature': ('temperature_external', None),
    'temp2': ('temperature_two', None),
    'temp1': ('temperature_one', None),
    'EXTTEMP': ('temperature_external', None),
    'TEMP': ('temperature', None),
    'Ext temperature': ('temperature_external', None),
    'temp_int2': ('temperature_internal_two', None),
    'COUNT': ('sentence_id', None),
    'pyro_battery_volts': ('battery_pyro', None),
    'Air Pressure': ('pressure', None),
    'external temperature': ('temperature_external', None),
    'battery_millivolts': ('battery', 0.001),
    'internal temperature': ('temperature_internal', None),
    'Temperature': ('temperature', None),
    'temp_cam': ('temperature_camera', None),
    'SATELLITES': ('satellites', None),
    'BATTERYMV': ('battery', 0.001),
    'External Temp': ('temperature_external', None),
    'Internal Temp': ('temperature_internal', None),
    'count': ('sentence_id', None),
    'BATMV': ('battery', 0.001),
    'ascent speed': ('ascentrate', None),
    'rate_of_ascent': ('ascentrate', None),
    'speed_vertical': ('ascentrate', None),
    'ascent_rate': ('ascentrate', None),
    'vertical_speed': ('ascentrate', None),
    'speed_east': ('speed', None),
    'sats': ('satellites', None),
    'temp_ext': ('temperature_external', None),
    'SATS': ('satellites', None),
    'ROA': ('ascentrate', None),
    'int_temp': ('temperature_internal', None),
    'Radio Temp': ('temperature_radio', None),
    'Speed': ('speed', None),
    'ext_temp': ('temperature_external', None),
    'internal_temperature': ('temperature_internal', None),
    'Batt Voltage': ('battery', None),
}

fields_seen = set()

fmt = "[%(asctime)s] %(levelname)s %(name)s: %(message)s"
logger = logging.getLogger()
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setFormatter(logging.Formatter(fmt))
ch.setLevel(logging.WARNING)
fh = logging.FileHandler('transition.log')
fh.setFormatter(logging.Formatter(fmt))
fh.setLevel(logging.INFO)
logger.addHandler(ch)
logger.addHandler(fh)
logging.getLogger("restkit").setLevel(logging.WARNING)

RED = "\033[1;31m"
CYAN = "\033[0;36m"
RESET = "\033[0m"

def update_flight(doc):
    logging.info("Doc {0}".format(doc['_id']))
    for payload in (doc['payloads'][p] for p in doc['payloads']):
        for field in payload['sentence']['fields']:
            k = field['name']
            if k in mapping:
                if mapping[k][1]:
                    f = {"type": "normal", "filter": "common.numeric_scale",
                         "source": k, "destination": mapping[k][0],
                         "factor": mapping[k][1]}
                    if "filters" not in payload:
                        payload['filters'] = {}
                    if "post" not in payload['filters']:
                        payload['filters']['post'] = []
                    payload['filters']['post'].append(f)
                    logging.info("Added filter {0}".format(str(f)))
                else:
                    field['name'] = mapping[k][0]
                    logging.info("{0} -> {1}".format(k, mapping[k][0]))
            elif k not in fields_seen:
                print CYAN + "New field: {0}".format(k) + RESET
                logging.info("New field: {0}".format(k))
                fields_seen.add(k)

def update_telem(doc):
    logging.info("Doc {0}".format(doc['_id']))
    for k in doc['data'].keys():
        if k in mapping:
            v = doc['data'][k]
            oldv = v
            if mapping[k][1]:
                v *= mapping[k][1]
                logging.info("{0} -> {1}".format(oldv, v))
            doc['data'][mapping[k][0]] = v
            del doc['data'][k]
            logging.info("{0} -> {1}".format(k, mapping[k][0]))
        elif k not in fields_seen:
            print CYAN + "New field: {0}".format(k) + RESET
            logging.info("New field: {0}".format(k))
            fields_seen.add(k)

for docid in db:
    doc = db[docid['id']]
    try:
        if doc['type'] == 'flight':
            update_flight(doc)
        elif doc['type'] == 'payload_telemetry':
            update_telem(doc)
        db.save_doc(doc)
    except KeyError:
        pass