Processing for PH2 site

gistfile1.py

import kplr
import json
import csv
import code
import numpy as np
import sys
from itertools import izip



client = kplr.API()
stars  = []


with open('Mdwarfs_keplerstellar_without_head.csv', 'r') as csvfile:
  reader = csv.reader(csvfile, delimiter=',', quotechar='#')
  for row in reader:
    stars.append(row)

lookup = json.load(open("zooniverse_id_lookup.json"))



quarter={0:"2009131105131", 1: '2009166043257', 2: "2009259160929", 3: "2009350155506", 4: "2010078095331", 5:"2010174085026", 6:"2010265121752", 7:"2010355172524", 8:"2011073133259", 9: "2011177032512", 10:"2011271113734", 11:"2012004120508", 12:"2012088054726", 13: "2012179063303" , 14:"2012277125453", 15: "2013011073258", 16:"2013098041711", 17:"2013131215648"}
inv_map = {v:k for k, v in quarter.items()}

manifest = []

count = 0

to_do = 100

for star_meta in stars:
  kepler_id = star_meta[2]
  star      = client.star(kepler_id)


  try:
    lcs       = star.get_light_curves(short_cadence=False)

    metadata  = {"star_kepler_id": kepler_id, "kepmag" : star_meta [3], "teff": star_meta [4], "logg" : star_meta [5], "feh":star_meta [6], "radius" : star_meta [7], "mass": star_meta [8], "nconfp": star_meta [9], "ra": star.kic_ra , "dec" : star.kic_dec, "lc_count" : len(lcs), "lcs": [] }
    count +=1

    if count == to_do:
      break

    print "doing " + kepler_id
    for lc in lcs:
      # import code
      # code.interact(local=locals())
      quarter = inv_map[lc.filename.split("-")[1].split("_")[0]]

      kid = "kplr" + kepler_id.zfill(9)


      with lc.open() as f:
        hdu_data = f[1].data


        flux = hdu_data["sap_flux"]
        time = hdu_data["time"]
        ferr = hdu_data["sap_flux_err"]

        mask = np.invert( np.isnan(flux))

        flux = flux[mask]
        time = time[mask]
        ferr = ferr[mask]

        total = len(flux)
        chunk = total/3

        time_range = time[-1] - time[0]
        t_chunk    = time_range/3.0
        t_start    = time[0]

        time_mask1 = [t < t_chunk + t_start  for t in time]
        time_mask2 = [t > t_chunk + t_start and t < t_chunk*2 + t_start for t in time]
        time_mask3 = [t > t_chunk*2 + t_start for t in time]


        flux_1  = [float(t) for t, s in izip(flux, time_mask1) if s]
        flux_2  = [float(t) for t, s in izip(flux, time_mask2) if s]
        flux_3  = [float(t) for t, s in izip(flux, time_mask3) if s]

        time_1  = [float(t) for t, s in izip(time, time_mask1) if s]
        time_2  = [float(t) for t, s in izip(time, time_mask2) if s]
        time_3  = [float(t) for t, s in izip(time, time_mask3) if s]

        ferr_1  = [float(t) for t, s in izip(ferr, time_mask1) if s]
        ferr_2  = [float(t) for t, s in izip(ferr, time_mask2) if s]
        ferr_3  = [float(t) for t, s in izip(ferr, time_mask3) if s]

        zooniverse_id_lu  = lookup[kid]

        q_key = str(quarter)+'.1'
        if q_key in zooniverse_id_lu:
          zoo_id1 = zooniverse_id_lu[q_key]
        else:
          zoo_id1 = "unknown"

        q_key = str(quarter)+'.2'
        if q_key in zooniverse_id_lu:
          zoo_id2 = zooniverse_id_lu[q_key]
        else:
          zoo_id2 = "unknown"

        q_key = str(quarter)+'.3'
        if q_key in zooniverse_id_lu:
          zoo_id3 = zooniverse_id_lu[q_key]
        else:
          zoo_id3 = "unknown"

        q_metadata_1 = {"start_time": time_1[0] , "quarter": quarter , "chunk": 1, "star_kepler_id": kepler_id, "old_zoonivers_id": zoo_id1 , "location" : "https://s3.amazonaws.com/demo.zooniverse.org/planet_hunter/subjects/"+kepler_id+"_"+str(quarter)+"-1.json" }
        q_metadata_2 = {"start_time": time_2[0] , "quarter": quarter , "chunk": 2, "star_kepler_id": kepler_id, "old_zoonivers_id": zoo_id2 , "location" : "https://s3.amazonaws.com/demo.zooniverse.org/planet_hunter/subjects/"+kepler_id+"_"+str(quarter)+"-2.json" }
        q_metadata_3 = {"start_time": time_3[0] , "quarter": quarter , "chunk": 3, "star_kepler_id": kepler_id, "old_zoonivers_id": zoo_id3 , "location" : "https://s3.amazonaws.com/demo.zooniverse.org/planet_hunter/subjects/"+kepler_id+"_"+str(quarter)+"-3.json" }

        jdata1 = {"x": time_1, "y": flux_1, "dy": ferr_1, "metadata": q_metadata_1 }
        jdata2 = {"x": time_2, "y": flux_2, "dy": ferr_2, "metadata": q_metadata_2 }
        jdata3 = {"x": time_3, "y": flux_3, "dy": ferr_3, "metadata": q_metadata_3 }


        with open("data_files/"+kepler_id+"_"+str(quarter)+"-1.json","w") as outfile:
          json.dump(jdata1, outfile)

        with open("data_files/"+kepler_id+"_"+str(quarter)+"-2.json","w") as outfile:
          json.dump(jdata2, outfile)

        with open("data_files/"+kepler_id+"_"+str(quarter)+"-3.json","w") as outfile:
          json.dump(jdata3, outfile)

        metadata["lcs"].append(q_metadata_1)
        metadata["lcs"].append(q_metadata_2)
        metadata["lcs"].append(q_metadata_3)

    manifest.append(metadata)
  except:
    print "error with "+ kepler_id

with open("manifest.json","w") as outfile:
  json.dump(manifest, outfile)