Planet Hunters K2 data ingest

PH2 ingest

# PlanetHunterSubject.destroy_all
# redis = Ouroboros.redis[Rails.env]
# redis.keys('planet_hunter*').each{ |key| redis.del(key) }

json = JSON.parse(File.read('/home/ubuntu/manifest_C1.json')); nil

project = PlanetHunterSubject.project

workflow = PlanetHunterSubject.first.workflows.first

project_id = project.id
workflow_ids = [workflow.id]

total = json.length
json.each.with_index do |source, index|
  puts "#{ index + 1 } / #{ total }"

  coords = source["coords"]
  magnitudes = source["magnitudes"]

  light_curves = source["light_curves"]

  subject = PlanetHunterSubject.create({
    project_id: project_id,
    workflow_ids: workflow_ids,
    coords: coords,
    zooniverse_id: source["zooniverse_id"],
    location: { },
    group_id: BSON::ObjectId('54f4c5ab8f165b6e85000001'),
    metadata: {
      k2: true,
      kepler_id: source["EPIC_number"],
      magnitudes: magnitudes,
      sdss_id: source["SDSS_ID"],
      sdss_ra: source["SDSS_RA"],
      sdss_dec: source["SDSS_DEC"],
      "2mass_id" =>  source["2mass_id"],
      "2mass_ra" =>  source["2mass_ra"],
      "2mass_dec" => source["2mass_dec"],
      spectral_type: source["spectral_type"],
      kepler_2_campaign_no: source["kepler_2_campaign_no"]

    }
  })

  light_curves.each do |lc|
    location = light_curve["location"]
    q        = light_curve["quarter"]
    data_location = location
    subject.add_light_curve! quarter: quarter, data_location: data_location, start_time: light_curve['start_time']
  end



end; nil

PH2 process

import kplr
import json
import csv
import code
import numpy as np
import sys
from itertools import izip



client = kplr.API()
stars  = []


with open('Mdwarfs_keplerstellar_without_head.csv', 'r') as csvfile:
  reader = csv.reader(csvfile, delimiter=',', quotechar='#')
  for row in reader:
    stars.append(row)

lookup = json.load(open("zooniverse_id_lookup.json"))



quarter={0:"2009131105131", 1: '2009166043257', 2: "2009259160929", 3: "2009350155506", 4: "2010078095331", 5:"2010174085026", 6:"2010265121752", 7:"2010355172524", 8:"2011073133259", 9: "2011177032512", 10:"2011271113734", 11:"2012004120508", 12:"2012088054726", 13: "2012179063303" , 14:"2012277125453", 15: "2013011073258", 16:"2013098041711", 17:"2013131215648"}
inv_map = {v:k for k, v in quarter.items()}

manifest = []

count = 0

to_do = 10000000000

for star_meta in stars:
  kepler_id = star_meta[2]
  star      = client.star(kepler_id)


  try:
    lcs       = star.get_light_curves(short_cadence=False)

    metadata  = {"star_kepler_id": kepler_id, "kepmag" : star_meta [3], "teff": star_meta [4], "logg" : star_meta [5], "feh":star_meta [6], "radius" : star_meta [7], "mass": star_meta [8], "nconfp": star_meta [9], "ra": star.kic_ra , "dec" : star.kic_dec, "lc_count" : len(lcs), "lcs": [] }
    count +=1

    if count == to_do:
      break

    print "doing " + kepler_id
    for lc in lcs:
      # import code
      # code.interact(local=locals())
      quarter = inv_map[lc.filename.split("-")[1].split("_")[0]]

      kid = "kplr" + kepler_id.zfill(9)


      with lc.open() as f:
        hdu_data = f[1].data


        flux = hdu_data["PDCsap_flux"]
        time = hdu_data["time"]
        ferr = hdu_data["PDCsap_flux_err"]

        mask = np.invert( np.isnan(flux))

        flux = flux[mask]
        time = time[mask]
        ferr = ferr[mask]

        total = len(flux)
        chunk = total/3

        time_range = time[-1] - time[0]
        t_chunk    = time_range/3.0
        t_start    = time[0]

        time_mask1 = [t < t_chunk + t_start  for t in time]
        time_mask2 = [t > t_chunk + t_start and t < t_chunk*2 + t_start for t in time]
        time_mask3 = [t > t_chunk*2 + t_start for t in time]


        flux_1  = [float(t) for t, s in izip(flux, time_mask1) if s]
        flux_2  = [float(t) for t, s in izip(flux, time_mask2) if s]
        flux_3  = [float(t) for t, s in izip(flux, time_mask3) if s]

        time_1  = [float(t) for t, s in izip(time, time_mask1) if s]
        time_2  = [float(t) for t, s in izip(time, time_mask2) if s]
        time_3  = [float(t) for t, s in izip(time, time_mask3) if s]

        ferr_1  = [float(t) for t, s in izip(ferr, time_mask1) if s]
        ferr_2  = [float(t) for t, s in izip(ferr, time_mask2) if s]
        ferr_3  = [float(t) for t, s in izip(ferr, time_mask3) if s]

        zooniverse_id_lu  = lookup[kid]

        q_key = str(quarter)+'.1'
        if q_key in zooniverse_id_lu:
          zoo_id1 = zooniverse_id_lu[q_key]
        else:
          zoo_id1 = "unknown"

        q_key = str(quarter)+'.2'
        if q_key in zooniverse_id_lu:
          zoo_id2 = zooniverse_id_lu[q_key]
        else:
          zoo_id2 = "unknown"

        q_key = str(quarter)+'.3'
        if q_key in zooniverse_id_lu:
          zoo_id3 = zooniverse_id_lu[q_key]
        else:
          zoo_id3 = "unknown"

        q_metadata_1 = {"start_time": time_1[0] , "quarter": quarter , "chunk": 1, "star_kepler_id": kepler_id, "old_zoonivers_id": zoo_id1 , "location" : "https://s3.amazonaws.com/demo.zooniverse.org/planet_hunter/subjects/"+kepler_id+"_"+str(quarter)+"-1.json" }
        q_metadata_2 = {"start_time": time_2[0] , "quarter": quarter , "chunk": 2, "star_kepler_id": kepler_id, "old_zoonivers_id": zoo_id2 , "location" : "https://s3.amazonaws.com/demo.zooniverse.org/planet_hunter/subjects/"+kepler_id+"_"+str(quarter)+"-2.json" }
        q_metadata_3 = {"start_time": time_3[0] , "quarter": quarter , "chunk": 3, "star_kepler_id": kepler_id, "old_zoonivers_id": zoo_id3 , "location" : "https://s3.amazonaws.com/demo.zooniverse.org/planet_hunter/subjects/"+kepler_id+"_"+str(quarter)+"-3.json" }

        jdata1 = {"x": time_1, "y": flux_1, "dy": ferr_1, "metadata": q_metadata_1 }
        jdata2 = {"x": time_2, "y": flux_2, "dy": ferr_2, "metadata": q_metadata_2 }
        jdata3 = {"x": time_3, "y": flux_3, "dy": ferr_3, "metadata": q_metadata_3 }


        with open("data_files/"+kepler_id+"_"+str(quarter)+"-1.json","w") as outfile:
          json.dump(jdata1, outfile)

        with open("data_files/"+kepler_id+"_"+str(quarter)+"-2.json","w") as outfile:
          json.dump(jdata2, outfile)

        with open("data_files/"+kepler_id+"_"+str(quarter)+"-3.json","w") as outfile:
          json.dump(jdata3, outfile)

        metadata["lcs"].append(q_metadata_1)
        metadata["lcs"].append(q_metadata_2)
        metadata["lcs"].append(q_metadata_3)

    manifest.append(metadata)
  except:
    print "error with "+ kepler_id
    e = sys.exc_info()[0]
    print e 
with open("manifest.json","w") as outfile:
  json.dump(manifest, outfile)