load_data

#!/bin/bash
########################
### Datasets loading ###
########################

import datetime
dsets_version = 0.0
import numpy as np
import sys
import hax
import pandas as pd

hax.__version__


hax.init(experiment='XENON1T',
         #pax_version_policy=pax_version,
         #sc_api_key='ssn1sslkvdhittoywjk88w9cmbxf4fmprn68r8hm',
         minitree_paths=[
                        '/project2/lgrandi/xenon1t/minitrees/pax_v6.8.0',
                        '/project/lgrandi/xenon1t/minitrees/pax_v6.8.0',
                       ] )

#dsets = hax.runs.datasets

# take all datasets in runs db
#datasets = hax.runs.datasets # this variable holds all dataset info
#print("ok")
dsets = hax.runs.tags_selection(include=['sciencerun1'])
#dsets = dsets[(dsets.source__type == 'none')| (dsets.source__type == 'None')] # 

dsets = dsets[(dsets.start > pd.to_datetime('02/02/2017')) & (dsets.end < pd.to_datetime('02/08/2017'))]
#dsets['start_date'] = dsets.start.dt.date

#dsets = '180204_2230'
#run_names = dsets.name

# get rid of problematic runs
bad_runs = ['161204_1517','170111_031','170111_0314','170630_1922','170403_0733','170403_0533','170705_1426','170717_0406','170726_0030',
           '170806_2306','170803_0059','170804_2156','170814_0403','170817_0406','170817_1508','170818_1231','170819_0735',
           '170818_2233','170819_0434','170819_1737','170820_0239','170820_0639','170820_1040','170820_1641','170820_1741',
           '170820_2242','170821_1946','170821_1545','170822','170822_0248','170822_0548','170822_0648','170823_0144',
           '170823_1647','170824_0649','170824_1150','170824_1451','170824_1751','170825_0554','170824_2252','170825_1006',
           '170825_1107','170825_1407','170825_1207','170825_1307'] # n'o minitree'

#for run in bad_runs:
#    run_names = run_names[run_names != run]
#    dsets  = dsets[dsets.name != run]
    
preselect=['cs1>0',
           'cs2>0',
           #'z<-9',
           #'z>-92.9',
           '0.0137*(cs2_bottom/11.7033907889 + cs1/0.142947753187)>1500',
           '0.0137*(cs2_bottom/11.7033907889 + cs1/0.142947753187)<3000', 
           #'s2_area_fraction_top < 0.74'
          ]






df = hax.minitrees.load(dsets.name, ['Corrections','Basics','Extended','Fundamentals', 'LargestPeakProperties','Proximity'], 
                        preselection=preselect,
                        force_reload=False, 
                        num_workers = 20,
                        )

df.to_pickle('/scratch/midway2/rocchetti/data_pax6.8.0_0nbb_MC_trya.p')