How to Export Datadog Metrics for Exploration in Jupyter Notebooks: extraction_logic.py

extraction_logic.py

#Extraction Logic :
# 1. take the datadog query 
# 2. pass the query to datadog api with a time span of time_delta milliseconds -> This would pull data in spans of T to T + time_delta
# 3. append this data to a pandas dataframe

def extract(query, start_at, end_at, time_delta):
    
    print('Running extraction for cpu utilization')
    
    '''
    datadog return data as a json blob with a field called 'series' that stores a list called 'pointlist' which has metrics like :
    * timestamp of the point for which the metric was extracted for.
    * value of the field at this timestamp.
    '''
    
    #initialise data lists : 
    
    datetime_list = []
    value_list = []
    metric_list = []
    
    # start reading till this timeframe and then increment timedelta window in the loop
    iteration_end = start_at + time_delta 
    
    while (iteration_end <= end_at):

        results = api.Metric.query(start=start_at, end=iteration_end, query=query)

        for datadog_result in results['series']:
            for time_value_pair_list in datadog_result['pointlist']:
                
                converted_datetime = datetime.fromtimestamp(time_value_pair_list[0]/1000)
                
                datetime_list.append(converted_datetime)
                value_list.append(time_value_pair_list[1])
                metric_list.append(datadog_result['metric']) # store the query that was executed in datadog.
         
        # increment the time spans 
        start_at = iteration_end # change start time as end of last iteration
        iteration_end = iteration_end + time_delta # increment reading frame
        
        #logging logic to see extraction progress
        #print('finished extracting data from ', datetime.fromtimestamp(start_at).strftime('%c') , ' to ', datetime.fromtimestamp(iteration_end).strftime('%c'))


    all_data = {
        'datetime': datetime_list,
        'value': value_list,
        'metric': metric_list
    }

    print('Finished extraction for cpu utilization')
    return pd.DataFrame.from_dict(all_data)
    
#extract data into a dataframe    
data_cpu_metrics = extract(query, start, end, time_delta)