App Insights Large Data Download Method - Python

app_insights_api_call.py

import json
import os
import requests
import pandas as pd

end_point = 'api.applicationinsights.io'


def download_data(product_n_data_type):
    start_time = "datetime(2021-10-13T00:00:00.001Z)" #Set a start time accoridng to your requirement
    download_finished = False
    complete_df = {}
    end_time = 'now()'
    is_first_time_download = True
    appId, API_KEY = get_keys()

    while not download_finished:
        query = get_query(product_n_data_type, start_time, end_time)
        params = {"query": query}
        headers = {'X-Api-Key': API_KEY, 'Accept': 'application/json'}
        url = f'https://api.applicationinsights.io/v1/apps/{appId}/query'
        response = requests.get(url, headers=headers, params=params)
        logs = json.loads(response.text)
        result = {}

        for table in logs['tables']:
            result[table['name']] = pd.DataFrame.from_records(table['rows'], columns=[col['name'] for col in table['columns']])
            df_final = result['PrimaryResult']

            if len(df_final) == 500000:
                start_time = 'datetime(' + df_final['timestamp'].max() + ')'
                if is_first_time_download:
                    complete_df = df_final
                    is_first_time_download = False
                else:
                    complete_df = complete_df.append(df_final)
            else:
                download_finished = True
                if is_first_time_download:
                    complete_df = df_final
                    is_first_time_download = False
                else:
                    complete_df = complete_df.append(df_final)

    return complete_df


def get_query():
    return "Add APP Insights Query Here"
    

def get_keys(product_n_data_type):
    # Assign app_Id and API_KEY values to variables or read from the pipeline secrets at run-time
    return app_Id, API_KEY