new_relic_data_fetcher.py

import datetime
import os
import json
import pandas as pd
import datetime as dt
import requests
import traceback


class NewRelicDataFetcher:
    def __init__(self, account_id, api_key, timeout=30):
        self.endpoint = "https://api.newrelic.com/graphql"
        self.account_id = account_id
        self.api_key = api_key
        self.timeout = timeout

    def _make_headers(self):
        return {
            "Content-Type": "application/json",
            "API-Key": self.api_key
        }

    def fetch(self, product, table, date, sampling_seconds):
        since = int(date.timestamp())
        until = int(since + sampling_seconds)

        try:
            query = f"SELECT * FROM {table} WHERE appName='{product}' SINCE {since} UNTIL {until} LIMIT MAX"
            placeholder = """
            {
              actor {
                account(id: __ACCOUNT__ID) {
                  nrql(timeout: __TIMEOUT__, query: "__QUERY__") {
                    results
                  }
                }
              }
            }
            """ \
                .replace("__ACCOUNT__ID", str(self.account_id)) \
                .replace("__TIMEOUT__", str(self.timeout)) \
                .replace("__QUERY__", query) \
                .strip()
            params = {"query": placeholder}
            response = requests.post(self.endpoint, headers=self._make_headers(), json=params, timeout=60)

            if response.status_code != 200:
                print("Invalid:",  response.status_code, response.text)
                return {}

            results = json.loads(response.text)["data"]["actor"]["account"]["nrql"]["results"]
            return results
        except:
            traceback.print_exc()
            return {}


if __name__ == '__main__':
    fetcher = NewRelicDataFetcher(account_id=os.environ['NEWRELIC_API_KEY'], api_key=os.environ['NEWRELIC_API_SECRET'])
    timezone = datetime.timezone(datetime.timedelta(hours=9))

    # 60秒毎にサンプリングする (1データ(60秒)で最大2000件の取得)
    sampling_seconds = 60

    tables = ['MobileSession', 'MobileRequest', 'MobileRequestError']
    apps = ['android', 'ios']

    # 並列化するとRateLimitにかかる可能性あり
    for table in tables:
        for app in apps:
            start = dt.datetime(2022, 12, 1, 0, 0, 0, tzinfo=timezone)
            end = dt.datetime(2022, 12, 1, 1, 0, 0, tzinfo=timezone)
            while end > start:
                indicators = fetcher.fetch(app, table, end, sampling_seconds)
                df = pd.DataFrame(indicators)
                df.to_csv(f'{table}_{app}.csv', header=False, mode='a')
                print(f"Done. {end}")
                end -= dt.timedelta(seconds=sampling_seconds)