how to pull yahoo finance data

yahoo_finance.py

def get_v1_url(symbol, period_type, crumb):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
    }

    period1 = 493590046
    period2 = 1913180947

    session = requests.Session()
    session.get('https://finance.yahoo.com', headers=headers)

    # Use the existing crumb if provided, otherwise fetch a new one
    if not crumb:
        crumb_response = session.get('https://query2.finance.yahoo.com/v1/test/getcrumb', headers=headers)
        if crumb_response.status_code == 200 and crumb_response.text:
            crumb = crumb_response.text
        else:
            print("Failed to retrieve crumb")

    base_url = 'https://query1.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/'

    if(period_type=='Annual'):
        annual_types = ",".join(["annual" + option for option in optionSet])
        url = f"{base_url}{symbol}?symbol={symbol}&type={annual_types}&period1={period1}&period2={period2}&crumb={crumb}"
    else:
        quarterly_types = ",".join(["quarterly" + option for option in optionSet])
        url = f"{base_url}{symbol}?symbol={symbol}&type={quarterly_types}&period1={period1}&period2={period2}&crumb={crumb}"

    #print("Annual URL:", annual_url)
    #print("Quarterly URL:", quarterly_url)
    return(url)

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
}

session = requests.Session()
session.get('https://finance.yahoo.com', headers=headers)

crumb = False
# Use the existing crumb if provided, otherwise fetch a new one
if not crumb:
    crumb_response = session.get('https://query2.finance.yahoo.com/v1/test/getcrumb', headers=headers)
    if crumb_response.status_code == 200 and crumb_response.text:
        crumb = crumb_response.text

symbol = 'AAPL'

#fundamentals
v7_url = f"""https://query1.finance.yahoo.com/v7/finance/quote?lang=en-US&region=US&corsDomain=finance.yahoo.com&symbols={symbol}&crumb={crumb}"""

v1_annual_url = get_v1_url('AAPL','Annual',crumb)
v1_quarterly_url = get_v1_url('AAPL','Quarterly',crumb)

#print(v1_annual_url)

v7_response = session.get(v7_url, headers=headers)

v1_a_response = session.get(v1_annual_url, headers=headers)
v1_q_response = session.get(v1_quarterly_url, headers=headers)

v7_data = {}
if v7_response.status_code == 200:
    v7_data = v7_response.json()

v7_payload = v7_data['quoteResponse']['result'][0].keys()
print('v7_payload')
print(v7_payload)

v1_q_data = {}
if v1_q_response.status_code == 200:
    v1_q_data = v1_q_response.json()

v1_a_data = {}
if v1_a_response.status_code == 200:
    v1_a_data = v1_a_response.json()

#financial statements
#
v1_q_payload = v1_q_data['timeseries']['result']
v1_a_payload = v1_a_data['timeseries']['result']

v1_q_payload = [p['meta']['type'] for p in v1_q_payload]
v1_a_payload = [p['meta']['type'] for p in v1_a_payload]
print('v1_q_payload')
print(v1_q_payload)

print('v1_a_payload')
print(v1_a_payload)
"""
[{'meta': {'symbol': ['AAPL'],
  'annualNetIncomeContinuousOperations': [{'dataId': 20094,
    'asOfDate': '2020-09-30',
    'periodType': '12M',
    'currencyCode': 'USD',
    'reportedValue': {'raw': 57411000000.0, 'fmt': '57.41B'}},
...
 {'meta': {'symbol': ['AAPL'], 'type': ['annualTaxEffectOfUnusualItems']},
  'timestamp': [1601424000, 1632960000, 1664496000, 1696032000],
  'annualTaxEffectOfUnusualItems': [{'dataId': 20419,
    'asOfDate': '2020-09-30',
    'periodType': '12M',
    'currencyCode': 'USD',
    'reportedValue': {'raw': 0.0, 'fmt': '0.00'}},
"""

# Save the results to JSON files
with open('v7_data.json', 'w') as f_data:
    json.dump(v7_data, f_data, ensure_ascii=False, indent=4)

# Save the results to JSON files
with open('v1_a_data.json', 'w') as f_data:
    json.dump(v1_a_data, f_data, ensure_ascii=False, indent=4)

with open('v1_q_data.json', 'w') as f_data:
    json.dump(v1_q_data, f_data, ensure_ascii=False, indent=4)