download_bybit_btcusd.py

#########################################
# Automate downloading BTCUSD data from
# https://public.bybit.com/trading/BTCUSD/
#########################################
import requests
import gzip
from datetime import datetime, timedelta
import requests
import gzip
import csv
from datetime import datetime, timedelta
from io import BytesIO
from tqdm import tqdm

start_date = datetime.strptime('2019-10-01', '%Y-%m-%d')
end_date = datetime.strptime('2023-06-14', '%Y-%m-%d')  # Change to the current date

date_list = []
while start_date <= end_date:
    date_list.append(start_date.strftime('%Y%m%d'))
    start_date += timedelta(days=1)

SYMBOL = "BTCUSD"  # margin - BTC for spot
BASE_URL = f'https://public.bybit.com/trading/{SYMBOL}/{SYMBOL}'

output_filename = f'bybit_all_{SYMBOL}_data.csv'

with open(output_filename, 'w', newline='', encoding='utf-8') as csvfile:
    csv_writer = csv.writer(csvfile)

    for d in tqdm(date_list):
        year = d[:4]
        month = d[4:6]
        day = d[6:8]
        url = f"{BASE_URL}{year}-{month}-{day}.csv.gz"
        r = requests.get(url, stream=True)

        if r.status_code == 200:
            # Decompress gzip data in memory
            gz_data = BytesIO(r.content)
            with gzip.open(gz_data, 'rt', encoding='utf-8') as gz_file:
                csv_reader = csv.reader(gz_file)
                for row in csv_reader:
                    csv_writer.writerow(row)
        else:
            print(r.status_code)
            print(r.text)
            print(f"Failed to fetch data for {d}")
            break