Skip to content

Instantly share code, notes, and snippets.

@sbarratt
Last active July 10, 2023 03:35
Show Gist options
  • Save sbarratt/18d23d0e442d2697065847a88b12db90 to your computer and use it in GitHub Desktop.
Save sbarratt/18d23d0e442d2697065847a88b12db90 to your computer and use it in GitHub Desktop.
A script to export all FTX history
import pandas as pd
import time
import requests
import time
import hmac
from requests import Request
import sys
import json
import os
try:
import pandas as pd
except Exception as e:
print("You need to install pandas! Try `python3 -m pip install pandas`")
raise e
if not os.path.exists("./data"):
os.mkdir("./data")
api_key = input("Enter api key:")
api_secret = input("Enter api secret:")
def get(url, subaccount=None):
ts = int(time.time() * 1000)
request = Request('GET', url)
prepared = request.prepare()
signature_payload = f'{ts}{prepared.method}{prepared.path_url}'
if prepared.body:
signature_payload += prepared.body
signature_payload = signature_payload.encode()
signature = hmac.new(api_secret.encode(), signature_payload, 'sha256').hexdigest()
headers = {
"accept": "application/json",
"Content-Type": "application/json",
"FTX-KEY": f"{api_key}",
"FTX-SIGN": f"{signature}",
"FTX-TS": f"{ts}",
}
if subaccount is not None:
headers["FTX-SUBACCOUNT"] = subaccount
r = requests.get(url, headers=headers)
if r.status_code != 200:
print("Request got error", r.json(), "status", r.status_code)
sys.exit("")
return r
def get_start_end(url, subaccount=None):
end_time = int(time.time())
df = pd.DataFrame()
while True:
shape_before = df.shape[0]
req_url = url + f"?start_time=0&end_time={end_time}"
l = get(req_url, subaccount=subaccount).json()["result"]
for x in l:
for k in x.keys():
if isinstance(x[k], dict):
x[k] = str(x[k])
df_new = pd.DataFrame(l)
df = pd.concat([df, df_new])
df = df.drop_duplicates()
shape_after = df.shape[0]
print(end_time, req_url, df_new.shape[0], shape_after - shape_before)
if shape_after == shape_before:
break
end_time = pd.to_datetime(df["time"]).min().timestamp()
return df
subaccounts = [None] + [x["nickname"] for x in get("https://ftx.com/api/subaccounts").json()["result"]]
print("Found subaccounts", subaccounts)
for subaccount in subaccounts:
if subaccount is None:
subaccount_prefix = "./data/"
else:
subaccount_prefix = f"./data/subaccount_{subaccount}_"
# account details
try:
account_info = get("https://ftx.com/api/account", subaccount=subaccount).json()["result"]
json.dump(account_info, open(subaccount_prefix + "account_info.json", "w"))
except Exception as e:
print("WARNING: On pulling", "account details got exception", e, "skipping")
# balances
try:
balances = get("https://ftx.com/api/wallet/all_balances", subaccount=subaccount).json()["result"]
json.dump(balances, open(subaccount_prefix + "balances.json", "w"))
except Exception as e:
print("WARNING: On pulling", "balances got exception", e, "skipping")
# Deposits
try:
deposit_history = get_start_end("https://ftx.com/api/wallet/deposits", subaccount=subaccount)
deposit_history.to_csv(subaccount_prefix + "deposit_history.csv")
except Exception as e:
print("WARNING: On pulling", "Deposits got exception", e, "skipping")
# Withdrawals
try:
withdrawal_history = get_start_end("https://ftx.com/api/wallet/withdrawals", subaccount=subaccount)
withdrawal_history.to_csv(subaccount_prefix + "withdrawal_history.csv")
except Exception as e:
print("WARNING: On pulling", "Withdrawals got exception", e, "skipping")
# borrow history
try:
borrow_history = get_start_end("https://ftx.com/api/spot_margin/borrow_history", subaccount=subaccount)
borrow_history.to_csv(subaccount_prefix + "borrow_history.csv")
except Exception as e:
print("WARNING: On pulling", "borrow history got exception", e, "skipping")
# lend history
try:
lending_history = get_start_end("https://ftx.com/api/spot_margin/lending_history", subaccount=subaccount)
lending_history.to_csv(subaccount_prefix + "lending_history.csv")
except Exception as e:
print("WARNING: On pulling", "lend history got exception", e, "skipping")
# referral history
try:
referral_history = get_start_end("https://ftx.com/api/referral_rebate_history", subaccount=subaccount)
referral_history.to_csv(subaccount_prefix + "referral_history.csv")
except Exception as e:
print("WARNING: On pulling", "referral history got exception", e, "skipping")
# fill history
try:
fills = get_start_end("https://ftx.com/api/fills", subaccount=subaccount)
fills.to_csv(subaccount_prefix + "fills.csv")
except Exception as e:
print("WARNING: On pulling", "fill history got exception", e, "skipping")
# funding payments
try:
funding_payments = get_start_end("https://ftx.com/api/funding_payments", subaccount=subaccount)
funding_payments.to_csv(subaccount_prefix + "funding_payments.csv")
except Exception as e:
print("WARNING: On pulling", "funding payments got exception", e, "skipping")
@ryanwebster90
Copy link

ryanwebster90 commented Nov 10, 2022

hey, I got an error on line:
df = df.drop_duplicates()
" Unhashable type: "dict" "
It's running the fix:
df = df.loc[df.astype('str').drop_duplicates().index]

@sbarratt
Copy link
Author

do you know which line from the main script generates this (eg is it fills/borrow/lending/etc)? Sorry, I don't have much FTX account history so probably didn't catch all corner cases.

@ryanwebster90
Copy link

yeah it's 53
df = df.drop_duplicates()
i didn't have the current revision

@hmate9
Copy link

hmate9 commented Nov 10, 2022

Thank you for this!
Note for others: To download for different subaccounts change the headers to:

headers = {
        "accept": "application/json",
        "Content-Type": "application/json",
        "FTX-KEY": f"{api_key}",
        "FTX-SIGN": f"{signature}",
        "FTX-TS": f"{ts}",
        "FTX-SUBACCOUNT": "subaccount_here"
    }

(and change some filenames otherwise they overlap)

@ryanwebster90
Copy link

Now the code is frozen though, so above fix isnt a good solution.

@sbarratt
Copy link
Author

I added looping through subaccounts, skipping pulls that throw errors, and better error messages, etc...

@philipperemy
Copy link

Thank you @sbarratt .

@toriving
Copy link

Thank you @sbarratt.

Line 1 seems superfluous because it overlaps lines 11-15, and line 1 provides more detailed error handling.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment