dummy_keys.py

API_KEYS = {
    'infura': "YOUR_INFURA_KEY",
    'thegraph': "YOUR_THEGRAPH_KEY",
    'etherscan': "YOUR_ETHERSCAN_KEY",
}

get_usd_prices.py

from collections import OrderedDict
import json
import time
import warnings
import requests
import argparse
from web3 import Web3
from web3.exceptions import ABIFunctionNotFound
from gql import gql, Client
from gql.transport.requests import RequestsHTTPTransport

from keys import API_KEYS

# Hardcoded unit conversion in case the ERC-20 interface doesn't implement `decimals`
wei = 18
units = {
    'WBTC': 8,
    'USDC': 6,
}

# see on etherscan https://etherscan.io/address/0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f
uniswap_factory_address = "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f"

# h/t https://towardsdatascience.com/exploring-decentraland-marketplace-sales-with-thegraph-and-graphql-2f5e8e7199b5
# Select your transport with a defined url endpoint
transport = RequestsHTTPTransport(
    url="https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2")
# Create a GraphQL client using the defined transport
client = Client(transport=transport, fetch_schema_from_transport=True)

# Use infura to fetch and call smart contracts via HTTP
web3 = Web3(Web3.HTTPProvider(
    f'https://mainnet.infura.io/v3/{API_KEYS["infura"]}'))


def load_tokens(tokens_file):
    with open(tokens_file, 'r') as f:
        tokens = f.read().splitlines()
    return tokens


def get_reserves(token0, token1, cache='reserves.json'):
    _token0, _token1, pool = get_uniswap_pool(token0, token1)
    reserve0, reserve1, blockTimeStampLast = pool.functions.getReserves().call()
    decimals0, decimals1 = map(get_decimals, (_token0, _token1))
    ret = OrderedDict({
        _token0: reserve0 / 10**decimals0,
        _token1: reserve1 / 10**decimals1,
    })

    # TODO switch to csv cache so you don't have to parse the whole file every time you want to append
    if cache:
        with open(cache, 'r') as f:
            data = json.load(f)
        data[f'{_token0}/{_token1}'] = ret
        with open(cache, 'w') as f:
            json.dump(data, f)

    return ret


def get_marginal_price(token0, token1):
    """Marginal price of `token0` in units of `token1`"""
    _token0, _token1, pool = get_uniswap_pool(token0, token1)
    reserve0, reserve1, blockTimeStampLast = pool.functions.getReserves().call()
    decimals0, decimals1 = map(get_decimals, (_token0, _token1))
    reserve0 /= 10**decimals0
    reserve1 /= 10**decimals1
    return reserve0 / reserve1, f"{_token0}/{_token1}"


def get_uniswap_pool(token0, token1):
    """
    Get contract object representing the Uniswap pool between `token0` and `token1`.
    @param token0, token1: String e.g. "DAI" or "WETH". Token symbols are searched on Uniswap's subgraph. The contract with that symbol and highest number of transactions is returned.
    @return (tokenA, tokenB, pool): `(tokenA, tokenB)` are the pool tokens _in the order they are defined by the contract_. `pool` is the contract object.
    """
    address0, address1 = map(get_token_address, (token0, token1))
    uniswap_factory = get_contract_from_address(uniswap_factory_address)
    pool_address = uniswap_factory.functions.getPair(address0, address1).call()
    pool = get_contract_from_address(pool_address)
    _address0, _address1 = pool.functions.token0().call(), pool.functions.token1().call()
    if _address0 == address0 and \
            _address1 == address1:
        return (token0, token1, pool)
    elif _address0 == address1 and \
            _address1 == address0:
        # print(
        #     f"WARNING: You requested {token0}/{token1} but this pool is {token1}/{token0}")
        return (token1, token0, pool)
    else:
        raise Exception("Pool symbols don't match")


def get_decimals(token, graphql_client=client, web3_provider=web3):
    contract = get_contract_from_address(get_token_address(
        token, graphql_client=graphql_client), web3_provider=web3_provider)
    try:
        return contract.functions.decimals().call()
    except ABIFunctionNotFound:
        warnings.warn(
            f"{token} ERC20 contract has no `decimals` function. Defaulting to {units.get(token, wei)}.")
        return units.get(token, wei)


def get_token_address(token, graphql_client=client):
    # graphQL magic
    # TODO use graphene, it's safer
    query = gql(f"""
  {{
    tokens(where: {{symbol:"{token}"}}, orderBy:txCount, orderDirection: desc, first: 1){{
      id,
      txCount,
      symbol,
    }}
  }}""")
    result = client.execute(query)
    assert len(result['tokens']
               ) == 1, f"Found more than one token with name {token}"
    # TODO addresses are not checksum
    # I don't know if this is TheGraph's or Uniswap's fault
    # I'm fixing it myself but this is a hack and it's not safe
    return Web3.toChecksumAddress(result['tokens'][0]['id'])


def get_contract_from_address(address, web3_provider=web3):
    """Get web3 contract object from address. Calls etherscan's API to retrieve ABI."""
    abi = get_abi(address)
    contract = web3.eth.contract(address=address, abi=abi)
    return contract


def get_abi(address):
    # TODO may need to work around API limit of 5 calls / sec
    response = requests.get(
        f'https://api.etherscan.io/api?module=contract&action=getabi&apikey={API_KEYS["etherscan"]}&address={address}')
    rjson = response.json()
    if rjson['status'] == '1' and rjson['message'] == 'OK':
        return rjson['result']
    print(rjson)
    raise Exception(rjson['message'])


def main(tokens, reference_token):
    prices = {}
    # Fetch prices
    for token in tokens:
        try:
            print(f"\n\nFetching {token} price in units of {reference_token}...")
            price, pool = get_marginal_price(token, reference_token)
            if pool == f"{token}/{reference_token}":
                price = 1/price
            elif pool == f"{reference_token}/{token}":
                pass
            else:
                raise ValueError(
                    f"Requested pool was {token}/{reference_token} but got {pool}.")
            prices[token] = price
            time.sleep(2)  # gentle on the APIs
        except Exception as e:
            warnings.warn(f"\tFailed to fetch price. Error: {e}")

    return prices


parser = argparse.ArgumentParser()

parser.add_argument("-t", "--tokens", default="./tokens.txt", type=str,
                    help="File containing list of token symbols, with one symbol per line", dest="input_file")
parser.add_argument("-r", "--ref", default='USDC', type=str,
                    help="Calculate prices with respect to this reference token", dest="reference_token")
parser.add_argument("-o", "--output", default="./prices.csv", type=str,
                    help="Saves output to a file with this name", dest="output_file")

if __name__ == "__main__":
    args = parser.parse_args()

    with open(args.input_file, 'r') as f:
        tokens = f.read().splitlines()

    prices = main(tokens, args.reference_token)

    with open(args.output_file, 'w') as f:
        f.write(f'asset,{args.reference_token}_price')
        f.write('\n')
        f.writelines([f'{k},{v}\n' for k, v in prices.items()])

requirements.txt

aniso8601==7.0.0
argon2-cffi==20.1.0
async-generator==1.10
attrs==20.3.0
autobahn==21.3.1
Automat==20.2.0
autopep8==1.5.6
backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
base58==2.1.0
bitarray==1.2.2
bleach==3.3.0
certifi==2020.12.5
cffi==1.14.5
chainlink-feeds==0.2.9
chardet==4.0.0
configparser==5.0.2
constantly==15.1.0
cryptography==3.4.7
cryptowatch-sdk==0.0.14
cycler==0.10.0
cytoolz==0.11.0
dateparser==1.0.0
decorator @ file:///home/ktietz/src/ci/decorator_1611930055503/work
defusedxml==0.7.1
entrypoints==0.3
eth-abi==2.1.1
eth-account==0.5.4
eth-hash==0.3.1
eth-keyfile==0.5.1
eth-keys==0.3.3
eth-rlp==0.2.1
eth-typing==2.2.2
eth-utils==1.10.0
gql==2.0.0
graphene==2.1.8
graphql-core==2.3.2
graphql-relay==2.0.1
hexbytes==0.2.1
hyperlink==21.0.0
idna==2.10
incremental==21.3.0
ipfshttpclient==0.7.0a1
ipykernel @ file:///tmp/build/80754af9/ipykernel_1607452791405/work/dist/ipykernel-5.3.4-py3-none-any.whl
ipython @ file:///tmp/build/80754af9/ipython_1614616449711/work
ipython-genutils @ file:///tmp/build/80754af9/ipython_genutils_1606773439826/work
ipywidgets==7.6.3
jedi==0.18.0
Jinja2==2.11.3
jsonschema==3.2.0
jupyter-client @ file:///tmp/build/80754af9/jupyter_client_1601311786391/work
jupyter-core @ file:///tmp/build/80754af9/jupyter_core_1612213314396/work
jupyterlab-pygments==0.1.2
jupyterlab-widgets==1.0.0
kiwisolver==1.3.1
lru-dict==1.1.7
MarkupSafe==1.1.1
marshmallow==3.11.1
matplotlib==3.3.4
mistune==0.8.4
multiaddr==0.0.9
nbclient==0.5.3
nbconvert==6.0.7
nbformat==5.1.3
nest-asyncio==1.5.1
netaddr==0.8.0
notebook==6.3.0
numpy==1.20.1
packaging==20.9
pandas==1.2.3
pandocfilters==1.4.3
parsimonious==0.8.1
parso==0.8.1
pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
Pillow==8.1.2
plotly==4.14.3
prometheus-client==0.9.0
promise==2.3
prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1616415428029/work
protobuf==3.15.5
ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.7.0
pycparser==2.20
pycryptodome==3.10.1
Pygments @ file:///tmp/build/80754af9/pygments_1615143339740/work
pyOpenSSL==20.0.1
pyparsing==2.4.7
pyrsistent==0.17.3
python-binance==0.7.9
python-coinmarketcap==0.2
python-dateutil @ file:///home/ktietz/src/ci/python-dateutil_1611928101742/work
python-dotenv==0.15.0
pytz==2021.1
PyYAML==5.4.1
pyzmq==20.0.0
regex==2021.3.17
requests==2.25.1
requests-cache==0.5.2
retrying==1.3.3
rlp==2.0.1
Rx==1.6.1
scipy==1.6.1
seaborn==0.11.1
Send2Trash==1.5.0
service-identity==18.1.0
six @ file:///tmp/build/80754af9/six_1605205306277/work
terminado==0.9.4
testpath==0.4.4
toml==0.10.2
toolz==0.11.1
tornado @ file:///tmp/build/80754af9/tornado_1606942317143/work
traitlets @ file:///home/ktietz/src/ci/traitlets_1611929699868/work
Twisted==21.2.0
txaio==21.2.1
tzlocal==2.1
ujson==4.0.2
urllib3==1.26.3
varint==1.0.2
wcwidth @ file:///tmp/build/80754af9/wcwidth_1593447189090/work
web3==5.17.0
webencodings==0.5.1
websocket-client==0.58.0
websockets==8.1
widgetsnbextension==3.5.1
zope.interface==5.3.0