Skip to content

Instantly share code, notes, and snippets.

@wlensinas
Last active February 8, 2022 03:10
Show Gist options
  • Save wlensinas/af799bcd204c8e7e7e2ed4e2b4557993 to your computer and use it in GitHub Desktop.
Save wlensinas/af799bcd204c8e7e7e2ed4e2b4557993 to your computer and use it in GitHub Desktop.
This is a simple scraper for get a list of crypto currency prices
import requests
import concurrent.futures
from bs4 import BeautifulSoup
from pprint import pprint
from re import sub
sites = [
('Bitcoin', 'https://coinmarketcap.com/currencies/bitcoin/'),
('Ethereum', 'https://coinmarketcap.com/currencies/ethereum/'),
('BNB', 'https://coinmarketcap.com/currencies/bnb/'),
('Cardano', 'https://coinmarketcap.com/currencies/cardano/'),
('XRP', 'https://coinmarketcap.com/currencies/xrp/'),
('Solana', 'https://coinmarketcap.com/currencies/solana/'),
('Terra (Luna)', 'https://coinmarketcap.com/currencies/terra-luna/'),
]
def sort_crypto_price(crypto_data):
return sorted(crypto_data, key= lambda k:k['price'], reverse=True)
def get_request(crypto_url, crypto_name):
res = requests.get(crypto_url)
return (crypto_name, res.text)
def get_data(crypto_sites):
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
cripto_data = []
for cs in crypto_sites:
futures.append(executor.submit(get_request, crypto_url=cs[1], crypto_name=cs[0]))
for future in concurrent.futures.as_completed(futures):
cripto_data.append(future.result())
return cripto_data
def convert_str_to_decimal(price: str) -> float:
return float(sub(r'[^\d.]', '', price))
def get_crypto_data(crypto_data):
values = []
CRYPTO_NAME = 0
TEXT_DATA = 1
for cd in crypto_data:
soup = BeautifulSoup(cd[TEXT_DATA], 'html.parser')
price = soup.select('.priceValue')
values.append(
{'cripto': cd[CRYPTO_NAME],
'price': convert_str_to_decimal(price[0].find('span').getText())})
return values
data_sites = get_data(sites)
sorted_data = sort_crypto_price(get_crypto_data(data_sites))
pprint(sorted_data)
beautifulsoup4==4.10.0
certifi==2021.10.8
charset-normalizer==2.0.11
idna==3.3
requests==2.27.1
soupsieve==2.3.1
urllib3==1.26.8
@wlensinas
Copy link
Author

Steps to use this code:

  1. Download files or copy the code
  2. Install dependencies: pip install -r requirements.txt
  3. Execute the file: python3 mini_crypto_scraper.py

Output:

[{'cripto': 'Bitcoin', 'price': 44031.8},
 {'cripto': 'Ethereum', 'price': 3149.67},
 {'cripto': 'BNB', 'price': 434.21},
 {'cripto': 'Solana', 'price': 117.81},
 {'cripto': 'Terra (Luna)', 'price': 58.97},
 {'cripto': 'Cardano', 'price': 1.24},
 {'cripto': 'XRP', 'price': 0.86}]

If you have a suggest, please write below :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment