Skip to content

Instantly share code, notes, and snippets.

@Aditya1001001

Aditya1001001/RSS_feed_aggregator_output.py

Last active April 14, 2022 08:54
Show Gist options
  • Save Aditya1001001/523631af6bde1bceaa07b6ab24df04e8 to your computer and use it in GitHub Desktop.
Save Aditya1001001/523631af6bde1bceaa07b6ab24df04e8 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
bitcoin_dot_com_string_example.py
{
"title": "Total Value Locked in Defi Takes a Hit Losing $21 Billion in 6 Days",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://news.bitcoin.com/feed/",
"value": "Total Value Locked in Defi Takes a Hit Losing $21 Billion in 6 Days"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://news.bitcoin.com/total-value-locked-in-defi-takes-a-hit-losing-21-billion-in-6-days/"
}
],
"link": "https://news.bitcoin.com/total-value-locked-in-defi-takes-a-hit-losing-21-billion-in-6-days/",
"authors": [
{
"name": "Jamie Redman"
}
],
"author": "Jamie Redman",
"author_detail": {
"name": "Jamie Redman"
},
"published": "Mon, 11 Apr 2022 18:00:05 +0000",
"published_parsed": [ 2022, 4, 11, 18, 0, 5, 0, 101, 0],
"tags": [
{
"term": "Defi",
"scheme": None,
"label": None
},
{
"term": "Aave",
"scheme": None,
"label": None
},
{
"term": "Anchor",
"scheme": None,
"label": None
},
{
"term": "Arbitrum",
"scheme": None,
"label": None
},
{
"term": "Avalanche",
"scheme": None,
"label": None
},
{
"term": "Binance Smart Chain",
"scheme": None,
"label": None
},
{
"term": "Bridges",
"scheme": None,
"label": None
},
{
"term": "BSC",
"scheme": None,
"label": None
},
{
"term": "Cross-chain Bridges",
"scheme": None,
"label": None
},
{
"term": "crypto assets",
"scheme": None,
"label": None
},
{
"term": "cryptoslam.io",
"scheme": None,
"label": None
},
{
"term": "Curve.finance",
"scheme": None,
"label": None
},
{
"term": "decentralized finance",
"scheme": None,
"label": None
},
{
"term": "DeFi",
"scheme": None,
"label": None
},
{
"term": "Defi Apps",
"scheme": None,
"label": None
},
{
"term": "Defi protocol",
"scheme": None,
"label": None
},
{
"term": "defi value locked",
"scheme": None,
"label": None
},
{
"term": "defillama.com",
"scheme": None,
"label": None
},
{
"term": "Ethereum",
"scheme": None,
"label": None
},
{
"term": "Fantom",
"scheme": None,
"label": None
},
{
"term": "lending app",
"scheme": None,
"label": None
},
{
"term": "Lido",
"scheme": None,
"label": None
},
{
"term": "makerdao",
"scheme": None,
"label": None
},
{
"term": "NFT sales",
"scheme": None,
"label": None
},
{
"term": "Polygon",
"scheme": None,
"label": None
},
{
"term": "ronin",
"scheme": None,
"label": None
},
{
"term": "Solana",
"scheme": None,
"label": None
},
{
"term": "Terra",
"scheme": None,
"label": None
}
],
"id": "https://news.bitcoin.com/?p=520507",
"guidislink": False,
"summary": "<img alt=\"Total Value Locked in Defi Takes a Hit Losing $21 Billion in 6 Days\" class=\"attachment-medium_large size-medium_large wp-post-image\" height=\"392\" src=\"https://static.news.bitcoin.com/wp-content/uploads/2022/04/image-2022-04-11-120341.jpg\" style=\"float: left; margin: 0 15px 15px 0;\" width=\"696\" />Over the last six days, the total value locked (TVL) in decentralized finance (defi) has dropped 9.21% from a high of $232.88 billion on April 5, to today&#x2019;s $211.43 billion. The value locked in defi protocols shed 4.40% of that value during the last 24 hours. Value Locked in Defi Shudders This Week While the [&#8230;]",
"summary_detail": {
"type": "text/html",
"language": None,
"base": "https://news.bitcoin.com/feed/",
"value": "<img alt=\"Total Value Locked in Defi Takes a Hit Losing $21 Billion in 6 Days\" class=\"attachment-medium_large size-medium_large wp-post-image\" height=\"392\" src=\"https://static.news.bitcoin.com/wp-content/uploads/2022/04/image-2022-04-11-120341.jpg\" style=\"float: left; margin: 0 15px 15px 0;\" width=\"696\" />Over the last six days, the total value locked (TVL) in decentralized finance (defi) has dropped 9.21% from a high of $232.88 billion on April 5, to today&#x2019;s $211.43 billion. The value locked in defi protocols shed 4.40% of that value during the last 24 hours. Value Locked in Defi Shudders This Week While the [&#8230;]"
},
"content": [
{
"type": "text/html",
"language": None,
"base": "https://news.bitcoin.com/feed/",
"value": "<img alt=\"Total Value Locked in Defi Takes a Hit Losing $21 Billion in 6 Days\" class=\"attachment-medium_large size-medium_large wp-post-image\" height=\"392\" src=\"https://static.news.bitcoin.com/wp-content/uploads/2022/04/image-2022-04-11-120341.jpg\" style=\"float: left; margin: 0 15px 15px 0;\" width=\"696\" /><p><strong>Over the last six days, the total value locked (TVL) in decentralized finance (defi) has dropped 9.21% from a high of $232.88 billion on April 5, to today&#x2019;s $211.43 billion. The value locked in defi protocols shed 4.40% of that value during the last 24 hours.</strong></p>\n<h2>Value Locked in Defi Shudders This Week</h2>\n<p>While the crypto economy has shed billions, the defi landscape has also felt the brunt of the storm. Last week, the TVL in defi <a href=\"https://news.bitcoin.com/total-value-locked-in-defi-nears-lifetime-high-ethereums-tvl-dominates-by-54/\">neared a record high</a> but most of that value has been erased since April 5. At the time of writing on Monday, April 11, 2022, the TVL in defi is coasting along at $211.43 billion which is 9.21% less than it was six days ago. Presently, the top defi protocol in terms of TVL is Curve Finance with a 9.52% dominance rating. Curve has a TVL of around $20.12 billion which is down 5.30% over the last week.</p>\n\n<p>The second-largest defi protocol, in terms of value locked, is Lido with $17.43 billion. Lido, however, has lost 13.95% during the last seven days. The third-largest defi protocol, Anchor, has $14.75 billion total value locked but has lost 6.79% this past week. Two defi protocols that have increased during the last week include Cbridge, up 35.34% and Platypus Finance up 22.44%. Out of the $211 billion locked on Monday, Ethereum commands 55.59% of the aggregate with $117.52 billion.</p>\n<p>Terra&#x2019;s (LUNA) defi TVL is the second largest with $26.68 billion or 12.62% of the $211.43 billion. The third-largest defi TVL by blockchain is Binance Smart Chain (BSC) as the network currently commands $13.16 billion or 6.23% of the TVL in defi today. Ethereum&#x2019;s, Terra&#x2019;s, and BSC&#x2019;s TVLs are followed by the TVLs held on Avalanche and Solana, respectively. The top 12 blockchains in terms of TVL are all down in value during the last week, except for the Near blockchain network which is up 22.57% over the last seven days.</p>\n\n<p>Overall, the top smart contract platform coins by market capitalization are all down in value during the last week. 24-hour stats show the $667 billion worth of the top smart contract platform coins is down 8.5%. As far as cross-chain bridges to Ethereum are concerned, there&#x2019;s $19.9 billion TVL across cross-chain bridges among 40,184 unique addresses. The figure is down 8.8% during the last 30 days.</p>\n<p><em><strong>What do you think about the week&#x2019;s defi action and the TVL losing $21 billion in six days? Let us know what you think about this subject in the comments section below.</strong></em></p>"
}
],
"bnmedia_url": "https://static.news.bitcoin.com/wp-content/uploads/2022/04/image-2022-04-11-120341.jpg",
"bnmedia_width": "760",
"bnmedia_height": "428",
"bnmedia_post-thumbnail": "",
"bnmedia_barker_title": "Over the last six days, the total value locked (TVL) in #DeFi has dropped 9.21% losing $21 billion.",
"bnmedia_late_barker_title": "Over the last six days, the total value locked (TVL) in #DeFi has dropped 9.21% losing $21 billion."
}
Raw
coindesk_tuple_example.py
{
"title": "IDEG Asset Management Partners With Coinbase Prime to Launch Actively Managed ETH Fund",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "IDEG Asset Management Partners With Coinbase Prime to Launch Actively Managed ETH Fund"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://www.coindesk.com/business/2022/04/12/ideg-asset-management-partners-with-coinbase-prime-to-launch-actively-managed-eth-fund/?utm_medium=referral&utm_source=rss&utm_campaign=headlines"
}
],
"link": "https://www.coindesk.com/business/2022/04/12/ideg-asset-management-partners-with-coinbase-prime-to-launch-actively-managed-eth-fund/?utm_medium=referral&utm_source=rss&utm_campaign=headlines",
"id": "SA34BZHVG5DGZGA73BJYRKNUPE",
"guidislink": False,
"authors": [
{
"name": "Aoyon Ashraf"
}
],
"author": "Aoyon Ashraf",
"author_detail": {
"name": "Aoyon Ashraf"
},
"summary": "The new fund will be part of IDEG\u2019s thematic product suite, \u201cTIMES.\u201d",
"summary_detail": {
"type": "text/html",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "The new fund will be part of IDEG\u2019s thematic product suite, \u201cTIMES.\u201d"
},
"published": "Tue, 12 Apr 2022 00:00:00 +0000",
"published_parsed": [ 2022, 4, 12, 0, 0, 0, 1, 102, 0],
"updated": "2022-04-12T02:24:10.777+00:00",
"updated_parsed": [ 2022, 4, 12, 2, 24, 10, 1, 102, 0],
"tags": [
{
"term": "Business",
"scheme": "https://www.coindesk.com/business/",
"label": None
}
],
"media_content": [
{
"url": "https://www.coindesk.com/resizer/OU8arxxyxaslFmjEoDkYJMoBb0s=/800x600/cloudfront-us-east-1.images.arcpublishing.com/coindesk/J4B5F34FYBBEHE2FHFP6MCQHWY.jpg",
"type": "image/jpeg",
"height": "600",
"width": "800"
}
],
"content": [
{
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "A digital screen displays the price of cryptocurrency Ethereum to U.S. dollar in Hong Kong, China, on Friday, March 25, 2022. Bitcoin\u00a0climbed to more than $44,000 for the first time in almost a month, breaking out of its recent narrow trading range amid a renewal of risk appetite.\u00a0Photographer: Paul Yeung/Bloomberg via Getty Images"
}
],
"media_credit": [
{
"role": "author",
"scheme": "urn:ebu",
"content": "Bloomberg"
}
],
"credit": "Bloomberg"
}
Raw
cointelegraph_dict_example.py
{
"title": "Price analysis 4/11: BTC, ETH, BNB, SOL, XRP, ADA, LUNA, AVAX, DOT, DOGE",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://cointelegraph.com/rss",
"value": "Price analysis 4/11: BTC, ETH, BNB, SOL, XRP, ADA, LUNA, AVAX, DOT, DOGE"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://cointelegraph.com/news/price-analysis-4-11-btc-eth-bnb-sol-xrp-ada-luna-avax-dot-doge"
},
{
"length": "840",
"type": "image/jpg",
"href": "https://images.cointelegraph.com/images/840_aHR0cHM6Ly9zMy5jb2ludGVsZWdyYXBoLmNvbS91cGxvYWRzLzIwMjItMDQvZWQzNzE1NDgtYmE5My00NGIyLTg2NGYtOGU3MzY2ZDU2OTFiLmpwZw==.jpg",
"rel": "enclosure"
}
],
"link": "https://cointelegraph.com/news/price-analysis-4-11-btc-eth-bnb-sol-xrp-ada-luna-avax-dot-doge",
"media_content": [
{
"url": "https://images.cointelegraph.com/images/840_aHR0cHM6Ly9zMy5jb2ludGVsZWdyYXBoLmNvbS91cGxvYWRzLzIwMjItMDQvZWQzNzE1NDgtYmE5My00NGIyLTg2NGYtOGU3MzY2ZDU2OTFiLmpwZw==.jpg",
"medium": "image"
}
],
"published": "Mon, 11 Apr 2022 19:23:50 +0100",
"published_parsed": [ 2022, 4, 11, 18, 23, 50, 0, 101, 0],
"authors": [
{
"name": "Cointelegraph By Rakesh Upadhyay"
}
],
"author": "Cointelegraph By Rakesh Upadhyay",
"author_detail": {
"name": "Cointelegraph By Rakesh Upadhyay"
},
"tags": [
{
"term": "Markets",
"scheme": None,
"label": None
},
{
"term": "Bitcoin",
"scheme": None,
"label": None
},
{
"term": "Ethereum",
"scheme": None,
"label": None
},
{
"term": "Binance Coin",
"scheme": None,
"label": None
},
{
"term": "Ripple",
"scheme": None,
"label": None
},
{
"term": "Cardano",
"scheme": None,
"label": None
},
{
"term": "Solana",
"scheme": None,
"label": None
},
{
"term": "Avalanche",
"scheme": None,
"label": None
},
{
"term": "Terra",
"scheme": None,
"label": None
},
{
"term": "Dogecoin",
"scheme": None,
"label": None
},
{
"term": "Polkadot",
"scheme": None,
"label": None
},
{
"term": "Price analysis",
"scheme": None,
"label": None
}
],
"id": "https://cointelegraph.com/news/price-analysis-4-11-btc-eth-bnb-sol-xrp-ada-luna-avax-dot-doge",
"guidislink": False,
"summary": "<p style=\"float: right; margin: 0 0 10px 15px; width: 240px;\"><img src=\"https://images.cointelegraph.com/images/840_aHR0cHM6Ly9zMy5jb2ludGVsZWdyYXBoLmNvbS91cGxvYWRzLzIwMjItMDQvZWQzNzE1NDgtYmE5My00NGIyLTg2NGYtOGU3MzY2ZDU2OTFiLmpwZw==.jpg\" /></p><p>BTC and altcoins broke below their lower support levels, signaling that further downside is the most likely short-term outcome.</p>",
"summary_detail": {
"type": "text/html",
"language": None,
"base": "https://cointelegraph.com/rss",
"value": "<p style=\"float: right; margin: 0 0 10px 15px; width: 240px;\"><img src=\"https://images.cointelegraph.com/images/840_aHR0cHM6Ly9zMy5jb2ludGVsZWdyYXBoLmNvbS91cGxvYWRzLzIwMjItMDQvZWQzNzE1NDgtYmE5My00NGIyLTg2NGYtOGU3MzY2ZDU2OTFiLmpwZw==.jpg\" /></p><p>BTC and altcoins broke below their lower support levels, signaling that further downside is the most likely short-term outcome.</p>"
}
}
Raw
create_streamlit_cols.py
def add_articles(articles):
columns = st.columns(2)
no_of_articles = len(articles)
article_index = 0
col_index = -1
while True:
article = articles[article_index]
with columns[0 if col_index < 0 else 1]:
title = article["title"]
# print(title)
link = article["link"]
date = article["pub_date"]
st.markdown(f"### [{title}]({link})")
if article["image"] == None:
if "nulltx" in link:
st.image("https://nulltx.com/wp-content/uploads/2018/10/nulltx-logo-red.png")
if "cryptoslate" in link:
st.image("https://cryptoslate.com/wp-content/themes/cryptoslate-2020/images/cs-media-logo-dark.png")
else:
st.image(article["image"])
st.caption(f"{date}")
st.write(article["summary"])
if no_of_articles == article_index + 1:
return
article_index += 1
col_index *= -1
Raw
dict_value_example.py
article_properties = {"https://cointelegraph.com/rss" : {"title": "title",
"link": "link", "pub_date": "published",
"summary" : {"summary": ("p", 1, "text")},
"image": {"summary" : ("img", "src")}}}
Raw
entries_example.py
{
"title": "IDEG Asset Management Partners With Coinbase Prime to Launch Actively Managed ETH Fund",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "IDEG Asset Management Partners With Coinbase Prime to Launch Actively Managed ETH Fund"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://www.coindesk.com/business/2022/04/12/ideg-asset-management-partners-with-coinbase-prime-to-launch-actively-managed-eth-fund/?utm_medium=referral&utm_source=rss&utm_campaign=headlines"
}
],
"link": "https://www.coindesk.com/business/2022/04/12/ideg-asset-management-partners-with-coinbase-prime-to-launch-actively-managed-eth-fund/?utm_medium=referral&utm_source=rss&utm_campaign=headlines",
"id": "SA34BZHVG5DGZGA73BJYRKNUPE",
"guidislink": False,
"authors": [
{
"name": "Aoyon Ashraf"
}
],
"author": "Aoyon Ashraf",
"author_detail": {
"name": "Aoyon Ashraf"
},
"summary": "The new fund will be part of IDEG\u2019s thematic product suite, \u201cTIMES.\u201d",
"summary_detail": {
"type": "text/html",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "The new fund will be part of IDEG\u2019s thematic product suite, \u201cTIMES.\u201d"
},
"published": "Tue, 12 Apr 2022 00:00:00 +0000",
"published_parsed": [ 2022, 4, 12, 0, 0, 0, 1, 102, 0],
"updated": "2022-04-12T02:24:10.777+00:00",
"updated_parsed": [ 2022, 4, 12, 2, 24, 10, 1, 102, 0],
"tags": [
{
"term": "Business",
"scheme": "https://www.coindesk.com/business/",
"label": None
}
],
"media_content": [
{
"url": "https://www.coindesk.com/resizer/OU8arxxyxaslFmjEoDkYJMoBb0s=/800x600/cloudfront-us-east-1.images.arcpublishing.com/coindesk/J4B5F34FYBBEHE2FHFP6MCQHWY.jpg",
"type": "image/jpeg",
"height": "600",
"width": "800"
}
],
"content": [
{
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "A digital screen displays the price of cryptocurrency Ethereum to U.S. dollar in Hong Kong, China, on Friday, March 25, 2022. Bitcoin\u00a0climbed to more than $44,000 for the first time in almost a month, breaking out of its recent narrow trading range amid a renewal of risk appetite.\u00a0Photographer: Paul Yeung/Bloomberg via Getty Images"
}
],
"media_credit": [
{
"role": "author",
"scheme": "urn:ebu",
"content": "Bloomberg"
}
],
"credit": "Bloomberg"
}
Raw
extract_attr_dict.py
if type(val) == dict:
nested_key = list(val.keys())[0]
nested_val = val[nested_key]
soup = BeautifulSoup(item[nested_key])
if val[nested_key] == "text":
article[key] = soup.text
elif type(nested_val) == tuple:
if nested_key == "summary":
if len(nested_val) == 2:
article["summary"] = soup.text
elif len(nested_val) == 3:
article["summary"] = soup.find_all(
[nested_val[0]])[nested_val[1]].text
if key == "image":
article["image"] = soup.find([nested_val[0]]).get(nested_val[1])
Raw
extract_attr_string.py
if type(val) == str:
article[key] = item[val]
Raw
extract_attr_tuple.py
if type(val) == tuple:
article[key] = item[val[0]][val[1]][val[2]]
Raw
feedparser_output.py
{
"bozo": False,
"entries": {
"title": "IDEG Asset Management Partners With Coinbase Prime to Launch Actively Managed ETH Fund",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "IDEG Asset Management Partners With Coinbase Prime to Launch Actively Managed ETH Fund"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://www.coindesk.com/business/2022/04/12/ideg-asset-management-partners-with-coinbase-prime-to-launch-actively-managed-eth-fund/?utm_medium=referral&utm_source=rss&utm_campaign=headlines"
}
],
"link": "https://www.coindesk.com/business/2022/04/12/ideg-asset-management-partners-with-coinbase-prime-to-launch-actively-managed-eth-fund/?utm_medium=referral&utm_source=rss&utm_campaign=headlines",
"id": "SA34BZHVG5DGZGA73BJYRKNUPE",
"guidislink": False,
"authors": [
{
"name": "Aoyon Ashraf"
}
],
"author": "Aoyon Ashraf",
"author_detail": {
"name": "Aoyon Ashraf"
},
"summary": "The new fund will be part of IDEG\u2019s thematic product suite, \u201cTIMES.\u201d",
"summary_detail": {
"type": "text/html",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "The new fund will be part of IDEG\u2019s thematic product suite, \u201cTIMES.\u201d"
},
"published": "Tue, 12 Apr 2022 00:00:00 +0000",
"published_parsed": [ 2022, 4, 12, 0, 0, 0, 1, 102, 0],
"updated": "2022-04-12T02:24:10.777+00:00",
"updated_parsed": [ 2022, 4, 12, 2, 24, 10, 1, 102, 0],
"tags": [
{
"term": "Business",
"scheme": "https://www.coindesk.com/business/",
"label": None
}
],
"media_content": [
{
"url": "https://www.coindesk.com/resizer/OU8arxxyxaslFmjEoDkYJMoBb0s=/800x600/cloudfront-us-east-1.images.arcpublishing.com/coindesk/J4B5F34FYBBEHE2FHFP6MCQHWY.jpg",
"type": "image/jpeg",
"height": "600",
"width": "800"
}
],
"content": [
{
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "A digital screen displays the price of cryptocurrency Ethereum to U.S. dollar in Hong Kong, China, on Friday, March 25, 2022. Bitcoin\u00a0climbed to more than $44,000 for the first time in almost a month, breaking out of its recent narrow trading range amid a renewal of risk appetite.\u00a0Photographer: Paul Yeung/Bloomberg via Getty Images"
}
],
"media_credit": [
{
"role": "author",
"scheme": "urn:ebu",
"content": "Bloomberg"
}
],
"credit": "Bloomberg"
},
"feed": {
"title": "CoinDesk",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "CoinDesk"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://www.coindesk.com"
},
{
"href": "https://www.coindesk.com/arc/outboundfeeds/rss/?outputType=xml",
"rel": "self",
"type": "application/rss+xml"
}
],
"link": "https://www.coindesk.com",
"subtitle": "Latest headlines from Coindesk.",
"subtitle_detail": {
"type": "text/html",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "Latest headlines from Coindesk."
},
"updated": "Tue, 12 Apr 2022 05:45:46 +0000",
"updated_parsed": [ 2022, 4, 12, 5, 45, 46, 1, 102, 0],
"language": "en-US",
"tags": [
{
"term": "News",
"scheme": "https://www.coindesk.com",
"label": None
}
],
"rights": "\u00a9 2022 CoinDesk, Inc.",
"rights_detail": {
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "\u00a9 2022 CoinDesk, Inc."
},
"generator_detail": {
"name": "Arc Publishing"
},
"generator": "Arc Publishing",
"ttl": "5",
"sy_updateperiod": "hourly",
"sy_updatefrequency": "12",
"image": {
"href": "https://www.coindesk.com/resizer/fTK3gATlyciJ-BZG2_OP12niDe0=/144x32/downloads.coindesk.com/arc/failsafe/feeds/coindesk-feed-logo.png",
"title": "CoinDesk: Bitcoin, Ethereum, Crypto News and Price Data",
"title_detail": {
"type": "text/plain",
"language": None,
"base": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"value": "CoinDesk: Bitcoin, Ethereum, Crypto News and Price Data"
},
"links": [
{
"rel": "alternate",
"type": "text/html",
"href": "https://www.coindesk.com"
}
],
"link": "https://www.coindesk.com"
}
},
"headers": {
"content-type": "application/xml; charset=utf-8",
"content-length": "14978",
"server": "openresty",
"content-encoding": "gzip",
"etag": "W/\"e67b-WoiTK6kIwyIrcoeTTMWcUFAjB4Y\"",
"last-modified": "Tue, 12 Apr 2022 05:45:46 GMT",
"vary": "Accept-Encoding",
"cache-control": "private, max-age=186",
"expires": "Tue, 12 Apr 2022 05:50:43 GMT",
"date": "Tue, 12 Apr 2022 05:47:37 GMT",
"connection": "close",
"set-cookie": "akaas_AS_coindesk_coindesk_prod=2147483647~rv=96~id=fb4fbaddffd162f74cf2fd438106dc0d; path=/; Expires=Tue, 19 Jan 2038 03:14:07 GMT; Secure; SameSite=None",
"content-security-policy": "upgrade-insecure-requests"
},
"etag": "W/\"e67b-WoiTK6kIwyIrcoeTTMWcUFAjB4Y\"",
"updated": "Tue, 12 Apr 2022 05:45:46 GMT",
"updated_parsed": [ 2022, 4, 12, 5, 45, 46, 1, 102, 0],
"href": "https://www.coindesk.com/arc/outboundfeeds/rss/",
"status": 200,
"encoding": "utf-8",
"version": "rss20",
"namespaces": {
"content": "http://purl.org/rss/1.0/modules/content/",
"wfw": "http://wellformedweb.org/CommentAPI/",
"dc": "http://purl.org/dc/elements/1.1/",
"": "http://www.w3.org/2005/Atom",
"sy": "http://purl.org/rss/1.0/modules/syndication/",
"slash": "http://purl.org/rss/1.0/modules/slash/",
"media": "http://search.yahoo.com/mrss/"
}
}
Raw
feedparser_output_example.py
from feedparser import parse
feed = parse("https://www.coindesk.com/arc/outboundfeeds/rss/")
print(feed)
Raw
init_api_and_get_articles.py
from newscatcherapi import NewsCatcherApiClient
API_KEY = 'YOUR-API-KEY'
newscatcherapi = NewsCatcherApiClient(x_api_key=API_KEY)
news_articles = newscatcherapi.get_search(q='*', sources = sources)
Raw
item_attr_map.py
item_attr_map = {"https://www.coindesk.com/arc/outboundfeeds/rss/" :
{"title": "title", "link": "link", "pub_date": "published",
"summary" : "summary", "image" : ("media_content", 0, "url")},
"https://cointelegraph.com/rss" : {"title": "title",
"link": "link", "pub_date": "published",
"summary" : {"summary": ("p", 1, "text")},
"image": {"summary" : ("img", "src")}},
"https://news.bitcoin.com/feed/" : {"title": "title",
"link": "link", "pub_date": "published",
"summary" : "bnmedia_barker_title",
"image" :"bnmedia_url"},
"https://cryptopotato.com/feed/" : {"title": "title",
"link": "link", "pub_date": "published",
"summary" : "summary",
"image": ("media_content", 0, "url")},
"https://zycrypto.com/category/news/feed/" : {"title": "title",
"link": "link", "pub_date": "published",
"summary": {"summary": "text"},
"image": {"summary" : ("img", "src")}},
"https://nulltx.com/feed/": {"title": "title",
"link": "link", "pub_date": "published",
"summary": {"summary": ("p", 0, "text")},
"image": None},
"https://coinquora.com/news/feed/": {"title": "title",
"link": "link", "pub_date": "published",
"summary": {"summary": "text"},
"image": {"summary" : ("img", "src")}},
"https://ambcrypto.com/feed/" : {"title": "title",
"link": "link", "pub_date": "published",
"summary": {"summary": "text"},
"image": {"summary" : ("img", "src")}},
"https://cryptoslate.com/feed/": {"title": "title",
"link": "link", "pub_date": "published",
"summary": {"summary": ("p", 0, "text")},
"image": None},
"https://crypto.news/feed/": {"title": "title",
"link": "link", "pub_date": "published",
"summary" : "summary",
"image" : ("media_content", 0, "url")}
}
Raw
list_of_sources.py
sources = ["coindesk.com", "cointelegraph.com", "bitcoin.com",
"cryptopotato.com", "zycrypto.com", "nulltx.com", "coinquora.com",
"ambcrypto.com", "cryptoslate.com", "crypto.news"]
Raw
newscatcherapi_output_loop.py
api_articles = []
news_articles = newscatcherapi.get_search(q='*', sources = sources, from_ = "01/02/2022")
for article in news_articles["articles"]:
temp_article = {}
temp_article["title"] = article["title"]
temp_article["link"] = article["link"]
temp_article["pub_date"] = article["published_date"]
temp_article["summary"] = article["excerpt"]
temp_article["image"] = article["media"]
api_articles.append(temp_article)
Raw
print_entries_example.py
print(feed['entries'][0])
Raw
RSS_feed_aggregator_output.py
{
"title": "DeFi Portal 1inch Launches Wallet App on Android",
"link": "https://www.coindesk.com/business/2022/03/31/defi-portal-1inch-launches-wallet-on-android/?utm_medium=referral&utm_source=rss&utm_campaign=headlines",
"pub_date": "31 Mar 2022 15:00:00 ",
"summary": "The move comes nearly one year after the app became available on Apple's iPhone.",
"image": "https://www.coindesk.com/resizer/QOu3JOV1i6UgnbwUc4nD1hXvaeo=/800x600/cloudfront-us-east-1.images.arcpublishing.com/coindesk/GXWHA5WEUJEFJJGAL44U5IXVKQ.png"
}
{
"title": "The state of Web3: Community talks about opportunities around the world",
"link": "https://cointelegraph.com/news/the-state-of-web3-community-talks-about-opportunities-around-the-world",
"pub_date": "01 Apr 2022 12:45:19 ",
"summary": "Web3 brings \u201cendless\u201d opportunities to Indonesia, changes lives in Nigeria, breaks barriers in India, changes skeptics\u2019 minds in the Philippines and makes its way to Uraguay.",
"image": "https://images.cointelegraph.com/images/840_aHR0cHM6Ly9zMy5jb2ludGVsZWdyYXBoLmNvbS91cGxvYWRzLzIwMjItMDQvZDYxYjNmNzktNTk1OC00NmRlLWJlZTgtYWQ1NDIxNjhiOTcxLmpwZw==.jpg"
}
{
"title": "Bitcoin, Ethereum Technical Analysis: ETH Climbs to $3,500 to Start the Weekend",
"link": "https://news.bitcoin.com/bitcoin-ethereum-technical-analysis-eth-climbs-to-3500-to-start-weekend/",
"pub_date": "02 Apr 2022 12:52:33 ",
"summary": "#Ethereum climbs above $3,500 on Saturday, which is its highest level since January.",
"image": "https://static.news.bitcoin.com/wp-content/uploads/2022/04/shutterstock_1962897382.jpg"
}
{
"title": "Intel Arc GPU, Moore\u2019s Law, and Crypto Mining in 2022",
"link": "https://cryptopotato.com/intel-arc-gpu-moores-law-and-crypto-mining-in-2022/",
"pub_date": "02 Apr 2022 10:15:11 ",
"summary": "Intel's new gaming chip, Moore's law, and how does this affect crypto mining in 2022. A closer look.",
"image": "https://cryptopotato.com/wp-content/uploads/2021/01/bitcoin_miningrig_cover.jpg"
}
{
"title": "ECB Executive Member Reveals Initial Findings Of Focus Group On A Digital Euro",
"link": "https://zycrypto.com/ecb-executive-member-reveals-initial-findings-of-focus-group-on-a-digital-euro/",
"pub_date": "01 Apr 2022 09:29:32 ",
"summary": "Fabio Panetta reveals that the EU needs to prioritize the acceptance of a digital euro in physical stores. He says the group aims to maintain high privacy standards to maintain users\u2019 trust. As per Christine Lagarde\u2019s statement last year, a digital euro can roll out in 2025. Fabio Panetta, a member of the ECB\u2019s executive [\u2026]",
"image": "https://zycrypto.com/wp-content/uploads/2021/09/ECB-President-Warns-Against-The-Highly-Speculative-Nature-Of-Bitcoin-Ether-Cardano-Solana-1-1024x597.jpg"
}
{
"title": "Top 3 Metaverse Crypto Coins Below $0.003 to Watch in April 2022",
"link": "https://nulltx.com/top-3-metaverse-crypto-coins-below-0-003-to-watch-in-april-2022/",
"pub_date": "02 Apr 2022 01:14:43 ",
"summary": "Metaverse crypto coins with a low unit price are some of the most popular options for new traders and investors looking to accumulate a large number of tokens for a relatively cheap price. There are dozens of low-priced projects out there, and for a new trader, it might be hard to figure out which ones [\u2026]",
"image": None
}
{
"title": "Controversial EU Cryptocurrency Law Unsettles Crypto Community",
"link": "https://coinquora.com/controversial-eu-cryptocurrency-law-unsettles-crypto-community/",
"pub_date": "01 Apr 2022 12:42:12 ",
"summary": "The EU Parliament has just passed a proposal that many in the cryptocurrency industry are concerned about. The proposal involves ...Read More...",
"image": "https://coinquora.com/wp-content/uploads/2022/04/eus-300x169.jpg"
}
{
"title": "Decoding what \u201919 million BTC in circulation\u2019 means for investors",
"link": "https://ambcrypto.com/after-19-million-btc-got-mined-heres-what-followed-next/",
"pub_date": "02 Apr 2022 11:30:12 ",
"summary": "Bitcoin is known for its deflationary nature. Its supply has a hard limit of 21 million coins. Therefore, the creation of a new Bitcoin is unlikely after the year 2140. Notably, since its inception in 2009, the value of Bitcoin has grown a million-fold over a decade. The limited supply makes it a scarce commodity [\u2026]",
"image": "https://files.ambcrypto.com/wp-content/uploads/2022/04/02120202/samuel-clara-yUWKDfPLp6w-unsplash.jpg"
}
{
"title": "19 millionth Bitcoin has just been mined, only 2 million BTC left",
"link": "https://cryptoslate.com/19000000th-bitcoin-has-just-been-mined-only-2-million-left/",
"pub_date": "01 Apr 2022 19:00:08 ",
"summary": "The 19,000,000th Bitcoin has just been mined at block #730003. ",
"image": None
}
{
"title": "Indian Parliament Passes Law Charging 30% Capital Gain Tax on Crypto",
"link": "https://crypto.news/indian-parliament-passes-law-charging-30-capital-gain-tax-on-crypto/",
"pub_date": "01 Apr 2022 06:00:00 ",
"summary": "Earlier today, the Indian Rajya Sabha approved a new rule that taxes 30% on digital currency holdings and transfers. This comes after the lower house passed the law to tax cryptocurrencies barely a few days ago. However, the new rule has been met with criticisms from crypto stakeholders.",
"image": "https://crypto.news/app/uploads/2022/01/Belgian-MP-becomes-first-European-politician-to-accept-salary-in-Bitcoin.jpg"
}
Raw
rss_feed_urls.py
rss_feed_urls = ["https://www.coindesk.com/arc/outboundfeeds/rss/", \
"https://cointelegraph.com/rss", "https://news.bitcoin.com/feed/", \
"https://cryptopotato.com/feed/", \
"https://zycrypto.com/category/news/feed/", \
"https://nulltx.com/feed/", "https://coinquora.com/news/feed/", \
"https://ambcrypto.com/feed/", "https://cryptoslate.com/feed/", \
"https://crypto.news/feed/"]
Raw
rss_test_loop.py
import json
rss_articles = []
for url in item_attr_map.keys():
feed = parse(url)
items = feed['entries']
for item in items:
attrs = item_attr_map[url]
article = {}
for key, val in attrs.items():
if key == "pub_date":
article[key] = item[val][5:-5]
elif type(val) == str:
article[key] = item[val]
elif val == None:
article[key] = None
elif type(val) == tuple:
article[key] = item[val[0]][val[1]][val[2]]
elif type(val) == dict:
nested_key = list(val.keys())[0]
nested_val = val[nested_key]
soup = BeautifulSoup(item[nested_key])
if val[nested_key] == "text":
article[key] = soup.text
elif type(nested_val) == tuple:
if nested_key == "summary":
if len(nested_val) == 2:
article["summary"] = soup.text
elif len(nested_val) == 3:
article["summary"] = soup.find_all(
[nested_val[0]])[nested_val[1]].text
if key == "image":
article["image"] = soup.find([nested_val[0]]).get(nested_val[1])
rss_articles.append(article)
print(json.dumps(rss_articles[-1], indent=4))
Raw
str_value_example.py
article_properties = {"https://news.bitcoin.com/feed/" : {"title": "title",
"link": "link", "pub_date": "published",
"summary" : "bnmedia_barker_title",
"image" :"bnmedia_url"}}
Raw
tuple_value_example.py
article_properties = {"https://www.coindesk.com/arc/outboundfeeds/rss/" :
{"title": "title", "link": "link", "pub_date": "published",
"summary" : "summary", "image" : ("media_content", 0, "url")}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment