Created
April 4, 2017 19:48
-
-
Save m-x-k/5c227d751ef7e6b681b0e2c82403d461 to your computer and use it in GitHub Desktop.
Web scrape example using python and BeautifulSoup
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import fire | |
import requests | |
from bs4 import BeautifulSoup | |
# Web scrape example using python and BeautifulSoup | |
# Setup: pip install requests beautifulsoup4 fire | |
# Run: python webscrape.py scrape | |
class WebScrape(object): | |
url = "https://www.themuse.com/advice/43-simple-habits-thatll-improve-your-life-even-if-you-just-pick-one" | |
def scrape(self): | |
response = requests.get(self.url, verify=False) | |
if response.status_code == 200: | |
data = response.text | |
lines = BeautifulSoup(data, "html.parser").find_all("h2") | |
for line in lines: | |
print(line.text) | |
else: | |
return "ERROR: %s" % response.text | |
if __name__ == '__main__': | |
fire.Fire(WebScrape) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment