Created
January 4, 2020 11:48
-
-
Save e96031413/66f0e52dfc5ff378b8f44b8eef898e56 to your computer and use it in GitHub Desktop.
Using urllib, requests, bs4 to scrape the search result
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#originally from https://github.com/getlinksc/scrape_google | |
import urllib | |
import requests | |
from bs4 import BeautifulSoup | |
# desktop user-agent | |
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0" | |
# mobile user-agent | |
MOBILE_USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36" | |
query = "tensorflow" | |
query = query.replace(' ', '+') | |
URL = f"https://google.com/search?q={query}" | |
headers = {"user-agent": USER_AGENT} | |
resp = requests.get(URL, headers=headers) | |
if resp.status_code == 200: | |
soup = BeautifulSoup(resp.content, "html.parser") | |
results = [] | |
for g in soup.find_all('div', class_='r'): | |
anchors = g.find_all('a') | |
if anchors: | |
link = anchors[0]['href'] | |
title = g.find('h3').text | |
item = { | |
"title": title, | |
"link": link | |
} | |
results.append(item) | |
print(results) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment