Dmitiry Zub☀️ dimitryzub
🇺🇦
dimitryzub
/ bs4_scrape_google_shopping_ads.py
Last active
May 13, 2021 10:46
medium_scrape_google_shopping_ads
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests, lxml, urllib.parse | |
| from bs4 import BeautifulSoup | |
| # Adding User-agent (default user-agent from requests library is 'python-requests') | |
| # https://github.com/psf/requests/blob/589c4547338b592b1fb77c65663d8aa6fbb7e38b/requests/utils.py#L808-L814 | |
| headers = { | |
| "User-Agent": | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3538.102 Safari/537.36 Edge/18.19582" | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests, lxml, urllib.parse | |
| from bs4 import BeautifulSoup | |
| # Adding user-agent to fake real user visit | |
| headers = { | |
| "User-Agent": | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3538.102 Safari/537.36 Edge/18.19582" | |
| } | |
| # Search query |
dimitryzub
/ serpapi_scrape_google_shopping_ads.py
Last active
May 13, 2021 10:52
serpapi_scrape_google_shopping_ads
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| from serpapi import GoogleSearch | |
| # Search query parameters | |
| params = { | |
| "engine": "google", | |
| "q": "cpu buy", | |
| "api_key": os.getenv("API_KEY"), | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| from serpapi import GoogleSearch | |
| params = { | |
| "engine": "google", | |
| "q": "kitchen table", | |
| "api_key": os.getenv("API_KEY"), | |
| "no_cache":"true" # add this param if it throws an error | |
| } |
dimitryzub
/ bs4_scrape_google_scholar_organic_results.py
Last active
April 18, 2023 22:31
Scrape Google Scholar Organic Results with Python
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from bs4 import BeautifulSoup | |
| import requests, lxml, os, json | |
| headers = { | |
| 'User-agent': | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582" | |
| } | |
| proxies = { | |
| 'http': os.getenv('HTTP_PROXY') # or just type proxy here without os.getenv() |
dimitryzub
/ serpapi_scrape_google_scholar_organic_results.py
Last active
May 20, 2021 09:43
Scrape Google Scholar Organic Results with SerpApi
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from serpapi import GoogleSearch | |
| import os, json | |
| params = { | |
| "api_key": os.getenv("API_KEY"), | |
| "engine": "google_scholar", | |
| "q": "samsung", | |
| } | |
| search = GoogleSearch(params) |
dimitryzub
/ scrape_google_scholar_profile_results.py
Last active
May 23, 2021 12:11
Scrape Google Scholar Profile Results with Python
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from bs4 import BeautifulSoup | |
| import requests, lxml, os | |
| headers = { | |
| 'User-agent': | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582" | |
| } | |
| proxies = { | |
| 'http': os.getenv('HTTP_PROXY') |
dimitryzub
/ serpapi_scrape_google_scholar_profile_results.py
Last active
May 23, 2021 12:13
Scrape Google Scholar Profile Results using SerpApi
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from serpapi import GoogleSearch | |
| import os | |
| params = { | |
| "api_key": os.getenv("API_KEY"), | |
| "engine": "google_scholar_profiles", | |
| "hl": "en", | |
| "mauthors": "samsung" | |
| } |
dimitryzub
/ python_scrape_google_scholar_profile_author_results.py
Last active
May 29, 2021 10:39
Scrape Google Scholar Profile-Author Results with Python
dimitryzub
/ serpapi_google_scholar_cite_results.py
Created
May 19, 2021 05:34
Scrape Google Scholar Cite Results with SerpApi
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from serpapi import GoogleSearch | |
| import os | |
| params = { | |
| "api_key": os.getenv("API_KEY"), | |
| "engine": "google_scholar_cite", | |
| "q": "FDc6HiktlqEJ" | |
| } | |
| search = GoogleSearch(params) |
OlderNewer