Skip to content

Instantly share code, notes, and snippets.

Avatar
😊

Dimitry Zub☀️ dimitryzub

😊
View GitHub Profile
@dimitryzub
dimitryzub / brave_search_organic_results.py
Created Oct 26, 2021
Scrape Brave Search Organic Results using Python
View brave_search_organic_results.py
from bs4 import BeautifulSoup
import requests, lxml, json
headers = {
'User-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582"
}
params = {'q': 'dune film', 'source': 'web'}
View duckduckgo_scrape_knowledge_graph.py
from selenium import webdriver
driver = webdriver.Chrome(executable_path='PATH/TO/chromedriver.exe')
# &iax=about - expanded knowledge graph
driver.get('https://duckduckgo.com/?q=elon musk&kl=us-en&ia=web&iax=about')
title = driver.find_element_by_css_selector('.module__title__link').text
try:
View bing_bs4_scrape_first_10_video_results.py
from bs4 import BeautifulSoup
import requests, lxml
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"
}
params = {
"q": "somebody toucha my spaghet",
View scrape_all_bing_video_results.py
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome(executable_path='path/to/chromedriver.exe')
driver.get('https://www.bing.com/videos/search?q=somebody+toucha+my+spaghet&FORM=HDRSC3&cc=us')
time.sleep(1)
# scrolls until "more videos" button is located
while True:
View duckduckgo_scrape_inline_images.py
from selenium import webdriver
import re, urllib.parse
driver = webdriver.Chrome(executable_path='path/to/chromedriver.exe')
driver.get('https://duckduckgo.com/?q=elon musk dogecoin&kl=us-en&ia=web')
for result in driver.find_elements_by_css_selector('.js-images-link'):
title = result.find_element_by_css_selector('.js-images-link a img').get_attribute('alt')
link = result.find_element_by_css_selector('.js-images-link a').get_attribute('href')
thumbnail_encoded = result.find_element_by_css_selector('.js-images-link a img').get_attribute('src')
View duckduckgo_scrape_inline_videos.py
from selenium import webdriver
import re, urllib.parse
driver = webdriver.Chrome(executable_path='path/to/chromedriver.exe')
driver.get('https://duckduckgo.com/?q=elon musk dogecoin&kl=us-en&ia=web')
for result in driver.find_elements_by_css_selector('#m3-0 .has-image'):
title = result.find_element_by_css_selector('#m3-0 .js-carousel-item-title').text
link = result.find_element_by_css_selector('#m3-0 .module--carousel__body a').get_attribute('href')
View scrape_duckduckgo_organic_news_results.py
from selenium import webdriver
import urllib.parse, re
driver = webdriver.Chrome(executable_path='path/tochromedriver.exe')
driver.get('https://duckduckgo.com/?q=elon musk&kl=us-en&ia=web')
for result in driver.find_elements_by_css_selector('#m1-0 .has-image'):
title = result.find_element_by_css_selector('#m1-0 .js-carousel-item-title').text.strip()
link = result.find_element_by_css_selector('#m1-0 .js-carousel-item-title').get_attribute('href')
source = result.find_element_by_css_selector('#m1-0 .result__url').text
@dimitryzub
dimitryzub / scrape_duckduckgo_ad_results.py
Last active Aug 6, 2021
DuckDuckGo Scrape Ad Results
View scrape_duckduckgo_ad_results.py
from selenium import webdriver
driver = webdriver.Chrome(executable_path='path/to/chromedriver.exe')
driver.get('https://duckduckgo.com/?q=rtx 3080&kl=us-en&ia=web')
for result in driver.find_elements_by_css_selector('.results--ads .result__body.links_main.links_deep'):
title = result.find_element_by_css_selector('.js-result-title-link').text
link = result.find_element_by_css_selector('.js-result-title-link').get_attribute('href')
source = result.find_element_by_css_selector('.js-result-extras-url').text
snippet = result.find_element_by_css_selector('.js-result-snippet > at').text
View bing_related_question.py
@dimitryzub
dimitryzub / serpapi_scrape_bing_related_searches.py
Created Jun 26, 2021
serpapi_scrape_bing_related_searches
View serpapi_scrape_bing_related_searches.py