DuckDuckGo Scrape Ad Results

scrape_duckduckgo_ad_results.py

from selenium import webdriver

driver = webdriver.Chrome(executable_path='path/to/chromedriver.exe')
driver.get('https://duckduckgo.com/?q=rtx 3080&kl=us-en&ia=web')

for result in driver.find_elements_by_css_selector('.results--ads .result__body.links_main.links_deep'):
    title = result.find_element_by_css_selector('.js-result-title-link').text
    link = result.find_element_by_css_selector('.js-result-title-link').get_attribute('href')
    source = result.find_element_by_css_selector('.js-result-extras-url').text
    snippet = result.find_element_by_css_selector('.js-result-snippet > at').text
    print(f'{title}\n{source}\n{snippet}\n{link}\n')

for sitelink in driver.find_elements_by_css_selector('.js-sitelinks-title'):
    sitelink_title = sitelink.text
    sitelink_url = sitelink.get_attribute('href')
    print(f'{sitelink_title}\n{sitelink_url}\n')

driver.quit()

Will result.find_elements_by_css_selector('.sitelink--small__title') work?

for result in driver.find_elements_by_css_selector('.result--ad'):
    title = result.find_element_by_css_selector('.results--ads .result__title .result__a').text
    link = result.find_element_by_css_selector('.results--ads .result__title .result__a').get_attribute('href')
    source = result.find_element_by_css_selector('.results--ads .result__extras__url').text
    snippet = result.find_element_by_css_selector('.results--ads .result__snippet').text
    print(f'{title}\n{link}\n{snippet}\n{source}\n')

    for sitelinks in result.find_elements_by_css_selector('.sitelink--small__title'):
        title = sitelinks.text
        link = sitelinks.get_attribute('href')
        print(f'{title}\n{link}\n')

@ilyazub thank you for pointing it out! It will work for inline site links but will skip expanded site links, e.g:

I updated the gist with different selectors since some of them don't work properly.

Here's a gif that demonstrates the output.

Will it scrape ads on the right side?

It will work for inline site links but will skip expanded site links.

Makes sense 👍