Skip to content

Instantly share code, notes, and snippets.

@milnomada
Last active August 24, 2021 08:14
Show Gist options
  • Save milnomada/b2e2391048ca2617f234c817954e4f74 to your computer and use it in GitHub Desktop.
Save milnomada/b2e2391048ca2617f234c817954e4f74 to your computer and use it in GitHub Desktop.
from selenium import webdriver
from scrapy.http import HtmlResponse
from selenium.webdriver.chrome.options import Options
"""
whatch chromedriver version supporting the chrome browser version in system
"""
class SeleniumMiddleware(object):
def process_request(self,request,spider):
options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--silent') # faster
options.add_argument('--log-level=3') # less verbosity
# options.binary_location = ''
capabilities = {}
# capabilities['platform'] = 'Linux'
# capabilities['version'] = '16.04'
driver = webdriver.Chrome(
executable_path='/usr/local/bin/chromedriver',
chrome_options=options,
desired_capabilities=capabilities)
try:
driver.get(request.url)
driver.switch_to.frame('g_iframe')
body = driver.page_source
return HtmlResponse(driver.current_url,body=body,encoding='utf-8',request=request)
except:
driver.quit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment