Use selenium with phantomjs (with custom capabilities) for screen scraping

selenium_phantom.py

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from bs4 import BeautifulSoup

# edit desired capabilities
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
    "(KHTML, like Gecko) Chrome/15.0.87"
)
dcap['pahntomjs.page.settings.loadImages'] = False

driver = webdriver.PhantomJS('/path/to/bin/phantomjs', desired_capabilities=dcap)
driver.get('http://scarpe.this.url/please')
soup = BeautifulSoup(driver.page_source)
# do stuff with your soup

# useful links 
# https://coderwall.com/p/9jgaeq/set-phantomjs-user-agent-string
# http://stackoverflow.com/a/15699761 # phantomjs + selenium example
# http://stackoverflow.com/a/6300672 # link for using selenium with xvfb (virtual display)