Created
January 15, 2019 12:31
-
-
Save mabelvj/18ae76d102cbd27f0ad9722f779d901f to your computer and use it in GitHub Desktop.
Lua Script for Scrapy Splash
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
script = """ | |
function main(splash) | |
local num_scrolls = 10 | |
assert(splash:go(splash.args.url)) | |
local get_dimensions = splash:jsfunc([[ | |
function () { | |
var rect = document.getElementsByClassName('css-vsuiox')[0] | |
.getElementsByTagName('button')[0] | |
.getClientRects()[0]; | |
return {"x": rect.left, "y": rect.top}; | |
} | |
]]) | |
local scroll_to = splash:jsfunc("window.scrollTo") | |
local get_body_height = splash:jsfunc( | |
"function() {return document.body.scrollHeight;}" | |
) | |
-- Loop and click `Show More` button multiple times | |
splash:set_viewport_full() | |
for i=1, num_scrolls do | |
scroll_to(0, get_body_height()) | |
splash:wait(0.5) -- to avoid errors like: "TypeError: null is not an object" | |
local dimensions = get_dimensions() | |
splash:mouse_click(dimensions.x, dimensions.y) | |
end | |
-- Wait split second to allow event to propagate. | |
splash:wait(0.1) | |
return splash:html() | |
end | |
""" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This is brilliant! although, I have a small question.
If
document.getElementsByClassName('css-vsuiox')[0]
is changed fordocument.querySelector('button')[0]
, where it's clicking on a button. Let's say that there are 5 buttons to click, how do I loop sequentially to click on these buttons using your script? It works well if and only if I manually change the index within the squared brackets, as it's written.