Last active
March 8, 2020 22:59
-
-
Save lauri-kaariainen/d9cce1cf92fec654d76693ad76c79e27 to your computer and use it in GitHub Desktop.
scrape links on page from developer console, calls separated by 1000ms
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var hrefs = | |
Array.from(document.querySelectorAll("a")) | |
.map(link=>link.href) | |
.sort() | |
.filter((el,i,arr)=>!arr.slice(i+1).includes(el)) | |
.filter(href=>href.match(/ads\/\d/)) | |
.filter((_,i,arr)=>i ? (console.log(arr.length),true):true) | |
.filter(href=>(console.log(href),true)).slice(0); | |
window.array = []; | |
hrefs.forEach((href,i)=> | |
setTimeout(()=> | |
Promise.all( | |
([href]) | |
.map(async (url,i)=>await fetch(url))) | |
.then(resolves=>resolves.map(async res=>await res.text())) | |
.then(lines=>Promise.all(lines)) | |
.then(lines=>( | |
console.log("done"), | |
window.array=window.array.concat( | |
lines | |
.map(line=>line.match(/Kansallisuus<\/div><div class="xs-50">([a-zA-ZäöåÄÖÅ]+)/)) | |
.filter(e=>e) | |
.map(e=>e[1]) | |
)) | |
), | |
i*1000 | |
) | |
) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment