Skip to content

Instantly share code, notes, and snippets.

@silvernoo
Created October 13, 2019 11:16
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save silvernoo/b998086fd2b933d2dcf70bee7460559a to your computer and use it in GitHub Desktop.
Save silvernoo/b998086fd2b933d2dcf70bee7460559a to your computer and use it in GitHub Desktop.
var chrome = (async (dbData, dbo) => {
browser = await puppeteer.launch({
executablePath: '/usr/bin/chromium-browser',
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--proxy-server=192.168.0.100:7890',
],
userDataDir: "/app/user_data"
})
const browserVersion = await browser.version()
console.log(`Started ${browserVersion}`)
page = await browser.newPage()
for (let i = 0; i < dbData.length; i++) {
if (!dbData[i]['downloaded']) {
const item = dbData[i]
console.log(url)
response = await page.goto(item['url'])
assert(response.ok())
await page.waitFor(1500);
await page._client.send('Page.setDownloadBehavior', { behavior: 'allow', downloadPath: '/screenshots' })
try {
await page.click('#download-link')
await page.click('#js-download')
} catch (err) {
console.log("not download!")
}
await page.screenshot({ path: `/screenshots/${i}.png` })
newvalues = JSON.parse(JSON.stringify(item))
newvalues['downloaded'] = true
console.log(item)
console.log(newvalues)
dbo.collection("niconico").updateOne(item, { $set: newvalues }, function (err, res) {
if (err) throw err;
console.log("1 document updated");
});
await page.waitFor(2000)
}
}
await browser.close();
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment