Skip to content

Instantly share code, notes, and snippets.

@niubl
Created July 1, 2020 01:39
Show Gist options
  • Save niubl/2c3855240b12c6d17b265374ca140411 to your computer and use it in GitHub Desktop.
Save niubl/2c3855240b12c6d17b265374ca140411 to your computer and use it in GitHub Desktop.
use puppeteer to get hackerone programs
const puppeteer = require('puppeteer');
const fs = require("fs");
const url = require('url');
(async () => {
const browser = await puppeteer.launch({
args: ['--no-sandbox'],
}
);
async function autoScroll(page) {
await page.evaluate(async () => {
await new Promise((resolve, reject) => {
var totalHeight = 0;
var distance = 100;
var timer = setInterval(() => {
var scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
//if (totalHeight >= 20850) {
clearInterval(timer);
resolve();
}
}, 1000);
});
});
}
var page = await browser.newPage();
await page.goto('https://hackerone.com/directory/programs', {
timeout: 0
});
await autoScroll(page);
var xp = '(//a[@class="daisy-link routerlink daisy-link daisy-link--major spec-profile-name"])'
await page.waitForXPath(xp, timeout=0)
var ar = await page.$x(xp);
var links = [];
for (let a of ar) {
links.push(await page.evaluate(e => e.href, a));
}
for (let link of links) {
await page.goto(link, {
timeout: 0
});
try {
xp = '(//span[@class="break-word"]/strong)'
await page.waitForXPath(xp, timeout=0)
var lks = await page.$x(xp);
for (let lk of lks) {
var data = await page.evaluate(e => e.innerText, lk);
fs.writeFile('hackerone.txt', (link + ' ' + data + '\n'), {
flag: 'a'
}, function (err) {
if (err) {
return console.error(err);
}
});
};
} catch (e) {
console.log('an expection: ', e);
continue;
}
};
await browser.close();
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment