skeptrunedev/yc-directory-scape.js

## yc-directory-scape.js
net::ERR_BLOCKED_BY_CLIENT
(anonymous) @ companies:18
(anonymous) @ companies:33
// i made this mini script because i am trying to collect the details of the existing public YC companies for a search demo with trieve
// with algolia, a search for "cloud storage" doesn't return Dropbox and a search for "bug monitoring" doesn't return PagerDuty, etc.
// find and build with trieve at github.com/devflowinc/trieve

// to use, navigate to https://www.ycombinator.com/companies and paste the following into the console

const sleepPromise = (ms) => {
  return new Promise((resolve) => {
    setTimeout(() => {
      resolve();
    }, ms);
  });
};

// get the div containing all the input checkboxes for each batch
const allSpans = document.getElementsByTagName('span');
let batchDiv = null;
for(var i = 0; i < allSpans.length; i++) {
    if(allSpans[i].textContent === "All batches") {
        batchDiv = allSpans[i].parentElement.parentElement;
        break;
    }
}

// click to expand the list to access all batches
// batchDiv.querySelector("a").click();

// convenience function to select a given input to get batch level filters
let allBatchInputs = batchDiv.querySelectorAll("input");
const clickIndex = (idxToClick) => {
    allBatchInputs.forEach((input, idx) => {
        const desiredChecked = idxToClick == idx;
        if (input.checked != desiredChecked) {
            input.click();
        }
    })
};

// convenience function to scroll down to expose all the companies in a given batch once it has been selected
const checkIfAllCompaniesExposed = () => {
    const showText = document.querySelector("._message_fj1ly_543").innerText;
    const numbers = showText.match(/\d+/g);
    return numbers[0] == numbers[1];
}

// scroll down once per 400ms until all companies exposed
const exposeAllCompanies = async () => {
    while(!checkIfAllCompaniesExposed()) {
        window.scrollTo(0, document.body.scrollHeight);
        await sleepPromise(400);
    }
}

let linksToCompanies = []
for(var i = 1; i < allBatchInputs.length; i++) {
    clickIndex(i);
    await sleepPromise(500);
    await exposeAllCompanies();
    window.scrollTo(0, 0);
    await sleepPromise(1000);
    let anchors = document.querySelectorAll('._company_fj1ly_339');
    console.log("adding following num of companies to list: ", anchors.length);
    anchors.forEach(function(anchor) {
      const url = anchor.href;
      linksToCompanies.push(url);
    });
}

console.log("n companies represented: ", linksToCompanies.length);
console.log(JSON.stringify(linksToCompanies));
	net::ERR_BLOCKED_BY_CLIENT
	(anonymous) @ companies:18
	(anonymous) @ companies:33
	// i made this mini script because i am trying to collect the details of the existing public YC companies for a search demo with trieve
	// with algolia, a search for "cloud storage" doesn't return Dropbox and a search for "bug monitoring" doesn't return PagerDuty, etc.
	// find and build with trieve at github.com/devflowinc/trieve

	// to use, navigate to https://www.ycombinator.com/companies and paste the following into the console

	const sleepPromise = (ms) => {
	return new Promise((resolve) => {
	setTimeout(() => {
	resolve();
	}, ms);
	});
	};

	// get the div containing all the input checkboxes for each batch
	const allSpans = document.getElementsByTagName('span');
	let batchDiv = null;
	for(var i = 0; i < allSpans.length; i++) {
	if(allSpans[i].textContent === "All batches") {
	batchDiv = allSpans[i].parentElement.parentElement;
	break;
	}
	}

	// click to expand the list to access all batches
	// batchDiv.querySelector("a").click();

	// convenience function to select a given input to get batch level filters
	let allBatchInputs = batchDiv.querySelectorAll("input");
	const clickIndex = (idxToClick) => {
	allBatchInputs.forEach((input, idx) => {
	const desiredChecked = idxToClick == idx;
	if (input.checked != desiredChecked) {
	input.click();
	}
	})
	};

	// convenience function to scroll down to expose all the companies in a given batch once it has been selected
	const checkIfAllCompaniesExposed = () => {
	const showText = document.querySelector("._message_fj1ly_543").innerText;
	const numbers = showText.match(/\d+/g);
	return numbers[0] == numbers[1];
	}

	// scroll down once per 400ms until all companies exposed
	const exposeAllCompanies = async () => {
	while(!checkIfAllCompaniesExposed()) {
	window.scrollTo(0, document.body.scrollHeight);
	await sleepPromise(400);
	}
	}

	let linksToCompanies = []
	for(var i = 1; i < allBatchInputs.length; i++) {
	clickIndex(i);
	await sleepPromise(500);
	await exposeAllCompanies();
	window.scrollTo(0, 0);
	await sleepPromise(1000);
	let anchors = document.querySelectorAll('._company_fj1ly_339');
	console.log("adding following num of companies to list: ", anchors.length);
	anchors.forEach(function(anchor) {
	const url = anchor.href;
	linksToCompanies.push(url);
	});
	}

	console.log("n companies represented: ", linksToCompanies.length);
	console.log(JSON.stringify(linksToCompanies));