Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Node.js script for querying GitHub Search API v3
// API docs: https://developer.github.com/v3/search/
// search docs: https://help.github.com/en/github/searching-for-information-on-github/searching-on-github
// you might have to limit the number of requests
// if trying to send too many at once
const request = require('request');
const _ = require('lodash');
const fs = require('fs');
const opts = {
headers: {
// user agent string can be something like 'Repository search'
'User-Agent': '<USER_AGENT_STRING>'
// optional: add your access token here
//
// providing an access token will allow you
// to send more requests per minute:
// https://developer.github.com/v3/search/#rate-limit
//
// Authorization: `token <ACCESS_TOKEN_HERE>`
}
};
// example url, adjust to your needs
// it's always a good idea to include per_page=100 (maximum allowed)
// to minimise number of requests
const initUrl = 'https://api.github.com/search/repositories?q=user:ilokhov&per_page=100';
sendRequest({ ...opts, ...{ url: initUrl } }, []);
function sendRequest(opts, results) {
console.log(`request: ${opts.url}`);
request(opts, (err, response, body) => {
if (err) throw err;
// add to results from current request
results.push(...getContent(body));
// check if we have more pages available
const responseHeader = response.headers;
const link = responseHeader.link;
// there are no addition pages
if (!link) {
writeOutput(results);
} else {
// check if we have next page
const nextRelIndex = link.indexOf('rel="next"');
// no next page
if (nextRelIndex === -1) {
writeOutput(results);
} else {
// get next page url
const nextStartIndex = link.lastIndexOf('<', nextRelIndex);
const nextEndIndex = link.lastIndexOf('>', nextRelIndex);
const nextUrl = link.substring(
nextStartIndex + 1,
nextEndIndex
);
// send next request
sendRequest({ ...opts, ...{ url: nextUrl } }, results);
}
}
});
}
function getContent(body) {
const items = JSON.parse(body).items;
// example value to pick, adjust to your needs
return items.map(item => _.pick(item, ['created_at']));
}
function writeOutput(results) {
console.log(results);
fs.writeFile('output.json', JSON.stringify(results), err => {
if (err) throw err;
console.log('file written!');
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment