Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Downloading tarballs (or zipballs) using JavaScript async/await, the `Get archive link` of the GitHub Contents API, the octokit/rest.js client and the request library
const Octokit = require('@octokit/rest')
const rp = require('request-promise')
const fs = require('fs')
function sleep (ms) {
return new Promise(resolve => setTimeout(resolve, ms))
}
const baseUrl = 'https://api.github.com'
const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN })
const perPage = 100
(async function downloader () {
// Let's get some repos
try {
const { data: { items: repos } } = await octokit.search.repos({
q: "stars:>=100 language:java sort:stars",
per_page: perPage,
page: 1
})
for (const repo of repos) {
let { name, owner: { login }, default_branch: ref } = repo
const options = {
headers: {
'User-Agent': 'downloader'
},
uri: `${baseUrl}/repos/${login}/${name}/zipball/${ref}`,
followAllRedirects: true
}
let downloaded = false
try {
const response = await rp(options)
.on('close', () => {
downloaded = true
})
.pipe(fs.createWriteStream(`${login}-${name}-${ref}.zip`))
/* This while loop is a hack to wait for the stream to finish
downloading before the next one starts. Otherwise the await will
wait only for the forwarded link and continue with the next link
and so on and so forth and all downloads will start at once. */
while (!downloaded) {
await sleep(2000)
}
console.log(`Repo ${login}/${name}$${ref} downloaded!`)
} catch (reqErr) {
console.error(reqErr)
}
}
} catch (err) {
console.error(err)
}
})()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.