Downloading tarballs (or zipballs) using JavaScript async/await, the `Get archive link` of the GitHub Contents API, the octokit/rest.js client and the request library

zipballs-downloader.js

const Octokit = require('@octokit/rest')
const rp = require('request-promise')
const fs = require('fs')

function sleep (ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

const baseUrl = 'https://api.github.com'
const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN })
const perPage = 100

(async function downloader () {
  // Let's get some repos
  try {
    const { data: { items: repos } } = await octokit.search.repos({
      q: "stars:>=100 language:java sort:stars",
      per_page: perPage,
      page: 1
    })

    for (const repo of repos) {
      let { name, owner: { login }, default_branch: ref } = repo
      const options = {
        headers: {
          'User-Agent': 'downloader'
        },
        uri: `${baseUrl}/repos/${login}/${name}/zipball/${ref}`,
        followAllRedirects: true
      }
      let downloaded = false
      try {
        const response = await rp(options)
          .on('close', () => {
            downloaded = true
          })
          .pipe(fs.createWriteStream(`${login}-${name}-${ref}.zip`))
        /* This while loop is a hack to wait for the stream to finish
           downloading before the next one starts. Otherwise the await will 
           wait only for the forwarded link and continue with the next link
           and so on and so forth and all downloads will start at once. */
        while (!downloaded) {
          await sleep(2000)
        }
        console.log(`Repo ${login}/${name}$${ref} downloaded!`)
      } catch (reqErr) {
        console.error(reqErr)
      }
    }
  } catch (err) {
    console.error(err)
  }
})()