Streaming data from the GitHub GraphQL API with an async generator

github-stream.js

function githubStream(delayTime = 2) {
  const delay = () => new Promise(resolve => setTimeout(resolve, delayTime * 1000))
  let repos = [];
  let maxIterations = 100;
  
  function getCursor() {
    if (repos.length) {
      return repos[repos.length - 1].cursor;
    }
    return '';
  }
  
  async function getRepos(variables, cursor = '') {
    const query = `query($q: String!) {
        search(type: REPOSITORY, query: $q, first: 100${cursor ? `, after: "${cursor}"` : ''}) {
          repositoryCount
          edges {
            node {
              ... on Repository {
                id
                createdAt
                updatedAt
                name
                nameWithOwner
                stargazers {
                  totalCount
                }
              }
            }
            cursor
          }
        }
      }`
    return fetch('https://api.github.com/graphql', {
      method: 'POST',
      body: JSON.stringify({
        query,
        variables
      }),
      headers: {
        Authorization: `bearer ${API_KEY}`,
        'Content-Type': 'application/json',
        Accept: 'application/vnd.github.machine-man-preview+json'
      }
    })
      .then(res => res.json())
      .then(json => {
        const { repositoryCount, edges } = json.data.search;
        maxIterations = repositoryCount;
        repos = [...repos, ...edges]
      })
  }
  
  return {
    [Symbol.asyncIterator]: async function*() {
      let iterationCount = 0;
      while (iterationCount <= maxIterations) {
        if (repos.length < 20) {
          await getRepos({ q: 'javascript' }, getCursor());
        }
        await delay();
        yield repos.shift();
        iterationCount += 1;
      }
    }
  }
}

(async () => {
  const stream = githubStream();
  for await (let repo of stream) {
    console.log(repo.node.nameWithOwner);
  }
})()