example-fetch.js

// Needed to fetch data from Twitch API
require('isomorphic-fetch');
require('dotenv').config();
// Needed for GraphQL queries/mutations
const { GraphQLClient } = require('graphql-request');

// Create an async-based version of setTimeout
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
// Number of streamers to be processed in each batch
const batchSize = 100;

// Endlessly update all streamers
const repeatUpdateStreamers = async () => {
  await updateAllStreamers(batchSize);
  await wait(5000);
  // Recurse to create endless loop
  repeatUpdateStreamers();
};

repeatUpdateStreamers();

async function updateAllStreamers(batchSize) {
  // fetch streamers from GraphQL DB
  const streamers = await getStreamers();
  const numStreamers = Math.ceil(streamers.allStreamers.length / 100);
  // Create an array filled with numbers 0-numStreamers
  const numStreamersIndexed = [...Array(streamersLen)];
  
  for (const i of numStreamersIndexed.keys()) {
    let graphIds = {};
    // Contains only current batch of streamers
    const batchStreamers = streamers.allStreamers.slice(
      i * batchSize,
      i * batchSize + batchSize
    );
    
    // Map through batch and create one string containing
    // a comma separated list of streamer IDs
    const twitchIds = batchStreamers
      .map(streamer => {
        // Side effect: Also populates graphIds
        graphIds[streamer.twitchId] = {
          id: streamer.id,
          streamType: streamer.streamType,
        };
        return streamer.twitchId;
      })
      .join(',');
    
    try {
      await updateData(twitchIds, graphIds);
    } catch (e) {
      console.log(`err in updateAllStreamers: ${e}`);
    }
    
    await wait(1000);
  }
}

async function updateData(twitchIds, graphIds) {
  // Fetch data from Twitch API
  const res = await fetch(
    `https://api.twitch.tv/kraken/streams/?channel=${twitchIds}&limit=100`,
    {
      method: 'GET',
      headers: {
        Accept: 'application/vnd.twitchtv.v5+json',
        'Client-ID': process.env.CLIENT_ID,
      },
    }
  );
  
  const data = res.json();
  const streams = data.streams;
  // Updates online streamers and deletes them from graphIds
  await updateOnlineStreamers(streams, graphIds);
  // Leftover, undeleted streamers in graphIds mean they're offline
  await updateOfflineStreamers(graphIds);
}

async function updateOnlineStreamers(streams, graphIds) {
  for (const stream of streams) {
    // Set up GraphQLClient vars
    const vars = {
      id: graphIds[stream.channel._id].id,
      twitchInfo: stream,
      streamType: 'online',
    };
    
    try {
      // Update GraphQL DB with streamer's current viewer count and more
      await gclient.request(updateStreamer, vars);
      // Delete successfully updated streamer from graphIds
      delete graphIds[stream.channel._id];
    } catch (e) {
      console.log(`err in updateOnlineStreamers: ${e}`);
    }
}

async function updateOfflineStreamers(graphIds) {
  for (const val of Object.entries(graphIds)) {
    if (val[1].streamType === 'online') {
      // Set up GraphQLClient vars
      const vars = {
        id: val[1].id,
        streamType: 'offline',
      };
      
      try {
        // Update GraphQL DB with streamer's offline status
        await gclient.request(updateStreamerStatus, vars);
      } catch(e) {
        console.log(`err in updateOnlineStreamers: ${e}`);
      }
    }
  }
}