github-tracker.js

const Apify = require('apify');
const cheerio = require('cheerio');
const requestPromise = require('request-promise');

// Utils functions to simplify the code
const { log } = console;
const pretty = object => JSON.stringify(object, null, 2);

// Get code line-by-line with cheerio into an Object
const arrayToObjectInContext = $ => (array, object = {}) => (
  array.reduce((acc, tr) => {
    const $tr = $(tr);
    const lineNumber = $tr.find('[data-line-number]').data('line-number');
    const code = $tr.find('.blob-code-inner').text();
    return Object.assign({}, acc, { [lineNumber]: code });
  }, object)
);

// Apify's main function to encapsule execution - optional
Apify.main(async () => {
  // payload from the WebHook
  const payload = await Apify.getValue('INPUT');
  const { repository, head_commit: headCommit } = payload;

  const { id: repositoryId } = repository;
  log('Repository ID:', repositoryId);

  // Create a repoStore to handle 'STATE' for each repo
  const repoStore = await Apify.openKeyValueStore(`Repository-${repositoryId}`);

  // Getting record 'STATE', if any
  let previousState = {};
  try {
    previousState = await repoStore.getValue('STATE');
  } catch (err) {
    // ignore this error
  }
  log('Previous STATE:', pretty(previousState));

  // Get Commit SHA, 7 characters are enough
  const commitId = headCommit.id.slice(0, 7);
  log('Current Commit ID:', commitId);

  const { added, removed, modified } = headCommit;
  const headCommitFiles = [].concat(added, removed, modified);
  log('Added/Removed/Modified Files:', headCommitFiles);

  // Add blob URL to each file
  const commitBlobUrl = headCommit.url.replace('commit', 'blob');
  const commitBlobUrls = headCommitFiles.map(file => ({
    uri: `${commitBlobUrl}/${file}`,
    file: file.toLowerCase(),
  }));

  // Magic here
  const stateChanges = await commitBlobUrls.reduce(async (acc, { uri, file }) => {
    log('Blob URL:', uri);
    const options = {
      uri,
      transform: body => cheerio.load(body),
    };
    const $ = await requestPromise(options);
    const getDiffIn = arrayToObjectInContext($);
    
    const $tableRows = $('table tr').toArray();
    log(`Lines found in '${file}': ${$tableRows.length}`);

    const fileState = previousState[file] || [];
    const currentState = fileState.filter(commitObject => !commitObject[commitId]);
    const commit = {
      [commitId]: {
        timestamp: headCommit.timestamp,
        committer: headCommit.committer,
        message: headCommit.message,
        url: headCommit.url,
      },
    };
    
    // Filtering non .js files
    if (file.includes('.js')) {
      Object.assign(commit[commitId], { code: getDiffIn($tableRows) });
    }
    return Object.assign({}, acc, { [file]: [commit, ...currentState] });
  }, {});

  const nextState = Object.assign({}, previousState, stateChanges);
  log('Next STATE:', pretty(nextState));

  log('Saving into repoStore:', repositoryId);
  await repoStore.setValue('STATE', nextState);

  // Save 'OUTPUT' of the current Act run 
  // (optional - necessary only if called from within an act to get its 'OUTPUT')
  await Apify.setValue('OUTPUT', nextState);
  return log('Done.');
});