Get complete list of repos and their respective details for a given GitHub user. Sorted by last updated, ascending. Output to CSV file.

configuration.js

// Holds pertinent authentiation
module.exports = {
  github: {
    token: '__your__token__here__'
  }
}

ibmcode.js

const fs = require( 'fs' );
const rp = require( 'request-promise-native' );

const config = require( '../configuration.js' );

aggregate();

async function aggregate() {
  let ibm = [];
  let last = 0;
  let page = 1;
  
  do {
    console.log( 'Page: ' + page );

    // Get the repository list
    // Will be paged
    let repos = await rp( {
      url: 'https://api.github.com/users/IBM/repos',
      method: 'GET',
      headers: {
        'User-Agent': 'IBM Developer'
      },
      qs: {
        access_token: config.github.token,
        page: page
      },
      resolveWithFullResponse: true,
      json: true
    } )
    .then( ( response ) => {
      // Page index in header
      // Parse total pages from header
      if( last === 0 ) {
        const link = response.headers['link'].trim();
        const page = link.lastIndexOf( 'page' ) + 5; 
        const close = link.lastIndexOf( '>' );
        last = parseInt( link.substring( page, close ) );        
        console.log( 'Last: ' + last );
      }

      // Retrun repositories
      return response.body;
    } )
    .catch( ( error ) => {
      console.log( 'Problem accessing API.' );
      console.log( error );
    } );

    // Debug
    // console.log( repos );

    // Repository list results missing some data
    // Need to get repository details specifically
    for( let r = 0; r < repos.length; r++ ) {
      let details = await rp( {
        url: `https://api.github.com/repos/${repos[r].full_name}`,
        method: 'GET',
        headers: {
          'User-Agent': 'IBM Developer'
        },
        qs: {
          access_token: config.github.token
        },
        json: true
      } );

      ibm.push( details );
    }

    // Next page
    page = page + 1;
  // } while( page <= 3 );
  } while( page <= last );

  console.log( 'Total: ' + ibm.length );

  // Sort by date updated
  ibm.sort( ( a, b ) => {
    let a_at = new Date( a.updated_at );
    let b_at = new Date( b.updated_at );

    if( a_at.getTime() > b_at.getTime() ) return 1;
    if( a_at.getTime() < b_at.getTime() ) return -1;

    return 0;
  } );

  // Output file
  const file = fs.createWriteStream( 'ibmcode.csv', {
    flags: 'a'
  } );

  // Line by line
  // Picking a few pertinent details
  for( let i = 0; i < ibm.length; i++ ) {
    const created_at = new Date( ibm[i].created_at );
    const created = `${created_at.getMonth() + 1}/${created_at.getDate()}/${created_at.getFullYear()}`;

    const updated_at = new Date( ibm[i].updated_at );
    const updated = `${updated_at.getMonth() + 1}/${updated_at.getDate()}/${updated_at.getFullYear()}`;    

    const pushed_at = new Date( ibm[i].pushed_at );
    const pushed = `${pushed_at.getMonth() + 1}/${pushed_at.getDate()}/${pushed_at.getFullYear()}`;        

    file.write( `${ibm[i].name},${created},${updated},${pushed},${ibm[i].subscribers_count},${ibm[i].stargazers_count},${ibm[i].forks_count},${ibm[i].open_issues}\n` );
  }

  // Close the file
  file.end();

  console.log( 'Done.' );
}

patterns.js

const Excel = require( 'exceljs' );
const fs = require( 'fs' );
const rp = require( 'request-promise-native' );

const config = require( './configuration.js' );

report();

async function report() {
  // Read list of repos
  // Split into array
  const matching = fs.readFileSync( __dirname + '/patterns.txt', 'utf8' ).trim();
  const patterns = matching.split( '\n');

  // Repository details
  let ibm = [];

  for( let p = 0; p < patterns.length; p++ ) {
    // Not on GitHub
    // This is for GitHub content only
    if( patterns[p].indexOf( 'github.com' ) === -1 ) {
      continue;
    }

    // No trailing slash, thank you
    if( patterns[p].charAt( patterns[p].length - 1 ) == '/' ) {
      patterns[p] = patterns[p].substring( 0, patterns[p].length - 1 );
    }

    // Repository full name
    // Full name includes user account
    const start = patterns[p].lastIndexOf( '.com/' ) + 5;
    const name = patterns[p].substring( start );

    console.log( name ); 

    // Get details from GitHub API
    let details = await rp( {
      url: `https://api.github.com/repos/${name}`,
      method: 'GET',
      headers: {
        'User-Agent': 'IBM Developer'
      },
      qs: {
        access_token: config.github.token
      },
      json: true
    } )
    .catch( ( error ) => {
      if( error.statusCode == 404 ) {
        return null;
      }
    } );

    if( details !== null ) {
      ibm.push( details );    
    }
  }

  console.log( 'Patterns: ' + ibm.length );

  // Sort by date updated
  ibm.sort( ( a, b ) => {
    let a_at = new Date( a.updated_at );
    let b_at = new Date( b.updated_at );

    if( a_at.getTime() > b_at.getTime() ) return 1;
    if( a_at.getTime() < b_at.getTime() ) return -1;

    return 0;
  } );

  // Generate statistics
  // Calculate manually
  // Must prepopulate values with formulas
  let statistics = {
    subscribers: {sum: 0, values: [], average: 0, median: 0},
    stargazers: {sum: 0, values: [], average: 0, median: 0},
    forks: {sum: 0, values: [], average: 0, median: 0},
    issues: {sum: 0, values: [], average: 0, median: 0},
    remove: 0,
    promote: 0
  };

  for( let i = 0; i < ibm.length; i++ ) {
    statistics.subscribers.sum = statistics.subscribers.sum + ibm[i].subscribers_count;
    statistics.subscribers.values.push( ibm[i].subscribers_count );

    statistics.stargazers.sum = statistics.stargazers.sum + ibm[i].stargazers_count;    
    statistics.stargazers.values.push( ibm[i].stargazers_count );

    statistics.forks.sum = statistics.forks.sum + ibm[i].forks_count;    
    statistics.forks.values.push( ibm[i].forks_count );

    statistics.issues.sum = statistics.issues.sum + ibm[i].open_issues;  
    statistics.issues.values.push( ibm[i].open_issues );    
  }

  // Average
  statistics.subscribers.average = Math.round( statistics.subscribers.sum / ibm.length );
  statistics.stargazers.average = Math.round( statistics.stargazers.sum / ibm.length );
  statistics.forks.average = Math.round( statistics.forks.sum / ibm.length );
  statistics.issues.average = Math.round( statistics.issues.sum / ibm.length );      

  // Median
  statistics.subscribers.values.sort( numeric );
  statistics.subscribers.median = median( statistics.subscribers.values );

  statistics.stargazers.values.sort( numeric );
  statistics.stargazers.median = median( statistics.stargazers.values );  

  statistics.forks.values.sort( numeric );
  statistics.forks.median = median( statistics.forks.values );

  statistics.issues.values.sort( numeric );
  statistics.issues.median = median( statistics.issues.values );  

  // Setup reporting
  let book = new Excel.Workbook();
  let sheet = book.addWorksheet( 'IBM Code' );
  sheet.columns = [
    {header: 'Name', width: 55, bold: true},
    {header: 'Created', width: 20},
    {header: 'Updated', width: 20},
    {header: 'Pushed', width: 20},    
    {header: 'Watchers', width: 10},        
    {header: 'Stars', width: 10},        
    {header: 'Forks', width: 10},        
    {header: 'Issues', width: 10},        
    {header: 'Candidate', width: 10},        
  ];  

  // Line by line
  // Picking a few pertinent details
  for( let i = 0; i < ibm.length; i++ ) {
    // Name
    sheet.getCell( `A${i + 2}` ).value = ibm[i].name;

    // Created
    const created_at = new Date( ibm[i].created_at );
    const created = `${created_at.getMonth() + 1}/${created_at.getDate()}/${created_at.getFullYear()}`;
    sheet.getCell( `B${i + 2}` ).value = created;

    // Updated
    const updated_at = new Date( ibm[i].updated_at );
    const updated = `${updated_at.getMonth() + 1}/${updated_at.getDate()}/${updated_at.getFullYear()}`;    
    sheet.getCell( `C${i + 2}` ).value = updated;    

    // Pushed
    const pushed_at = new Date( ibm[i].pushed_at );
    const pushed = `${pushed_at.getMonth() + 1}/${pushed_at.getDate()}/${pushed_at.getFullYear()}`;        
    sheet.getCell( `D${i + 2}` ).value = pushed;

    // Watchers
    // Also called subscribers in the API
    sheet.getCell( `E${i + 2}` ).value = ibm[i].subscribers_count;

    if( ibm[i].subscribers_count < statistics.subscribers.average ) {
      sheet.getCell( `E${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFFFFD54'}};    
    }

    if( ibm[i].subscribers_count < statistics.subscribers.median ) {
      sheet.getCell( `E${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFF6C9CF'}};          
    }

    // Stargazers
    sheet.getCell( `F${i + 2}` ).value = ibm[i].stargazers_count;

    if( ibm[i].stargazers_count < statistics.stargazers.average ) {
      sheet.getCell( `F${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFFFFD54'}};    
    }

    if( ibm[i].stargazers_count < statistics.stargazers.median ) {
      sheet.getCell( `F${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFF6C9CF'}};          
    }

    // Forks
    sheet.getCell( `G${i + 2}` ).value = ibm[i].forks_count;    

    if( ibm[i].forks_count < statistics.forks.average ) {
      sheet.getCell( `G${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFFFFD54'}};    
    }

    if( ibm[i].forks_count < statistics.forks.median ) {
      sheet.getCell( `G${i + 2}` ).fill = {type: 'pattern', pattern: 'solid', fgColor: {argb: 'FFF6C9CF'}};          
    }    

    // Issues
    sheet.getCell( `H${i + 2}` ).value = ibm[i].open_issues;

    // Candidate for removal
    // All measures fall below
    if( ibm[i].subscribers_count < statistics.subscribers.median &&
        ibm[i].stargazers_count < statistics.stargazers.median &&
        ibm[i].forks_count < statistics.forks.median ) {
      sheet.getCell( `I${i + 2}` ).value = 'X';
      statistics.remove = statistics.remove + 1;
    }

    // Outperforming
    // All measures fall above
    if( ibm[i].subscribers_count >= Math.round( statistics.subscribers.average ) &&
        ibm[i].stargazers_count >= Math.round( statistics.stargazers.average ) &&
        ibm[i].forks_count >= Math.round( statistics.forks.average ) ) {
      sheet.getCell( `I${i + 2}` ).value = 'Y';
      statistics.promote = statistics.promote + 1;
    }    
  }

  // Watchers summary
  sheet.getCell( `E${ibm.length + 2}` ).value = {
    formula: `=MEDIAN(E2:E${ibm.length + 1})`, 
    result: statistics.subscribers.median
  };
  sheet.getCell( `E${ibm.length + 3}` ).value = {
    formula: `=ROUND(SUM(E2:E${ibm.length + 1})/${ibm.length},0)`, 
    result: statistics.subscribers.average
  };

  // Stars summary
  sheet.getCell( `F${ibm.length + 2}` ).value = {
    formula: `=MEDIAN(F2:F${ibm.length + 1})`, 
    result: statistics.stargazers.median
  };
  sheet.getCell( `F${ibm.length + 3}` ).value = {
    formula: `=ROUND(SUM(F2:F${ibm.length + 1})/${ibm.length},0)`, 
    result: statistics.stargazers.average
  };  

  // Forks summary
  sheet.getCell( `G${ibm.length + 2}` ).value = {
    formula: `=MEDIAN(G2:G${ibm.length + 1})`, 
    result: statistics.forks.median
  };
  sheet.getCell( `G${ibm.length + 3}` ).value = {
    formula: `=ROUND(SUM(G2:G${ibm.length + 1})/${ibm.length},0)`, 
    result: statistics.forks.average
  };    

  // Issues summary
  sheet.getCell( `H${ibm.length + 2}` ).value = {
    formula: `=MEDIAN(H2:H${ibm.length + 1})`, 
    result: statistics.issues.median
  };
  sheet.getCell( `H${ibm.length + 3}` ).value = {
    formula: `=ROUND(SUM(H2:H${ibm.length + 1})/${ibm.length},0)`, 
    result: statistics.issues.average
  };      

  // Candidate summary
  sheet.getCell( `I${ibm.length + 2}` ).value = {
    formula: `=COUNTIF(I2:I${ibm.length + 1},"X")`, 
    result: statistics.remove
  };
  sheet.getCell( `I${ibm.length + 3}` ).value = {
    formula: `=COUNTIF(I2:I${ibm.length + 1},"Y")`, 
    result: statistics.promote
  };

  // Summary labels
  sheet.getCell( `D${ibm.length + 2}` ).value = 'Median';
  sheet.getCell( `D${ibm.length + 3}` ).value = 'Average';  
  sheet.getCell( `J${ibm.length + 2}` ).value = 'Underperform';  
  sheet.getCell( `J${ibm.length + 3}` ).value = 'Outperform';  
  sheet.getCell( `E${ibm.length + 4}` ).value = 'Watchers';    
  sheet.getCell( `F${ibm.length + 4}` ).value = 'Stars';
  sheet.getCell( `G${ibm.length + 4}` ).value = 'Forks';
  sheet.getCell( `H${ibm.length + 4}` ).value = 'Issues';            

  // Write the file
  await book.xlsx.writeFile( 'patterns.xlsx' );

  console.log( 'Done.' );
}

function median( values ) {
  // Odd length: Just the middle number
  // Equal number of digits on both sides
  let result = values[Math.floor( values.length / 2 )];

  // Even length: Average of two middle numbers
  if( ( values.length % 2 ) === 0 ) {
    const low = values.length / 2;
    const high = low + 1;
    result = ( values[low] + values[high] ) / 2;
  }

  return result;
}

// Ascending
function numeric( a, b ) {
  if( a < b ) return -1;
  if( a > b ) return 1;
  return 0;
}

patterns.txt

# List of links
# One per line
# That point to a repository hosted on GitHub
https://github.com/IBM/powerai-market-sentiment
https://github.com/IBM/connector-for-storediq/
https://github.com/IBM/query-knowledge-base-with-domain-specific-documents/
https://github.com/IBM/image-analysis-iot-alert
https://github.com/IBM/Analyze-Investment-Portfolio
https://github.com/IBM/xgboost-financial-predictions