Find (package|bower|component|composer).json commonality

fetch.js

var fs = require('fs'),
  path = require('path'),
  request = require('request'),
  diff = require('jsondiffpatch'),
  
  cJSON = require('canonical-json'),
  
  _ = require('lodash'),
  async = require('async'),

  gh = 'https://raw.githubusercontent.com/',
  
  repos = {
    'jashkenas/underscore': 'master',
    'jashkenas/backbone': 'master',
    'visionmedia/mocha': 'master',
    'twbs/bootstrap': 'master',
    'jquery/jquery': 'master',
    'lodash/lodash': 'master',
    'caolan/async': 'master',
    'noflo/noflo': 'master',
    'Stuk/jszip': 'master',
    'mbostock/d3': 'master',
    'wout/svg.js': 'master',
    'kriskowal/q': 'v1'
  },

  files = [
    'component',
    'composer',
    'bower'
  ];

function cache_remote(file, remote, cached, done){
  request(remote,
    function (error, response, body){
      var result;
      if(!error && response.statusCode < 400){
        try {
          result = {};
          result[file] = JSON.parse(body);
        }catch(err){
        }
      }
      
      if(result && result[file]){
        fs.mkdir(path.dirname(cached), function(){
          fs.writeFile(cached, body, function(){
            done(null, result);
          });
        });
        return;
      }
      
      done(null, result);
    }
  );
}


function handle_file(repo){
  return function(file, done){
    var cached = "cached/" + repo[0].split("/")[1] + "/" + file + ".json",
      remote = gh + repo[0] + '/' + repo[1] + '/' + file + '.json';
    
    fs.readFile(cached, {encoding: "utf-8"}, function(err, content){
      if(err){
        cache_remote(file, remote, cached, done);
      }else{
        var result = {};
        result[file] = JSON.parse(content);
        done(null, result);
      }
    });
  };
}


function handle_repo(repo, done){
  var repo_files = {};
  async.map(
    ["package"].concat(files), handle_file(repo),
    function(err, repo_files){
      repo_files = _.merge.apply(null, _.chain(repo_files)
        .compact()
        .value());

      var pkg = cJSON(repo_files.package);
      _(files).map(function(file){
        console.log('}},\n"' + repo[0].split("/")[1] +
          "_" + file + '": {\n  fields:{');
        _.chain(repo_files[file])
          .keys()
          .map(function(key){
            var val = cJSON(repo_files[file][key]);
            if(pkg.indexOf(val) !== -1){
              var match = pkg.split(val)[0].match(/"([^"]*?)":\s*$/);
              if(match){
                console.log("    " + key + ': "' + match[1] + '",');
              }
            }
          });
      });
      done(null);
    }
  );
}


async.map(_.pairs(repos), handle_repo);

output.js

{
"d3_component": {
  fields:{
    name: "exports",
    description: "description",
    keywords: "keywords",
    version: "version",
    main: "browserify",
    scripts: "files",
    license: "type",
}},
"d3_composer": {
  fields:{
    description: "description",
}},
"d3_bower": {
  fields:{
    name: "exports",
    version: "version",
    main: "browserify",
    scripts: "files",
}},
"underscore_component": {
  fields:{
    name: "name",
    description: "description",
    keywords: "keywords",
    version: "version",
    license: "type",
}},
"underscore_bower": {
  fields:{
    name: "name",
    version: "version",
    keywords: "keywords",
}},
"backbone_component": {
  fields:{
    name: "name",
    version: "version",
    description: "description",
    keywords: "keywords",
    main: "main",
    license: "license",
}},
"backbone_bower": {
  fields:{
    name: "name",
    version: "version",
    main: "main",
    dependencies: "dependencies",
}},
"lodash_component": {
  fields:{
    name: "name",
    repo: "repository",
    version: "version",
    description: "description",
    license: "license",
    main: "browser",
}},
"lodash_bower": {
  fields:{
    name: "name",
    version: "version",
    main: "browser",
}},
"async_component": {
  fields:{
    name: "name",
    description: "description",
}},
"noflo_component": {
  fields:{
    description: "description",
    keywords: "keywords",
    version: "version",
    license: "type",
}},
"mocha_component": {
  fields:{
    version: "version",
    description: "description",
    keywords: "keywords",
}},
"mocha_bower": {
  fields:{
    version: "version",
}},
"jszip_component": {
  fields:{
    name: "name",
    description: "description",
    version: "version",
    keywords: "keywords",
    license: "license",
}},
"jszip_bower": {
  fields:{
    name: "name",
    version: "version",
    description: "description",
    keywords: "keywords",
    license: "license",
}},
"svg.js_component": {
  fields:{
    name: "name",
    version: "version",
    author: "author",
    license: "type",
}},
"svg.js_bower": {
  fields:{
    name: "name",
    version: "version",
    description: "description",
    license: "type",
}},
"bootstrap_composer": {
  fields:{
    description: "description",
    homepage: "homepage",
    license: "type",
}},
"bootstrap_bower": {
  fields:{
    name: "name",
    description: "description",
    version: "version",
    homepage: "homepage",
}},
"jquery_bower": {
  fields:{
    main: "main",
    license: "type",
    keywords: "keywords",
}}
}