nithub-sort.js

var fs = require("fs")
var sprintf = require('sprintf').sprintf


fs.readFile(__dirname+"/npmdump.json", function (err, data) {
    if (err) throw err
    var packages = JSON.parse(data)

    fs.readFile("../output.json", function (err, data) {
        if (err) throw err
        var objects = JSON.parse(data)

        var tagMap = {}
        var tags = []

        objects.forEach(function(pack) {
            pack.score = (pack.watchers-1) + (pack.forks-1) * 5
            pack.tags = packages[pack.name].keywords

            if (pack.tags) {
                if (!pack.tags.forEach) {
                    if (/,/.test(pack.tags)) {
                        pack.tags = pack.tags.split(/\s*,\s*/)
                    } else {
                        pack.tags = pack.tags.split(/\s+/)
                    }
                }
                if (!pack.tags.some(function(tag){return tag == pack.name})) {
                    pack.tags.push(pack.name)
                }
            } else {
                pack.tags = [pack.name]
            }

            pack.tags.forEach(function(tag) {
                tag = tag.toLowerCase()
                if (!tagMap[tag]) {
                    tags.push({
                        name: tag,
                        packages: tagMap[tag] = [pack]
                    })
                } else {
                    tagMap[tag].push(pack)
                }
            })
        })


        function show(fieldName) {
            objects.sort(function(a,b) {
                return b[fieldName] - a[fieldName]
            })
            console.log("Top 50 githubbed npm modules by "+fieldName+":")
            objects.slice(0,50).forEach(function(pack, index) {
                console.log(sprintf("%-3s %-50s %5d %s - %s", index+1+".", pack.name, pack[fieldName], fieldName, pack.url))
            })
            console.log()
        }
        show("watchers")
        show("forks")
        show("score")

        tags.sort(function(a,b) {
            if (a.name<b.name) return -1
            if (a.name>b.name) return 1
            return 0
        })

        tags.forEach(function (tag) {
            if (tag.packages.length > 1) {
                var vs = tag.packages.map(function(p){return p.name}).join("  VS  ")
                tag.packages.sort(function(a,b) {
                    return b.score - a.score
                })
                var winner = tag.packages[0]
                console.log(tag.name+" KEYWORD DEATHMATCH")
                tag.packages.slice(0,5).forEach(function(pack, index) {
                    console.log(sprintf("   %-3s %-50s %5d %s - %s", index+1+".", pack.name, pack.score, "score", pack.url))
                })
                console.log()
            }
        })

    })

})

nithub.js

var fs = require("fs")
var http = require('http')
var sprintf = require('sprintf').sprintf

var packageJsonFile = __dirname+'/npmdump.json'

function getHttp(host,port,path, callback) {
    http.get({
        host:host,
        port:port,
        path:path
    }, function(res) {
        var data = ''
        res.on('data', function(chunk) {
            data += chunk
        })
        res.on('end', function() {
            if (res.statusCode != 200) {
                callback(new Error("Status code "+res.statusCode+"\n"+data))
            } else {
                callback(null, data)
            }
        })
    }).on('error', callback)
}


if (!fs.statSync(packageJsonFile).isFile()) {
    getHttp('registry.npmjs.org', 80, '/', function(err, json) {
        if (err) throw err
        fs.writeFile(packageJsonFile, json)
        inspectPackages(JSON.parse(json))
    })
} else {
    fs.readFile(packageJsonFile, function (err, data) {
        if (err) throw err
        inspectPackages(JSON.parse(data))
    });
}

var gitUrlMatcher = /github\.com(?:\/|:)([^/]+)\/([^/"]+)(.git)?/i

function inspectPackages(packages) {
    var githubbed = 0
    var total = 0
    var pool = []
    for (var name in packages) {
        total++
        var pack = packages[name]

        var githubUser = null
        var githubName = null
        var gitUrl = null

        function checkForGitUrl(url) {
            var match = gitUrlMatcher.exec(url)
            if (match) {
                gitUrl = match[0].replace(/\.git$/,'')
                githubUser = match[1]
                githubName = match[2].replace(/\.git$/,'')
            }
        }

        if (pack.repository && pack.repository.url) {
            checkForGitUrl(pack.repository.url)
        } else {
            checkForGitUrl(pack.repository)
        }
        if (!githubUser) {
            checkForGitUrl(JSON.stringify(pack))
        }
        if (githubUser && githubName) {
            githubbed++
            console.log(sprintf("%30s is on github! %-50s - %s",name,githubUser+"/"+githubName,gitUrl))
            if (name > "yui3-bare")
            pool.push({
                name: name,
                githubUser: githubUser,
                githubName: githubName,
                gitUrl: gitUrl
            })
        }
    }
    console.log(githubbed+"/"+total+" npm packages are on github!")

    var output = []
    var errorCount = 0

    function readNext() {
        if (pool.length) {
            var pack = pool.shift()
            console.log("connecting to ", pack)
            getHttp("github.com",80,"/api/v2/json/repos/show/"+pack.githubUser+"/"+pack.githubName, function(err, json) {
                if (err) {
                    console.warn("Error retrieving github info for:"+pack.name,err.stack)
                    errorCount++
                }
                if (json) {
                    var info = JSON.parse(json)
                    if (info.repository) {
                        info = info.repository
                    } else {
                        console.error("unexpected json:",info)
                    }
                    output.push({
                        name: pack.name,
                        url: info.url,
                        description: info.description,
                        watchers: info.watchers,
                        forks: info.forks
                    })
                    fs.writeFile(__dirname+"/../output.json", JSON.stringify(output,null,2))
                }
                setTimeout(readNext,1000)
            })
        }
    }
    readNext()
}