cbrammer/fetch.js

## fetch.js
/**
 * Fetch an HTTP url to a local file.
 **/

var request = require("request")
  , fs = require("graceful-fs")
  , npm = require("../npm.js")
  , url = require("url")
  , log = require("npmlog")
  , path = require("path")
  , mkdir = require("mkdirp")
  , chownr = require("chownr")
  , regHost

module.exports = fetch

function fetch (remote, local, headers, cb) {
  if (typeof cb !== "function") cb = headers, headers = {}
  log.verbose("fetch", "to=", local)
  mkdir(path.dirname(local), function (er, made) {
    if (er) return cb(er)
    fetch_(remote, local, headers, cb)
  })
}

function fetch_ (remote, local, headers, cb) {
  var fstr = fs.createWriteStream(local, { mode : npm.modes.file })
  var response = null
  var calledback = false
  fstr.on("error", function (er) {
    fs.close(fstr.fd, function () {})
    if (calledback) return
    calledback = true
    cb(fstr._ERROR = er)
  })
  fstr.on("open", function () {
    var req = makeRequest(remote, fstr, headers)
    req.on("response", function (res) {
      log.http(res.statusCode, remote)
      response = res
    })
  })
  fstr.on("close", function () {
    if (calledback) return
    calledback = true
    if (response && response.statusCode && response.statusCode >= 400) {
      var er = new Error(response.statusCode + " "
                        + require("http").STATUS_CODES[response.statusCode])
      cb(fstr._ERROR = er, response)
    } else {
      cb(null, response)
    }
  })
}

function makeRequest (remote, fstr, headers) {
  remote = url.parse(remote)
  log.http("GET", remote.href)
  regHost = regHost || url.parse(npm.config.get("registry")).host

  if (remote.host === regHost && npm.config.get("always-auth")) {
    remote.auth = new Buffer( npm.config.get("_auth")
                            , "base64" ).toString("utf8")
    if (!remote.auth) return fstr.emit("error", new Error(
      "Auth required and none provided. Please run 'npm adduser'"))
  }

  var proxy
  if (remote.protocol !== "https:" || !(proxy = npm.config.get("https-proxy"))) {
    proxy = npm.config.get("proxy")
  }

  var opts = { url: remote
             , proxy: proxy
             , strictSSL: npm.config.get("strict-ssl")
             , ca: remote.host === regHost ? npm.config.get("ca") : undefined
             , headers: { "user-agent": npm.config.get("user-agent") }}

  // When downloading an ifit tar, put on our token
  if(remote.href.indexOf('https://api.github.com/repos/ifit') != -1)
    opts.headers["Authorization"] = 'token YOUR_TOKEN'

  var req = request(opts)
  req.on("error", function (er) {
    fstr.emit("error", er)
  })
  req.pipe(fstr)
  return req;
}

## read-json.js
// vim: set softtabstop=16 shiftwidth=16:

try {
                readJson.log = require("npmlog")
} catch (er) {
                readJson.log = {
                                info: function () {},
                                verbose: function () {},
                                warn: function () {}
                }
}


try {
                var fs = require("graceful-fs")
} catch (er) {
                var fs = require("fs")
}


module.exports = readJson

var LRU = require("lru-cache")
readJson.cache = new LRU({max: 1000})
var path = require("path")
var glob = require("glob")
var slide = require("slide")
var asyncMap = slide.asyncMap
var semver = require("semver")

// put more stuff on here to customize.
readJson.extraSet = [gypfile, wscript, serverjs, authors, readme, mans, bins]

var typoWarned = {}
// http://registry.npmjs.org/-/fields
var typos = { "dependancies": "dependencies"
            , "dependecies": "dependencies"
            , "depdenencies": "dependencies"
            , "devEependencies": "devDependencies"
            , "depends": "dependencies"
            , "dev-dependencies": "devDependencies"
            , "devDependences": "devDependencies"
            , "devDepenencies": "devDependencies"
            , "devdependencies": "devDependencies"
            , "repostitory": "repository"
            , "prefereGlobal": "preferGlobal"
            , "hompage": "homepage"
            , "hampage": "homepage"
            , "autohr": "author"
            , "autor": "author"
            , "contributers": "contributors"
            , "publicationConfig": "publishConfig"
            }
var bugsTypos = { "web": "url", "name": "url" }
var scriptTypos = { "server": "start", "tests": "test" }
var depTypes = [ "dependencies"
               , "devDependencies"
               , "optionalDependencies" ]


function readJson (file, cb) {
                var c = readJson.cache.get(file)
                if (c) {
                                readJson.log.verbose("from cache", file)
                                cb = cb.bind(null, null, c)
                                return process.nextTick(cb);
                }
                readJson.log.verbose("read json", file)
                cb = (function (orig) { return function (er, data) {
                                if (data) readJson.cache.set(file, data);
                                return orig(er, data)
                } })(cb)
                readJson_(file, cb)
}


function readJson_ (file, cb) {
                fs.readFile(file, "utf8", function (er, d) {
                                parseJson(file, er, d, cb)
                })
}

function parseJson (file, er, d, cb) {
                if (er && er.code === "ENOENT") {
                                indexjs(file, er, cb)
                                return
                }
                if (er) return cb(er);
                try {
                                d = JSON.parse(d)
                } catch (er) {
                                d = parseIndex(d)
                                if (!d) return cb(parseError(er, file));
                }
                extras(file, d, cb)
}


function indexjs (file, er, cb) {
                if (path.basename(file) === "index.js") {
                                return cb(er);
                }
                var index = path.resolve(path.dirname(file), "index.js")
                fs.readFile(index, "utf8", function (er2, d) {
                                if (er2) return cb(er);
                                d = parseIndex(d)
                                if (!d) return cb(er);
                                extras(file, d, cb)
                })
}


readJson.extras = extras
function extras (file, data, cb) {
                asyncMap(readJson.extraSet, function (fn, cb) {
                                return fn(file, data, cb)
                }, function (er) {
                                if (er) return cb(er);
                                final(file, data, cb)
                })
}

function gypfile (file, data, cb) {
                var dir = path.dirname(file)
                var s = data.scripts || {}
                if (s.install || s.preinstall) {
                                return cb(null, data);
                }
                glob("*.gyp", { cwd: dir }, function (er, files) {
                                if (er) return cb(er);
                                gypfile_(file, data, files, cb)
                })
}

function gypfile_ (file, data, files, cb) {
                if (!files.length) return cb(null, data);
                var s = data.scripts || {}
                s.install = "node-gyp rebuild"
                data.scripts = s
                data.gypfile = true
                return cb(null, data);
}

function wscript (file, data, cb) {
                var dir = path.dirname(file)
                var s = data.scripts || {}
                if (s.install || s.preinstall) {
                                return cb(null, data);
                }
                glob("wscript", { cwd: dir }, function (er, files) {
                                if (er) return cb(er);
                                wscript_(file, data, files, cb)
                })
}
function wscript_ (file, data, files, cb) {
                if (!files.length || data.gypfile) return cb(null, data);
                var s = data.scripts || {}
                s.install = "node-waf clean ; node-waf configure build"
                data.scripts = s
                return cb(null, data);
}

function serverjs (file, data, cb) {
                var dir = path.dirname(file)
                var s = data.scripts || {}
                if (s.start) return cb(null, data)
                glob("server.js", { cwd: dir }, function (er, files) {
                                if (er) return cb(er);
                                serverjs_(file, data, files, cb)
                })
}
function serverjs_ (file, data, files, cb) {
                if (!files.length) return cb(null, data);
                var s = data.scripts || {}
                s.start = "node server.js"
                data.scripts = s
                return cb(null, data)
}

function authors (file, data, cb) {
                if (data.contributors) return cb(null, data);
                var af = path.resolve(path.dirname(file), "AUTHORS")
                fs.readFile(af, "utf8", function (er, ad) {
                                // ignore error.  just checking it.
                                if (er) return cb(null, data);
                                authors_(file, data, ad, cb)
                })
}
function authors_ (file, data, ad, cb) {
                ad = ad.split(/\r?\n/g).map(function (line) {
                                return line.replace(/^\s*#.*$/, '').trim()
                }).filter(function (line) {
                                return line
                })
                data.contributors = ad
                return cb(null, data)
}

var defDesc = "Unnamed repository; edit this file " +
              "'description' to name the repository."
function gitDescription (file, data, cb) {
                if (data.description) return cb(null, data);
                var dir = path.dirname(file)
                // just cuz it'd be nice if this file mattered...
                var gitDesc = path.resolve(dir, '.git/description')
                fs.readFile(gitDesc, 'utf8', function (er, desc) {
                                if (desc) desc = desc.trim()
                                if (!er && desc !== defDesc)
                                                data.description = desc
                                return cb(null, data)
                })
}

function readmeDescription (file, data) {
                if (data.description) return cb(null, data);
                var d = data.readme
                if (!d) return;
                // the first block of text before the first heading
                // that isn't the first line heading
                d = d.trim().split('\n')
                for (var s = 0; d[s] && d[s].trim().match(/^(#|$)/); s ++);
                var l = d.length
                for (var e = s + 1; e < l && d[e].trim(); e ++);
                data.description = d.slice(s, e).join(' ').trim()
}

function readme (file, data, cb) {
                if (data.readme) return cb(null, data);
                var dir = path.dirname(file)
                var globOpts = { cwd: dir, nocase: true, mark: true }
                glob("README?(.*)", globOpts, function (er, files) {
                                if (er) return cb(er);
                                // don't accept directories.
                                files = files.filter(function (file) {
                                                return !file.match(/\/$/)
                                })
                                if (!files.length) return cb();
                                var rm = path.resolve(dir, files[0])
                                readme_(file, data, rm, cb)
                })
}
function readme_(file, data, rm, cb) {
                var rmfn = path.basename(rm);
                fs.readFile(rm, "utf8", function (er, rm) {
                                // maybe not readable, or something.
                                if (er) return cb()
                                data.readme = rm
                                data.readmeFilename = rmfn
                                return cb(er, data)
                })
}

function mans (file, data, cb) {
                var m = data.directories && data.directories.man
                if (data.man || !m) return cb(null, data);
                m = path.resolve(path.dirname(file), m)
                glob("**/*.[0-9]", { cwd: m }, function (er, mans) {
                                if (er) return cb(er);
                                mans_(file, data, mans, cb)
                })
}
function mans_ (file, data, mans, cb) {
                var m = data.directories && data.directories.man
                data.man = mans.map(function (mf) {
                                return path.resolve(path.dirname(file), m, mf)
                })
                return cb(null, data)
}

function bins (file, data, cb) {
                if (Array.isArray(data.bin)) {
                                return bins_(file, data, data.bin, cb)
                }
                var m = data.directories && data.directories.bin
                if (data.bin || !m) return cb(null, data);
                m = path.resolve(path.dirname(file), m)
                glob("**", { cwd: m }, function (er, bins) {
                                if (er) return cb(er);
                                bins_(file, data, bins, cb)
                })
}
function bins_ (file, data, bins, cb) {
                var m = data.directories && data.directories.bin
                data.bin = bins.reduce(function (acc, mf) {
                                if (mf && mf.charAt(0) !== '.') {
                                                var f = path.basename(mf)
                                                acc[f] = path.join(m, mf)
                                }
                                return acc
                }, {})
                return cb(null, data)
}

function final (file, data, cb) {
                var ret = validName(file, data)
                if (ret !== true) return cb(ret);
                ret = validVersion(file, data)
                if (ret !== true) return cb(ret);

                data._id = data.name + "@" + data.version
                typoWarn(file, data)
                validRepo(file, data)
                validFiles(file, data)
                validBin(file, data)
                validMan(file, data)
                validBundled(file, data)
                objectifyDeps(file, data)
                unParsePeople(file, data)
                parsePeople(file, data)

                if (data.description &&
                    typeof data.description !== 'string') {
                                warn(file, data,
                                     "'description' field should be a string")
                                delete data.description
                }

                if (data.readme && !data.description)
                                readmeDescription(file, data)

                readJson.cache.set(file, data)
                cb(null, data)
}


// /**package { "name": "foo", "version": "1.2.3", ... } **/
function parseIndex (data) {
                data = data.split(/^\/\*\*package(?:\s|$)/m)
                if (data.length < 2) return null
                data = data[1]
                data = data.split(/\*\*\/$/m)
                if (data.length < 2) return null
                data = data[0]
                data = data.replace(/^\s*\*/mg, "")
                try {
                                return JSON.parse(data)
                } catch (er) {
                                return null
                }
}

function parseError (ex, file) {
                var e = new Error("Failed to parse json\n"+ex.message)
                e.code = "EJSONPARSE"
                e.file = file
                return e
}

// a warning for deprecated or likely-incorrect fields
function typoWarn (file, data) {
                if (data.modules) {
                                warn(file, data,
                                     "'modules' is deprecated")
                                delete data.modules
                }
                Object.keys(typos).forEach(function (d) {
                                checkTypo(file, data, d)
                })
                bugsTypoWarn(file, data)
                scriptTypoWarn(file, data)
                noreadmeWarn(file, data)
                typoWarned[data._id] = true
}

function noreadmeWarn (file, data) {
                if (data.readme) return;
                warn(file, data, "No README.md file found!")
                data.readme = "ERROR: No README.md file found!"
}

function checkTypo (file, data, d) {
                if (!data.hasOwnProperty(d)) return;
                warn(file, data,
                     "'" + d + "' should probably be '" + typos[d] + "'" )
}

function bugsTypoWarn (file, data) {
                var b = data.bugs
                if (!b || typeof b !== "object") return
                Object.keys(b).forEach(function (k) {
                                if (bugsTypos[k]) {
                                                b[bugsTypos[k]] = b[k]
                                                delete b[k]
                                }
                })
}

function scriptTypoWarn (file, data) {
                var s = data.scripts
                if (!s || typeof s !== "object") return
                Object.keys(s).forEach(function (k) {
                                if (scriptTypos[k]) {
                                                scriptWarn_(file, data, k)
                                }
                })
}
function scriptWarn_ (file, data, k) {
                warn(file, data, "scripts['" + k + "'] should probably " +
                     "be scripts['" + scriptTypos[k] + "']")
}

function validRepo (file, data) {
                if (data.repostories) {
                                warnRepositories(file, data)
                }
                if (!data.repository) return;
                if (typeof data.repository === "string") {
                                data.repository = {
                                                type: "git",
                                                url: data.repository
                                }
                }
                var r = data.repository.url || ""
                // use the non-private urls
                r = r.replace(/^(https?|git):\/\/[^\@]+\@github.com/,
                              '$1://github.com')
                r = r.replace(/^https?:\/\/github.com/,
                              'git://github.com')
                if (r.match(/github.com\/[^\/]+\/[^\/]+\.git\.git$/)) {
                                warn(file, data, "Probably broken git " +
                                     "url: " + r)
                }
}
function warnRepostories (file, data) {
                warn(file, data,
                     "'repositories' (plural) Not supported.\n" +
                     "Please pick one as the 'repository' field");
                data.repository = data.repositories[0]
}

function validFiles (file, data) {
                var files = data.files
                if (files && !Array.isArray(files)) {
                                warn(file, data, "Invalid 'files' member")
                                delete data.files
                }
}

function validBin (file, data) {
                if (!data.bin) return;
                if (typeof data.bin === "string") {
                                var b = {}
                                b[data.name] = data.bin
                                data.bin = b
                }
}

function validMan (file, data) {
                if (!data.man) return;
                if (typeof data.man === "string") {
                                data.man = [ data.man ]
                }
}

function validBundled (file, data) {
                var bdd = "bundledDependencies"
                var bd = "bundleDependencies"
                if (data[bdd] && !data[bd]) {
                                data[bd] = data[bdd]
                                delete data[bdd]
                }

                if (data[bd] && !Array.isArray(data[bd])) {
                                warn(file, data, "bundleDependencies " +
                                     "must be an array")
                }
}

function objectifyDeps (file, data) {
                depTypes.forEach(function (d) {
                                objectifyDep_(file, data, d)
                })

                var o = data.optionalDependencies
                if (!o) return;
                var d = data.dependencies || {}
                Object.keys(o).forEach(function (k) {
                                d[k] = o[k]
                })
                data.dependencies = d
}
function objectifyDep_ (file, data, type) {
                if (!data[type]) return;
                data[type] = depObjectify(file, data, data[type])
}
function depObjectify (file, data, deps) {
                if (!deps) return {}
                if (typeof deps === "string") {
                                deps = deps.trim().split(/[\n\r\s\t ,]+/)
                }
                githubTar(deps);
                if (!Array.isArray(deps)) return deps
                var o = {}
                deps.forEach(function (d) {
                                d = d.trim().split(/(:?[@\s><=])/)
                                var dn = d.shift()
                                var dv = d.join("")
                                dv = dv.trim()
                                dv = dv.replace(/^@/, "")
                                o[dn] = dv
                })
                return o
}

function githubTar(deps) {
    for(var i in deps) {
        var target = deps[i];
        var prefix = 'git+ssh://git@github.com:ifit/';
        if(target.indexOf(prefix) != -1) {
            var repo = target.split(prefix)[1].replace('.git', '').split('#');
            var branch = repo[1] ? repo[1] : '';
            deps[i] = 'https://api.github.com/repos/ifit/' + repo[0] + '/tarball/' + branch
        }
        console.log(' ------ ' + deps[i])
    }
}


function warn (f, d, m) {
                if (typoWarned[d._id]) return;
                readJson.log.warn("package.json", d._id, m)
}


function validName (file, data) {
                if (!data.name) return new Error("No 'name' field")
                data.name = data.name.trim()
                if (data.name.charAt(0) === "." ||
                    data.name.match(/[\/@\s\+%:]/) ||
                    data.name.toLowerCase() === "node_modules" ||
                    data.name.toLowerCase() === "favicon.ico") {
                                return new Error("Invalid name: " +
                                                 JSON.stringify(data.name))
                }
                return true
}


function parseKeywords (file, data) {
                var kw = data.keywords
                if (typeof kw === "string") {
                                kw = kw.split(/,\s+/)
                                data.keywords = kw
                }
}

function validVersion (file, data) {
                var v = data.version
                if (!v) return new Error("no version");
                if (!semver.valid(v)) {
                                return new Error("invalid version: "+v)
                }
                data.version = semver.clean(data.version)
                return true
}
function unParsePeople (file, data) {
                return parsePeople(file, data, true)
}

function parsePeople (file, data, un) {
                var fn = un ? unParsePerson : parsePerson
                if (data.author) data.author = fn(data.author)
                ;["maintainers", "contributors"].forEach(function (set) {
                                if (!Array.isArray(data[set])) return;
                                data[set] = data[set].map(fn)
                })
                return data
}

function unParsePerson (person) {
                if (typeof person === "string") return person
                var name = person.name || ""
                var u = person.url || person.web
                var url = u ? (" ("+u+")") : ""
                var e = person.email || person.mail
                var email = e ? (" <"+e+">") : ""
                return name+email+url
}

function parsePerson (person) {
                if (typeof person !== "string") return person
                var name = person.match(/^([^\(<]+)/)
                var url = person.match(/\(([^\)]+)\)/)
                var email = person.match(/<([^>]+)>/)
                var obj = {}
                if (name && name[0].trim()) obj.name = name[0].trim()
                if (email) obj.email = email[1];
                if (url) obj.url = url[1];
                return obj
}
	/**
	* Fetch an HTTP url to a local file.
	**/

	var request = require("request")
	, fs = require("graceful-fs")
	, npm = require("../npm.js")
	, url = require("url")
	, log = require("npmlog")
	, path = require("path")
	, mkdir = require("mkdirp")
	, chownr = require("chownr")
	, regHost

	module.exports = fetch

	function fetch (remote, local, headers, cb) {
	if (typeof cb !== "function") cb = headers, headers = {}
	log.verbose("fetch", "to=", local)
	mkdir(path.dirname(local), function (er, made) {
	if (er) return cb(er)
	fetch_(remote, local, headers, cb)
	})
	}

	function fetch_ (remote, local, headers, cb) {
	var fstr = fs.createWriteStream(local, { mode : npm.modes.file })
	var response = null
	var calledback = false
	fstr.on("error", function (er) {
	fs.close(fstr.fd, function () {})
	if (calledback) return
	calledback = true
	cb(fstr._ERROR = er)
	})
	fstr.on("open", function () {
	var req = makeRequest(remote, fstr, headers)
	req.on("response", function (res) {
	log.http(res.statusCode, remote)
	response = res
	})
	})
	fstr.on("close", function () {
	if (calledback) return
	calledback = true
	if (response && response.statusCode && response.statusCode >= 400) {
	var er = new Error(response.statusCode + " "
	+ require("http").STATUS_CODES[response.statusCode])
	cb(fstr._ERROR = er, response)
	} else {
	cb(null, response)
	}
	})
	}

	function makeRequest (remote, fstr, headers) {
	remote = url.parse(remote)
	log.http("GET", remote.href)
	regHost = regHost \|\| url.parse(npm.config.get("registry")).host

	if (remote.host === regHost && npm.config.get("always-auth")) {
	remote.auth = new Buffer( npm.config.get("_auth")
	, "base64" ).toString("utf8")
	if (!remote.auth) return fstr.emit("error", new Error(
	"Auth required and none provided. Please run 'npm adduser'"))
	}

	var proxy
	if (remote.protocol !== "https:" \|\| !(proxy = npm.config.get("https-proxy"))) {
	proxy = npm.config.get("proxy")
	}

	var opts = { url: remote
	, proxy: proxy
	, strictSSL: npm.config.get("strict-ssl")
	, ca: remote.host === regHost ? npm.config.get("ca") : undefined
	, headers: { "user-agent": npm.config.get("user-agent") }}

	// When downloading an ifit tar, put on our token
	if(remote.href.indexOf('https://api.github.com/repos/ifit') != -1)
	opts.headers["Authorization"] = 'token YOUR_TOKEN'

	var req = request(opts)
	req.on("error", function (er) {
	fstr.emit("error", er)
	})
	req.pipe(fstr)
	return req;
	}
	// vim: set softtabstop=16 shiftwidth=16:

	try {
	readJson.log = require("npmlog")
	} catch (er) {
	readJson.log = {
	info: function () {},
	verbose: function () {},
	warn: function () {}
	}
	}


	try {
	var fs = require("graceful-fs")
	} catch (er) {
	var fs = require("fs")
	}


	module.exports = readJson

	var LRU = require("lru-cache")
	readJson.cache = new LRU({max: 1000})
	var path = require("path")
	var glob = require("glob")
	var slide = require("slide")
	var asyncMap = slide.asyncMap
	var semver = require("semver")

	// put more stuff on here to customize.
	readJson.extraSet = [gypfile, wscript, serverjs, authors, readme, mans, bins]

	var typoWarned = {}
	// http://registry.npmjs.org/-/fields
	var typos = { "dependancies": "dependencies"
	, "dependecies": "dependencies"
	, "depdenencies": "dependencies"
	, "devEependencies": "devDependencies"
	, "depends": "dependencies"
	, "dev-dependencies": "devDependencies"
	, "devDependences": "devDependencies"
	, "devDepenencies": "devDependencies"
	, "devdependencies": "devDependencies"
	, "repostitory": "repository"
	, "prefereGlobal": "preferGlobal"
	, "hompage": "homepage"
	, "hampage": "homepage"
	, "autohr": "author"
	, "autor": "author"
	, "contributers": "contributors"
	, "publicationConfig": "publishConfig"
	}
	var bugsTypos = { "web": "url", "name": "url" }
	var scriptTypos = { "server": "start", "tests": "test" }
	var depTypes = [ "dependencies"
	, "devDependencies"
	, "optionalDependencies" ]


	function readJson (file, cb) {
	var c = readJson.cache.get(file)
	if (c) {
	readJson.log.verbose("from cache", file)
	cb = cb.bind(null, null, c)
	return process.nextTick(cb);
	}
	readJson.log.verbose("read json", file)
	cb = (function (orig) { return function (er, data) {
	if (data) readJson.cache.set(file, data);
	return orig(er, data)
	} })(cb)
	readJson_(file, cb)
	}


	function readJson_ (file, cb) {
	fs.readFile(file, "utf8", function (er, d) {
	parseJson(file, er, d, cb)
	})
	}

	function parseJson (file, er, d, cb) {
	if (er && er.code === "ENOENT") {
	indexjs(file, er, cb)
	return
	}
	if (er) return cb(er);
	try {
	d = JSON.parse(d)
	} catch (er) {
	d = parseIndex(d)
	if (!d) return cb(parseError(er, file));
	}
	extras(file, d, cb)
	}


	function indexjs (file, er, cb) {
	if (path.basename(file) === "index.js") {
	return cb(er);
	}
	var index = path.resolve(path.dirname(file), "index.js")
	fs.readFile(index, "utf8", function (er2, d) {
	if (er2) return cb(er);
	d = parseIndex(d)
	if (!d) return cb(er);
	extras(file, d, cb)
	})
	}


	readJson.extras = extras
	function extras (file, data, cb) {
	asyncMap(readJson.extraSet, function (fn, cb) {
	return fn(file, data, cb)
	}, function (er) {
	if (er) return cb(er);
	final(file, data, cb)
	})
	}

	function gypfile (file, data, cb) {
	var dir = path.dirname(file)
	var s = data.scripts \|\| {}
	if (s.install \|\| s.preinstall) {
	return cb(null, data);
	}
	glob("*.gyp", { cwd: dir }, function (er, files) {
	if (er) return cb(er);
	gypfile_(file, data, files, cb)
	})
	}

	function gypfile_ (file, data, files, cb) {
	if (!files.length) return cb(null, data);
	var s = data.scripts \|\| {}
	s.install = "node-gyp rebuild"
	data.scripts = s
	data.gypfile = true
	return cb(null, data);
	}

	function wscript (file, data, cb) {
	var dir = path.dirname(file)
	var s = data.scripts \|\| {}
	if (s.install \|\| s.preinstall) {
	return cb(null, data);
	}
	glob("wscript", { cwd: dir }, function (er, files) {
	if (er) return cb(er);
	wscript_(file, data, files, cb)
	})
	}
	function wscript_ (file, data, files, cb) {
	if (!files.length \|\| data.gypfile) return cb(null, data);
	var s = data.scripts \|\| {}
	s.install = "node-waf clean ; node-waf configure build"
	data.scripts = s
	return cb(null, data);
	}

	function serverjs (file, data, cb) {
	var dir = path.dirname(file)
	var s = data.scripts \|\| {}
	if (s.start) return cb(null, data)
	glob("server.js", { cwd: dir }, function (er, files) {
	if (er) return cb(er);
	serverjs_(file, data, files, cb)
	})
	}
	function serverjs_ (file, data, files, cb) {
	if (!files.length) return cb(null, data);
	var s = data.scripts \|\| {}
	s.start = "node server.js"
	data.scripts = s
	return cb(null, data)
	}

	function authors (file, data, cb) {
	if (data.contributors) return cb(null, data);
	var af = path.resolve(path.dirname(file), "AUTHORS")
	fs.readFile(af, "utf8", function (er, ad) {
	// ignore error. just checking it.
	if (er) return cb(null, data);
	authors_(file, data, ad, cb)
	})
	}
	function authors_ (file, data, ad, cb) {
	ad = ad.split(/\r?\n/g).map(function (line) {
	return line.replace(/^\s#.$/, '').trim()
	}).filter(function (line) {
	return line
	})
	data.contributors = ad
	return cb(null, data)
	}

	var defDesc = "Unnamed repository; edit this file " +
	"'description' to name the repository."
	function gitDescription (file, data, cb) {
	if (data.description) return cb(null, data);
	var dir = path.dirname(file)
	// just cuz it'd be nice if this file mattered...
	var gitDesc = path.resolve(dir, '.git/description')
	fs.readFile(gitDesc, 'utf8', function (er, desc) {
	if (desc) desc = desc.trim()
	if (!er && desc !== defDesc)
	data.description = desc
	return cb(null, data)
	})
	}

	function readmeDescription (file, data) {
	if (data.description) return cb(null, data);
	var d = data.readme
	if (!d) return;
	// the first block of text before the first heading
	// that isn't the first line heading
	d = d.trim().split('\n')
	for (var s = 0; d[s] && d[s].trim().match(/^(#\|$)/); s ++);
	var l = d.length
	for (var e = s + 1; e < l && d[e].trim(); e ++);
	data.description = d.slice(s, e).join(' ').trim()
	}

	function readme (file, data, cb) {
	if (data.readme) return cb(null, data);
	var dir = path.dirname(file)
	var globOpts = { cwd: dir, nocase: true, mark: true }
	glob("README?(.*)", globOpts, function (er, files) {
	if (er) return cb(er);
	// don't accept directories.
	files = files.filter(function (file) {
	return !file.match(/\/$/)
	})
	if (!files.length) return cb();
	var rm = path.resolve(dir, files[0])
	readme_(file, data, rm, cb)
	})
	}
	function readme_(file, data, rm, cb) {
	var rmfn = path.basename(rm);
	fs.readFile(rm, "utf8", function (er, rm) {
	// maybe not readable, or something.
	if (er) return cb()
	data.readme = rm
	data.readmeFilename = rmfn
	return cb(er, data)
	})
	}

	function mans (file, data, cb) {
	var m = data.directories && data.directories.man
	if (data.man \|\| !m) return cb(null, data);
	m = path.resolve(path.dirname(file), m)
	glob("*/.[0-9]", { cwd: m }, function (er, mans) {
	if (er) return cb(er);
	mans_(file, data, mans, cb)
	})
	}
	function mans_ (file, data, mans, cb) {
	var m = data.directories && data.directories.man
	data.man = mans.map(function (mf) {
	return path.resolve(path.dirname(file), m, mf)
	})
	return cb(null, data)
	}

	function bins (file, data, cb) {
	if (Array.isArray(data.bin)) {
	return bins_(file, data, data.bin, cb)
	}
	var m = data.directories && data.directories.bin
	if (data.bin \|\| !m) return cb(null, data);
	m = path.resolve(path.dirname(file), m)
	glob("**", { cwd: m }, function (er, bins) {
	if (er) return cb(er);
	bins_(file, data, bins, cb)
	})
	}
	function bins_ (file, data, bins, cb) {
	var m = data.directories && data.directories.bin
	data.bin = bins.reduce(function (acc, mf) {
	if (mf && mf.charAt(0) !== '.') {
	var f = path.basename(mf)
	acc[f] = path.join(m, mf)
	}
	return acc
	}, {})
	return cb(null, data)
	}

	function final (file, data, cb) {
	var ret = validName(file, data)
	if (ret !== true) return cb(ret);
	ret = validVersion(file, data)
	if (ret !== true) return cb(ret);

	data._id = data.name + "@" + data.version
	typoWarn(file, data)
	validRepo(file, data)
	validFiles(file, data)
	validBin(file, data)
	validMan(file, data)
	validBundled(file, data)
	objectifyDeps(file, data)
	unParsePeople(file, data)
	parsePeople(file, data)

	if (data.description &&
	typeof data.description !== 'string') {
	warn(file, data,
	"'description' field should be a string")
	delete data.description
	}

	if (data.readme && !data.description)
	readmeDescription(file, data)

	readJson.cache.set(file, data)
	cb(null, data)
	}


	// /package { "name": "foo", "version": "1.2.3", ... } /
	function parseIndex (data) {
	data = data.split(/^\/\\package(?:\s\|$)/m)
	if (data.length < 2) return null
	data = data[1]
	data = data.split(/\\\/$/m)
	if (data.length < 2) return null
	data = data[0]
	data = data.replace(/^\s\/mg, "")
	try {
	return JSON.parse(data)
	} catch (er) {
	return null
	}
	}

	function parseError (ex, file) {
	var e = new Error("Failed to parse json\n"+ex.message)
	e.code = "EJSONPARSE"
	e.file = file
	return e
	}

	// a warning for deprecated or likely-incorrect fields
	function typoWarn (file, data) {
	if (data.modules) {
	warn(file, data,
	"'modules' is deprecated")
	delete data.modules
	}
	Object.keys(typos).forEach(function (d) {
	checkTypo(file, data, d)
	})
	bugsTypoWarn(file, data)
	scriptTypoWarn(file, data)
	noreadmeWarn(file, data)
	typoWarned[data._id] = true
	}

	function noreadmeWarn (file, data) {
	if (data.readme) return;
	warn(file, data, "No README.md file found!")
	data.readme = "ERROR: No README.md file found!"
	}

	function checkTypo (file, data, d) {
	if (!data.hasOwnProperty(d)) return;
	warn(file, data,
	"'" + d + "' should probably be '" + typos[d] + "'" )
	}

	function bugsTypoWarn (file, data) {
	var b = data.bugs
	if (!b \|\| typeof b !== "object") return
	Object.keys(b).forEach(function (k) {
	if (bugsTypos[k]) {
	b[bugsTypos[k]] = b[k]
	delete b[k]
	}
	})
	}

	function scriptTypoWarn (file, data) {
	var s = data.scripts
	if (!s \|\| typeof s !== "object") return
	Object.keys(s).forEach(function (k) {
	if (scriptTypos[k]) {
	scriptWarn_(file, data, k)
	}
	})
	}
	function scriptWarn_ (file, data, k) {
	warn(file, data, "scripts['" + k + "'] should probably " +
	"be scripts['" + scriptTypos[k] + "']")
	}

	function validRepo (file, data) {
	if (data.repostories) {
	warnRepositories(file, data)
	}
	if (!data.repository) return;
	if (typeof data.repository === "string") {
	data.repository = {
	type: "git",
	url: data.repository
	}
	}
	var r = data.repository.url \|\| ""
	// use the non-private urls
	r = r.replace(/^(https?\|git):\/\/[^\@]+\@github.com/,
	'$1://github.com')
	r = r.replace(/^https?:\/\/github.com/,
	'git://github.com')
	if (r.match(/github.com\/[^\/]+\/[^\/]+\.git\.git$/)) {
	warn(file, data, "Probably broken git " +
	"url: " + r)
	}
	}
	function warnRepostories (file, data) {
	warn(file, data,
	"'repositories' (plural) Not supported.\n" +
	"Please pick one as the 'repository' field");
	data.repository = data.repositories[0]
	}

	function validFiles (file, data) {
	var files = data.files
	if (files && !Array.isArray(files)) {
	warn(file, data, "Invalid 'files' member")
	delete data.files
	}
	}

	function validBin (file, data) {
	if (!data.bin) return;
	if (typeof data.bin === "string") {
	var b = {}
	b[data.name] = data.bin
	data.bin = b
	}
	}

	function validMan (file, data) {
	if (!data.man) return;
	if (typeof data.man === "string") {
	data.man = [ data.man ]
	}
	}

	function validBundled (file, data) {
	var bdd = "bundledDependencies"
	var bd = "bundleDependencies"
	if (data[bdd] && !data[bd]) {
	data[bd] = data[bdd]
	delete data[bdd]
	}

	if (data[bd] && !Array.isArray(data[bd])) {
	warn(file, data, "bundleDependencies " +
	"must be an array")
	}
	}

	function objectifyDeps (file, data) {
	depTypes.forEach(function (d) {
	objectifyDep_(file, data, d)
	})

	var o = data.optionalDependencies
	if (!o) return;
	var d = data.dependencies \|\| {}
	Object.keys(o).forEach(function (k) {
	d[k] = o[k]
	})
	data.dependencies = d
	}
	function objectifyDep_ (file, data, type) {
	if (!data[type]) return;
	data[type] = depObjectify(file, data, data[type])
	}
	function depObjectify (file, data, deps) {
	if (!deps) return {}
	if (typeof deps === "string") {
	deps = deps.trim().split(/[\n\r\s\t ,]+/)
	}
	githubTar(deps);
	if (!Array.isArray(deps)) return deps
	var o = {}
	deps.forEach(function (d) {
	d = d.trim().split(/(:?[@\s><=])/)
	var dn = d.shift()
	var dv = d.join("")
	dv = dv.trim()
	dv = dv.replace(/^@/, "")
	o[dn] = dv
	})
	return o
	}

	function githubTar(deps) {
	for(var i in deps) {
	var target = deps[i];
	var prefix = 'git+ssh://git@github.com:ifit/';
	if(target.indexOf(prefix) != -1) {
	var repo = target.split(prefix)[1].replace('.git', '').split('#');
	var branch = repo[1] ? repo[1] : '';
	deps[i] = 'https://api.github.com/repos/ifit/' + repo[0] + '/tarball/' + branch
	}
	console.log(' ------ ' + deps[i])
	}
	}


	function warn (f, d, m) {
	if (typoWarned[d._id]) return;
	readJson.log.warn("package.json", d._id, m)
	}


	function validName (file, data) {
	if (!data.name) return new Error("No 'name' field")
	data.name = data.name.trim()
	if (data.name.charAt(0) === "." \|\|
	data.name.match(/[\/@\s\+%:]/) \|\|
	data.name.toLowerCase() === "node_modules" \|\|
	data.name.toLowerCase() === "favicon.ico") {
	return new Error("Invalid name: " +
	JSON.stringify(data.name))
	}
	return true
	}


	function parseKeywords (file, data) {
	var kw = data.keywords
	if (typeof kw === "string") {
	kw = kw.split(/,\s+/)
	data.keywords = kw
	}
	}

	function validVersion (file, data) {
	var v = data.version
	if (!v) return new Error("no version");
	if (!semver.valid(v)) {
	return new Error("invalid version: "+v)
	}
	data.version = semver.clean(data.version)
	return true
	}
	function unParsePeople (file, data) {
	return parsePeople(file, data, true)
	}

	function parsePeople (file, data, un) {
	var fn = un ? unParsePerson : parsePerson
	if (data.author) data.author = fn(data.author)
	;["maintainers", "contributors"].forEach(function (set) {
	if (!Array.isArray(data[set])) return;
	data[set] = data[set].map(fn)
	})
	return data
	}

	function unParsePerson (person) {
	if (typeof person === "string") return person
	var name = person.name \|\| ""
	var u = person.url \|\| person.web
	var url = u ? (" ("+u+")") : ""
	var e = person.email \|\| person.mail
	var email = e ? (" <"+e+">") : ""
	return name+email+url
	}

	function parsePerson (person) {
	if (typeof person !== "string") return person
	var name = person.match(/^([^\(<]+)/)
	var url = person.match(/\(([^\)]+)\)/)
	var email = person.match(/<([^>]+)>/)
	var obj = {}
	if (name && name[0].trim()) obj.name = name[0].trim()
	if (email) obj.email = email[1];
	if (url) obj.url = url[1];
	return obj
	}