npm retry on 408 or 5xx only: npm/lib/cache.js and fetch.js changes

no-retry-on-404.log

$ touch proxy.js && sleep 1 && rm -rf tmp/cache node_modules/bunyan && node ~/tm/npm/cli.js --loglevel=info --cache=`pwd`/tmp/cache --registry=http://localhost:8000/ install bunyan@0.6.8npm info it worked if it ends with ok
npm info using npm@1.1.32
npm info using node@v0.6.19
npm info retry registry request attempt 1 at 12:21:47
npm http GET http://localhost:8000/bunyan/0.6.8
npm http 200 http://localhost:8000/bunyan/0.6.8
npm info retry fetch attempt 1 at 12:21:48
npm http GET http://localhost:8000/bunyan/-/bunyan-0.6.8-BOGUS.tgz
npm http 404 http://localhost:8000/bunyan/-/bunyan-0.6.8-BOGUS.tgz
npm ERR! fetch failed http://localhost:8000/bunyan/-/bunyan-0.6.8-BOGUS.tgz
npm ERR! Error: 404 Not Found
npm ERR!     at [object Object].<anonymous> (/Users/trentm/tm/npm/lib/utils/fetch.js:45:16)
npm ERR!     at [object Object].emit (events.js:67:17)
npm ERR!     at Request.<anonymous> (/Users/trentm/tm/npm/lib/utils/fetch.js:81:10)
npm ERR!     at Request.emit (events.js:67:17)
npm ERR!     at ClientRequest.<anonymous> (/Users/trentm/tm/npm/node_modules/request/main.js:487:12)
npm ERR!     at ClientRequest.g (events.js:156:14)
npm ERR!     at ClientRequest.emit (events.js:67:17)
npm ERR!     at HTTPParser.parserOnIncomingClient (http.js:1256:7)
npm ERR!     at HTTPParser.parserOnHeadersComplete [as onHeadersComplete] (http.js:91:29)
npm ERR!     at Socket.socketOnData (http.js:1288:20)
npm ERR!  [Error: 404 Not Found]
npm ERR! You may report this log at:
npm ERR!     <http://github.com/isaacs/npm/issues>
npm ERR! or email it to:
npm ERR!     <npm-@googlegroups.com>

npm ERR! System Darwin 10.8.0
npm ERR! command "node" "/Users/trentm/tm/npm/cli.js" "--loglevel=info" "--cache=/Users/trentm/tm/npm-registry-proxy/tmp/cache" "--registry=http://localhost:8000/" "install" "bunyan@0.6.8"
npm ERR! cwd /Users/trentm/tm/npm-registry-proxy
npm ERR! node -v v0.6.19
npm ERR! npm -v 1.1.32
npm ERR! message 404 Not Found
npm ERR! 
npm ERR! Additional logging details can be found in:
npm ERR!     /Users/trentm/tm/npm-registry-proxy/npm-debug.log
npm ERR! not ok code undefined
npm ERR! not ok code 1

npm-lib.diff

diff --git a/lib/cache.js b/lib/cache.js
index bd07d17..269b197 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -274,10 +274,10 @@ function add (args, cb) {
 }
 
 function fetchAndShaCheck (u, tmp, shasum, cb) {
-  fetch(u, tmp, function (er) {
+  fetch(u, tmp, function (er, response) {
     if (er) {
       log.error("fetch failed", u)
-      return cb(er)
+      return cb(er, response)
     }
     if (!shasum) return cb()
     // validate that the url we just downloaded matches the expected shasum.
@@ -323,8 +323,11 @@ function addRemoteTarball (u, shasum, name, cb_) {
       operation.attempt(function (currentAttempt) {
         log.info("retry", "fetch attempt " + currentAttempt
           + " at " + (new Date()).toLocaleTimeString())
-        fetchAndShaCheck(u, tmp, shasum, function (er) {
-          if (operation.retry(er)) {
+        fetchAndShaCheck(u, tmp, shasum, function (er, response) {
+          // Only retry on 408, 5xx or no `response`.
+          var statusCode = response && response.statusCode
+          var statusRetry = !statusCode || (statusCode === 408 || statusCode >= 500)
+          if (er && statusRetry && operation.retry(er)) {
             log.info("retry", "will retry, error on last attempt: " + er)
             return
           }
diff --git a/lib/utils/fetch.js b/lib/utils/fetch.js
index b489c21..834cf74 100644
--- a/lib/utils/fetch.js
+++ b/lib/utils/fetch.js
@@ -25,18 +25,30 @@ function fetch (remote, local, headers, cb) {
 
 function fetch_ (remote, local, headers, cb) {
   var fstr = fs.createWriteStream(local, { mode : npm.modes.file })
+  var calledback = false;
   fstr.on("error", function (er) {
     fs.close(fstr.fd, function () {})
-    if (fstr._ERROR) return
+    if (calledback) return
+    calledback = true
     cb(fstr._ERROR = er)
   })
   fstr.on("open", function () {
     makeRequest(remote, fstr, headers)
   })
   fstr.on("close", function () {
-    if (fstr._ERROR) return
+    if (calledback) return
+    calledback = true
     cb()
   })
+  fstr.on("response", function (res) {
+    if (res && res.statusCode && res.statusCode >= 400) {
+      var er = new Error(res.statusCode + " "
+                        + require("http").STATUS_CODES[res.statusCode])
+      if (calledback) return
+      calledback = true
+      cb(fstr._ERROR = er, res)
+    }
+  })
 }
 
 function makeRequest (remote, fstr, headers) {
@@ -66,6 +78,7 @@ function makeRequest (remote, fstr, headers) {
   })
   req.on("response", function (res) {
     log.http(res.statusCode, remote.href)
+    fstr.emit("response", res)
   })
   req.pipe(fstr)
 }

proxy.js

// Proxy for registry.npmjs.org.
// The relevant change is:
//     data = data.replace(/bunyan-0.6.8.tgz/g, "bunyan-0.6.8-BOGUS.tgz")
// to make the package request a 404.

var http = require('http')
var log = console.log
var counters = {};


var server = http.createServer(function(req, res) {
    log("proxy:", req.method, req.url)

    if (req.method !== "GET") {
        res.statusCode = 400;
        res.write('only handle GETs\n')
        res.end()
        return
    }

    // TODO: Inject optional "server" failures here.
    // - Fail twice for 'GET /$package'
    /*
    if (counters.a === undefined) counters.a = 0;
    if (req.url.split('/').length - 1 === 1 && counters.a < 2) {
        log('500 this request')
        res.statusCode = 500
        res.end()
        counters.a++
    }
    */
    /*
    // - Fail once for 'GET /$package/-/$package-$version.tgz'
    if (counters.b === undefined) counters.b = 0;
    var packageRe = new RegExp('^/([^/]+)/-/\\1-[^/]+\.tgz$')
    if (packageRe.test(req.url) && counters.b < 1) {
        log('500 this request')
        res.statusCode = 500
        res.end()
        counters.b++
    }
    */

    var options = {
        host: 'registry.npmjs.org',
        port: 80,
        path: req.url,
        method: req.method
    };

    var preq = http.request(options, function(pres) {
        // The registry json for a module hardcodes "registry.npmjs.org"
        // for the package URLs. If you want `npm install` to download
        // via the proxy, we need to replace those URLs here.
        if (pres.headers['content-type'] === "application/json") {
            var chunks = [];
            pres.on('data', function (chunk) {
                //log('PROXY: got a chunk (%d bytes)', chunk.length)
                chunks.push(chunk);
            });
            pres.on('end', function () {
                var data = chunks.join('')
                data = data.replace(/registry.npmjs.org/g, "localhost:8000")
                data = data.replace(/bunyan-0.6.8.tgz/g, "bunyan-0.6.8-BOGUS.tgz")
                res.write(data);
                res.end();
            });
        } else {
            pres.on('data', function (chunk) {
                res.statusCode = pres.statusCode
                res.write(chunk)
            });
            pres.on('end', function () {
                res.end()
            });
        }
    });

    preq.on('error', function(e) {
        res.statusCode = 500;
        res.write('error proxying: ' + e)
        res.end();
    });

    preq.end();
})
server.listen(8000)

retry-on-500.log

$ touch proxy.js && sleep 1 && rm -rf tmp/cache node_modules/bunyan && node ~/tm/npm/cli.js --loglevel=info --cache=`pwd`/tmp/cache --registry=http://localhost:8000/ install bunyan@0.6.8npm info it worked if it ends with ok
npm info using npm@1.1.32
npm info using node@v0.6.19
npm info retry registry request attempt 1 at 12:22:45
npm http GET http://localhost:8000/bunyan/0.6.8
npm http 200 http://localhost:8000/bunyan/0.6.8
npm info retry fetch attempt 1 at 12:22:46
npm http GET http://localhost:8000/bunyan/-/bunyan-0.6.8.tgz
npm http 500 http://localhost:8000/bunyan/-/bunyan-0.6.8.tgz
npm ERR! fetch failed http://localhost:8000/bunyan/-/bunyan-0.6.8.tgz
npm info retry will retry, error on last attempt: Error: 500 Internal Server Error
npm info retry fetch attempt 2 at 12:22:56
npm http GET http://localhost:8000/bunyan/-/bunyan-0.6.8.tgz
npm http 200 http://localhost:8000/bunyan/-/bunyan-0.6.8.tgz
npm info shasum d64d19ce20d780a2ef237f3dd67bcf9ee9fc50e9
npm info shasum /var/folders/a1/a1q548caE+ytwn0liWfg7E+++TI/-Tmp-/npm-48413/1340911366115-0.7085440659429878/tmp.tgz
npm info shasum 2f0d81fa1c6c8c2d591719d5478ae7b2e5986697
npm info shasum /Users/trentm/tm/npm-registry-proxy/tmp/cache/bunyan/0.6.8/package.tgz
npm info install bunyan@0.6.8 into /Users/trentm/tm/npm-registry-proxy
npm info installOne bunyan@0.6.8
npm info /Users/trentm/tm/npm-registry-proxy/node_modules/bunyan unbuild
npm info preinstall bunyan@0.6.8
npm info build /Users/trentm/tm/npm-registry-proxy/node_modules/bunyan
npm info linkStuff bunyan@0.6.8
npm info install bunyan@0.6.8
npm info postinstall bunyan@0.6.8
bunyan@0.6.8 node_modules/bunyan
npm info ok