Rough Node.js Hierarchical Queue

gistfile1.js

var http = require('http');

function RequestQueue() {
  this._maxRequests = 3;
  this._pendingCount = 0;
  this._queue = [];
  this.count = 0;
  this._uriInQueue = {};
}

RequestQueue.prototype._digest = function () {
  while (this._pendingCount < this._maxRequests && this._queue.length) {
    this._pendingCount++;
    var request = this._queue.shift();
    this._sendRequest(request);
  }
};

RequestQueue.prototype._sendRequest = function (request) {
  var self = this;
  http.get(request.uri)
    .on('response', function (response) {

      self.count++;
      console.log(self.count + ':  ' + request.uri);
      if (request.parent && request.parent.completeDependency(request)) {
        var parent = request.parent;
        while (parent && self._uriInQueue[parent.uri]) {
          self._reportDuplicate(parent.uri);
          parent = parent.parent;

        }
        if (parent) {
          self._queue.push(parent);
          self._uriInQueue[parent.uri] = true;
        }
      }
      self._pendingCount--;
      self._digest();
    })
    .on('error', function () {
      self._pendingCount--;
      console.log('error:', request.uri);
      self._digest();
    });

};

RequestQueue.prototype.addRequest = function (request) {
  if (!this._uriInQueue[request.uri]) {
    this._queue.push(request);
    this._uriInQueue[request.uri] = true;
    this._digest();
  } else {
    this._reportDuplicate(request.uri);
  }
};

RequestQueue.prototype._reportDuplicate = function (uri) {
  console.log('dup: ', uri)
};

RequestQueue.prototype.addRequests = function (requests) {
  var simpleRequests = [];

  for (var requestIndex = 0; requestIndex < requests.length; requestIndex++) {
    var request = requests[requestIndex];

    if (request.hasDependencies) {
      this.addRequests(request.dependencies);
    } else {
      simpleRequests.push(request);
    }
  }
  for (var simpleIndex = 0; simpleIndex < simpleRequests.length; simpleIndex++) {
    this.addRequest(simpleRequests[simpleIndex]);
  }
};



function Request(options) {
  this.parent = options.parent;
  this.hasDependencies = !!options.dependencies.length;
  this.dependencies = [];
  this.uri = "http://www.majgis.com?_=" + options.url;
  this._loadDependencies(options.dependencies);
  this.pendingDependencies = options.dependencies.length;
}

Request.prototype.completeDependency = function (request) {
  this.pendingDependencies--;
  return this.pendingDependencies === 0;
};

Request.prototype._loadDependencies = function (dependencies) {
  for (var depIndex = 0; depIndex < dependencies.length; depIndex++) {
    var requestOptions = dependencies[depIndex];
    requestOptions.parent = this;
    this.dependencies.push(new Request(requestOptions));
  }
};



var requestQueue = new RequestQueue();
function sendRequests(requests) {
  var requestObjs = [];
  for (var requestIndex = 0; requestIndex < requests.length; requestIndex++) {
    requestObjs.push(new Request(requests[requestIndex]));
  }
  requestQueue.addRequests(requestObjs);
}

var requests = [];
var totalRequests = 0;
for (var i = 0; i < 10; i++) {
  var deps = [];
  while (i && Math.random() > 0.5) {
    deps.push(requests[Math.floor(Math.random() * i)]);
  }
  requests.push({
    url: i + ".json",
    dependencies: deps
  });
  totalRequests++;
  totalRequests += deps.length;
}
sendRequests(requests);