Skip to content

Instantly share code, notes, and snippets.

@ealeksandrov
Created April 24, 2015 19:39
Show Gist options
  • Save ealeksandrov/eed152005cdbaa5b4e5a to your computer and use it in GitHub Desktop.
Save ealeksandrov/eed152005cdbaa5b4e5a to your computer and use it in GitHub Desktop.
Web API crawler on node.js
var Q = require('q');
var request = Q.denodeify(require('request'));
var domain = "http://challenge.shopcurbside.com/";
var localSessionId;
function getSession() {
var response = request({
uri: domain + 'get-session',
method: 'GET'
})
return response.then(function (res) {
return res[0].body;
})
}
function getNode(nodeId,sessionId) {
var response = request({
uri: domain + nodeId,
method: 'GET',
headers: {"Session" : sessionId}
})
return response.then(function (res) {
var jsonobj = JSON.parse(res[0].body);
if(jsonobj.error) {
throw new Error(jsonobj.error)
} else {
return jsonobj;
}
}).then(function (res) {
if(res.secret) {
process.stdout.write(res.secret);
}
return res;
}, function (error) {
if(error.message == "\"Session\" header is missing. \"/get-session\" to get a session id." || error.message == "Invalid session id, a token is valid for 10 requests.") {
var session = getSession();
return session.then(function (newSessionId) {
localSessionId = newSessionId;
return getNode(nodeId,newSessionId);
})
} else {
//console.log('Unhandled error: ' + error);
}
});
}
function getChildren(nodeArray) {
var idx = 0;
var childrenArray = [];
function getChildNode(nodeId) {
return getNode(nodeId,localSessionId).then(function(node) {
var promise1;
var promise2;
var childNodes = node['next'];
if(!childNodes) {
childNodes = node['nExt'];
}
if(!childNodes) {
childNodes = node['neXT'];
}
if(childNodes) {
childrenArray = childrenArray.concat(childNodes);
}
idx++;
if(idx < nodeArray.length) {
getChildNode(nodeArray[idx]);
} else {
getChildren(childrenArray);
}
});
}
return getChildNode(nodeArray[0]);
}
var nodeArray = ["start"];
getChildren(nodeArray);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment