Client_V2.js

/**
 * Runescape's Grand Exchange update script
 *
 * This is a client that you will run on servers with different IPs
 * 
 * This script provides a base of to update via creating DigitalOcean
 * Virtual Machines with an image already created and item prices from
 * Runescape's Grand Exchange.
 *
 * To run the update script simply main();
 *
 * @author	Michael Pivonka (Codedninja) <m.pivonka@codedninja.com>
 * @copyright	Michael Pivonka 02/02/14
 * @version 1.0
 * @link	http://codedninja.com
 *
 */
 var client = require("socket.io-client"),
	request = require('request'),
	os = require('os');

// Change to master server server infomation
var io = client.connect("http://localhost:8000");

io.on('connect', function () {
	console.log('connected.');
	console.log(os.hostname());
	io.emit('auth', {hostname:os.hostname()});
});

io.on('newUrl', function(data) {
	console.log(data);
	request(data, function(err, res, body) {
		console.log(body);
		io.emit('returnData', {url: data, body:body});
	});
});

Server_v1.js

/**
 * Runescape's Grand Exchange update script
 * 
 * This script provides a base of to update
 * item prices from Runescape's Grand Exchange.
 *
 * To run the update script simply main();
 *
 * @author	Michael Pivonka (Codedninja) <m.pivonka@codedninja.com>
 * @copyright	Michael Pivonka 02/02/14
 * @version 1.0
 * @link	http://codedninja.com
 *
 */
var request = require('request'),
	fs = require('fs'),
	Sync = require('sync'),
	sleep = require('sleep');


// Decalre Semi Global varibles
var last_updated = 0,
	page_requests = 0,
	error_urls = [],
	item_urls = [],
	item_output = [],
	category_urls = [],
	max_categories = 36,
	items_per_page = 12,
	ge_cat_url = "http://services.runescape.com/m=itemdb_rs/api/catalogue/category.json?category=",
	ge_items_url = "http://services.runescape.com/m=itemdb_rs/api/catalogue/items.json?category=";

var checkUpdateTimer;


// Get the highest Key out of an Object
function newestTimestamp(times) {
	var top = null;
	Object.keys(times).forEach(function(key) {
		if (key > top) {
			top = key;
		}
	});
	return top;
}

// Trim extra unused data from a string
function trim(string) {
    return string.replace(/^\s*|\s*$/g, '')
}

function checkForUpdate() {
	console.log('Checking update...');
	Sync(function(){
		var data = request.sync(null, 'http://services.runescape.com/m=itemdb_rs/api/graph/2.json');
		var times = data[0].body;
		if(times != null) {
			try {
				prices = JSON.parse(times);
				var timestamp = newestTimestamp(prices.daily);
				if(timestamp > last_updated) {
					clearTimeout(checkUpdateTimer);
					startUpdate();
				}
				else {
					console.log('No update needed...');
				}
			}
			catch(err) {console.log(err);}
		}
	});
}

function getCategoryLinks(category_urls) {
	var letterList, data, letters, max_page;
	for(x in category_urls) {
		letterList = null;
		letters = [];
		data = request.sync(null, category_urls[x]);
		data = data[0].body;
		try {
			data = JSON.parse(data);
			letterList = data.alpha;
			
			for(var letter = 0; letter < letterList.length; letter++) {
				if(letterList[letter]['items'] != 0) {
					letters.push(letterList[letter]);
				}
			}

			for(var itemList = 0; itemList < letters.length; itemList++) {
				letter = letters[itemList]['letter'];
				max_page = Math.ceil(letters[itemList]['items'] / items_per_page);
				
				for(var page = 1; page <= max_page; page++) {
					item_urls.push(ge_items_url+x+"&alpha="+letter+"&page="+page);
				}
			}
		}
		catch(err) {
			error_urls.push(category_urls[x]);
		}

		sleep.sleep(6);
	}
}

function getItemLinks(item_urls) {
	for(x in item_urls) {
		data = request.sync(null, item_urls[x]);
		data = data[0].body;
		try {
			var items = JSON.parse(data);
			item_output = item_output.concat(items['items']);
		}
		catch(err) {
			error_urls.push(item_urls[x]);
		}

		console.log(x);
		
		sleep.sleep(6);
	}
}

function checkErrors(callback) {
	if(error_urls.length > 0) {
		errors = true
	}

	while(errors) {
		urls = error_urls;
		error_urls = [];

		console.log('Some pages got errors...');

		callback(urls);

		if(error_urls.length == 0) {
			errors = false;
		}
	}
}

function startUpdate() {
		var errors = false, data;
		item_output = [];

		console.log('Starting update...');
	
		for (var cat = 0; cat <= 0; cat++) {
			category_urls.push(ge_cat_url+cat);
		}

		getCategoryLinks(category_urls);

		checkErrors(getCategoryLinks);

		console.log('Starting to request item pages...');

		getItemLinks(item_urls);

		checkErrors(getItemLinks);

		fs.writeFileSync('output.json', JSON.stringify(item_output));
		
		console.log(error_urls.length);
		
		setTimeout(checkForUpdate(), 5000);
}

function main() {
	console.log('Starting...')
	checkUpdateTimer = setTimeout(checkForUpdate(), 5000);
}

main();

Server_v2.js

/**
 * Runescape's Grand Exchange update script
 * 
 * This is the server you run to be the brains of updating
 *
 * This script provides a base of to update via creating DigitalOcean
 * Virtual Machines with an image already created and item prices from
 * Runescape's Grand Exchange.
 *
 * To run the update script simply main();
 *
 * @author	Michael Pivonka (Codedninja) <m.pivonka@codedninja.com>
 * @copyright	Michael Pivonka 02/02/14
 * @version 1.0
 * @link	http://codedninja.com
 *
 */
var request = require('request'),
	fs = require('fs'),
	Sync = require('sync'),
	io = require('socket.io').listen(8000);


// Decalre Semi Global varibles
var last_updated = 0,
	page_requests = 0,
	urlOffset = 0,
	urls = [],
	item_output = [],
	clients = {},
	max_categories = 36,
	items_per_page = 12,
	ge_cat_url = "http://services.runescape.com/m=itemdb_rs/api/catalogue/category.json?category=",
	ge_items_url = "http://services.runescape.com/m=itemdb_rs/api/catalogue/items.json?category=";

// DigitalOcean infomation
var client_id = "",
	api_key = "",
	serverAmount = 5,
	size_id = 43,
	image_id = 1606,
	region_id = 2,
	dropletHostnames = {};

var checkUpdateTimer;


// Get the highest Key out of an Object
function newestTimestamp(times) {
	var top = null;
	Object.keys(times).forEach(function(key) {
		if (key > top) {
			top = key;
		}
	});
	return top;
}

// Trim extra unused data from a string
function trim(string) {
    return string.replace(/^\s*|\s*$/g, '')
}

function checkForUpdate() {
	console.log('Checking update...');
	Sync(function(){
		var data = request.sync(null, 'http://services.runescape.com/m=itemdb_rs/api/graph/2.json');
		var times = data[0].body;
		if(times != null) {
			try {
				prices = JSON.parse(times);
				var timestamp = newestTimestamp(prices.daily);
				if(timestamp > last_updated) {
					clearTimeout(checkUpdateTimer);
					startUpdate();
				}
				else {
					console.log('No update needed...');
				}
			}
			catch(err) {console.log(err);}
		}
	});
}

function createServer() {
	var name = Math.floor(Math.random()*11111111111);

	request("http://api.digitalocean.com/droplets/new?name="+name+"&size_id="+size_id+"&image_id="+image_id+"&region_id="+region_id+"&client_id="+client_id+"&api_key="+api_key,
		function(err, res, body) {
			try {
				var data = JSON.parse(body);
				if(data.status == "OK") {
					dropletHostnames[name] = data.droplet.id;
					console.log("Created new server with id of "+data.droplet.id);
				}
			}
			catch(err){
				
			}
		}
	);
}

function deleteServer(dropletID) {
	request("https://api.digitalocean.com/droplets/"+dropletID+"/destroy?client_id="+client_id+"&api_key="+api_key,
		function(err, res, body){
			try {
				var data = JSON.parse(body);
				if(data.status == "OK") {
					console.log(dropletID+" has been destroy.");
				}
				else {
					console.log("Could not destroy server by the id: "+dropletID);
				}
			}
			catch(err){
				console.log(body);
			}
		}
	);
}

function destroyServers()
{
	for(droplet in dropletHostnames)
	{
		deleteServer(droplet);
	}

	resetVars();
}

function resetVars()
{
	dropletHostnames = {};
	urls = [];
	clients = [];
}

function startUpdate() {
	console.log('Starting update...');
	urlOffset = 0;
	item_output = [];

	console.log('Spawning servers...');

	// Spawn servers
	for(var servers = 1; servers <= serverAmount; servers++) {
		createServer();
	}

	console.log('Generating category urls...');
	// Generate Category links
	for(var cat = 0; cat <= max_categories; cat++) {
		urls.push(ge_cat_url+cat);
	}

	console.log('Giving out urls to clients already connected...');
	// Give out urls to clients that are still connected
	for(client in clients) {
		client.emit('newUrl', urls[urlOffset]);
		urlOffset++;
	}
}

function parseData(data) {
	if(/category\.json/.exec(data.url) != null) {
		// Parse out data for category urls
		var letterList, letters, max_page;
		
		letterList = data.body.alpha;
		
		for(var letter = 0; letter < letterList.length; letter++) {
			if(letterList.letter.items > 0) {
				letters.push(letterList[letter]);
			}
		}
		
		for(var itemList = 0; itemList < letters.length; itemList++) {
			letter = letters.itemList.letter;
			max_page = Math.ceil(letters.itemList.items / items_per_page);
			
			for(var page = 1; page <= max_page; page++) {
				urls.push(ge_items_url+x+"&alpha="+letter+"&page="+page);
			}
		}
	}
	else if(/items\.json/.exec(data.url) != null) {
		// Parse data for items
		item_output = item_output.concat(data.body.items);
	}
}

io.sockets.on('connection', function (socket) {

	socket.on('auth', function(data) {
		if(dropletHostnames[data.hostname] != null) {
			clients[socket.id] = socket;
			io.sockets.socket(socket.id).emit('newUrl', urls[urlOffset]);
			urlOffset++;
		}
	});

	socket.on('returnData', function(data) {
		if(clients[socket.id] != null) {
			try {
				data.body = JSON.parse(data.body);
				
				parseData(data);

				if(urls[urlOffset] != null) {
					io.sockets.socket(socket.id).emit('newUrl', urls[urlOffset]);
					urlOffset++;
				}
				else {
					console.log(output_data);
					console.log('Socket number: '+socket.id+' server is done.');
				}
			}
			catch(err) {
				io.sockets.socket(socket.id).emit('newUrl', data.url);
			}

			// If there are no more URLs then shutdown all the servers.
			if(urlOffset == urls.length)
			{
				destroyServers();
				saveItems();
				checkUpdateTimer = setTimeout(checkForUpdate(), 5000);
			}
		}
	});
});

function saveItems(){
	// Write something to do with item data
}

function main() {
	checkUpdateTimer = setTimeout(checkForUpdate(), 5000);
}

console.log('Starting...');
main();