[php] Runescape price update script

ge_update.php

<?php
/**
 * Runescape's Grand Exchange update script
 * 
 * This class provides Main functionality to get, update, and save
 * item prices from Runescape's Grand Exchange.
 *
 * To run the update script simply run $this->update_prices()
 *
 * @author	Michael Pivonka (Codedninja) <m.pivonka@codedninja.com>
 * @copyright	Michael Pivonka 03/28/12
 * @version 2.0
 * @link	http://codedninja.com
 *
 * You can view info about script at the following link
 * @link http://codedninja.com/blog/?p=1
 */

class Ge_update {
	
	/**
 	 * @access public
     * @var int 
     */
	public $ips = array('33.33.33.33', '44.44.44.44', '55.55.55.55', '66.66.66.66');
	
	/**
	 * @access private
     * @var int 
     */
	private $page_requests = 0;
	
	/**
	 * @access private
     * @var array
     */
	private $error_urls = array();
	
	/**
	 * @access private
     * @var array
     */
	private $item_urls = array();	
	
	/**
	 * @access private
     * @var array
     */
	private $item_output = array();
	
	/**
	 * @access private
     * @var int
     */
	private $max_categories = 36;
	
	/**
	 * @access private
     * @var int
     */
	private $items_per_page = 12;
	
	/**
	 * @access private
     * @var string
     */
	private $ge_cat_url = "http://services.runescape.com/m=itemdb_rs/api/catalogue/category.json?category=";
	
	/**
	 * @access private
     * @var string
     */
	private $ge_items_url = "http://services.runescape.com/m=itemdb_rs/api/catalogue/items.json?category=";

	/**
	 * Creates a CLI progress bar
	 *
	 * @access	private
	 * @param	int [$done] This is the amount done out of $total
	 * @param  	int [$total] This is the total amount need to be done
	 * @param  	int [$size] How long you want the bar to be
	 * @return	void
	 */
	private function _show_status($done, $total, $size=30)
	{
	    static $start_time;

	    // if we go over our bound, just ignore it
	    if($done > $total) return;

	    if(empty($start_time)) $start_time=time();
	    $now = time();

	    $perc=(double)($done/$total);

	    $bar=floor($perc*$size);

	    $status_bar="\r[";
	    $status_bar.=str_repeat("=", $bar);
	    if($bar<$size)
		{
	        $status_bar.=">";
	        $status_bar.=str_repeat(" ", $size-$bar);
	    }
		else
		{
	        $status_bar.="=";
	    }

	    $disp=number_format($perc*100, 0);

	    $status_bar.="] $disp%  $done/$total";

	    $rate = ($now-$start_time) / $done;
	    $left = $total - $done;
	    $eta = round($rate * $left, 2);

	    $elapsed = $now - $start_time;

	    $status_bar.= " remaining: ".number_format($eta)." sec.  elapsed: ".number_format($elapsed)." sec.";

	    echo "$status_bar  ";

	    flush();

	    // when done, send a newline
	    if($done == $total)
		{
	        echo "\n";
	    }

	}

	/**
	 * Complex Mutli-CURL request, CURLs multiple URLs at one time
	 * 
	 * @access	private
	 * @param	array [$data] Array of urls
	 * @return	array
	 */
	private function _multi_request($data)
	{

		// array of curl handles
	  	$curly = array();

	  	// data to be returned
	  	$result = array();

	  	// multi handle
	  	$mh = curl_multi_init();

	  	// loop through $data and create curl handles
	  	// then add them to the multi-handle
	  	foreach ($data as $id => $d)
		{

	    	$curly[$id] = curl_init();

	    	$url = (is_array($d) && !empty($d['url'])) ? $d['url'] : $d;
	    	curl_setopt($curly[$id], CURLOPT_URL,            $url);
	    	curl_setopt($curly[$id], CURLOPT_HEADER,         0);
	    	curl_setopt($curly[$id], CURLOPT_RETURNTRANSFER, 1);
	    	curl_setopt($curly[$id], CURLOPT_TIMEOUT, 500);
	    	curl_setopt($curly[$id], CURLOPT_CONNECTTIMEOUT, 500);

	    	// Has options?
	    	if (is_array($d))
			{
	      		if (!empty($d['post']))
				{
	        		curl_setopt($curly[$id], CURLOPT_POST, 1);
	        		curl_setopt($curly[$id], CURLOPT_POSTFIELDS, $d['post']);
	      		}
	      		if(!empty($d['options']))
	      		{
	        		curl_setopt_array($curly[$id], $d['options']);
	      		}
	    	}

	    	curl_multi_add_handle($mh, $curly[$id]);
	    	$this->page_requests++;
	  	}

	  	// execute the handles
	  	$running = null;
	  	do
		{
	    	curl_multi_exec($mh, $running);
	  	}
		while($running > 0);

	  	// get content and remove handles
	  	foreach($curly as $id => $c)
		{
	    	$results[$id] = curl_multi_getcontent($c);
	    	curl_multi_remove_handle($mh, $c);
	  	}
	
		foreach($results as $id => $result) {
			if(empty($result))
				$this->error_urls[] = $d[$id]['url'];
		}

	  	// all done
	  	curl_multi_close($mh);

	  	return $results;
	}

	/**
	 * Returns the last key inside an array
	 * 
	 * @access	private
	 * @param	array [$array] Array you want last key of
	 * @return	string
	 */
	private function _end_key($array)
	{
		end($array);
		return key($array);
	}
	
	/**
	 * Splits an array evenly based on how many ips
	 * 
	 * @access	private
	 * @param	array [$urls] Array of urls
	 * @return	array
	 */
	private function _split_url_array($urls)
	{
		$split_at = ceil(count($urls) / count($this->ips));
		$new_urls = array_chunk($urls, $split_at);
		return $new_urls;
	}
	
	/**
	 * Sets up urls for each ip
	 * 
	 * @access	private
	 * @param	array [$urls] Array of urls
	 * @return	array
	 */
	private function _ip_foreach_urls($urls)
	{
		for($x = 0; $x < count($this->ips); $x++)
		{
			for($i = 0; $i < count($urls[$x]); $i++)
			{
				$temp_array['url'] = $urls[$x][$i];
				$temp_array['options'] = array( CURLOPT_INTERFACE=>$this->ips[$x]);

				$urls[$x][$i] = $temp_array;
				unset($temp_array);
			}
		}
		return $urls;
	}
	
	/**
	 * Find out how many requests are need for url array.
	 * 
	 * @access	private
	 * @param	array [$urls] Array of urls
	 * @return	int
	 */
	private function _max_requests($urls)
	{
		for($x = 0; $x < count($urls); $x++)
		{
			if($x == 0)
				$max_requests = count($urls[$x]);
			$max_requests = ($max_requests>count($urls[$x+1])) ? $max_requests : count($urls[$x+1]);
		}
		return $max_requests;
	}
	
	/**
	 * Handle all the category data
	 * 
	 * @access	private
	 * @param	array [$letter_lists] Array of data
	 * @param	array [$request] Array of requested data
	 * @return	void
	 */
	private function _handle_category_data($letter_lists, $request)
	{
		foreach($letter_lists as $key => $letter_list)
		{
			if(!empty($letter_list))
			{
				preg_match('/category=(\d+)/i', $request[$key]['url'], $category);
				
				$letter_list = json_decode($letter_list, TRUE);
			
				// Only want the letters
				$letter_list = $letter_list['alpha'];
			
				// Store letters that have items to loop through
				$letters = array();
			
				// Loop though Letters from the CURL request
				for($letter = 0; $letter < count($letter_list); $letter++)
				{
					// Letter has items so push them to array for later.
					if($letter_list[$letter]['items'] != 0)
				 	{
				 		array_push($letters, $letter_list[$letter]);	
				 	}
				}
			
				// Loop through Letters that has Items
				for($item_list = 0; $item_list < count($letters); $item_list++)
				{
					$letter = $letters[$item_list]['letter'];

					// Max page to go to.
			 		$max_page = ceil($letters[$item_list]['items'] / $this->items_per_page);

					// Loop through pages and generate the urls need to grab all the item prices
			 		for($page = 1; $page <= $max_page; $page++)
			 		{
			 			$this->item_urls[] = $this->ge_items_url.$category[1]."&alpha=".$letter."&page=".$page;
			 		}
				}
			}
			
		}
	}
	
	/**
	 * Find out how many requests are need for url array.
	 * 
	 * @access	private
	 * @param	array [$items] Array of data
	 * @param	array [$request] Array of requested data
	 * @return	void
	 */
	private function _handle_item_data($items, $request)
	{
		foreach($items as $item)
		{
			$item = json_decode($item, TRUE);
			$this->item_output = array_merge($this->item_output, (array)$item['items']);
		}
	}
	
	/**
	 * Go through an array of urls and multi curl them
	 * 
	 * @access	private
	 * @param	array [$urls] Array of urls
	 * @return	string
	 */
	private function _multicurl_urls($urls, $callback = NULL)
	{
		$ge_delay = 6;
		
		// Split up the urls evenly over the IPs
		$new_urls = $this->_split_url_array($urls);
		
		// Set up each url differently with IPs for $this->_multiRequest()
		$new_urls = $this->_ip_foreach_urls($new_urls);
			
		// Figure out max requests for the for loop
		$max_requests = $this->_max_requests($new_urls);
		
		// Loop through $newUrls and grab url assaigned an IP
		for($x = 0; $x < $max_requests; $x++)
		{
			// Make temp request array for $this->_multiRequest()
			for($i = 0; $i < count($new_urls); $i++)
			{
				if(isset($new_urls[$i][$x]))
					$temp_request[] = $new_urls[$i][$x];
			}
			
			// Sleep since Jagex wants a 6 second delay
			// If you remove this you might get blank pages
			sleep($ge_delay);
	
			$data = $this->_multi_request($temp_request);
	
			// Call back function
			if(method_exists($this, $callback))
				call_user_func_array(array($this, $callback), array($data, $temp_request));
			elseif(function_exists($callback))
				call_user_func_array($callback, array($data, $temp_request));
	
			unset($temp_request);
	
			$this->_show_status($x+1, $max_requests);
		}
	}

	/**
	 * Main function to get all the items from Runescape's Grand Exchange
	 * 
	 * @access	public
	 * @return	void
	 */
	public function update_prices($force=FALSE)
	{
		
		// Get graph data of an item to  check if there has been an update.
		$update_data = $this->_multi_request(array('http://services.runescape.com/m=itemdb_rs/api/graph/1337.json'));
		$update = json_decode($update_data[0], TRUE);

		// Need to sort the timestamps (keys) to make sure they are in order
		ksort($update['daily']);
		
		// Grab the last (highest) timestamps (keys)
		$update = $this->_end_key($update['daily']);

		echo "Checking for update...\n";

		if(!empty($update))
		{
			$last_update = fopen('lastUpdate.txt', 'r');
			$time = fread($last_update, filesize('lastUpdate.txt'));
			fclose($last_update);
	
			// Check the time saved from last update to current timestamp
			if ($update > $time || $force)
			{
				echo "Starting to update prices...\n";
		
				echo "Getting Categories...\n";
		
				// Loop through categories Ammuo, Weps, Melee, etc.
				for ($cat = 0; $cat <= $this->max_categories; $cat++)
				{
					$category_urls[] = $this->ge_cat_url.$cat;
				}
				
				$this->_multicurl_urls($category_urls, '_handle_category_data');
				
				if(count($this->error_urls)>0)
					$errors = TRUE;
				else
					$errors = FALSE;
					
				while($errors)
				{
					echo "Found blank requests doing them...\n";
					
					$this->_multicurl_urls($this->error_urls, '_handle_category_data');
					
					if(count($this->error_urls)===0)
						$errors = FALSE;
				}
		
		
				// ---------- Items ----------

				echo "Grabbing item pages...\n";

				$this->_multicurl_urls($this->item_urls, '_handle_item_data');
				
				if(count($this->error_urls)>0)
					$errors = TRUE;
				else
					$errors = FALSE;
					
				while($errors)
				{
					echo "Found blank requests doing them again...\n";
					
					$this->_multicurl_urls($this->error_urls, '_handle_item_data');
					
					if(count($this->error_urls)===0)
						$errors = FALSE;
				}
				

				$output_items = fopen('output.json', "w");
				fputs($output_items, json_encode($this->item_output));
				fclose($output_items);
		
				$update_file = fopen('lastUpdate.txt', "w");
				fputs($update_file, $update);
				fclose($update_file);

				echo $this->page_requests." page requests.\n";
				echo count($this->item_output)." items found and saved.\n";
			}
			else
			{
				echo "No update at this time.\n";
			}
		}
	}
}

$class = new Ge_update;
$class->update_prices(true);

/* End of file ge_update.php */