Spacega.me data collector

collator.py

# spacega.me data collector

import re
import mechanize
import json
import pprint
import sys
import time
import logging

pp = pprint.PrettyPrinter(indent=4)
errpp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
br = mechanize.Browser()

# Log information about HTTP redirects and Refreshes.
#br.set_debug_redirects(True)
# Log HTTP response bodies (ie. the HTML, most of the time).
#br.set_debug_responses(True)
# Print HTTP headers.
#br.set_debug_http(True)

# To make sure you're seeing all debug output:
#logger = logging.getLogger("mechanize")
#logger.addHandler(logging.StreamHandler(sys.stderr))
#logger.setLevel(logging.INFO)

errpp.pprint("Trying to log in. Opening Signin page.")

resp = br.open("http://spacega.me/accounts/signin/")

errpp.pprint("Success on opening Signin page. Filling form.")

## first: we log in

#print resp.info()

br.select_form(nr=0)

#scrape the token (I hope this stays constant for a session?)
csrftoken = br["csrfmiddlewaretoken"]

br["identification"] = "Yourname"
br["password"] = "Yourpassword"

resp = br.submit()

errpp.pprint("Logged in. Joining Game...")

br.addheaders.append(( "X-CSRFToken", csrftoken))

br.open_novisit("http://spacega.me/join/1")

errpp.pprint("Joined the game. Scraping...")

##second: we scrape the markets

# system data first

resp = br.open_novisit("http://spacega.me/data/systems?game=1&page=1&start=0&limit=1000")

#errpp.pprint(resp)

systems = json.loads(resp.read())

if systems["success"]:
	errpp.pprint("Systems loaded")
else:
	errpp.pprint("System load failure:")
	errpp.pprint(systems)

#pp.pprint(systems)

habitableSystems = {}

for system in systems["data"]:
	#pp.pprint(system["hasHabitable"])
	if system["hasHabitable"]:
		habitableSystems[system["name"]] = system
		habitableSystems[system["name"]]["sold"] = {}
		habitableSystems[system["name"]]["bought"] = {}
		
errpp.pprint({ "Habitable Systems:", len(habitableSystems) })
		
##now the trade data

#time = str((int)(time.time()*1000))

#sale orders

resp = br.open_novisit("http://spacega.me/data/sellorders?game_id=1&item=food&page=1&start=0&limit=1000")
sell = json.loads(resp.read())



for sale in sell["data"]:
	habitableSystems[sale["system"]]["sold"]["food"] = { }
	habitableSystems[sale["system"]]["sold"]["food"]["price"] = sale["price"]	

resp = br.open_novisit("http://spacega.me/data/sellorders?game_id=1&item=goods")
sell = json.loads(resp.read())

for sale in sell["data"]:
	habitableSystems[sale["system"]]["sold"]["goods"] = { }
	habitableSystems[sale["system"]]["sold"]["goods"]["price"] = sale["price"]
	

resp = br.open_novisit("http://spacega.me/data/sellorders?game_id=1&item=luxurygoods")
sell = json.loads(resp.read())

for sale in sell["data"]:
	habitableSystems[sale["system"]]["sold"]["luxurygoods"] = { }
	habitableSystems[sale["system"]]["sold"]["luxurygoods"]["price"] = sale["price"]

#buy orders	

resp = br.open_novisit("http://spacega.me/data/buyorders?game_id=1&item=food&page=1&start=0&limit=1000")
buys = json.loads(resp.read())


for buy in buys["data"]:
	habitableSystems[buy["system"]]["bought"]["food"] = { }
	habitableSystems[buy["system"]]["bought"]["food"]["price"] = buy["price"]	

resp = br.open_novisit("http://spacega.me/data/buyorders?game_id=1&item=goods")
buy = json.loads(resp.read())

for buy in buys["data"]:
	habitableSystems[buy["system"]]["bought"]["goods"] = { }
	habitableSystems[buy["system"]]["bought"]["goods"]["price"] = buy["price"]
	

resp = br.open_novisit("http://spacega.me/data/buyorders?game_id=1&item=luxurygoods")
buy = json.loads(resp.read())

for buy in buys["data"]:
	habitableSystems[buy["system"]]["bought"]["luxurygoods"] = { }
	habitableSystems[buy["system"]]["bought"]["luxurygoods"]["price"] = buy["price"]
	
errpp.pprint("Done scraping. Calculating routes...")
		
#pp.pprint(habitableSystems)

routes = {}

for buySystemKey, buySystemValue in enumerate(habitableSystems):
	#routes[buySystemValue] = {}
	for sellSystemKey, sellSystemValue in enumerate(habitableSystems):
		#routes[buySystemValue][sellSystemValue] = {}
		if buySystemValue != sellSystemValue:
			for soldGoodKey, soldGoodValue in enumerate(habitableSystems[buySystemValue]["sold"]):
				if soldGoodValue in habitableSystems[sellSystemValue]["bought"]:
					profit = habitableSystems[sellSystemValue]["bought"][soldGoodValue]["price"] - habitableSystems[buySystemValue]["sold"][soldGoodValue]["price"]
					if profit > 0:
						distanceSquared = abs(habitableSystems[sellSystemValue]["x"] - habitableSystems[buySystemValue]["x"])**2 + abs(habitableSystems[sellSystemValue]["y"] - habitableSystems[buySystemValue]["y"])**2
						
						ratio = profit/distanceSquared
						
						if soldGoodValue == "goods":
							ratio /= 1.2
							
						if soldGoodValue == "luxurygoods":
							ratio /= 1.6
						
						intratio = (int)(ratio)
						
						if not intratio in routes:
							routes[intratio] = []
						
						routes[intratio].append({ "buy_in": buySystemValue, "sell_in":  sellSystemValue, "good": soldGoodValue, "profit" : profit, "distance" : distanceSquared, "ratio" : ratio })
					
					
pp.pprint(routes)

Updated for current game version.