badp/Pipfile

## Pipfile
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true

[dev-packages]
ipython = "*"
pylint = "*"

[packages]
vowpal-porpoise = "*"
requests = "*"
vowpalwabbit = "*"

[requires]
python_version = "3.6"

[pipenv]
allow_prereleases = true

## predict.py
import requests
import json
import re
import os
import gzip
from vowpalwabbit import pyvw

currency_label = {
    "alch": 1,
    "chaos": 2,
    "exa": 3,
}

price_re = re.compile("""
    ~
    (?:b/o|price)
    [ ]
    ([0-9]+)
    [ ]
    (alch|chaos|exa)
""", re.X)

mod_re = re.compile("-?[0-9]+-?")

def featurize(item):
    descs = {"name": {}, "requirement": {}, "properties": {}, "mod": {}}
    for word in item["typeLine"].split(" "):
        descs["name"][word] = 1
    for property in item.get("properties", []) + item.get("additionalProperties", []):
        for prop in property["name"].split(", "):
            values = property.get("progress") or property.get("values")
            if "StackSize" in prop: continue
            # Limited to: 1 Historic
            if prop == "Limited to": continue
            if values:
                value = values[0][0] if type(values) is list else values
                if type(value) is str:
                    value = value.replace("%", "")
                    value = value.replace("+", "")
                    # cast time
                    if value == "Instant": value = "0.001"
                    value = value.replace(" sec", "")
                    value = value.replace(" of base", "")
                    value = re.sub("/[0-9]+", "", value)
                    value = re.sub(" [(].+", "", value)
                    # radius
                    if value == "Small": value = "1"
                    if value == "Medium": value = "2"
                    if value == "Large": value = "4"

                    # Genus: wolves ...
                    if not re.match("-?[0-9.-]+", value):
                        # print(f"ignoring {property}, value is {value}")
                        continue

                    if "-" in value:
                        numbers = re.findall(mod_re, value)
                        if numbers:
                            #bleh
                            value = 0
                            for num in numbers:
                                if num.endswith("-"): num = num[:-1]
                                num = float(num)
                                value += num / len(numbers)
                value = float(value)
                descs["properties"][prop] = value
                descs["properties"][f"1/prop"] = 1/value if value != 0 else 999
            else:
                descs["properties"][prop] = 1
    if "sockets" in item:
        # grab the size of the biggest group
        sockets = item["sockets"]
        descs["properties"]["_white sockets"] = sum(1 for s in sockets if s["sColour"] == "w")
        descs["properties"]["_linked sockets"] = max(sum(1 for s in sockets if s["group"] == i) for i in range(6))
    descs["properties"]["_identified"] = 1 if item.get("identified") else 0
    descs["properties"]["_corrupted"] = 1 if item.get("corrupted") else 0
    descs["properties"]["_vaal"] = 1 if "hybrid" in item and item["hybrid"].get("isVaalGem") else 0
    descs["properties"]["_abyss"] = 1 if "abyssJewel" in item else 0
    descs["properties"]["_elder"] = 1 if "elder" in item else 0
    descs["properties"]["_shaper"] = 1 if "shaper" in item else 0
    for requirement in item.get("requirements", []):
        name = requirement["name"]
        value = requirement["values"][0][0]
        descs["requirement"][name] = float(value)
    for mod in item.get("explicitMods", []) + item.get("implicitMods", []) + item.get("craftedMods", []):
        numbers = re.findall(mod_re, mod)
        if numbers:
            #bleh
            value = 0
            for num in numbers:
                if num.endswith("-"): num = num[:-1]
                num = float(num)
                value += num / len(numbers)
        else:
            value = 1
        name = re.sub(mod_re, "", mod)
        descs["mod"][name] = value
        descs["mod"][f"1/name"] = 1/value if value != 0 else 9999
    return descs

which_vw = pyvw.vw(quiet=True, loss_function='logistic', save_resume=True, i="which.vw")
alch_vw = pyvw.vw(quiet=True, save_resume=True, i="alch.vw")
chaos_vw = pyvw.vw(quiet=True, save_resume=True, i="chaos.vw")
exalt_vw = pyvw.vw(quiet=True, save_resume=True, i="exa.vw")

currency_vw = {
    1: alch_vw,
    2: chaos_vw,
    3: exalt_vw,
}

POESESSID = os.environ["POESESSID"]
accountName = "bp_"
league = "Legion"
stashes = range(1,14)
base_url = f"https://pathofexile.com/character-window/get-stash-items?league={league}&tabs=1&accountName={accountName}"

prices = {x: {} for x in currency_label}
for stash in stashes:
    if os.path.isfile(f"stash_{stash}.json"):
        with open(f"stash_{stash}.json", "r") as f:
            data = f.read()
    else:
        data = requests.get(f"{base_url}&tabIndex={stash}", cookies={"POESESSID": POESESSID}).text
        with open(f"stash_{stash}.json", "w") as f:
            f.write(data)
    contents = json.loads(data)
    stash_name = contents["tabs"][stash]["n"]
    for item in contents["items"]:
        desc = featurize(item)

        ex = which_vw.example()
        for ns in desc:
            #import pprint
            #pprint.pprint({ns: list(desc[ns].items())})
            ex.push_features(ns, list(desc[ns].items()))
        currency = which_vw.predict(ex)
        which_vw.finish_example(ex)

        ex = currency_vw[currency].example()
        for ns in desc:
            ex.push_features(ns, list(desc[ns].items()))
        amount = currency_vw[currency].predict(ex)
        currency_vw[currency].finish_example(ex)

        currency_english = {y:x for (x,y) in currency_label.items()}[currency]
        prices[currency_english][f"{stash_name:20}\t {item['name']:>20} {item['typeLine']:20}\t{item.get('note') or stash_name:20}"] = amount

for currency in currency_label:
    for item, worth in sorted(prices[currency].items(), key = lambda x: x[1]):
        print(f"{item}\t{worth:.1f} {currency}")

## train.py
import requests
import json
import re
import os
import gzip
from vowpalwabbit import pyvw

currency_label = {
    "alch": 1,
    "chaos": 2,
    "exa": 3,
}

price_re = re.compile("""
    ~
    (?:b/o|price)
    [ ]
    ([0-9]+)
    [ ]
    (alch|chaos|exa)
""", re.X)

mod_re = re.compile("-?[0-9]+-?")

def featurize(item):
    descs = {"name": {}, "requirement": {}, "properties": {}, "mod": {}}
    for word in item["typeLine"].split(" "):
        descs["name"][word] = 1
    for property in item.get("properties", []) + item.get("additionalProperties", []):
        for prop in property["name"].split(", "):
            values = property.get("progress") or property.get("values")
            if "StackSize" in prop: continue
            # Limited to: 1 Historic
            if prop == "Limited to": continue
            if values:
                value = values[0][0] if type(values) is list else values
                if type(value) is str:
                    value = value.replace("%", "")
                    value = value.replace("+", "")
                    # cast time
                    if value == "Instant": value = "0.001"
                    value = value.replace(" sec", "")
                    value = value.replace(" of base", "")
                    value = re.sub("/[0-9]+", "", value)
                    value = re.sub(" [(].+", "", value)
                    # radius
                    if value == "Small": value = "1"
                    if value == "Medium": value = "2"
                    if value == "Large": value = "4"

                    # Genus: wolves ...
                    if not re.match("-?[0-9.-]+", value):
                        # print(f"ignoring {property}, value is {value}")
                        continue

                    if "-" in value:
                        numbers = re.findall(mod_re, value)
                        if numbers:
                            #bleh
                            value = 0
                            for num in numbers:
                                if num.endswith("-"): num = num[:-1]
                                num = float(num)
                                value += num / len(numbers)
                value = float(value)
                descs["properties"][prop] = value
                descs["properties"][f"1/prop"] = 1/value if value != 0 else 999
            else:
                descs["properties"][prop] = 1
    if "sockets" in item:
        # grab the size of the biggest group
        sockets = item["sockets"]
        descs["properties"]["_white sockets"] = sum(1 for s in sockets if s["sColour"] == "w")
        descs["properties"]["_linked sockets"] = max(sum(1 for s in sockets if s["group"] == i) for i in range(6))
    descs["properties"]["_identified"] = 1 if item.get("identified") else 0
    descs["properties"]["_corrupted"] = 1 if item.get("corrupted") else 0
    descs["properties"]["_vaal"] = 1 if "hybrid" in item and item["hybrid"].get("isVaalGem") else 0
    descs["properties"]["_abyss"] = 1 if "abyssJewel" in item else 0
    descs["properties"]["_elder"] = 1 if "elder" in item else 0
    descs["properties"]["_shaper"] = 1 if "shaper" in item else 0
    for requirement in item.get("requirements", []):
        name = requirement["name"]
        value = requirement["values"][0][0]
        descs["requirement"][name] = float(value)
    for mod in item.get("explicitMods", []) + item.get("implicitMods", []) + item.get("craftedMods", []):
        numbers = re.findall(mod_re, mod)
        if numbers:
            #bleh
            value = 0
            for num in numbers:
                if num.endswith("-"): num = num[:-1]
                num = float(num)
                value += num / len(numbers)
        else:
            value = 1
        name = re.sub(mod_re, "", mod)
        descs["mod"][name] = value
        descs["mod"][f"1/name"] = 1/value if value != 0 else 9999
    return descs

i = 0
changeid = 0

from collections import Counter
counters = {"curr": Counter(), "league": Counter(), "skip": Counter()}

if not os.path.isfile("which.wv"):
    # train from scratch
    which_vw = pyvw.vw(loss_function='logistic', oaa=3, save_resume=True, progress=2, final_regressor="which.vw")
    alch_vw = pyvw.vw(quiet=True, save_resume=True, final_regressor="alch.vw")
    chaos_vw = pyvw.vw(quiet=True, save_resume=True, final_regressor="chaos.vw")
    exalt_vw = pyvw.vw(quiet=True, save_resume=True, final_regressor="exa.vw")
else:
    # do a new epoch
    which_vw = pyvw.vw(loss_function='logistic', oaa=3, save_resume=True, progress=2, i="which.vw", final_regressor="which.vw")
    alch_vw = pyvw.vw(quiet=True, save_resume=True, i="alch.vw", final_regressor="alch.vw")
    chaos_vw = pyvw.vw(quiet=True, save_resume=True, i="chaos.vw", final_regressor="chaos.vw")
    exalt_vw = pyvw.vw(quiet=True, save_resume=True, i="exa.vw", final_regressor="exa.vw")

currency_vw = {
    "alch": alch_vw,
    "chaos": chaos_vw,
    "exa": exalt_vw,
}

while True:
    if os.path.isfile(f"cache/{changeid}.json"):
        with open(f"cache/{changeid}.json", "rb") as f:
            with open(f"cache/{changeid}.json.gz", "wb") as g:
                g.write(gzip.compress(f.read()))
                os.unlink(f)
    elif not os.path.isfile(f"cache/{changeid}.json.gz"):
            response = requests.get(f"http://api.pathofexile.com/public-stash-tabs?id={changeid}")
            data = response.json()
            with open(f"cache/{changeid}.json.gz", "wb") as f:
                f.write(gzip.compress(response.content))
    else:
        with open(f"cache/{changeid}.json.gz", "rb") as f:
            r = gzip.decompress(f.read())
            data = json.loads(r)

    for stash in data["stashes"]:
        if stash["league"] not in ("Legion",):
            continue
        for item in stash["items"]:
            price = item.get("note", stash["stash"])
            if not price: counters["skip"].update([0]); continue
            match = price_re.search(price)
            if not match: counters["skip"].update([1]); continue
            amount, currency = match.groups()
            desc = featurize(item)

            ex = which_vw.example()
            for ns in desc:
                #import pprint
                #pprint.pprint({ns: list(desc[ns].items())})
                ex.push_features(ns, list(desc[ns].items()))
            ex.set_label_string(str(currency_label[currency]))
            which_vw.learn(ex)
            which_vw.finish_example(ex)

            ex = currency_vw[currency].example()
            for ns in desc:
                ex.push_features(ns, list(desc[ns].items()))
            ex.set_label_string(amount)
            currency_vw[currency].learn(ex)
            currency_vw[currency].finish_example(ex)

            counters["league"].update(stash["league"][0])
            counters["curr"].update(currency[0])
            print(counters, end="\r")

    which_vw.save("which.vw")
    counters["w"] = which_vw.get_sum_loss()
    alch_vw.save("alch.vw")
    chaos_vw.save("chaos.vw")
    exalt_vw.save("exa.vw")

    changeid = data["next_change_id"]

#do not cache the final response so we can see newer items in newer runs
os.unlink(f"cache/{changeid}.json.gz")

raise "Nope"
	[[source]]
	name = "pypi"
	url = "https://pypi.org/simple"
	verify_ssl = true

	[dev-packages]
	ipython = "*"
	pylint = "*"

	[packages]
	vowpal-porpoise = "*"
	requests = "*"
	vowpalwabbit = "*"

	[requires]
	python_version = "3.6"

	[pipenv]
	allow_prereleases = true
	import requests
	import json
	import re
	import os
	import gzip
	from vowpalwabbit import pyvw

	currency_label = {
	"alch": 1,
	"chaos": 2,
	"exa": 3,
	}

	price_re = re.compile("""
	~
	(?:b/o\|price)
	[ ]
	([0-9]+)
	[ ]
	(alch\|chaos\|exa)
	""", re.X)

	mod_re = re.compile("-?[0-9]+-?")

	def featurize(item):
	descs = {"name": {}, "requirement": {}, "properties": {}, "mod": {}}
	for word in item["typeLine"].split(" "):
	descs["name"][word] = 1
	for property in item.get("properties", []) + item.get("additionalProperties", []):
	for prop in property["name"].split(", "):
	values = property.get("progress") or property.get("values")
	if "StackSize" in prop: continue
	# Limited to: 1 Historic
	if prop == "Limited to": continue
	if values:
	value = values[0][0] if type(values) is list else values
	if type(value) is str:
	value = value.replace("%", "")
	value = value.replace("+", "")
	# cast time
	if value == "Instant": value = "0.001"
	value = value.replace(" sec", "")
	value = value.replace(" of base", "")
	value = re.sub("/[0-9]+", "", value)
	value = re.sub(" [(].+", "", value)
	# radius
	if value == "Small": value = "1"
	if value == "Medium": value = "2"
	if value == "Large": value = "4"

	# Genus: wolves ...
	if not re.match("-?[0-9.-]+", value):
	# print(f"ignoring {property}, value is {value}")
	continue

	if "-" in value:
	numbers = re.findall(mod_re, value)
	if numbers:
	#bleh
	value = 0
	for num in numbers:
	if num.endswith("-"): num = num[:-1]
	num = float(num)
	value += num / len(numbers)
	value = float(value)
	descs["properties"][prop] = value
	descs["properties"][f"1/prop"] = 1/value if value != 0 else 999
	else:
	descs["properties"][prop] = 1
	if "sockets" in item:
	# grab the size of the biggest group
	sockets = item["sockets"]
	descs["properties"]["_white sockets"] = sum(1 for s in sockets if s["sColour"] == "w")
	descs["properties"]["_linked sockets"] = max(sum(1 for s in sockets if s["group"] == i) for i in range(6))
	descs["properties"]["_identified"] = 1 if item.get("identified") else 0
	descs["properties"]["_corrupted"] = 1 if item.get("corrupted") else 0
	descs["properties"]["_vaal"] = 1 if "hybrid" in item and item["hybrid"].get("isVaalGem") else 0
	descs["properties"]["_abyss"] = 1 if "abyssJewel" in item else 0
	descs["properties"]["_elder"] = 1 if "elder" in item else 0
	descs["properties"]["_shaper"] = 1 if "shaper" in item else 0
	for requirement in item.get("requirements", []):
	name = requirement["name"]
	value = requirement["values"][0][0]
	descs["requirement"][name] = float(value)
	for mod in item.get("explicitMods", []) + item.get("implicitMods", []) + item.get("craftedMods", []):
	numbers = re.findall(mod_re, mod)
	if numbers:
	#bleh
	value = 0
	for num in numbers:
	if num.endswith("-"): num = num[:-1]
	num = float(num)
	value += num / len(numbers)
	else:
	value = 1
	name = re.sub(mod_re, "", mod)
	descs["mod"][name] = value
	descs["mod"][f"1/name"] = 1/value if value != 0 else 9999
	return descs

	which_vw = pyvw.vw(quiet=True, loss_function='logistic', save_resume=True, i="which.vw")
	alch_vw = pyvw.vw(quiet=True, save_resume=True, i="alch.vw")
	chaos_vw = pyvw.vw(quiet=True, save_resume=True, i="chaos.vw")
	exalt_vw = pyvw.vw(quiet=True, save_resume=True, i="exa.vw")

	currency_vw = {
	1: alch_vw,
	2: chaos_vw,
	3: exalt_vw,
	}

	POESESSID = os.environ["POESESSID"]
	accountName = "bp_"
	league = "Legion"
	stashes = range(1,14)
	base_url = f"https://pathofexile.com/character-window/get-stash-items?league={league}&tabs=1&accountName={accountName}"

	prices = {x: {} for x in currency_label}
	for stash in stashes:
	if os.path.isfile(f"stash_{stash}.json"):
	with open(f"stash_{stash}.json", "r") as f:
	data = f.read()
	else:
	data = requests.get(f"{base_url}&tabIndex={stash}", cookies={"POESESSID": POESESSID}).text
	with open(f"stash_{stash}.json", "w") as f:
	f.write(data)
	contents = json.loads(data)
	stash_name = contents["tabs"][stash]["n"]
	for item in contents["items"]:
	desc = featurize(item)

	ex = which_vw.example()
	for ns in desc:
	#import pprint
	#pprint.pprint({ns: list(desc[ns].items())})
	ex.push_features(ns, list(desc[ns].items()))
	currency = which_vw.predict(ex)
	which_vw.finish_example(ex)

	ex = currency_vw[currency].example()
	for ns in desc:
	ex.push_features(ns, list(desc[ns].items()))
	amount = currency_vw[currency].predict(ex)
	currency_vw[currency].finish_example(ex)

	currency_english = {y:x for (x,y) in currency_label.items()}[currency]
	prices[currency_english][f"{stash_name:20}\t {item['name']:>20} {item['typeLine']:20}\t{item.get('note') or stash_name:20}"] = amount

	for currency in currency_label:
	for item, worth in sorted(prices[currency].items(), key = lambda x: x[1]):
	print(f"{item}\t{worth:.1f} {currency}")