get_dota_data.py

def request_(req_url, sleep_time=1):
    succeeded = False
    while not succeeded:
        try:
            print("Requesting: %s" % req_url)
            request = Request(req_url)
            request.add_header('User-Agent',
                               'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36')
            response = urlopen(request)
            out = response.read().decode(
                'utf8')  # cos python3 is kind of stupid http://stackoverflow.com/questions/6862770/python-3-let-json-object-accept-bytes-or-let-urlopen-output-strings
            time.sleep(sleep_time)  # obey api rate limits
            succeeded = True
        except:
            sleep_time += 1
            traceback.print_exc()
            continue
    return out


def get_matches():
    # holy crap this league has lots of games 4122
    skip = 0
    leagues_json = []
    while True:
        new = json.loads(request_("https://api.stratz.com/api/v1/league?take=100&skip=%s" % skip))
        if not len(new):
            break
        leagues_json.extend(new)
        skip += 100
    matches = []

    for league in (
            l["id"] for l in leagues_json if l["id"] > 10):
        league_games = json.loads(request_(
            "https://api.stratz.com/api/v1/match?leagueId=%s&include=pickBan,GameVersionId&take=250" % league,
        ))
        left = league_games["total"] - 250
        skip = 250
        matches.extend([l["pickBans"] for l in league_games["results"] if l.get("gameVersionId") >= 75])
        while left >= 0:
            league_games = json.loads(request_(
                "https://api.stratz.com/api/v1/match?leagueId=%s&include=pickBan,GameVersionId&take=250&skip=%s" % (league, skip),
            ))
            matches.extend([l["pickBans"] for l in league_games["results"] if l.get("gameVersionId") >= 75])
            skip += 250
            left -= 250
    return matches


def load_data():
# only data is really necessary. you could infer/construct inputs(data2) and outputs(data3) from jsut data. but lazy.
    data = []
    inputs = []
    outputs = []
    if os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data.txt")):
        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data.txt")) as f:
            data = ast.literal_eval(f.read())

        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data2.txt")) as f:
            inputs = ast.literal_eval(f.read())

        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data3.txt")) as f:
            outputs = ast.literal_eval(f.read())
    else:
        matches = get_matches()
        for match in matches:
            input_ = []
            if len(match) != 20:
                continue
            for i, pick in enumerate(match):
                hero_id = pick["heroId"]
                data.append(hero_id)
                if i == 19:  # lets just focus on testing last pick
                    outputs.append(hero_id)
                    inputs.append(input_)
                else:
                    input_.append(hero_id)

        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data.txt"), "w+") as f:
            f.write(str(data))

        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data2.txt"), "w+") as f:
            f.write(str(inputs))

        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data3.txt"), "w+") as f:
            f.write(str(outputs))

return data, inputs, outputs