compore_bot.py

import glob
import json
import os
import re
import shutil
import subprocess
import zipfile
from contextlib import contextmanager
from tempfile import TemporaryDirectory

import math


@contextmanager
def cd(newdir):
    prevdir = os.getcwd()
    os.chdir(os.path.expanduser(newdir))
    try:
        yield
    finally:
        os.chdir(prevdir)


def unzip(name):
    with zipfile.ZipFile(name) as zfile:
        for name in zfile.namelist():
            zfile.extract(name)


def clear_dir_files(directory):
    for file in os.listdir(directory):
        file_path = os.path.join(directory, file)
        if os.path.isfile(file_path):
            os.unlink(file_path)


def binomial_confidence(successes, failures, ratio=None):
    if ratio is None:
        ratio = successes / (successes + failures)

    var = ratio * (1.0 - ratio)

    sd = math.sqrt(var / (successes + failures))

    m = 1.96 * sd

    return max(ratio - m, 0), min(ratio + m, 1)


def compare(current_bot, other_bots, binary, num_games, map_sizes):
    assert current_bot
    assert len(other_bots) == 1 or len(other_bots) == 3

    two_game = len(other_bots) == 1
    four_game = len(other_bots) == 3

    os.makedirs("lost_replays", exist_ok=True)
    os.makedirs("won_replays", exist_ok=True)
    os.makedirs("error_logs", exist_ok=True)

    clear_dir_files("won_replays")
    clear_dir_files("lost_replays")
    clear_dir_files("error_logs")

    current_dir = os.getcwd()
    lost_replays_dir = os.path.join(current_dir, "lost_replays")
    won_replays_dir = os.path.join(current_dir, "won_replays")
    error_logs_dir = os.path.join(current_dir, "error_logs")

    bot_wins = [0, 0]
    if four_game:
        bot_wins = [0, 0, 0, 0]

    with TemporaryDirectory() as t:
        print("Running in tempdir {}".format(t))
        with cd(t):
            def prepare_bot(name, bot_path):
                os.makedirs(name)
                with cd(name):
                    unzip(bot_path)

                return os.path.join(t, name + os.sep + "MyBot.py")

            current_bot_wins = 0
            other_bots_wins = 0

            print("Starting tournament with {} games".format(num_games))

            tmp_current_path = prepare_bot("current_bot", current_bot)
            tmp_bot_paths = [prepare_bot("bot_" + str(i), bot) for i, bot in enumerate(other_bots, start=1)]

            def did_bot_win(output_data, bot_number):
                # matching_string = "Player #{}(.*)came in rank #(\d)".format(str(bot_number))
                return output_data['stats'][str(bot_number)]["rank"] == 1

            def get_replay(output_data):
                replay = output_data['replay']
                replay = os.path.join(t, replay)
                return replay

            for i in range(num_games):
                python_string = '"python3 {} bot_name current"'.format(tmp_current_path)
                for tmp_bot_path in tmp_bot_paths:
                    python_string += ' "python3 {}"'.format(tmp_bot_path)

                current_map_index = i % len(map_sizes)
                map_size = map_sizes[current_map_index]

                cmd = '{} -q -d "{}" {}'.format(binary, map_size, python_string)
                print(cmd)
                # sys.stdin.readline().rstrip('\n')
                output = subprocess.check_output(cmd, shell=True).decode()
                print(output)
                output_data = json.loads(output)
                map_seed = output_data['map_seed']
                print("map_seed", map_seed)

                if output_data['error_logs']:
                    print("There are errors - saving logs")
                    for error_file in output_data['error_logs'].values():
                        print(error_file)
                        for file in glob.glob('*.log'):
                            dest_file = os.path.join(error_logs_dir, map_seed + file)
                            print(dest_file)
                            shutil.copy(file, dest_file)

                        shutil.copy(error_file, error_logs_dir)
                        shutil.copy("", error_logs_dir)

                # they use player 0 and player 1 instead of 1 and 2 as we do
                current_bot_win = did_bot_win(output_data, 0)
                replay_file = get_replay(output_data)
                if current_bot_win:
                    current_bot_wins += 1
                    bot_wins[0] += 1
                    shutil.copy(replay_file, won_replays_dir)
                else:
                    print("Current bot lost - saving replay in: {}".format(lost_replays_dir))
                    print(replay_file)
                    shutil.copy(replay_file, lost_replays_dir)

                    other_bots_wins += 1
                    for j, bot in enumerate(other_bots, start=1):
                        bot_wins[j] += 1 if did_bot_win(output_data, j) else 0

                min_win_rate, max_win_rate = binomial_confidence(current_bot_wins, other_bots_wins)
                print("Current bot win ratio range: {:1.2f} - {:1.2f}".format(
                    min_win_rate, max_win_rate,
                ))

                if two_game:
                    print("{}:{}".format(*bot_wins))
                if four_game:
                    print("{}:{}:{}:{}".format(*bot_wins))


if __name__ == '__main__':
    current_bot = "bot_zips/bot-20171101-003652.zip"

    second_bot = "bot_zips/bot-20171030-233406.zip"
    third_bot = "bot_zips/bot-20171030-233406.zip"
    fourth_bot = "bot_zips/bot-20171030-233406.zip"

    other_bots = [
        second_bot,
        # third_bot,
        # fourth_bot,
    ]


    def abs_bot_path(bot_path):
        current_dir = os.getcwd()
        bot_path = os.path.abspath(bot_path)
        bot_path = os.path.join(current_dir, bot_path)
        return bot_path


    map_sizes = [
        "160 120",
        "240 160",
        "312 208",
        "264 176",
    ]

    compare(
        current_bot=abs_bot_path(current_bot),
        other_bots=[abs_bot_path(bot) for bot in other_bots],
        binary=os.path.abspath("halite"),
        map_sizes=map_sizes,
        num_games=30,
    )