A simple neural network for tic-tac-toe game

mcts.py

import numpy as np
from Net import Net
import play

import sys
sys.setrecursionlimit(100000)  # to temporarily solve Recursion Depth Limit issue

# Reference :
# https://www.reddit.com/r/learnmachinelearning/comments/fmx3kv/empirical_example_of_mcts_calculation_puct_formula/

# PUCT formula : https://colab.research.google.com/drive/14v45o1xbfrBz0sG3mHbqFtYz_IrQHLTg#scrollTo=1VeRCpCSaHe3

# https://en.wikipedia.org/wiki/Monte_Carlo_tree_search#Exploration_and_exploitation
cfg_puct = np.sqrt(2)  # to balance between exploitation and exploration
puct_array = []  # stores puct ratio for every child nodes for argmax()


# determined by PUCT formula
def find_best_path(parent):
    print("find_best_path()")
    if (parent == root) | (len(parent.nodes) == 0):
        return 0

    for N in parent.nodes:
        puct_array.append(N.puct)

    max_index = np.argmax(puct_array)
    puct_array.clear()  # resets the list so that other paths could reuse it

    #  leaf node has 0 child node
    is_leaf_node = (len(parent.nodes[max_index].nodes) == 0)
    if is_leaf_node:
        return max_index

    else:
        return -1


# for play.py inference coding
is_simulation_stage = None  # initialized to None because game had not started yet


def is_mcts_in_simulate_stage():
    return is_simulation_stage


class Mcts:
    def __init__(self, parent):
        # https://www.tutorialspoint.com/python_data_structure/python_tree_traversal_algorithms.htm
        # https://www.geeksforgeeks.org/sum-parent-nodes-child-node-x/

        self.parent = parent  # this is the parent node
        self.nodes = []  # creates an empty list with no child nodes initially
        self.data = 0  # can be of any value, but just initialized to 0
        self.visit = 1  # when a node is first created, it is counted as visited once
        self.win = 0  # because no play/simulation had been performed yet
        self.loss = 0  # because no play/simulation had been performed yet
        self.puct = 0  # initialized to 0 because game had not started yet

    # this function computes W/N ratio for each node
    def compute_total_win_and_visits(self, total_win=0, visits=0):
        print("compute_total_win_and_visits()")

        if self.win:
            total_win = total_win + 1

        if self.visit:
            visits = visits + 1

        if self.nodes:  # if there is/are child node(s)
            for n in self.nodes:  # traverse down the entire branch for each child node
                n.compute_total_win_and_visits(total_win, visits)

        return total_win, visits  # same order (W/N) as in
        # https://i.imgur.com/uI7NRcT.png inside each node

    # Selection stage of MCTS
    # https://www.reddit.com/r/reinforcementlearning/comments/kfg6qo/selection_phase_of_montecarlo_tree_search/
    def select(self):
        print("select()")
        # traverse recursively all the way down from the root node
        # to find the path with the highest W/N ratio (this ratio is determined using PUCT formula)
        # and then select that leaf node to do the new child nodes insertion
        leaf = find_best_path(self)  # returns a reference pointer to the desired leaf node
        parent_node = self

        while leaf == -1:
            parent_node = parent_node.nodes[leaf]
            leaf = find_best_path(parent_node)  # keeps looping in case it is not the leaf yet

        parent_node.insert()  # this leaf node is selected to insert child nodes under it

    # Expansion stage of MCTS
    # Insert Child Nodes for a leaf node
    def insert(self):
        print("insert()")
        # assuming that we are playing tic-tac toe
        # we subtract number of game states already played from the total possible game states
        num_of_possible_game_states = play.TOTAL_NUM__OF_BOXES - play.num_of_play_rounds

        for S in range(num_of_possible_game_states):
            self.nodes.append(Mcts(self))  # inserts child nodes

        self.simulate()

    # Simulation stage of MCTS
    def simulate(self):
        print("simulate()")
        best_child_node = find_best_path(self)

        # Instantiates neural network inference coding (play.py) here
        play.mcts_play(is_mcts_in_simulate_stage=1, ongoing_game=play.game_is_on, best_child_node=best_child_node)
        print("after one round of game")

        if play.game_is_on == 1:  # game not yet finished
            # predicted "intermediate" score during each step of the game,
            # so it is either win (1) or draw (0) or lose (-1)
            print("intermediate out_score = ", play.out_score)

            if play.out_score == 1:
                print("win")
                self.win = 1
                self.loss = 0

            if play.out_score == -1:
                print("lose")
                self.win = 0
                self.loss = 1

            if play.out_score == 0:
                print("draw")
                self.win = 0
                self.loss = 0

            self.backpropagation(self.win, self.loss)

        else:  # game finished
            print(root.print_tree(root))  # for verifying MCTS logic correctness

    # Backpropagation stage of MCTS
    def backpropagation(self, win, loss):
        print("backpropagation()")
        # traverses upwards to the root node
        # and updates PUCT ratio for each parent nodes
        # computes the PUCT expression Q+U https://slides.com/crem/lc0#/9

        if self.parent == 0:
            num_of_parent_visits = 0
        else:
            num_of_parent_visits = self.parent.visit

        total_win_for_all_child_nodes, num_of_child_visits = self.compute_total_win_and_visits(0, 0)

        self.visit = num_of_child_visits

        # traverses downwards all branches (only for those branches involved in previous play/simulation)
        # and updates PUCT values for all their child nodes
        self.puct = (total_win_for_all_child_nodes / num_of_child_visits) + \
            cfg_puct * np.sqrt(num_of_parent_visits) / (num_of_child_visits + 1)

        if self.parent == 0:  # already reached root node
            self.select()

        else:
            self.parent.visit = self.parent.visit + 1
            if win:
                if self.parent.parent:  # grandparent node (same-coloured player) exists
                    self.parent.parent.win = self.parent.parent.win + 1

            if (win == 0) & (loss == 0):  # tie is between loss (0) and win (1)
                self.parent.win = self.parent.win + 0.5  # parent node (opponent player)

                if self.parent.parent:  # grandparent node (same-coloured player) exists
                    self.parent.parent.win = self.parent.parent.win + 0.5

            self.parent.backpropagation(win, loss)

    # Print the Tree
    def print_tree(self, child):
        for x in child.nodes:
            print(x.data)
            if x.nodes:
                self.print_tree(x.nodes)


root = Mcts(0)  # we use parent=0 because this is the head/root node
root.select()

Net.py

import torch
import torch.utils.data
import torch.nn as nn
import torch.optim as optim

import numpy as np
import pandas as pd
from collections import Counter
from sklearn.model_selection import train_test_split

TEST_DATASET_RATIO = 0.05  # 5 percent of the dataset is dedicated for testing purpose
NUM_OF_BOARD_FEATURES = 18
PLAYER_TURN_COLUMN = -2  # second to last column of the csv file
SCORE_COLUMN = -1  # last column of the csv file
NUM_OF_BOARD_FEATURES_AND_TURN = NUM_OF_BOARD_FEATURES + 1  # add 1 because of player turn
NUM_OF_POSSIBLE_MOVES = 9  # a normal 3x3 tic-tac-toe has 9 input boxes
NUM_OF_POSSIBLE_SCORES = 3  # -1, 0, 1 == loss, draw, win
# POSSIBLE_SCORES = [-1, 0, 1]
SIZE_OF_HIDDEN_LAYERS = 64
NUM_EPOCHS = 6000
LEARNING_RATE = 0.7
MOMENTUM = 0.9


class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        self.linears = nn.Sequential(
            nn.Linear(NUM_OF_BOARD_FEATURES_AND_TURN, SIZE_OF_HIDDEN_LAYERS),
            nn.ReLU(),
            nn.Dropout(),

            nn.Linear(SIZE_OF_HIDDEN_LAYERS, SIZE_OF_HIDDEN_LAYERS),
            nn.ReLU(),
            nn.Dropout(),

            nn.Linear(SIZE_OF_HIDDEN_LAYERS, SIZE_OF_HIDDEN_LAYERS),
            nn.ReLU(),
            nn.Dropout(),

            nn.Linear(SIZE_OF_HIDDEN_LAYERS, NUM_OF_POSSIBLE_MOVES)
        )

        self.linears2 = nn.Sequential(
            nn.Linear(NUM_OF_BOARD_FEATURES_AND_TURN, SIZE_OF_HIDDEN_LAYERS),
            nn.ReLU(),
            nn.Dropout(),

            nn.Linear(SIZE_OF_HIDDEN_LAYERS, SIZE_OF_HIDDEN_LAYERS),
            nn.ReLU(),
            nn.Dropout(),

            nn.Linear(SIZE_OF_HIDDEN_LAYERS, SIZE_OF_HIDDEN_LAYERS),
            nn.ReLU(),
            nn.Dropout(),

            nn.Linear(SIZE_OF_HIDDEN_LAYERS, NUM_OF_POSSIBLE_SCORES),
            nn.Softmax(1)
        )

    def forward(self, x):
        policy = self.linears(x)
        value = self.linears2(x)

        return policy, value  # move, score


USE_CUDA = torch.cuda.is_available()


def train():
    # part of the code is referenced from
    # https://github.com/bsamseth/tictacNET/blob/master/tictacnet.py and
    # https://github.com/bsamseth/tictacNET/blob/master/tictactoe-data.csv

    df = pd.read_csv("tictactoe-data.csv")
    print("Scores:", Counter(df["score"]))

    # Input is all the board features (2x9 squares) plus the turn.
    board_features_and_turn = df.iloc[:, list(range(NUM_OF_BOARD_FEATURES)) + [PLAYER_TURN_COLUMN]]

    # To predict score instead, use this as the target:
    # score = pd.get_dummies(df['score'])
    # print(score)

    # split into training dataset (80%) and validation dataset (20%)
    # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html and
    # https://scikit-learn.org/stable/glossary.html#term-random_state show that splitting is randomized.
    # Since the dataset involving 1 input and 2 outputs, need to combine the 2 outputs first before splitting
    # in order to preserve data alignment

    # Target variables are the possible move squares as well as the predicted output scores
    moves_score = df.iloc[:, list(range(NUM_OF_BOARD_FEATURES, NUM_OF_BOARD_FEATURES + NUM_OF_POSSIBLE_MOVES)) +
                          [SCORE_COLUMN]]
    # print(moves_score)

    board_train, board_test, moves_score_train, moves_score_test = \
        train_test_split(board_features_and_turn, moves_score, test_size=TEST_DATASET_RATIO)
    # print(board_test)

    moves_test = moves_score_test.iloc[:, list(range(0, NUM_OF_POSSIBLE_MOVES))]
    score_test = moves_score_test.iloc[:, [NUM_OF_POSSIBLE_MOVES]]

    moves_train = moves_score_train.iloc[:, list(range(0, NUM_OF_POSSIBLE_MOVES))]
    score_train = moves_score_train.iloc[:, [NUM_OF_POSSIBLE_MOVES]]

    print(len(score_train))
    print(len(score_test))

    net = Net()
    if USE_CUDA:
        net = net.cuda()
    print(net)

    params = list(net.parameters())
    print(len(params))

    # for x in range(10):
    #    print(params[x].size())  # conv1's .weight

    optimizer = optim.SGD(net.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM)

    loss_function2 = nn.CrossEntropyLoss()
    loss_function = nn.MSELoss()

    TRAIN_BATCH_SIZE = int(len(moves_score) * (1 - TEST_DATASET_RATIO))

    for epoch in range(NUM_EPOCHS):
        loss_ = 0

        # See https://github.com/bsamseth/tictacNET/issues/2
        # for a description of the inputs and output of the neural network
        train_loader = zip(np.array(board_train, dtype='float32'),
                           np.array(moves_train, dtype='float32'),
                           np.array(score_train, dtype='float32'))
        train_loader = torch.utils.data.DataLoader(
            list(train_loader),
            batch_size=TRAIN_BATCH_SIZE,
        )

        for _board_features_and_turn, move, _score in train_loader:
            if USE_CUDA:
                _board_features_and_turn = _board_features_and_turn.cuda()
                move = move.cuda()
                _score = _score.cuda()

            # Forward Pass
            policy_output, value_output = net(_board_features_and_turn)

            # Loss at each iteration by comparing to target(moves)
            loss1 = loss_function(policy_output, move)

            # Loss at each iteration by comparing to target(score)
            # adds 1 to _score because cross_entrophy loss cannot accept input value of -1
            # https://github.com/pytorch/pytorch/issues/1204#issuecomment-292746566
            loss2 = loss_function2(value_output, (_score + 1).squeeze(1).long())

            loss = loss1 + loss2

            # Backpropagating gradient of loss
            optimizer.zero_grad()
            loss.backward()

            # Updating parameters(weights and bias)
            optimizer.step()

            loss_ += loss.item()
        print("Epoch{}, Training loss:{}".format(epoch, loss_ / len(moves_train)))

    print('Finished Training')

    path = './tictactoe_net.pth'
    torch.save(net, path)

    print("############################################")
    print("Doing train_accuracy check")

    train_correct = 0
    train_total = 0

    train_loader = zip(np.array(board_train, dtype='float32'),
                       np.array(moves_train, dtype='float32'),
                       np.array(score_train, dtype='float32'))
    train_loader = torch.utils.data.DataLoader(
        list(train_loader),
        batch_size=TRAIN_BATCH_SIZE,
    )

    with torch.no_grad():
        for _board_train, _moves_train, _score_train in train_loader:
            if USE_CUDA:
                _board_train = _board_train.cuda()
                _moves_train = _moves_train.cuda()
                _score_train = _score_train.cuda()

            model_input = _board_train
            _policy_output, _value_output = net(model_input)
            predicted = torch.argmax(_policy_output, 1)

            # print("_policy_output = ", _policy_output)
            # print("predicted = ", predicted)
            # print("_moves_train = ", _moves_train)

            for train_index in range(len(_moves_train)):
                # print("move testing for train_index = ", train_index)

                # print("_moves_train[train_index][predicted[train_index]] = ",
                #      _moves_train[train_index][predicted[train_index]], '\n')

                if _moves_train[train_index][predicted[train_index]]:
                    # print("predicted == _moves_train")
                    train_correct = train_correct + 1

            train_total = train_total + len(_moves_train)

    print('Accuracy of the network on train move: %d %%' % (
            100 * train_correct / train_total))

    train_correct = 0
    train_total = 0

    with torch.no_grad():
        for _board_train, _moves_train, _score_train in train_loader:
            if USE_CUDA:
                _board_train = _board_train.cuda()
                _moves_train = _moves_train.cuda()
                _score_train = _score_train.cuda()

            model_input = _board_train
            _policy_output, _value_output = net(model_input)
            predicted = torch.argmax(_value_output, 1)

            # print("_value_output = ", _value_output)
            # print("predicted = ", predicted)
            # print("_score_train = ", _score_train)

            for train_index in range(len(_score_train)):
                # print("score testing for train_index = ", train_index)

                # print("_score_train[train_index] = ", _score_train[train_index], '\n')

                # substract 1 because score is one of these [-1, 0, 1] values
                if (predicted[train_index] - 1) == _score_train[train_index]:
                    # print("predicted == _score_train")
                    train_correct = train_correct + 1

            train_total = train_total + len(_score_train)

    print('Accuracy of the network on train score: %d %%' % (
            100 * train_correct / train_total))

    print("############################################")
    print("Doing test_accuracy check")

    # validate the trained NN model for both predicted recommended move
    # and its corresponding predicted score
    TEST_BATCH_SIZE = int(len(moves_score) * TEST_DATASET_RATIO)  # 4520*0.8

    test_correct = 0
    test_total = 0

    test_loader = zip(np.array(board_test, dtype='float32'),
                      np.array(moves_test, dtype='float32'),
                      np.array(score_test, dtype='float32'))
    test_loader = torch.utils.data.DataLoader(
        list(test_loader),
        batch_size=TEST_BATCH_SIZE,
    )
    # print(test_loader)

    with torch.no_grad():
        for _board_test, _moves_test, _score_test in test_loader:
            if USE_CUDA:
                _board_test = _board_test.cuda()
                _moves_test = _moves_test.cuda()
                _score_test = _score_test.cuda()

            model_input = _board_test
            _policy_output, _value_output = net(model_input)
            predicted = torch.argmax(_policy_output, 1)

            # print("_policy_output = ", _policy_output)
            # print("predicted = ", predicted)
            # print("_moves_test = ", _moves_test)

            for test_index in range(len(_moves_test)):
                # print("move testing for test_index = ", test_index)

                # print("_moves_test[test_index][predicted[test_index]] = ",
                #      _moves_test[test_index][predicted[test_index]], '\n')

                if _moves_test[test_index][predicted[test_index]]:
                    # print("predicted == _moves_test")
                    test_correct = test_correct + 1

            test_total = test_total + len(_moves_test)

    print('Accuracy of the network on test move: %d %%' % (
            100 * test_correct / test_total))

    test_correct = 0
    test_total = 0

    with torch.no_grad():
        for _board_test, _moves_test, _score_test in test_loader:
            if USE_CUDA:
                _board_test = _board_test.cuda()
                _moves_test = _moves_test.cuda()
                _score_test = _score_test.cuda()

            model_input = _board_test
            _policy_output, _value_output = net(model_input)
            predicted = torch.argmax(_value_output, 1)

            # print("_value_output = ", _value_output)
            # print("predicted = ", predicted)
            # print("_score_test = ", _score_test)

            for test_index in range(len(_score_test)):
                # print("move testing for test_index = ", test_index)

                # print("_score_test[test_index][predicted[test_index]] = ",
                #      _score_test[test_index][predicted[test_index]], '\n')

                # substract 1 because score is one of these [-1, 0, 1] values
                if (predicted[test_index] - 1) == _score_test[test_index]:
                    # print("predicted == _score_test")
                    test_correct = test_correct + 1

            test_total = test_total + len(_score_test)

    print('Accuracy of the network on test score: %d %%' % (
            100 * test_correct / test_total))


if __name__ == "__main__":
    train()

play.py

# This file is the test/inference coding
# See https://github.com/bsamseth/tictacNET/issues/2
# for a description of the inputs and output of the neural network

import numpy as np
import torch
import random
from Net import Net


CROSS, NOUGHT = 0, 1
PLAYERS = [CROSS, NOUGHT]
TOTAL_NUM__OF_BOXES = 9  # 3x3 tic-tac-toe


# Winning patterns encoded in bit patterns.
# E.g. three in a row in the top row is
#   448 = 0b111000000
WINNING_PATTERNS = [448, 56, 7, 292, 146, 73, 273, 84]  # Row  # Columns  # Diagonals


def binary_to_string(input_binary):
    mask = 0b1
    output = ""
    for i in range(TOTAL_NUM__OF_BOXES):
        if mask & input_binary:
            output += '1'
        else:
            output += '0'
        mask <<= 1
    return output[::-1]


def update_move(_next_move, _next_move_probabilities, _player_turn, _cross_positions, _nought_positions):
    print("we will update the next_move accordingly inside this function")

    _next_move = np.binary_repr(_next_move_probabilities.argmax())

    # What if the square box (next_move) is already filled by the other player ?
    # then we will have to go for the next less preferred next_move
    # During actual project in later stage, we will use Monte-Carlo Tree Search
    # instead of the following logic
    cross_positions_str = binary_to_string(int(_cross_positions, 2))
    nought_positions_str = binary_to_string(int(_nought_positions, 2))
    _next_move_in_integer = int(_next_move, 2)

    print("next_move in integer = ", _next_move_in_integer)
    print("cross_positions_str = ", cross_positions_str)
    print("nought_positions_str = ", nought_positions_str)

    # if opponent player had filled the square box position
    #    OR the same player had filled the same square box position
    if ((_player_turn == CROSS) and (nought_positions_str[_next_move_in_integer] == '1')) or \
       ((_player_turn == NOUGHT) and (cross_positions_str[_next_move_in_integer] == '1')) or \
       ((_player_turn == CROSS) and (cross_positions_str[_next_move_in_integer] == '1')) or \
       ((_player_turn == NOUGHT) and (nought_positions_str[_next_move_in_integer] == '1')):

        print("going for second preferred next move")
        print("next_move_probabilities = ", _next_move_probabilities)
        _next_move_probabilities[0, _next_move_in_integer] = 0  # makes way for less preferred next_move
        print("after setting certain bit to 0, next_move_probabilities = ", _next_move_probabilities)
        _next_move = np.binary_repr(_next_move_probabilities.argmax())

        # checks again whether this less preferred next_move had already been played before
        _next_move = update_move(_next_move, _next_move_probabilities, _player_turn,
                                 _cross_positions, _nought_positions)

    return _next_move


def players_have_winning_patterns(_cross_positions, _nought_positions):
    # needs to match every bits in the WINNING_PATTERNS
    if any(
            np.bitwise_and(win, int(_cross_positions, 2)) == win or
            np.bitwise_and(win, int(_nought_positions, 2)) == win
            for win in WINNING_PATTERNS
    ):
        return 1

    else:
        return 0


def initialize():
    # initial game config
    random_player_start_turn_ = random.randint(CROSS, NOUGHT)
    cross_positions_ = '000000000'
    nought_positions_ = '000000000'
    player_turn_ = random_player_start_turn_
    model_input_ = cross_positions_ + nought_positions_ + str(player_turn_)
    next_move_ = 99999999999999  # just for initialization
    # next_move_probabilities = np.zeros(TOTAL_NUM__OF_BOXES)  # 9 boxes choice
    # predicted_score = np.zeros(3)  # loss, draw, win
    # out = [next_move_probabilities, predicted_score]

    trained_model_path = './tictactoe_net.pth'

    # Load
    model_ = torch.load(trained_model_path)
    model_.eval()

    return cross_positions_, nought_positions_, model_, model_input_, next_move_, player_turn_


def play(using_mcts, best_child_node, model, model_input, next_move, player_turn, cross_positions, nought_positions):
    USE_CUDA = torch.cuda.is_available()

    if USE_CUDA:
        out_policy, out_value = model(torch.from_numpy(
            np.array([int(v) for v in model_input], dtype='float32')[np.newaxis]
        ).cuda())

    else:
        out_policy, out_value = model(torch.from_numpy(
            np.array([int(v) for v in model_input], dtype='float32')[np.newaxis]
        ))

    print("out_policy = ", out_policy)
    print("out_value = ", out_value)
    next_move_probabilities = out_policy

    if using_mcts:  # will determine next_move according to highest PUCT values of child nodes
        next_move = best_child_node
        next_move_in_integer = next_move
    else:
        # updates next_move
        next_move = update_move(next_move, next_move_probabilities, player_turn, cross_positions, nought_positions)
        next_move_in_integer = int(next_move, 2)

    print("Confirmed next_move = ", next_move_in_integer)

    # updates cross_positions or NAUGHT_POSITIONS (based on next_move output from NN)
    # depending on which player turn
    if player_turn == CROSS:
        # bitwise OR (cross_positions, next_move)
        cross_positions = binary_to_string(int(cross_positions, 2) |
                                           (1 << (TOTAL_NUM__OF_BOXES-next_move_in_integer-1)))

    else:
        # bitwise OR (nought_positions, next_move)
        nought_positions = binary_to_string(int(nought_positions, 2) |
                                            (1 << (TOTAL_NUM__OF_BOXES-next_move_in_integer-1)))

    print("cross_positions = ", cross_positions)
    print("nought_positions = ", nought_positions)

    print("player_turn = ", player_turn)

    # updates model_input for next player turn
    model_input = cross_positions + nought_positions + str(player_turn)

    print("model_input = ", model_input)
    print("\n")

    return out_value, cross_positions, nought_positions


if __name__ == '__main__':

    print("standalone inference coding")
    game_is_on = 1
    num_of_play_rounds = 0
    _cross_positions, _nought_positions, _model, _model_input, __next_move, _player_turn = initialize()

    # while (cross_positions != WINNING_PATTERNS) | (nought_positions != WINNING_PATTERNS):
    while game_is_on:  # game is still ON
        num_of_play_rounds = num_of_play_rounds + 1

        __out_value, __cross_positions, __nought_positions \
            = play(0, 0, _model, _model_input, __next_move, _player_turn, _cross_positions, _nought_positions)

        out_score = torch.argmax(__out_value)
        game_is_on = (players_have_winning_patterns(__cross_positions, __nought_positions) == 0)

        _cross_positions = __cross_positions
        _nought_positions = __nought_positions

        # switches player turn after each step
        if _player_turn == CROSS:
            _player_turn = NOUGHT

        else:
            _player_turn = CROSS

    game_is_on = 0
    print("game finished")

else:  # executed from mcts.py

    print("using mcts")
    _cross_positions = 0
    _nought_positions = 0
    _model = 0
    _model_input = 0
    __next_move = 0
    _player_turn = 0
    num_of_play_rounds = 0
    game_is_on = 0  # initialized to 0 because game has not started yet
    out_score = 0

    def mcts_play(is_mcts_in_simulate_stage=0, ongoing_game=0, best_child_node=0):

        global _cross_positions
        global _nought_positions
        global _model
        global _model_input
        global __next_move
        global _player_turn
        global num_of_play_rounds
        global game_is_on
        global out_score

        if is_mcts_in_simulate_stage:
            print("is_mcts_in_simulate_stage")

            if ongoing_game == 0:  # first step of the game
                print("ongoing_game == 0")
                game_is_on = 1
                num_of_play_rounds = 0
                _cross_positions, _nought_positions, _model, _model_input, __next_move, _player_turn = initialize()

            print("ongoing_game == 1")
            num_of_play_rounds = num_of_play_rounds + 1

            __out_value, __cross_positions, __nought_positions \
                = play(1, best_child_node, _model, _model_input, __next_move, _player_turn,
                       _cross_positions, _nought_positions)

            out_score = torch.argmax(__out_value)
            game_is_on = (players_have_winning_patterns(__cross_positions, __nought_positions) == 0)

            _cross_positions = __cross_positions
            _nought_positions = __nought_positions

            # switches player turn after each step
            print("switches player turn")
            if _player_turn == CROSS:
                _player_turn = NOUGHT

            else:
                _player_turn = CROSS

            if game_is_on == 0:
                print("game finished")

        return None

tictactoe-data.csv