lc0 q to cp eval fitting

download_tcec_evals.py

#!/usr/bin/python3
from os import path
import urllib.request

base_url = 'https://www.tcec-chess.com/json/TCEC_Season_17_-_Superfinal_{}.pgjson'
games = list(range(1, 17+1))

for game in games:
    url = base_url.format(game)
    basename = url.split('/')[-1]
    if path.exists(basename):
        print('Already have game {}'.format(game))
        continue
    print('Downloading {}'.format(url))
    response = urllib.request.urlopen(url).read()
    with open(basename, 'wb') as f:
        f.write(response)

lc0_eval.fit.py

#!/usr/bin/python3
import numpy as np
import glob
import matplotlib.pyplot as plt
import json
from scipy.optimize import root, curve_fit, minimize
from scipy import stats
plt.style.use('ggplot')

def move_to_float(m):
    if '-M' in m or '-#' in m:
        return -128
    if 'M' in m or '#' in m:
        return 128
    m = float(m)
    if m > 128:
        return 128
    if m < -128:
        return -128
    return m

def get_evals(game):
    with open(game, 'r') as f:
        game = json.loads(f.read())

    lc0_black = 'LCZero' in game['Headers']['Black']
    black_eval = []
    white_eval = []
    for e, move in enumerate(game['Moves']):
        if move['book']:
            continue
        if e % 2 == 0:
            white_eval.append(move_to_float(move['wv']))
        else:
            black_eval.append(move_to_float(move['wv']))

    # Returns [lc0_eval, sf_eval]
    if lc0_black:
        lc0_eval, sf_eval = black_eval, white_eval
    else:
        lc0_eval, sf_eval = white_eval, black_eval

    if len(lc0_eval) > len(sf_eval):
        lc0_eval = lc0_eval[:-1]
    if len(sf_eval) > len(lc0_eval):
        sf_eval = sf_eval[:-1]
    assert len(lc0_eval) == len(sf_eval)

    return lc0_eval, sf_eval

games = glob.glob('*.pgjson')
lc0_eval, sf_eval = [], []
for game in games:
    lc0, sf = get_evals(game)
    lc0_eval.extend(lc0)
    sf_eval.extend(sf)

def q_to_cp(q):
    return 295 * q / (1 - 0.976953126 * q**14)

def q_to_cp_2018(q):
    return 290.680623072 * np.tan(1.548090806 * q)

def q_to_cp_new(q):
    return 111.7 * np.tan(1.5620688421 * q)

def cp_to_q(cp):
    f = lambda q: 295 * q / (1 - 0.976953126 * q**14) - cp
    return root(f, x0=0).x[0]

def abs_loss(f_cp, qs, evals):
    l = 0
    for e in range(len(evals)):
        if qs[e] < 0:
            continue
        if np.isnan(evals[e]):
            continue
        l += np.abs(0.01 * f_cp(qs[e]) - evals[e])
    return l / len(evals)

sf_eval10 = [min(10, max(-10, e)) for e in sf_eval]
q = [cp_to_q(100 * cp) for cp in lc0_eval]
q10 = [cp_to_q(100 * min(10, max(-10, cp))) for cp in lc0_eval]

sf_eval10_bins = stats.binned_statistic(q, sf_eval, 'median', bins=40, range=(-1,1))
bin_centers = 0.5 * (sf_eval10_bins.bin_edges[1:] + sf_eval10_bins.bin_edges[:-1])

plt.figure()
plt.scatter(q, sf_eval10, label='SF eval')
qs = np.linspace(-1, 1, 100)
cps = [0.01 * cp for cp in q_to_cp(qs)]
cps_2018 = [0.01 * cp for cp in q_to_cp_2018(qs)]
cps_new = [0.01 * cp for cp in q_to_cp_new(qs)]
print('centipawn', abs_loss(q_to_cp, bin_centers, sf_eval10_bins.statistic))
print('centipawn_2018', abs_loss(q_to_cp_2018, bin_centers, sf_eval10_bins.statistic))
print('centipawn_pr841', abs_loss(q_to_cp_new, bin_centers, sf_eval10_bins.statistic))
plt.plot(bin_centers, sf_eval10_bins.statistic, label='Median SF eval', color='y')
plt.plot(qs, cps, color='b', linestyle='--', label='centipawn')
plt.plot(qs, cps_2018, color='g', linestyle='--', label='centipawn_2018')
plt.plot(qs, cps_new, color='g', linestyle='--', label='centipawn_pr841')
plt.legend(loc='upper left')

plt.xlabel('Lc0 Q')
plt.ylabel('SF eval')
plt.xlim([-1, 1])
plt.ylim([-10, 10])

plt.show()