Testing openskill, specifically whether rating can go down on a win

openskillTesting.js

const util = require('util');
const { range, random, chain, min, max, mean, takeRight, cloneDeep, orderBy } = require('lodash');
const { rating, ordinal, rate } = require('openskill');

const PLAYER_COUNT = 100;
const START_MU = 25;
const START_SIGMA = 25.0 / 3.0;
const MATCH_COUNT = PLAYER_COUNT * 1000; // Creates ~200 games per player
// const TAU = 25.0 / 300.0;
const TAU = START_SIGMA / 100.0;
const ORDINAL_SCALING = 25.0;
const ORDINAL_OFFSET = 1100.0;
// const ORDINAL_SCALING = 1;
// const ORDINAL_OFFSET = 0;

// const PLAYER_COUNT = 100;
// const START_MU = 1200.0;
// const START_SIGMA = 1200.0 / 3.0;
// const MATCH_COUNT = PLAYER_COUNT * 1000; // Creates ~200 games per player
// // const TAU = 25.0 / 300.0;
// const TAU = 14.4;
// const ORDINAL_SCALING = 0.5;
// const ORDINAL_OFFSET = 600.0;

const logDeep = (obj, depth=10) => {
  console.log(util.inspect(obj, { colors: true, depth: depth, maxArrayLength: null }));
}

const slippiOrdinal = (rating) => {
  // The constants used get a range fairly close to glicko's range
  // https://docs.google.com/spreadsheets/d/1rDPjiYX8JaoGebTlIK_2OYeRzEVm2fcRTJOsMYKY-1Q/edit#gid=0
  return ORDINAL_SCALING * (rating.mu - 3 * rating.sigma) + ORDINAL_OFFSET
  // return rating.mu - 3 * rating.sigma;
};

// https://stackoverflow.com/a/49434653/1249024
const randn_bm = () => {
  let u = 0, v = 0;
  while(u === 0) u = Math.random(); //Converting [0,1) to (0,1)
  while(v === 0) v = Math.random();
  let num = Math.sqrt( -2.0 * Math.log( u ) ) * Math.cos( 2.0 * Math.PI * v );
  num = num / 10.0 + 0.5; // Translate to 0 -> 1
  if (num > 1 || num < 0) return randn_bm() // resample between 0 and 1
  return num
}

const playMatches = (players, count, tau = 0) => {
  const games = [];
  const changes = [];

  let ratingDropOnWinCount = 0;

  const skillSortedPlayers = orderBy(players, 'skill');
  const playerToFollow = skillSortedPlayers[PLAYER_COUNT / 2];

  const getRating = r => ({
    mu: r.mu,
    sigma: r.sigma,
    ordinal: slippiOrdinal(r)
  });

  range(0, count, 1).forEach(() => {
    const p1Idx = random(PLAYER_COUNT - 1);
    let p2Idx = random(PLAYER_COUNT - 1);
    while (p2Idx === p1Idx) {
      p2Idx = random(PLAYER_COUNT - 1);
    }

    const p1 = players[p1Idx];
    const p2 = players[p2Idx];

    p1.gamesPlayed += 1;
    p2.gamesPlayed += 1;

    // This is probably a shit way to decide who wins
    const lossProbability = 0.5 - 0.4 * (p1.skill - p2.skill);
    const gameResultRoll = Math.random();
    const result = gameResultRoll > lossProbability ? [1, 2] : [2, 1];

    const game = {
      result: result,
      player1: {
        change: {
          previous: getRating(p1.rating),
        },
        change2: {
          previous: getRating(p1.rating2),
        },
      },
      player2: {
        change: {
          previous: getRating(p2.rating),
        },
        change2: {
          previous: getRating(p2.rating2),
        },
      },
    };

    // console.log({
    //   p1Index: p1.index,
    //   p1Skill: p1.skill,
    //   p2Index: p2.index,
    //   p2Skill: p2.skill,
    //   lossProbability: lossProbability,
    //   gameResultRoll: gameResultRoll,
    //   result: result,
    // });

    const [[r1], [r2]] = rate([[p1.rating], [p2.rating]], {
      rank: result,
    });

    p1.rating = r1;
    p2.rating = r2;

    const [[r1_2], [r2_2]] = rate([[p1.rating2], [p2.rating2]], {
      rank: result,
      tau: TAU,
      // preventSigmaIncrease: true,
    });

    p1.rating2 = r1_2;
    p2.rating2 = r2_2;

    game.player1.change.new = getRating(p1.rating);
    game.player1.change2.new = getRating(p1.rating2);
    game.player1.change.amount = game.player1.change.new.ordinal - game.player1.change.previous.ordinal;
    game.player1.change2.amount = game.player1.change2.new.ordinal - game.player1.change2.previous.ordinal;
    game.player2.change.new = getRating(p2.rating);
    game.player2.change2.new = getRating(p2.rating2);
    game.player2.change.amount = game.player2.change.new.ordinal - game.player2.change.previous.ordinal;
    game.player2.change2.amount = game.player2.change2.new.ordinal - game.player2.change2.previous.ordinal;

    if (result[0] == 1 && (game.player1.change.amount < 0 || game.player1.change2.amount < 0)) {
      ratingDropOnWinCount++;
    }

    if (result[1] == 1 && (game.player2.change.amount < 0 || game.player2.change2.amount < 0)) {
      ratingDropOnWinCount++;
    }

    games.push(game);

    if (p1 === playerToFollow) {
      changes.push({
        gameNum: p1.gamesPlayed,
        ratingChange: Math.abs(game.player1.change.amount),
        ratingChangeWithTau: Math.abs(game.player1.change2.amount),
        sigma: p1.rating.sigma,
        sigmaWithTau: p1.rating2.sigma,
      });
    } else if (p2 === playerToFollow) {
      changes.push({
        gameNum: p2.gamesPlayed,
        ratingChange: Math.abs(game.player2.change.amount),
        ratingChangeWithTau: Math.abs(game.player2.change2.amount),
        sigma: p2.rating.sigma,
        sigmaWithTau: p2.rating2.sigma,
      });
    }
  });

  return {
    games: games,
    trackedPlayerChanges: changes,
    ratingDropOnWinCount: ratingDropOnWinCount,
  };
};

const printPlayers = (players) => {
  console.log("Index, Skill, GamesPlayed, Rating, Mu, Sigma, Rating2, Mu2, Sigma2");
  players.forEach((p) => {
    console.log(`${p.index}, ${p.skill}, ${p.gamesPlayed}, ${slippiOrdinal(p.rating)}, ${p.rating.mu}, ${p.rating.sigma}, ${slippiOrdinal(p.rating2)}, ${p.rating2.mu}, ${p.rating2.sigma}`);
  });
};

const printDistribution = (players) => {
  console.log("Rating, Count");
  const results = chain(players).groupBy(p => Math.floor(slippiOrdinal(p.rating) / 50))
    .map((v, k) => ({ rating: parseInt(k) * 50 + 25, count: v.length }))
    .orderBy((v) => v.rating)
    .value();

  results.forEach(r => {
    console.log(`${r.rating}, ${r.count}`);
  });
};

const printAggregates = (players) => {
  const ratings = players.map(p => slippiOrdinal(p.rating));
  console.log({
    minRating: min(ratings),
    maxRating: max(ratings),
    averageRating: mean(ratings),
  });
};

const printChanges = (changes) => {
  console.log("GameNum, RatingChange, RatingChangeWithTau, Sigma, SigmaWithTau");
  changes.forEach(c => {
    console.log(`${c.gameNum}, ${c.ratingChange}, ${c.ratingChangeWithTau}, ${c.sigma}, ${c.sigmaWithTau}`)
  });
}

// Sheet with testing data:
// https://docs.google.com/spreadsheets/d/1KDnA_w_RwGDzS_5ErhrIwxQHiChAM3JBdMqCXhJ6qUs/edit#gid=1174040382
(async () => {
  console.log("Initializing Players...");
  const players = range(0, PLAYER_COUNT, 1).map((idx) => {
    const skill = randn_bm() * 10;
    // const skill = 5;
    return {
      skill: skill,
      index: idx,
      rating: rating({ mu: START_MU, sigma: START_SIGMA }),
      rating2: rating({ mu: START_MU, sigma: START_SIGMA }),
      gamesPlayed: 0,
    };
  });

  // Run 10000 matches
  // TODO: Perhaps add matchmaking requirements?
  console.log(`Playing ${MATCH_COUNT} matches...`);
  const results = playMatches(players, MATCH_COUNT, TAU);

  console.log("Printing players...");
  printPlayers(players);

  console.log("Printing aggregates...");
  printAggregates(players);

  console.log("Printing rating changes...");
  printChanges(results.trackedPlayerChanges);

  console.log(`Rating dropped on a win: ${results.ratingDropOnWinCount} times`);

  // logDeep(takeRight(games, 5));

  // console.log("Printing distribution...");
  // printDistribution(players);

  // const newSeasonPlayers = cloneDeep(players);

  // // Modify rating values to simulate season reset
  // newSeasonPlayers.forEach(p => {
  //   // Increase uncertainty
  //   p.rating.sigma += 3;
  //   if (p.rating.sigma > START_SIGMA) {
  //     p.rating.sigma = START_SIGMA;
  //   }

  //   // Scrunch mu's around 23
  //   p.rating.mu = 0.8 * (p.rating.mu - 23) + 23
  // });

  // console.log(`Playing ${MATCH_COUNT} matches after season reset...`);
  // playMatches(newSeasonPlayers, MATCH_COUNT);

  // console.log("Printing players after season reset and games...");
  // printPlayers(newSeasonPlayers);

  // console.log(`Playing ${MATCH_COUNT} matches without rating reset...`);
  // playMatches(players, MATCH_COUNT);

  // console.log("Printing players with additional games but no rating reset...");
  // printPlayers(players);
})();