gdq historical statistics

data.txt

- - - EVENT: AGDQ 2016 (id: 17) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 221
Total runners: 185
New runners: 185 (100.00%)
Duplicate runners: 36 (16.29%)
- - - EVENT: SGDQ 2016 (id: 18) - - -
(ignored 3 preshow-only IDs)
Runners IDs assigned to runs: 229
Total runners: 188
New runners: 153 (81.38%)
Duplicate runners: 41 (17.90%)
- - - EVENT: AGDQ 2017 (id: 19) - - -
(ignored 4 preshow-only IDs)
Runners IDs assigned to runs: 239
Total runners: 186
New runners: 106 (56.99%)
Duplicate runners: 53 (22.18%)
- - - EVENT: SGDQ 2017 (id: 20) - - -
(ignored 4 preshow-only IDs)
Runners IDs assigned to runs: 175
Total runners: 147
New runners: 72 (48.98%)
Duplicate runners: 27 (15.43%)
- - - EVENT: AGDQ 2018 (id: 22) - - -
(ignored 3 preshow-only IDs)
Runners IDs assigned to runs: 198
Total runners: 161
New runners: 86 (53.42%)
Duplicate runners: 37 (18.69%)
- - - EVENT: SGDQ 2018 (id: 23) - - -
(ignored 5 preshow-only IDs)
Runners IDs assigned to runs: 183
Total runners: 162
New runners: 85 (52.47%)
Duplicate runners: 21 (11.48%)
- - - EVENT: AGDQ 2019 (id: 25) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 168
Total runners: 153
New runners: 76 (49.67%)
Duplicate runners: 15 (8.93%)
- - - EVENT: SGDQ 2019 (id: 26) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 181
Total runners: 161
New runners: 79 (49.07%)
Duplicate runners: 20 (11.05%)
- - - EVENT: AGDQ 2020 (id: 28) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 205
Total runners: 182
New runners: 87 (47.80%)
Duplicate runners: 23 (11.22%)
- - - EVENT: SGDQ 2020 (id: 30) - - -
(ignored 6 preshow-only IDs)
Runners IDs assigned to runs: 164
Total runners: 151
New runners: 94 (62.25%)
Duplicate runners: 13 (7.93%)
- - - EVENT: AGDQ 2021 (id: 34) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 194
Total runners: 161
New runners: 91 (56.52%)
Duplicate runners: 33 (17.01%)
- - - EVENT: SGDQ 2021 (id: 35) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 172
Total runners: 160
New runners: 95 (59.38%)
Duplicate runners: 12 (6.98%)
- - - EVENT: AGDQ 2022 (id: 37) - - -
(ignored 0 preshow-only IDs)
Runners IDs assigned to runs: 202
Total runners: 174
New runners: 96 (55.17%)
Duplicate runners: 28 (13.86%)
- - - EVENT: SGDQ 2022 - - -
Total runners: 149
New runners: 70 (46.98%)
Duplicate runners: 15  (9.15%)

New Runners:
[
  'legrandgrand',       'juh0',            'jaekrock',
  'space coast gaming', 'glitchiness',     'jimmy_diamonds',
  'deceptix_',          'groshiken',       'mrmightymouse_',
  'groggydog',          'demonchildelise', 'potty',
  'muted',              'hyp3rsomniac',    'doctordt',
  'jexvrok',            'seto kiaba',      'drewlith',
  'foxyjira',           'ace delusional',  'artful hobbes',
  'ritzblues',          'sanchopanda',     'bramhallthefifth',
  'danthevp',           'cicada',          'amoser',
  'hibnotix',           'aciidz',          'juraspatryk',
  'rskiller',           'nes_',            'josephjoestar316',
  'kekumanshoyu',       'jalbagel',        'pochilin',
  'the retro runner',   'mega retroman',   'starsmiley',
  'katie4',             'mckiddy',         'zero_if',
  'achocolateorange',   'peace egg',       'astralspiff',
  'kaos_wulf',          'epiphane',        'rubiehart',
  'maddyinc',           'nave357',         'leviern',
  'windeu',             'leadostrich',     'drunkendraconian',
  'isofrieze',          'carlsagan42',     'shoujo',
  'tanukidan',          'raysfire',        '-ganon-',
  'shovda',             'thirdwall',       'nukkuler',
  'savestate',          'sauraen',         'stevezero01',
  'maethedkc',          'eldritchwolfie',  'kamacrimson',
  'enel'
]

package.json

{
  "name": "gdq-scraper",
  "version": "1.0.0",
  "main": "index.js",
  "scripts": {
    "start": "node index.js"
  },
  "license": "MIT",
  "dependencies": {
    "isomorphic-fetch": "^3.0.0",
    "lodash": "^4.17.21"
  }
}

script.js

const fetch = require('isomorphic-fetch');
const _ = require('lodash');

const EVENT_IDS = {
  // 1: "SGDQ 2012",
  // 2: "AGDQ 2012",
  // 3: "SGDQ 2011",
  // 5: "AGDQ 2011",
  // 6: "CGDQ",
  // 7: "AGDQ 2013",
  // 8: "SGDQ 2013",
  // 9: "AGDQ 2014",
  // 10: "SGDQ 2014",
  // 11: "AGDQ 2015",
  // 16: "SGDQ 2015",
  17: "AGDQ 2016",
  18: "SGDQ 2016",
  19: "AGDQ 2017",
  20: "SGDQ 2017",
  22: "AGDQ 2018",
  23: "SGDQ 2018",
  25: "AGDQ 2019",
  26: "SGDQ 2019",
  28: "AGDQ 2020",
  30: "SGDQ 2020",
  34: "AGDQ 2021",
  35: "SGDQ 2021",
  37: "AGDQ 2022",
};

const IGNORED_RUNNER_IDS = [
  1434, // Interview Crew
  367, // Tech Crew
  885, // GDQ Staff
];

// runners that used different aliases in the past
const RUNNER_ALIASES = {
  'cheese05': 'cheese',
  'ghost king': 'ghostkumo',
};

// taken from https://gamesdonequick.com/submission/games-list and running:
//   const runners = [...document.querySelectorAll('#runTable tbody tr > td:first-child')].map(x => x.textContent.trim())
//   JSON.stringify(runners.map(x => x.replace('[Remote Runner]', '').trim()))
const sgdq2022Runners = JSON.parse("[\"legrandgrand\",\"tbcr\",\"juh0\",\"JaekRock\",\"Space Coast Gaming\",\"Glitchiness\",\"duck\",\"Jimmy_Diamonds\",\"UraniumAnchor\",\"davidtki\",\"The Blacktastic\",\"Deceptix_\",\"JaekRock\",\"Aquas\",\"JupiterClimb\",\"GrosHiken\",\"MrMightyMouse_\",\"grandpoobear\",\"groggydog\",\"Obiyo\",\"DemonchildElise\",\"DemonicRobots\",\"Potty\",\"Shockwve\",\"WaifuRuns\",\"Tonkotsu\",\"spikevegeta\",\"MuteD\",\"ZELLLOOO\",\"catalystz\",\"HYP3RSOMNIAC\",\"Vynn\",\"DoctorDT\",\"Jexvrok\",\"seto kiaba\",\"drewlith\",\"FoxyJira\",\"Ace Delusional\",\"Artful Hobbes\",\"Ritzblues\",\"KZ_FREW\",\"SaberaMesia\",\"Benja\",\"Buffet Time\",\"WaifuRuns\",\"teddyras\",\"SanchoPanda\",\"bramhallthefifth\",\"DanTheVP\",\"Ghoul02\",\"GhostKumo\",\"Barbarian\",\"cicada\",\"davidtki\",\"Mr_Shasta\",\"swordsmankirby\",\"amoser\",\"Hibnotix\",\"aciidz\",\"JurasPatryk\",\"RsKiller\",\"Ozmourn\",\"nes_\",\"Elipsis\",\"Breakdown\",\"Kefka15\",\"Biglaw\",\"Omnigamer\",\"duck\",\"MikeWave\",\"Lizstar\",\"Ppotdot1\",\"WhiteHat94\",\"The Blacktastic\",\"FocusSight\",\"FocusSight\",\"JosephJoestar316\",\"Mekarazium\",\"dlimes13\",\"kekumanshoyu\",\"Illumina\",\"Bullets\",\"JalBagel\",\"pochilin\",\"The Retro Runner\",\"Mega RetroMan\",\"starsmiley\",\"sharkhat87\",\"Katie4\",\"McKiddy\",\"Zero_IF\",\"AChocolateOrange\",\"NPC\",\"FlannelKat\",\"Peace Egg\",\"Brossentia\",\"eddaket\",\"360Chrism\",\"Keizaron\",\"Shenanagans\",\"adef\",\"quo\",\"AstralSpiff\",\"Can't Even\",\"Kaos_Wulf\",\"Xem\",\"Helix\",\"Epiphane\",\"RUBIEHART\",\"DanTheVP\",\"AnEternalEnigma\",\"MaddyInc\",\"Nave357\",\"bryonato\",\"Kirbymastah\",\"Katie4\",\"Allegro\",\"leviern\",\"Dayoman\",\"SpootyBiscuit\",\"WinDEU\",\"LeadOstrich\",\"DrunkenDraconian\",\"Crunan\",\"Bullets\",\"davidtki\",\"Simply\",\"cheese\",\"SethBling\",\"IsoFrieze\",\"CarlSagan42\",\"grandpoobear\",\"Shoujo\",\"PangaeaPanga\",\"TanukiDan\",\"raysfire\",\"Aurateur\",\"-ganon-\",\"CarlSagan42\",\"juzcook\",\"kezcade\",\"Shoujo\",\"shovda\",\"ThirdWall\",\"Oatsngoats\",\"Glan\",\"Nukkuler\",\"TASBot\",\"dwangoAC\",\"spikevegeta\",\"Savestate\",\"Sauraen\",\"gymnast86\",\"SteveZero01\",\"Maxylobes\",\"MaeTheDKC\",\"sharif\",\"Sunnymuffin\",\"EldritchWolfie\",\"KamaCrimson\",\"Jaxler\",\"Mr_Shasta\",\"Enel\",\"Froob\"]");

let previousRunners = [];

function asPercent(a, b) {
  return `${((a / b) * 100).toFixed(2)}%`;
}

(async () => {
  const eventIds = Object.keys(EVENT_IDS).map(Number);

  eventIds.sort((a, b) => a - b);


  for (let id of eventIds) {
    const runnerData = await fetch(`https://gamesdonequick.com/tracker/search?type=runner&event=${id}`);
    const runnerDataJson = await runnerData.json();

    const runData = await fetch(`https://gamesdonequick.com/tracker/search?type=run&event=${id}`);
    const runDataJson = await runData.json();

    const preshowRuns = runDataJson.filter(x => x.fields.name === 'Pre-Show');
    const nonPreshowRuns = runDataJson.filter(x => x.fields.name !== 'Pre-Show');

    const allNonPreshowIds = _.difference(nonPreshowRuns.reduce((acc, x) => [...acc, ...x.fields.runners], []), IGNORED_RUNNER_IDS);
    const allPreshowIds = _.difference(preshowRuns.reduce((acc, x) => [...acc, ...x.fields.runners], []), IGNORED_RUNNER_IDS);

    const preshowOnlyIds = allPreshowIds.filter(x => allNonPreshowIds.indexOf(x) === -1);
    
    // const runnersByPronouns = runnerDataJson.reduce((pronounsAcc, runner) => {
    //   const pronouns = (runner.fields.pronouns || '').toLowerCase() || '<none>';

    //   return 
    //     ...pronounsAcc,
    //     [pronouns]: (pronounsAcc[pronouns] || 0) + 1,
    //   };
    // }, {});

    console.log(`- - - EVENT: ${EVENT_IDS[id]} (id: ${id}) - - -`);

    console.log(`(ignored ${preshowOnlyIds.length} preshow-only IDs)`);

    console.log(`Runners IDs assigned to runs: ${allNonPreshowIds.length}`);
    
    // console.log('Pronoun weighting:');

    // Object.entries(runnersByPronouns).forEach(([pronouns, count]) => {
    //   console.log(`  ${pronouns}: ${count}`);
    // });

    const relevantRunnerData = runnerDataJson.filter(x => IGNORED_RUNNER_IDS.indexOf(x.pk) === -1 && preshowOnlyIds.indexOf(x.pk) === -1);
    const eventRunnerNames = _.uniq(relevantRunnerData.map(runner => runner.fields.name.toLowerCase()));
    const aliasedRunnerNames = eventRunnerNames.map(name => {
      return RUNNER_ALIASES[name] ?? name;
    })
    const totalRunners = aliasedRunnerNames.length;

    console.log(`Total runners: ${totalRunners}`);

    const eventDiff = _.difference(aliasedRunnerNames, previousRunners);

    console.log(`New runners: ${eventDiff.length} (${asPercent(eventDiff.length, totalRunners)})`);

    const uniqueIds = _.uniq(allNonPreshowIds).length;
    console.log(`Duplicate runners: ${allNonPreshowIds.length - uniqueIds} (${asPercent(allNonPreshowIds.length - uniqueIds, allNonPreshowIds.length)})`);

    previousRunners = _.uniq([
      ...previousRunners,
      ...aliasedRunnerNames,
    ]);
  }

  const normalizedSGDQ2022Runners = sgdq2022Runners.map(x => x.toLowerCase());
  const uniqNormalized2022Runners = _.uniq(normalizedSGDQ2022Runners);

  const diff = _.difference(uniqNormalized2022Runners, previousRunners);
  const dupeCount = normalizedSGDQ2022Runners.length - uniqNormalized2022Runners.length;

  console.log(`- - - EVENT: SGDQ 2022 - - -`);
  console.log(`Total runners: ${uniqNormalized2022Runners.length}`);
  console.log(`New runners: ${diff.length} (${asPercent(diff.length, uniqNormalized2022Runners.length)})`);
  console.log(`Duplicate runners: ${dupeCount}  (${asPercent(dupeCount, sgdq2022Runners.length)})`)
  console.log('\nNew Runners:')
  console.log(diff);
  console.log(`(calculated from ${previousRunners.length} previous runners)`);
})().then(() => console.log('Done!'));