TRN API Warzone Scraper

README.md

Tracker.gg API Warzone Scraper

This was once used in Stagg but was removed for a few reasons:

1. Data was slow to update and fetch
2. No authentication leads to easily exceeding rate limits
3. Match data was missing player loadouts among other things
4. Profiles cannot be fetched until they are initialized (must visit profile URL before fetching data)

How-to

If you're still interesting in interacting with this API, transactions are simple and straightforward. The next param is a microsecond timestamp that can be pulled from the last match returned in the list. For Warzone matches, you must first fetch the match list and subsequently fetch match details for each corresponding match in the list from the previous request.

All URLs use the same platform identifiers and username constraints. All usernames must be URL encoded (eg: # = %23). Platform identifiers are listed below.

Activision = atvi
Battle.net = battlenet

Warzone/Multiplayer Matches URL

warzone: { version: 'v1', game: 'warzone', type: 'wz', next: null }
multiplayer: { version: 'v1', game: 'modern-warfare', type: 'mp', next: null }
https://api.tracker.gg/api/:version/:game/matches/:platform/:username?type=:type&next=:next

Warzone Match Details URL

{ version: 'v1', game: 'warzone' }
https://api.tracker.gg/api/:version/:game/matches/:matchId

api.ts

import axios, { Method } from 'axios'

const BaseURL = 'https://api.tracker.gg/api/v1'
const getStat = (stats:any, stat:string) => stats && stats[stat] ? stats[stat].value : 0

export namespace Warzone {
    export const Initialize = async (platform:string, player:string) => 
        GenericRequest(Requests.Warzone.Initialize(platform, player))
    export const MatchList = async (platform:string, player:string, next:number=null):Promise<Types.Warzone.Matches> => 
        GenericRequest(Requests.Warzone.MatchList(platform, player, next))
    export const MatchDetails = async (matchId:string|number):Promise<Types.Warzone.MatchDetails> => 
        GenericRequest(Requests.Warzone.MatchDetails(matchId))
}
export namespace Normalize {
    export const Match = (match:Types.Warzone.MatchSummary):any => ({
        matchId: match.attributes.id,
        mapId: match.attributes.mapId,
        modeId: match.attributes.modeId,
        utcSecStart: new Date(match.metadata.timestamp).getSeconds(),
        utcSecEnd: new Date(match.metadata.timestamp).getSeconds() + match.metadata.duration.value,
    })
    export const Teams = (matchDetails:Types.Warzone.MatchDetails) => {
        const teams:any = []
        for(const seg of matchDetails.segments) {
            let team = teams.find((team:any) => team.name === seg.attributes.team)
            if (!team) {
                teams.push({
                    name: seg.attributes.team,
                    placement: seg.metadata.placement.value,
                    time: 0,
                    players: [],
                })
                team = teams[teams.length-1]
            }
            team.placement = seg.metadata.placement.value
            team.players.push({
                username: seg.metadata.platformUserHandle,
                clantag: seg.metadata.clanTag,
                platform: seg.attributes.platformSlug,
                rank: null, // rank information is not available via TRN (rank is currently useless anyway - 05/26/2020)
                loadouts: [], // loadout information is not available via TRN
                stats: {
                    kills: getStat(seg.stats, 'kills'),
                    deaths: getStat(seg.stats, 'deaths'),
                    score: getStat(seg.stats, 'score'),
                    assists: getStat(seg.stats, 'assists'),
                    headshots: getStat(seg.stats, 'headshots'),
                    executions: getStat(seg.stats, 'executions'),
                    damageDone: getStat(seg.stats, 'damageDone'),
                    damageTaken: getStat(seg.stats, 'damageTaken'),
                    longestStreak: getStat(seg.stats, 'longestStreak'),
                    timePlayed: getStat(seg.stats, 'timePlayed'),
                    distanceTraveled: getStat(seg.stats, 'distanceTraveled'),
                    percentTimeMoving: getStat(seg.stats, 'percentTimeMoving'),
                },
            })
        }
        return teams
    }
    export const Performance = (match:Types.Warzone.MatchSummary, player:{ _id:string }) => {
        const normalizedPerformance:any = {}
        normalizedPerformance.matchId = match.attributes.id
        normalizedPerformance.player = {
            _id: player._id,
            username: match.segments[0].metadata.platformUserHandle,
            clantag: match.segments[0].metadata.clanTag,
        }
        // Count downs
        let downs = []
        const downKeys = Object.keys(match.segments[0].stats).filter(key => key.includes('objectiveBrDownEnemyCircle'))
        for(const key of downKeys) {
            const circleIndex = Number(key.replace('objectiveBrDownEnemyCircle', ''))
            downs[circleIndex] = getStat(match.segments[0].stats, key)
        }
        try {
            normalizedPerformance.stats = {
                kills: getStat(match.segments[0].stats, 'kills'),
                deaths: getStat(match.segments[0].stats, 'deaths'),
                downs,
                gulagKills: getStat(match.segments[0].stats, 'gulagKills'),
                gulagDeaths: getStat(match.segments[0].stats, 'gulagDeaths'),
                revives: getStat(match.segments[0].stats, 'objectiveReviver'),
                contracts: getStat(match.segments[0].stats, 'objectiveBrMissionPickupTablet'),
                teamWipes: getStat(match.segments[0].stats, 'objectiveTeamWiped'),
                lootCrates: getStat(match.segments[0].stats, 'objectiveBrCacheOpen'),
                buyStations: getStat(match.segments[0].stats, 'objectiveBrKioskBuy'),
                teamPlacement: match.segments[0].metadata.placement,
                teamSurvivalTime: getStat(match.segments[0].stats, 'teamSurvivalTime'),
                xp: {
                    score: getStat(match.segments[0].stats, 'scoreXp'),
                    match: getStat(match.segments[0].stats, 'matchXp'),
                    bonus: getStat(match.segments[0].stats, 'bonusXp'),
                    medal: getStat(match.segments[0].stats, 'medalXp'),
                    misc: getStat(match.segments[0].stats, 'miscXp'),
                    challenge: getStat(match.segments[0].stats, 'challengeXp'),
                }
            }
        } catch(e) {
            console.log(`    Error parsing performance stats: ${e}`)
        }
        return normalizedPerformance
    }
}

interface Request {
    options: {
        method: Method
        url: string
    }
}
namespace Requests {
    export namespace Warzone {
        export const Initialize = (platform:string, player:string):Request => ({
            options: {
                method: 'GET',
                url: `https://cod.tracker.gg/warzone/profile/${platform}/${encodeURIComponent(player)}/matches`
            }
        })
        export const MatchList = (platform:string, player:string, next:number):Request => ({
            options: {
                method: 'GET',
                url: `${BaseURL}/warzone/matches/${platform}/${encodeURIComponent(player)}?type=wz&next=${next}`
            }
        })
        export const MatchDetails = (matchId:string|number):Request => ({
            options: {
                method: 'GET',
                url: `${BaseURL}/warzone/matches/${matchId}`
            }
        })
    }
}

const GenericRequest = async ({ options }:Request) => {
    try {
        console.log(`    Requesting ${options.url}`)
        const { data:res, status } = await axios({
            ...options,
            headers: {
                'Cache-Control': 'no-cache'
            }
        })
        if (status !== 200) {
            console.log(res.errors[0].code)
            throw res.errors[0].code
        }
        return res.data
    } catch(e) {
        throw { code: e.response.data.errors[0].code, status: Number(e.message.replace('Request failed with status code ', '')) }
    }
}

export namespace Types {
    // Segments are basically just users
    export namespace Segment {
        export interface Stat {
            rank: number
            percentile: number
            displayName: string
            displayCategory: string
            category: string
            metdata: {}
            value: number
            displayValue: string
            displayType: string
        }
    }
    export interface Segment {
        type: string
        attributes: {
            platformUserIdentifier: string
            platformSlug: string // platform name
            team: string
        }
        metadata: {
            platformUserHandle: string // player username
            clanTag: string
            placement: number | any
        }
        expiryDate: Date
        // Stat interfaces can differ for match details vs match summary
        stats: Types.Warzone.MatchDetails.Segment.Stats | Types.Warzone.MatchSummary.Segment.Stats
    }
    export namespace Warzone {
        export interface Matches {
            matches: MatchSummary[]
            paginationType: number
            metadata: {
                next: number
            }
            requestingPlayerAttributes: {
                platformUserIdentifier: string
            }
        }
        export interface MatchSummary {
            attributes: {
                id: string
                mapId: string
                modeId: string
            }
            metadata: {
                duration: {
                    value: number
                    displayValue: string
                    displayType: string
                }
                timestamp: string
                playerCount: number
                teamCount: number
                mapName: string
                mapImageUrl: string
                modeName: string
            }
            segments: Types.Warzone.MatchSummary.Segment[]
        }
        export namespace MatchSummary {
            export interface Segment extends Types.Segment {
                metadata: {
                    platformUserHandle: string // player username
                    clanTag: string
                    placement: number
                }
                stats: Types.Warzone.MatchSummary.Segment.Stats
            }
            export namespace Segment {
                export interface Stats extends Types.Warzone.MatchDetails.Segment.Stats {
                    medalXp: Types.Segment.Stat
                    matchXp: Types.Segment.Stat
                    scoreXp: Types.Segment.Stat
                    totalXp: Types.Segment.Stat
                    miscXp: Types.Segment.Stat
                    bonusXp: Types.Segment.Stat
                    challengeXp: Types.Segment.Stat
                    placement: Types.Segment.Stat
                    teamPlacement: Types.Segment.Stat
                    teamSurvivalTime: Types.Segment.Stat
                    gulagKills?: Types.Segment.Stat
                    gulagDeaths?: Types.Segment.Stat
                    objectiveReviver?: Types.Segment.Stat
                    objectiveTeamWiped?: Types.Segment.Stat
                    objectiveBrKioskBuy?: Types.Segment.Stat
                    objectiveBrCacheOpen?: Types.Segment.Stat
                    objectiveLastStandKill?: Types.Segment.Stat
                    objectiveBrMissionPickupTablet?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle1?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle2?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle3?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle4?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle5?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle6?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle7?: Types.Segment.Stat
                    objectiveBrDownEnemyCircle8?: Types.Segment.Stat
                }
            }
        }
        export interface MatchDetails {
            attributes: {
                id: string
                mapId: string
                modeId: string
            }
            metadata: {
                duration: {
                    value: number
                    displayValue: string
                    displayType: string
                }
                timestamp: string
                playerCount: number
                teamCount: number
                mapName: string
                mapImageUrl: string
                modeName: string
            }
            segments: Types.Warzone.MatchDetails.Segment[]
        }
        export namespace MatchDetails {
            export interface Segment extends Types.Segment {
                metadata: {
                    platformUserHandle: string // player username
                    clanTag: string
                    placement: {
                        value: number
                        displayValue: string
                        displayType: string
                    }
                }
                stats: Types.Warzone.MatchDetails.Segment.Stats
            }
            export namespace Segment {
                export interface Stats {
                    kills: Types.Segment.Stat
                    deaths: Types.Segment.Stat
                    damageTaken: Types.Segment.Stat
                    damageDone: Types.Segment.Stat
                    headshots: Types.Segment.Stat
                    executions: Types.Segment.Stat
                    assists: Types.Segment.Stat
                    kdRatio: Types.Segment.Stat
                    score: Types.Segment.Stat
                    timePlayed: Types.Segment.Stat
                    longestStreak: Types.Segment.Stat
                    scorePerMinute: Types.Segment.Stat
                    damageDonePerMinute: Types.Segment.Stat
                    distanceTraveled: Types.Segment.Stat
                    percentTimeMoving: Types.Segment.Stat
                }
            }
        }
    }
}

scrape.ts

import config from '../config'
import * as Mongo from '../mongo'
import * as TrackerAPI from '../api/trn'

const delay = (ms:number) => new Promise(resolve => setTimeout(() => resolve(), ms))

export namespace COD {
    export class Warzone {
        private db:any
        private next:number
        private attempts:number
        private complete:boolean
        private logger:Function
        public profile:Mongo.UserParams
        public player:Mongo.Schema.Player
        public platform:Mongo.Schema.Platform
        public history:TrackerAPI.Types.Warzone.Matches[]
        constructor(logger?:Function) {
            this.logger = logger
            this.Run()
        }
        async Run() {
            this.db = await Mongo.Client()
            await this.InfiniteLoop()
        }
        async InfiniteLoop() {
            while(true) {
                this.logger('[>] Looking for players to initialize...')
                try {
                    await this.SelectProfile()
                } catch(e) { // no players found, wait...
                    this.logger(`    ${e}`)
                    this.logger(`    Waiting ${config.wait}ms`)
                    await delay(config.wait)
                    continue
                }
                this.logger(`[>] Scrape Tracker.gg for ${this.profile.platform}<${this.profile.username}>`)
                await this.DownloadMatchHistory()
                await this.UpdatePlayer()
            }
        }
        async SelectProfile() {
            let uninitializedPlayer = await this.db.collection('cod.players').findOne({ 'api.trn.updated': { $exists: false } })
            if (!uninitializedPlayer) {
                [ uninitializedPlayer ] = await this.db.collection('cod.players').find({ 'api.trn.updated': { $gt: -1 } }).sort({ 'api.trn.updated': 1 }).toArray()
            }
            if (!uninitializedPlayer) {
                throw 'No players found to initialize'
            }
            this.player = uninitializedPlayer
            const [platform] = Object.keys(this.player.profiles)
            const [username] = Object.values(this.player.profiles)
            this.profile = { platform, username }
            this.platform = await this.db.collection('cod.platforms').findOne({ tag: this.profile.platform })
            // Initialize player in TRN API - without this new players will never be accessible
            try {
                await TrackerAPI.Warzone.Initialize(this.platform.api, this.profile.username)
            } catch(e) {
                console.log(e.response)
            }
        }
        async DownloadMatchHistory() {
            this.next = null
            this.attempts = 0
            this.history = []
            this.complete = false
            if (this.player.api && this.player.api && this.player.api.next) {
                this.next = this.player.api.next
            }
            while(!this.complete && this.attempts <= config.api.retry) {
                await this.FetchMatchHistorySegment()
                try {
                    await this.RecordMatchHistory()
                } catch(e) {
                    this.logger(`    ${e}`)
                }
            }
            // We reached the end or exhausted all our attempts, process the data and move on...
            this.logger(`    Exiting loop...`)
        }
        async FetchMatchHistorySegment() {
            this.history = []
            try {
                const res = await TrackerAPI.Warzone.MatchList(this.platform.api, this.profile.username, this.next)
                this.attempts = 0 // reset attempts on success
                this.history.push(res)
                this.next = res.metadata.next
                this.logger(`    Found ${res.matches.length} matches...`)
                // If we get less than 20 matches we know we are at the end of their match history
                // Without a definitive end to their match history, we wait for attempts to trigger close
                if (res.matches.length < 20) {
                    this.complete = true
                }
            } catch(e) {
                if (e.status === 400 && e.code.toLowerCase().includes('noaccount')) {
                    // no records found, abort
                    this.logger('    Player not found')
                    this.complete = true
                    return
                }
                this.attempts++
                const msg = this.attempts >= config.api.retry 
                    ? 'exiting loop' 
                    : `waiting ${config.api.delay}ms for attempt #${this.attempts + 1}`
                this.logger(`    ${e.status} Error \"${e.code}\"... ${msg}...`)
                if (this.attempts < config.api.retry) {
                    await delay(config.api.delay)
                }
            }
        }
        async RecordMatchHistory() {
            const allMatches:TrackerAPI.Types.Warzone.MatchSummary[] = [].concat.apply([], this.history.map(res => res.matches))
            if (!allMatches.length && !this.complete) {
                throw 'No matches found'
            }
            for(const match of allMatches) {
                // Check if performance exists, if so we can skip everything
                const performanceRecord = await this.db.collection('cod.performances').findOne({ matchId: match.attributes.id, "player._id": this.player._id })
                if (performanceRecord) {
                    this.logger(`    Performance found for ${match.attributes.id}... skipping...`)
                    continue
                }
                this.logger(`    Saving performance for match ${match.attributes.id}`)
                const normalizedPerformance = TrackerAPI.Normalize.Performance(match, this.player)
                // Store performance in db
                await this.db.collection('cod.performances').insertOne(normalizedPerformance)
                const normalizedMatch = TrackerAPI.Normalize.Match(match)
                // If we don't have this match in the db fetch details to compile teams
                const matchRecord = await this.db.collection('cod.matches').findOne({ matchId: normalizedMatch.matchId })
                if (!matchRecord) {
                    this.logger(`    Creating record for match ${match.attributes.id}`)
                    try {
                        const matchDetails = await this.FetchMatchDetails(normalizedMatch.matchId)
                        if (!matchDetails || !matchDetails.segments) { // most likely joined and left immediately - not quite sure about these
                            this.logger(matchDetails)
                            continue
                        }
                        normalizedMatch.teams = TrackerAPI.Normalize.Teams(matchDetails)
                        await this.db.collection('cod.matches').insertOne(normalizedMatch)
                    } catch(e) {
                        console.log(match)
                        throw new Error(e)
                    }
                }
            }
        }
        async FetchMatchDetails(matchId:string) {
            this.attempts = 0
            this.logger(`    Fetching match details for id ${matchId}`)
            while(true) {
                try {
                    return await TrackerAPI.Warzone.MatchDetails(matchId)
                } catch(e) {
                    this.attempts++
                    const msg = this.attempts >= config.api.retry 
                        ? 'exiting loop' 
                        : `waiting ${config.api.delay}ms for attempt #${this.attempts + 1}`
                    this.logger(`    ${e.status} Error \"${e.code}\"... ${msg}...`)
                    if (this.attempts >= config.api.retry) {
                        throw e
                    }
                    await delay(config.api.delay)
                }
            }
        }
        async UpdatePlayer() {
            const trn = { ...this.player.api, next: this.next, updated: Math.floor(Date.now() / 1000) }
            if (!this.history.length) {
                trn.failures = (trn.failures || 0) + 1
            }
            if (trn.failures >= config.api.failures) {
                trn.next = 0
                trn.failures = 0
                console.log(`    Resetting ${this.profile.username}...`)
            }
            await this.db.collection('cod.players').updateOne({ _id: this.player._id }, { $set: {
                ...this.player,
                api: {
                    ...this.player.api,
                    trn
                }
            } })
            console.log(`    Updated ${this.profile.username}...`)
        }
    }
}