example_sequence_scraper.groovy

gistfile1.txt

package gg.abilitydraft

import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.node.ArrayNode
import com.google.common.util.concurrent.RateLimiter
import grails.core.GrailsApplication
import grails.gorm.transactions.Transactional


import java.lang.reflect.Array

@Transactional
class ValveWebAPIService {

    static lazyInit = false

    def jsonService

    Long sequenceNum = 4602912814 // first sequenceId as an initializer

    List<String> apikeys
    int apiIndex = 0
    RateLimiter rateLimiter
    Random rng = new Random()

    final Integer ABILITY_DRAFT_GAME_MODE = 18

    def poke() {
        if (!readyToGo) {
            log.warn("Not ready yet!")
            return
        }

        String valveUrl = "https://api.steampowered.com/IDOTA2Match_570/GetMatchHistoryBySequenceNum/V001/?" +
                "key=${apikeys[apiIndex]}&matches_requested=100&start_at_match_seq_num=${sequenceNum}"
        JsonNode node = getJsonUrl(valveUrl)

        apiIndex = (1 + apiIndex) % apikeys.size()

        if (!((node != null) && node.has('result') && node.get('result').has('matches'))) return // sometimes junky response, thanks Valve
        List<GameDTO> tempGames = node.get('result').get('matches').collect { JsonNode m ->
            return new GameDTO(
                    matchId: m.get('match_id').longValue(),
                    gameStart: new Date(1000L * m.get('start_time').longValue()),
                    duration: m.get('duration').intValue(),
                    sequenceNum: m.get('match_seq_num').longValue(),
                    isAbilityDraft: m.get('game_mode').intValue() == ABILITY_DRAFT_GAME_MODE,
                    radiantVictory: m.get('radiant_win').booleanValue(),
                    players: ((ArrayNode) m.get('players')).collect { pg ->
                        return new PlayerGameDTO(
                                faction: (pg.get('player_slot').intValue() < 5) ? Faction.RADIANT : Faction.DIRE,
                                hero: pg.get('hero_id').intValue(),
                                kills: pg.get('kills').intValue(),
                                deaths: pg.get('deaths').intValue(),
                                assists: pg.get('assists').intValue(),
                                gpm: pg.get('gold_per_min').intValue(),
                                xpm: pg.get('xp_per_min').intValue(),
                                lastHits: pg.get('last_hits').intValue(),
                                heroDamage: pg.get('hero_damage').intValue(),
                                heroHealing: pg.get('hero_healing').intValue(),
                                abilities: stripAbilities(pg.get('ability_upgrades')),
                                items: stripItems(pg)
                        )
                    }
            )
        }

        if (tempGames) {
            log.info("Have [${tempGames.size()}] temp games to persist")
            // persist new games
            Set<Long> seenMatchIds = Game.findAllByMatchIdInList(tempGames*.matchId)*.matchId as Set
            tempGames.findAll { it.isAbilityDraft }.each { tg ->
                if (!seenMatchIds.contains(tg.matchId)) {
                    Game g = new Game(
                            matchId: tg.matchId,
                            gameStart: tg.gameStart,
                            radiantVictory: tg.radiantVictory,
                            duration: tg.duration,
                            sequenceNum: tg.sequenceNum
                    ).save(failOnError: true)
                    tg.players.each { pg ->
                        new PlayerGame(
                                game: g,
                                faction: pg.faction,
                                hero: pg.hero,
                                kills: pg.kills,
                                deaths: pg.deaths,
                                assists: pg.assists,
                                gpm: pg.gpm,
                                xpm: pg.xpm,
                                lastHits: pg.lastHits,
                                heroDamage: pg.heroDamage,
                                heroHealing: pg.heroHealing,
                                abilities: pg.abilities,
                                items: pg.items
                        ).save(failOnError: true)
                    }
                }
            }

            // update sequence_num even if it's not ability draft
            sequenceNum = tempGames.last().sequenceNum
            if (sequenceNum == 4718271902L) sequenceNum = 4718271993L
            log.info("Update sequence index to ${sequenceNum}")
        }
        else {
            log.warn("No games found :(")
        }
    }
  
}