A Willie IRC Bot (http://willie.dftba.net/) module that scrapes character info off of Fallout IRC RPG (http://falloutirc.webs.com/)Requires: Python 2.7, Willie, fuzzywuzzy, and lxml

fobot_stats.py

"""
fobot_stats.py - New Reno Fallout Stat Module
Copryight 2014 Chase Colman, https://gist.github.com/chase/8554277
Licensed under the MIT License
"""

# TODO: Reorganize into Classes

import re

from datetime import datetime, timedelta

from lxml import html
from lxml.etree import XPath

from fuzzywuzzy import process

from willie import web
from willie.module import commands, example, priority
from willie.tools import WillieMemory

# Separator (Only affects formatting)
SHEET = '==='
FIELD = '\\\\\\'
BLOCK = '#'

# Passive Update Intervals
CHAR_INDEX_UPDATE = timedelta(minutes=30)
CHAR_UPDATE = timedelta(minutes=30)
ARMOR_UPDATE = timedelta(days=1)

# URLs
CHAR_INDEX='http://falloutirc.webs.com/apps/forums/show/6340597-character-sheet-index'
INACTIVE_CHAR_INDEX='http://falloutirc.webs.com/apps/forums/show/6621254-inactive-characters'
ARMOR_POST='http://falloutirc.webs.com/apps/forums/topics/show/6630902-index'

# Overrides
ARMOR_OVERRIDES = {
    'Full Armor':
        { 'Power Armor Mark VI':
            { '_MODIFIERS':
                { 'Stats': { 'Strength': 10, 'Stamina': 10, 'Perception': 10, 'Intelligence': 10, 'Tech': 10, 'Agility': 10} }
            }
        }
}

# XPaths
BODY_PATH = "/html/body/div[@id='fw-container']/div[@id='fw-blockContainer']/div[@id='fw-bigcontain']/div[@id='fw-columnContainer']/div[@id='fw-mainColumn']/div/div"
TOPICS_PATH = BODY_PATH + "/table/tr/td/b/a"
PAGES_PATH = BODY_PATH + "//div[@class='pagination']/a[last()-1]/text()"

SHEET_PATH = BODY_PATH + "/table[@id='topicDisplay']/tr[@id]/td[2]/span[1]"
BLOCK_PATH = SHEET_PATH + "//td/*[text()=$title]/../../.."
BLOCK_TEXT_PATH = BLOCK_PATH + "/tr[2]/td//text()"

ROW_TEXT_PATH = "./tr[$row]/td//text()"
COLUMN_TEXT_PATH = "./td[$col]//text()"

POSTS_PATH = BODY_PATH + "/table[@id='topicDisplay']/tr[@id]/td/span/table"
HEADERLESS_ROWS_PATH = "./tr[position()>1]"

_topics = XPath(TOPICS_PATH)
_page_count = XPath(PAGES_PATH)
_block = XPath(BLOCK_PATH)
_block_text = XPath(BLOCK_TEXT_PATH)

_column_text = XPath(COLUMN_TEXT_PATH)
_row_text = XPath(ROW_TEXT_PATH)

_posts = XPath(POSTS_PATH)
_headerless_rows = XPath(HEADERLESS_ROWS_PATH)

_specials_re=re.compile(r'([+-]\d+) to (\S+)( rolls)?')
_itemsplit_re=re.compile(r'\s*(\([^)]+\))\s*|,\s+')

def _deep_update_dictionary(original, update, add_numbers=False):
    for key in update.iterkeys():
        if key in original:
            if add_numbers and isinstance(original[key], int) and isinstance(update[key], int):
                original[key] += update[key]
                continue
            if isinstance(original[key], dict) and isinstance(update[key], dict):
                _deep_update_dictionary(original[key], update[key], add_numbers)
                continue

        original[key] = update[key]

    return original

def _normalize_text(text):
    return text.encode('ascii', 'ignore').strip(',: \t\n\r\f\v').replace('\n\n','').replace('  ',' ')

def _item_dictionary_from_tables(tables):
    result = {}

    categories = tables[0::2]
    items = iter(tables[1::2])

    for category in categories:
        category_name = _row_text(category, row=1)[0]
        result[category_name] = {}

        category_items = _headerless_rows(items.next())

        for item in category_items:
            item_name = _normalize_text(_column_text(item, col=1)[0])
            item_description = _column_text(item, col=2)
            result[category_name][item_name] = {'Description': _normalize_text(item_description[-1])}

            at_name = False
            attribute = ""
            for part in item_description[:-1]:
                value = _normalize_text(part)
                if value == "":
                    continue

                at_name ^= True
                if at_name:
                    attribute = value
                    continue

                result[category_name][item_name][attribute] = value

    return result

def _dictionary_from_block_rows(header,data,to_int=False):
    items = iter(data)
    if to_int:
        return {category:int(items.next()) for category in header}
    else:
        return {category:items.next() for category in header}

def _dictionary_from_block_array(array):
    result={}
    key="Miscellaneous"
    for element in array:
        value=element.strip()
        if value == '':
            continue
        if value.endswith(':'):
            key=_normalize_text(value).replace('Miscellanious','Miscellaneous')
            continue
        if key not in result:
            result[key] = []
        result[key]+=[item for item in _itemsplit_re.split(_normalize_text(value)) if item]
    return result

# ITEM METHODS

def _get_modifiers(item):
    result = {'Stats': {'HP': 0}, 'Rolls': {}}

    if 'Special' in item:
        tuples = _specials_re.findall(item['Special'])
        for element in tuples:
            # If there is a value in the 3rd group, it is a roll
            entry = result['Rolls' if element[2] else 'Stats']
            entry[element[1].title()] = int(element[0])

    if 'AR' in item:
        result['Stats']['HP'] = int(item['AR'])

    if '_MODIFIERS' in item:
        _deep_update_dictionary(result, item['_MODIFIERS'])

    return result


def _gather_armor(bot):
    if 'armor' not in bot.memory:
        bot.memory['armor'] = WillieMemory()

    armor = bot.memory['armor']
    if '_LAST_UPDATE' in armor and datetime.utcnow() - armor['_LAST_UPDATE'] < ARMOR_UPDATE:
        return

    armor_posts = _posts(html.fromstring(web.get(ARMOR_POST)))
    armor['_LAST_UPDATE'] = datetime.utcnow()

    raw_dictionary = _item_dictionary_from_tables(armor_posts)
    bot.memory['armor'] = WillieMemory(_deep_update_dictionary(raw_dictionary, ARMOR_OVERRIDES))

def _get_armor(bot, name, kind=None):
    if 'armor' not in bot.memory:
        _gather_armor(bot)

    armor = bot.memory['armor']
    if kind:
        fuzzy_kind = process.extractOne(kind, armor.keys())
        if fuzzy_kind[1] > 70:
            fuzzy_name = process.extractOne(name, armor[fuzzy_kind[0]].keys())
            if fuzzy_name[1] > 70:
                return armor[fuzzy_kind[0]][fuzzy_name[0]]

    for items in armor.itervalues():
        fuzzy_name = process.extractOne(name, armor[fuzzy_kind[0]].keys())
        if fuzzy_name[1] > 70:
            return items[fuzzy_name[0]]

    return None

# CHARACTER METHODS

def _gather_characters(bot,active=True):
    if 'characters' not in bot.memory:
        bot.memory['characters'] = WillieMemory()

    characters = bot.memory['characters']
    if '_LAST_UPDATE' in characters and active:
        if datetime.utcnow() - characters['_LAST_UPDATE'] < CHAR_INDEX_UPDATE:
            return

    idx = html.fromstring(web.get(CHAR_INDEX if active else INACTIVE_CHAR_INDEX))
    characters['_LAST_UPDATE'] = datetime.utcnow()

    page_count = int(_page_count(idx)[0])
    current_page = 1
    while True:
        for character in _topics(idx):
            name = character.text.strip()

            # Skip Stickied threads and the Sample sheet
            if name.startswith("Sticky:") or name == "Sample Sheet":
                continue

            bot.memory['characters'][name] = {'_URL': character.get('href'), '_ACTIVE': active}

        if current_page == page_count:
            break
        current_page += 1
        idx = html.fromstring(web.get(CHAR_INDEX + "?page=" + str(current_page)))

def _update_character(bot, name):
    character = bot.memory['characters'][name]
    if '_LAST_UPDATE' in character and datetime.utcnow() - character['_LAST_UPDATE'] < CHAR_UPDATE:
        return character
    char_page = html.fromstring(web.get(character['_URL']))
    character['_LAST_UPDATE'] = datetime.utcnow()

    posts=_posts(char_page)
    if not posts:
        return None

    # Name
    character['Name']=_row_text(posts[0], row=1)[0]

    # Main Stats Block
    stats=_block(char_page,title='HP')[0]
    character['Stats'] =_dictionary_from_block_rows(_row_text(stats,row=1), _row_text(stats,row=2), to_int=True)

    # Secondary Info Block
    bio=_block(char_page,title='Level')[0]
    character['Bio'] =_dictionary_from_block_rows(_row_text(bio,row=1), _row_text(bio,row=2))

    # Armor
    raw_armor = _dictionary_from_block_array(_block_text(char_page,title='Armor'))
    character['Armor'] = raw_armor

    # Weapons
    raw_weapons = _dictionary_from_block_array(_block_text(char_page,title='Weapons'))
    character['Weapons'] = raw_weapons

    # Items
    character['Items'] = _dictionary_from_block_array(_block_text(char_page,title='Items'))

    # Skills
    character['Skills'] = _dictionary_from_block_array(_block_text(char_page,title='Skills'))

    # Modifiers
    character['_MODIFIERS'] = {'Stats': {'AR': 0}, 'Rolls': {}}

    # +-Armor
    for kind, items in raw_armor.iteritems():
        for item in items:
            raw_item = _get_armor(bot, item, kind)
            if not raw_item:
                continue
            raw_modifiers =_get_modifiers(raw_item)
            _deep_update_dictionary(character['_MODIFIERS'], raw_modifiers, add_numbers=True)

    # TODO: +-Stats (for Rolls)

    return character

def _get_character(bot, fuzzy_name, update_index=True):
    if 'characters' not in bot.memory:
        _gather_characters(bot)

    fuzzy_match = process.extractOne(fuzzy_name, bot.memory['characters'].keys())
    if fuzzy_match[1] < 70:
        if not update_index:
            return None

        # Try again after refreshing the active character index and getting the inactive character index
        _gather_characters(bot)
        _gather_characters(bot, active=False)
        return _get_character(bot, fuzzy_name, False)

    character_name = fuzzy_match[0]
    return _update_character(bot, character_name)

# IRC Formatting

def _bold(string):
    return '\x02' + string + '\x02'

def _underline(string):
    return '\x1F' + string + '\x1F'

def _italic(string):
    return '\x16' + string + '\x16'

# Formatting

def _line(string):
    return BLOCK + ' ' + string + ' ' + BLOCK

def _limit_list(items, limit=250):
    counter = limit
    result = []
    sublist = []

    for item in items:
        if '(' in item:
            continue
        item_len = len(item)
        if counter < item_len:
            counter = limit
            result.append(", ".join(sublist))
            sublist = []
        sublist.append(item)
        counter -= item_len

    result.append(", ".join(sublist))
    return result

def _stat(character, stat):
    if stat == 'Level':
        original = character['Bio'][stat]
    else:
        original = character['Stats'][stat]
    modifiers = character['_MODIFIERS']['Stats']

    result = _bold(stat + ": ") + str(original)

    if stat in modifiers:
        modifier = modifiers[stat]
        result += '+' + str(modifiers[stat]) + ' = ' + _italic(str(modifier+original))
    return result

def _say_categories(bot, character, field):
    categories = character[field]
    prefix = BLOCK + ' ' + field + ' - '
    suffix = ' ' + BLOCK
    padding_len = len(prefix) + len(suffix)
    for category, elements in categories.iteritems():
        line = prefix + _bold(category) + ' ' + FIELD + ' '
        offset = len(line) + padding_len
        for element in _limit_list(elements, limit=250-offset):
            bot.say(line + element + suffix)

def _say_stats(bot, character):
    modifier_stat=character['_MODIFIERS']['Stats']
    stat_order=['Level', 'HP', 'Strength', 'Stamina', 'Perception', 'Agility', 'Intelligence', 'Tech', 'Luck']
    bot.say(_line((' ' + FIELD + ' ').join([_stat(character, stat) for stat in stat_order])))


def _say_sheet(bot, character):
    bot.say(_italic("{} Start of {}'s Stat Sheet {}").format(SHEET, character['Name'], SHEET))
    if not character['_ACTIVE']:
        bot.say(_bold('!!! WARNING: THIS CHARACTER IS CURRENTLY MARKED AS INACTIVE !!!'))
    _say_stats(bot, character)
    _say_categories(bot, character, 'Skills')
    _say_categories(bot, character, 'Weapons')
    _say_categories(bot, character, 'Armor')
    _say_categories(bot, character, 'Items')
    bot.say(_italic("{} End of {}'s Stat Sheet {}").format(SHEET, character['Name'], SHEET))

# Bot
def setup(bot):
    _gather_armor(bot)
    _gather_characters(bot)

@commands('stats')
def stats(bot, trigger):
    """ Retrieves a stat sheet for a name provided. """

    args = trigger.group(2)
    char = _get_character(bot, args)
    if not char:
        bot.say("I could not find a character named: "+ args)
        return
    _say_sheet(bot, char)

Lots of spaghetti code and repetition. Needs some serious refactoring, but works... mostly.

Need to think about handling the customized item characteristics in parenthesis.

Old character sheets are reported as not being found. Perhaps it should fail gently with something like, Please notify a GM to have your character's sheet updated.