Quick and Dirty Parser for D&D Poisons

poisonParse.py

from bs4 import BeautifulSoup
import sqlite3
from collections import defaultdict

conn = sqlite3.connect("poison.db")
c = conn.cursor()
insert = conn.cursor()
c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='poison'")

if c.fetchone() == None:
    c.execute('''CREATE TABLE poison (id int PRIMARY KEY, name text, level int, flavor text, 
                 cost int, atkBonus int, defense text, effect text, firstFail text, 
                 secondFail text, afterEffect text, special text, published text)''')

# Assume you have a sqlite database with the raw html poison data in it already...
results = c.execute("SELECT id, title, raw FROM data")
for row in results:

    soup = BeautifulSoup(row[2])

    poison = defaultdict(lambda: None)
    poison["id"] = row[0]
    poison["name"] = row[1]
    print row[1]
    poison["level"] = int(soup.span.string.replace("Poison", "").replace("Level", "").strip())
    poison["flavor"] = soup.find("p", {"class":"flavor"}).string

    details = soup.p.next_sibling

    index = ""

    # quick and dirty parsing of the poison properties
    for detail in details.contents:
        if index != "":
            poison[index] = detail.strip()
            index = ""
            continue
        
        contents = detail.string
        
        if contents == "Poison":
            index = "cost"
        elif contents == "Attack:":
            index = "effect"
        elif contents == "Aftereffect:":
            index = "afterEffect"
        elif contents == "Special:":
            index = "special"
        elif contents == "First Failed Saving Throw:":
            index = "firstFail"
        elif contents == "Second Failed Saving Throw:":
            index = "secondFail"
        else:
            index = ""
         
    poison["published"] =  str(details.next_sibling.contents[1]) + str(details.next_sibling.contents[2])

    # We've picked out the data we want, but it's not quite formatted yet
    poison["cost"] = int(poison["cost"].replace("gp","").strip())

    atkInfo = poison["effect"].split(";", 2)[0] # "+5 vs. Fortitude"

    poison["atkBonus"] = int(atkInfo.split("vs.")[0].strip()) # 5
    poison["defense"] = atkInfo.split("vs.")[1].strip() # Fortitude

    poison["effect"]= poison["effect"].split(";", 2)[1].strip().capitalize()
    
    #id int PRIMARY KEY, name text, level int, flavor text, 
    #cost int, atkBonus int, defense text, effect text, firstFail text, 
    #secondFail text, afterEffect text, special text, published text)'
    plist = [poison["id"], poison["name"], poison["level"], poison["flavor"], 
        poison["cost"], poison["atkBonus"], poison["defense"], poison["effect"], poison["firstFail"], 
        poison["secondFail"], poison["afterEffect"], poison["special"], poison["published"]]
    
    query = "INSERT INTO poison VALUES (%s)" % ','.join(['?'] * len(plist))
    insert.execute(query , plist)
    
conn.commit()
insert.close()
c.close()