Process raw Wikipedia data and create Anki deck for collective nouns

collective-deck-import.py

import sys
import csv

###################################################################
# Handling inputs
###################################################################

def getInput(csvFileName):
    with file(csvFileName) as f:
        reader = csv.DictReader(f, fieldnames=['Subject', 'CollectiveNoun', 'Notes', 'Source'])

        next(reader, None) # Skip the first line

        subToNoun = {}
        nounToSub = {}
        hints = {}
        allPairs = set()

        for row in reader:
            sub = row['Subject']

            # Select only the first collective noun, if many
            coll = row['CollectiveNoun'].split()[0]

            notes = row['Notes']

            if notes is not None and len(notes) > 0:
                hints[(sub, coll)] = notes

            if sub not in subToNoun: subToNoun[sub] = []
            if coll not in nounToSub: nounToSub[coll] = []
            subToNoun[sub].append(coll)
            nounToSub[coll].append(sub)

            # Only select those pairs where the column vector is more than a
            # single character long (exclude single alphabet columns)
            if len(coll) > 1:
                allPairs.add((sub, coll))

        return (subToNoun, nounToSub, hints, allPairs)


###################################################################
# Processing
###################################################################

def cleanStr(s):
    return s.strip().strip('*').strip()

def minPrefixIdx(targetStr, strs):
    # Remove the targetStr from strs, if it is present
    strs = [ s for s in strs if s != targetStr ]

    # If there is only one string, then no prefix is needed
    if len(strs) == 0:
        return 0

    idx = 1
    while idx < len(targetStr):
        conflict = False
        for s in strs:
            if targetStr[0:idx] == s[0:idx]:
                conflict = True
                break

        if conflict:
            idx += 1
        else:
            return idx

    print >> sys.stderr, "Something went wrong: {0}, {1}".format(targetStr, str(strs))

    # One string was completely a prefix of another!
    return None

def createCloze(subject, collectiveNoun, nounToSub, subToNoun, hintsDict):
    colls = [cleanStr(s) for s in subToNoun[subject]]
    subs = [cleanStr(s) for s in nounToSub[collectiveNoun]]

    hint = cleanStr(hintsDict.get((subject, collectiveNoun), ''))
    sub = cleanStr(subject)
    coll = cleanStr(collectiveNoun)

    collsPrefixIdx = minPrefixIdx(coll, colls)
    subsPrefixIdx = minPrefixIdx(sub, subs)

    if collsPrefixIdx is not None and subsPrefixIdx is not None:
        cloze = '{collPrefix}{{{{c2::{restColl}}}}} of {subPrefix}{{{{c1::{restSubject}}}}}' \
                .format( subPrefix   = sub[:subsPrefixIdx]
                       , restSubject = sub[subsPrefixIdx:]
                       , collPrefix  = coll[:collsPrefixIdx]
                       , restColl    = coll[collsPrefixIdx:]
                       )

        return (cloze, hint)
    else:
        # Unfortunately, could not find a proper cloze deletion for 
        # this item. Skipping it.
        print >> sys.stderr, "Skipping ({0}, {1}).".format(sub, coll)
        return None


###################################################################
# Execution and output
###################################################################

def process(inputFileName, outputFileName):
    (subToNoun, nounToSub, hints, allPairs) = getInput(inputFileName)
    opAll = [ createCloze(x, y, nounToSub, subToNoun, hints) for x, y in allPairs ]
    op = [ x for x in opAll if x is not None ]
    with file(outputFileName, 'w') as f:
        w = csv.writer(f, )
        w.writerows(op)

input.csv