This script transforms the tab seperated files to ARFF to allow us to use String Kernel from SVM. There are some issues with WEKA and UTF-8 yet to be resolved.

tostringkernelarff.py

#!/usr/bin/env python
# encoding: utf-8
import codecs
from collections import Counter 
class Card: 
    def __init__(self, data):
        self.idnr= data[0].strip("\n")
        self.category = data[1].strip("\n")
        self.star = data[2].strip("\n")
        self.name = data[3].strip("\n")
        self.questions = list()
        self.addQuestion(data[4].strip("\n"), data[5], data[6].strip("\n")) 
    
    def addQuestion(self,value, text, answer):
        value = value.strip()
        if value== "250":  nr = 1
        elif value == "500": nr = 2
        elif value == "1000": nr = 3
        elif value == "2000": nr = 4
        elif value == "5000": nr = 5
        elif value == "10000": nr = 6
        else: raise Exception("Parse error")
        self.questions.append([nr, value.strip("\n"), text.strip("\n"), answer.strip("\n")])
    def __str__(self):
        s =  u"kvitt:card{0} rdf:type kvitt:Card;\n"
        s += u"\tkvitt:header [\n"
        s += u"\t\tkvitt:id\t{0};\n"
        s += u'\t\tkvitt:category\t"{1}";\n'
        s += u'\t\tkvitt:star\t"{2}";\n'
        s += u'\t\tkvitt:name\t"{3}";\n'
        s += u'\t\tkvitt:questions \n'
        for q in self.questions:
            if q[0] == 6: 
                last = u"]." 
            else: 
                last = u","
            s += u'\t\t\t\t[kvitt:line\t{0}; kvitt:value\t{1}; kvitt:text\t"{2}"; kvitt:answer\t"{3}"]{4}\n'.format(q[0],q[1],q[2],q[3],last)
        return s.format(self.idnr, self.category, self.star, self.name) 

def toSVM(): 
    src = codecs.open('fragor.txt', 'r', "utf-8-sig")
    output = open('bockerfilm_training_stringkernel.arff', 'w')
    output.write("""@RELATION book
@ATTRIBUTE  data   string
@ATTRIBUTE class  {book, notbook}
@DATA
""")

    cards = dict()
    for line in src:
        data = line.split('\t')
        if len(data) < 7: print data
        if data[0] in cards:
            cards[data[0]].addQuestion(data[4],data[5],data[6])
        else:
            cards[data[0]] = Card(data)
    for card in cards.values():
        for q in card.questions:
            row = "'"+q[2].replace("'", '')+"', "
            if card.category == u"Böcker och film": row +="book"
            else: row += "notbook"
            row = row.encode('ascii', 'ignore')
            output.write(row+"\n")


if __name__ == "__main__": 
    toSVM()
    

It does some horrible to ascii conversion since I was unable to get WEKA reading UTF-8. This ought to get fixed, and will probably yield some small improvements.