Skip to content

Instantly share code, notes, and snippets.

@melpomene
Last active December 16, 2015 09:49
Show Gist options
  • Save melpomene/5415838 to your computer and use it in GitHub Desktop.
Save melpomene/5415838 to your computer and use it in GitHub Desktop.
This script transforms the tab seperated files to ARFF to allow us to use String Kernel from SVM. There are some issues with WEKA and UTF-8 yet to be resolved.
#!/usr/bin/env python
# encoding: utf-8
import codecs
from collections import Counter
class Card:
def __init__(self, data):
self.idnr= data[0].strip("\n")
self.category = data[1].strip("\n")
self.star = data[2].strip("\n")
self.name = data[3].strip("\n")
self.questions = list()
self.addQuestion(data[4].strip("\n"), data[5], data[6].strip("\n"))
def addQuestion(self,value, text, answer):
value = value.strip()
if value== "250": nr = 1
elif value == "500": nr = 2
elif value == "1000": nr = 3
elif value == "2000": nr = 4
elif value == "5000": nr = 5
elif value == "10000": nr = 6
else: raise Exception("Parse error")
self.questions.append([nr, value.strip("\n"), text.strip("\n"), answer.strip("\n")])
def __str__(self):
s = u"kvitt:card{0} rdf:type kvitt:Card;\n"
s += u"\tkvitt:header [\n"
s += u"\t\tkvitt:id\t{0};\n"
s += u'\t\tkvitt:category\t"{1}";\n'
s += u'\t\tkvitt:star\t"{2}";\n'
s += u'\t\tkvitt:name\t"{3}";\n'
s += u'\t\tkvitt:questions \n'
for q in self.questions:
if q[0] == 6:
last = u"]."
else:
last = u","
s += u'\t\t\t\t[kvitt:line\t{0}; kvitt:value\t{1}; kvitt:text\t"{2}"; kvitt:answer\t"{3}"]{4}\n'.format(q[0],q[1],q[2],q[3],last)
return s.format(self.idnr, self.category, self.star, self.name)
def toSVM():
src = codecs.open('fragor.txt', 'r', "utf-8-sig")
output = open('bockerfilm_training_stringkernel.arff', 'w')
output.write("""@RELATION book
@ATTRIBUTE data string
@ATTRIBUTE class {book, notbook}
@DATA
""")
cards = dict()
for line in src:
data = line.split('\t')
if len(data) < 7: print data
if data[0] in cards:
cards[data[0]].addQuestion(data[4],data[5],data[6])
else:
cards[data[0]] = Card(data)
for card in cards.values():
for q in card.questions:
row = "'"+q[2].replace("'", '')+"', "
if card.category == u"Böcker och film": row +="book"
else: row += "notbook"
row = row.encode('ascii', 'ignore')
output.write(row+"\n")
if __name__ == "__main__":
toSVM()
@melpomene
Copy link
Author

It does some horrible to ascii conversion since I was unable to get WEKA reading UTF-8. This ought to get fixed, and will probably yield some small improvements.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment