Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import bz2
import os
import re
from collections import defaultdict
from enum import Enum, auto
from conllu import parse
from spacy.lang.sv import Swedish
def get_num_sentences_from_corpus(path):
with bz2.open(path, "rt", encoding="utf-8") as fp:
return len(fp.read().split("\n\n"))
def get_sentence_from_corpus(path):
with bz2.open(path, "rt", encoding="utf-8") as fp:
lines = ""
for i, line in enumerate(fp.readlines()):
if line != "\n":
lines += line
else:
yield lines
lines = ""
def get_orginal_sentence(parsed_sentence):
out = []
for token in parsed_sentence:
spaceafter = False
if "misc" in token and token["misc"] and \
"SpaceAfter" in token["misc"] and token["misc"]["SpaceAfter"] == "No":
spaceafter = True
out.append(token["form"] + (" " if not spaceafter else ""))
return "".join(out)
def print_stats(stats):
print(f"""
Correct sentences: {stats["sentence_correct"]}
Incorrect sentences: {stats["sentence_incorrect"]}
""")
ABBREVIATIONS = [
# From UD_Swedish-Talbanken README about exceptions
"bl a", "d v s", "e d", "f n", "fr o m", "m fl",
"m m", "o s v", "s k", "t ex", "t o m", "t v",
# From UD_Swedish-Talbanken abbreviations
"tel", "sid", "kungl", "prof", "proc", "doc", "f", "milj", "fig",
"kap", "mt", "mos", "kor", "t h", "vol",
# From SpaCy tokenizer_exceptions.py
"ang", "anm", "bil", "bl a", "dvs", "e kr", "el", "e d", "eng",
"etc", "exkl", "f d", "fid", "f kr", "forts", "fr o m", "f ö",
"förf", "inkl", "jur", "kl", "kr", "lat", "m a o", "max", "m fl",
"min", "m m", "obs", "o d", "osv", "p g a", "ref", "resp", "s a s",
"s k", "st", "s t", "t ex", "t o m", "ung", "äv", "övers"
]
ABBREVIATIONS += [abbr.replace(" ", ".") for abbr in ABBREVIATIONS]
ABBREVIATIONS += [abbr + "." for abbr in ABBREVIATIONS]
class ERROR_TYPES(Enum):
UNKNOWN = auto()
DASH = auto()
ABBR = auto()
GENITIVE = auto()
SINGLELETTER = auto()
LISTS = auto()
PARENTESISEQUAL = auto()
def categorize_error(spacy_token, ud_token):
if "-" in ud_token and ud_token.split("-")[0] == spacy_token:
return ERROR_TYPES.DASH
if ":" in ud_token and ud_token.split(":")[0] == spacy_token or \
ud_token[-1] == "'":
return ERROR_TYPES.GENITIVE
if ud_token.lower() in ABBREVIATIONS:
return ERROR_TYPES.ABBR
if len(spacy_token) == 2 and spacy_token[1] == ".":
return ERROR_TYPES.SINGLELETTER
if re.match(r"^\(?[1-9a-z](\)|\.)?$", ud_token):
return ERROR_TYPES.LISTS
if spacy_token == "(=":
return ERROR_TYPES.PARENTESISEQUAL
return ERROR_TYPES.UNKNOWN
def main():
nlp = Swedish()
stats = {"sentence_correct": 0, "sentence_incorrect": 0}
incorrect_types = defaultdict(lambda: 0)
base_path = os.path.expanduser("~/Downloads/")
corpus_paths = [
base_path + "sv_talbanken-ud-train.conllu.bz2",
base_path + "sv_talbanken-ud-dev.conllu.bz2",
base_path + "sv_talbanken-ud-test.conllu.bz2",
]
num_sentences = sum(get_num_sentences_from_corpus(corpus_path) for corpus_path in corpus_paths)
sentence_count = 0
for corpus_path in corpus_paths:
for conllu_sentence in get_sentence_from_corpus(corpus_path):
sentence_count += 1
if sentence_count % 100 == 0:
print(f"{sentence_count}/{num_sentences} sentences parsed.")
sentence = parse(conllu_sentence)[0]
doc = nlp(get_orginal_sentence(sentence))
sentence_correct = True
for j, token in enumerate(doc):
spacy_token = token.text
ud_token = sentence[j]["form"]
if spacy_token != ud_token:
sentence_correct = False
error_type = categorize_error(spacy_token, ud_token)
# assert spacy_token != "gälleräven", (ud_token, error_type)
incorrect_types[error_type] += 1
if error_type == ERROR_TYPES.UNKNOWN:
print(f"UNKNOWN TYPE. SpaCy: '{spacy_token}', UD: '{ud_token}'")
break
stats["sentence_correct" if sentence_correct else "sentence_incorrect"] += 1
print("Done.")
print_stats(stats)
print(f"{100*stats['sentence_correct']/num_sentences:.2f}% where correctly tokenized.")
print("\n".join([f"{key}: {value}" for key, value in incorrect_types.items()]))
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.