Gets the diffs direct from XML

diff_from_xml.py

# NOTE: Currently this prints the diff for the second data element and exits

import nltk
import xml.etree.ElementTree as ET
import copy
import difflib

tree = ET.parse('xmlsdev.xml')
root = tree.getroot()

def custom_tokenize(sentence):
    s = "".join(sentence.itertext()).replace('-', ' - ')
    return(nltk.word_tokenize(s))

def remove_all_nodes(parent, childname):
    for child in parent.findall(childname):
            tail = child.tail
            child.clear()
            child.tail = tail

with open('corrected', 'w', encoding='utf-8') as file:
    i = 0
    for sentence in root.iter('sentence'):
        sentence_c = copy.deepcopy(sentence)

        remove_all_nodes(sentence, 'ins')
        remove_all_nodes(sentence_c, 'del')

        s1 = custom_tokenize(sentence)
        s2 = custom_tokenize(sentence_c)

        diff = difflib.ndiff(s1, s2)

        if (i==1):
            print('\n'.join(diff))
            exit()

        file.write('\n')

        if i%100 == 0:
            print(i)
        i += 1