extract themes from the Gdelt SET_EVENTPATTERNS.xml file

themes.py

"""
The patterns file is here: https://github.com/ahalterman/GKG-Themes/blob/master/SET_EVENTPATTERNS.xml

It is not valid XML so using regex

There are non-theme entries in this file not considered here. The globals section at the top of the
file should be taken into account when processing documents with pattern matches. 
"""
import re

p = re.compile(r'^<CATEGORY NAME="([^"]+)" TYPE="THEME">\s*<TERMS>([^<]+)</TERMS>', re.M|re.S)

themes = {}

with open('SET_EVENTPATTERNS.xml') as f:
    for theme, terms in p.findall(f.read()):
        terms = [tuple(t.split('\t')) for t in terms.split('\n') if t and len(t.split('\t')) == 2]
        if terms:
            themes[theme] = terms

print(themes)