data_format.py

# Pulls all the data from the manually generated imparatives dataset   
with open('data/imperatives.csv', 'r') as imperative_file:
  for row in imperative_file:
    tagged_comments[imperative] = "command"
    
# Pulls all data from the SPAADIA dataset, adds to our dataset                            
for doc in os.listdir('data/SPAADIA'):
    with open('data/SPAADIA/' + doc, 'r') as handle:
        conversations = BeautifulSoup(handle, features="xml")
        for imperative in conversations.findAll("imp"):
          tagged_comments[imperative.get_text()] = "command"
        for declarative in conversations.findAll("decl"):
          tagged_comments[declarative.get_text()] = "statement"
        for question in conversations.findAll("q-yn"):
          tagged_comments[question.get_text()] = "question"
        for question in conversations.findAll("q-wh"):
          tagged_comments[question.get_text()] = "question"