crf_model_usage.py

#convert raw sentences into list of tuples (token and empty)
def sents2tuples(sents):
      res = []
      for sent in sents:
        tokens = word_tokenize(sent)
        res.append([(token,'') for token in tokens])
      return res

#with sent2tuples, preprocessing will work just fine with new text
def preprocess( texts):
      texts = [res for res in sents2tuples(texts)]
      X = [sent2features(s) for s in texts]
      return X

samples = ["Facebook has a price target of $ 20 for this quarter",
         "$ AAPL is gaining a new momentum"]


processed = preprocess(samples)

pred = crf.predict(processed)
for i in range(len(samples)):
  sentence = samples[i].split()
  for j in range(len(sentence)):
    print(sentence[j],'-->',pred[i][j])
  print()