ngrams.clj

(defn lemmatize-text
  "Apply a lucene tokenizer to cleaned text content as a lazy-seq"
  [page-text]
  (let [reader (java.io.StringReader. page-text)
        analyzer (->
                  (resource-to-temp-file
                   "stanford_nlp_models/bidirectional-distsim-wsj-0-18.tagger"
                   ".tagger")
                  (.getAbsolutePath)
                  (MaxentTagger.)
                  (EnglishLemmaAnalyzer.))
        tokenizer (.tokenStream analyzer nil reader)
        term-att (.addAttribute tokenizer TermAttribute)]
    (tokenizer-seq tokenizer term-att)))