gistfile1.clj

(defn parse-row
  [^String row]
  (let [[k v] (.split row \tab)
        [k1 k2] (map (partial string/join " ") (json/read-str k))]
    [k1 k2 (Long/parseLong (.trim v))]))

(defn co-occurrence
  [dir]
  (let [source (hfs-textline dir)]
    (<- [?k1 ?k2 ?count]
      (source ?line)
      (parse-row ?line :> ?k1 ?k2 ?count))))

(defn discrim-probs
  [dir]
  (let [occurrences (co-occurrence dir)
        global-total (<- [?total]
                      (occurrences _ _ ?c)
                      (c/sum ?c :> ?total))
        total (<- [?k1 ?sum]
                (occurrences ?k1 _ ?c)
                (c/sum ?c :> ?sum))]
      (<- [?k1 ?k2 ?prob]
        (global-total ?total)
        (total ?k1 ?dict-total)
        (total ?k2 ?ngram-count)
        (occurrences ?k1 ?k2 ?count)
        (correlation-prob ?total ?dict-total ?count :> ?prob))))