other_words.clj

(defn generate-others [root]
  (let [ga (select-fields (gadata-tap root) ["!kw"])
        kw-stats (select-fields (kw-stats-tap root) "?ngram")]
    (<- [?ngram ?other ?kw]
	(ga ?kw)
        (kw-stats ?ngram) ;join on kw-stats to filter
        (not= ?kw "")
	(other-ngrams ?kw 4 :> ?ngram ?other))))

(defn generate-other-words [output-tap root date]
  (let [others (generate-others root)
        landings (landings-by-kw root)
        sq (<- [?ngram ?other ?total-l ?total-pv ?total-b ?total-nv ?total-rv
                ?b-rate ?nv-rate ?rv-rate ?avg-depth]
               (others ?ngram ?other ?kw)
               (landings ?kw ?l ?pv ?b ?nv ?rv)
               (ybot-stats ?l ?pv ?b ?nv ?rv :>
                           ?total-l ?total-pv ?total-b ?total-nv ?total-rv
                           ?b-rate ?nv-rate ?rv-rate ?avg-depth))]
    (?<- "other words" output-tap [?json]
         (sq ?ngram ?other ?l ?pv ?b ?nv ?rv ?br ?nvr ?rvr ?ad)
         (write-json-map
          [[:d :w :ow :l :pv :b :nV :rV :br :nVr :rVr :ad]]
          (mongo-date date) ?ngram ?other ?l ?pv ?b ?nv ?rv ?br ?nvr ?rvr ?ad :> ?json))))

(defn generate-other-words-url-stats [output-tap root date]
  (let [others (generate-others root)
        landings-by-kw-url (landings-by-kw-url root)
        sq (<- [?ngram ?other !url ?total-l ?total-pv ?total-b ?total-nv ?total-rv
                ?b-rate ?nv-rate ?rv-rate ?avg-depth]
               (others ?ngram ?other ?kw)
               (landings-by-kw-url ?kw !url ?l ?pv ?b ?nv ?rv)
               (ybot-stats ?l ?pv ?b ?nv ?rv :>
                           ?total-l ?total-pv ?total-b ?total-nv ?total-rv
                           ?b-rate ?nv-rate ?rv-rate ?avg-depth))]
    (?<- "other words url stats" output-tap [?json]
         (sq ?ngram ?other !url ?l ?pv ?b ?nv ?rv ?br ?nvr ?rvr ?ad)
         (write-json-map
          [[:d :w :ow :url :l :pv :b :nV :rV :br :nVr :rVr :ad]]
          (mongo-date date) ?ngram ?other !url ?l ?pv ?b ?nv ?rv ?br ?nvr ?rvr ?ad :> ?json))))