gistfile1.clj

(ns importer.core
  (:require [clojure.data.json :as json]
            [monger.collection :as monger-coll]
            [monger.core :as monger]
            [clojure.java.io :as io]
            [clojure.tools.cli :as cli])
  (:import (com.mongodb Mongo DB DBObject)))

(defn update-unique-devices [uuid app]
  (monger-coll/update
   "unique_devices",
   {:_id uuid},
   {"$addToSet" {:install app}},
   :upsert true))

(defn process-line
  "Executed for each line in the big file"
  [line]
  (let [decoded (json/read-json line)
        uuid (:uuid decoded)
        app (:app decoded)]
    (update-unique-devices uuid app)
    (println "added uuid: " uuid "app: " app "!") nil)) ; log and return nil

(def connection-info
  {:host "ec2-23-21-72-55.compute-1.amazonaws.com", :port 27017})

(defn import-file
  "Import file"
  [file]
  (monger/connect! connection-info) ; connect to mongo
  (monger/set-db! (monger.core/get-db "chartboost"))
  (with-open [r (io/reader file)]
    (doseq [line (line-seq r)]
      (process-line line))))

(defn -main [& args]
  (let [[options args banner]
        (cli/cli args
          ["--file" "which file to process"])]
    ;(import-file "installs.3_24.json")
    (import-file "tail.json")
    (shutdown-agents)))