gistfile1.clj

(ns importer.core
  (:require [clojure.data.json :as json]
            [monger.collection :as monger-coll]
            [monger.core :as monger]
            [clojure.java.io :as io]
            [clojure.tools.cli :as cli])
  (:import (com.mongodb Mongo DB DBObject)))

(defn update-unique-devices [uuid app]
  (monger-coll/update
   "unique_devices",
   {:_id uuid},
   {"$addToSet" {:install app}},
   :upsert true))

(defn process-line
  "Executed for each line in the big file"
  [line]
  (let [decoded (json/read-json line)
        uuid (:uuid decoded)
        app (:app decoded)]
    (update-unique-devices uuid app)
    (println "added uuid: " uuid "app: " app "!") nil)) ; log and return nil

(def connection-info
  {:host "ec2-23-21-72-55.compute-1.amazonaws.com", :port 27017})

(defn read-parallel [f]
  (->> (line-seq (io/reader f))
       (partition 10)
       (pmap (fn [l] (map process-line l)))
       dorun))

(defn import-file
  "Import file"
  [file]
  (monger/connect! connection-info) ; connect to mongo
  (monger/set-db! (monger.core/get-db "chartboost"))
  (read-parallel file)) ; read file in parallel

(defn -main [& args]
  (let [[options args banner]
        (cli/cli args
          ["--file" "which file to process"])]
    (import-file "head.json")))

;(import-file "head.json")

(with-open [r (io/reader file)]
  (pmap each-line (line-seq r)))