company_count.clj

(ns company-count
  (:use [clojure.contrib.http.agent :only (http-agent stream)])
  (:use [clojure.contrib.json])
  (:use [clojure.contrib.io :only (read-lines)]))


(defonce companies (read-json (slurp "http://api.crunchbase.com/v/1/companies.js")))


(def emp-ranges (agent {}))


(defn which-range [n]
  (cond (nil? n) :unknown

        (< n 15) :under15

        (< n 50) :15to49

        (< n 100) :50to99

        (< n 501) :100to500

        :else :over500))


(defn api-url [company]
  (str "http://api.crunchbase.com/v/1/company/" (:permalink company) ".js"))


(defn update-emp-range! [name emp-cnt]
  (send emp-ranges
        (fn [m k] (let [v (get m k {})
                        v (assoc v name emp-cnt)]
                    (println (str "handled " name))
                    (assoc m k v)))
        (which-range emp-cnt)))


(defn process-company [co]
  (http-agent
   (api-url co)
   :handler (fn [agnt]
              (let [res (slurp (stream agnt))
                    data (read-json res)
                    emps (:number_of_employees data)]
                (update-emp-range! (:name co) emps))))

  nil)

exceptions.log

(filter (complement nil?) (pmap process-company (take 1000 companies)))

Exception in thread "pool-1-thread-3" java.lang.IllegalStateException: Pop without matching push
Exception in thread "pool-2-thread-477" java.lang.IllegalStateException: Pop without matching push
Exception in thread "pool-2-thread-964" java.lang.IllegalStateException: Pop without matching push
	at clojure.lang.Var.popThreadBindings(Var.java:297)
	at clojure.lang.Agent$Action.doRun(Agent.java:145)
	at clojure.lang.Agent$Action.run(Agent.java:150)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Exception in thread "pool-2-thread-955" java.lang.IllegalStateException: Pop without matching push
	at clojure.lang.Var.popThreadBindings(Var.java:297)
	at clojure.lang.Agent$Action.doRun(Agent.java:145)
	at clojure.lang.Agent$Action.run(Agent.java:150)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Exception in thread "pool-2-thread-554" java.lang.IllegalStateException: Pop without matching push
	at clojure.lang.Var.popThreadBindings(Var.java:297)
	at clojure.lang.Agent$Action.doRun(Agent.java:145)
	at clojure.lang.Agent$Action.run(Agent.java:150)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)
Exception in thread "pool-2-thread-160" java.lang.IllegalStateException: Pop without matching push
	at clojure.lang.Var.popThreadBindings(Var.java:297)
	at clojure.lang.Agent$Action.doRun(Agent.java:145)
	at clojure.lang.Agent$Action.run(Agent.java:150)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:680)