multi-curl.clj

(defn process-urls [callback chunk-size stream]
  (let [mcurl (MCurl.)]
    (letfn [(add-handle
             [next-thunk]
             (when-let [[url cbdata] (next-thunk)]                       
               (.add mcurl (Curl. url inner-callback cbdata))))
            (inner-callback
             [info content data]
             (add-handle #(or (apply callback args)
                              (stream))))])
    (dotimes [_ chunk-size]
      (add-handle stream))
    (.finish mcurl)))

(defn simple-multi-curl [callback cbdata-fn chunk-size url-seq]
  (let [url-ref (atom [url-seq nil])] ; remaining urls, next work item
    (process-urls callback
                  chunk-size
                  (fn []
                    (when-let [url (second (swap! url-ref
                                                  (comp (juxt rest first)
                                                        first)))]
                      [url (cbdata-fn url)])))))