Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
(defn process-urls [callback chunk-size stream]
(let [mcurl (MCurl.)]
(letfn [(add-handle
[next-thunk]
(when-let [[url cbdata] (next-thunk)]
(.add mcurl (Curl. url inner-callback cbdata))))
(inner-callback
[info content data]
(add-handle #(or (apply callback args)
(stream))))])
(dotimes [_ chunk-size]
(add-handle stream))
(.finish mcurl)))
(defn simple-multi-curl [callback cbdata-fn chunk-size url-seq]
(let [url-ref (atom [url-seq nil])] ; remaining urls, next work item
(process-urls callback
chunk-size
(fn []
(when-let [url (second (swap! url-ref
(comp (juxt rest first)
first)))]
[url (cbdata-fn url)])))))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment