reports3.clj

(defn download-report [{:keys [client date] :as state}]
  ;; snip
  (assoc state :file downloaded-file))

(defn upload-to-s3 [{:keys [file] :as state}]
  ;; snip
  (assoc state :bucket bucket :key key))

(defrecord Transition [op next-state])

(defn completed? [{:keys [state]}]
  (= :completed state))

(defn transition
  [{:keys [state] :as current-state}]
  (condp = state
    :downloadable (Transition. download-report :uploadable)
    :uploadable   (Transition. upload-to-s3 :completed)))

(defn step [current-state]
  (let [{:keys [op next-state]} (transition current-state)]
    (try (assoc (op current-state) :state next-state)
         (catch Exception e
           (assoc current-state :error e)))))

(defn progressor [states-ch completed-ch]
  (go-loop [current-state (<! states-ch)]
    (if (completed? current-state)
      (>! completed-ch current-state)
      (let [{:keys [error] :as new-state} (<! (thread (step current-state)))]
        (if error
          (do (log/error (:error new-state))
              (<! (timeout 5000))             ; delay the retry for 5s
              (>! states-ch current-state))
          (>! states-ch new-state)))))
    (recur (<! states-ch))))


(defn initial-state [date client]
  {:date   date
   :client client
   :state  :downloadable})

(defn -main []
  (let [states-ch (chan 1024)]
    (doseq [_ (range 10)]
      (progressor options states-ch completed-ch))

    ;; snip- creates sequence of initial states
    ;; using initial-state fn.
    (doseq [state initial-states]
      (>! states-ch state))

    ;; snip- wait for all operations to complete
    ))

Hi, nice post !

I tried testing your skeleton and removed some bugs/added some logging.
Maybe you'd want to update your blog-entry and/or this Gist, so people have an easier time executing it.

https://gist.github.com/clojj/f69094a851939103407a

Best,
Joerg