gistfile1.clj

(ns webscraper.core
  (:gen-class)
  (:require [clojure.core.async :as async]))

(defn process [line]
  line)

(def stdin-reader
  (java.io.BufferedReader. *in*))

(def in-chan (async/chan))
(def out-chan (async/chan))

(defmacro async/threads 
  [num-consumers & body]
  `(dotimes [_ num-consumers]
    (async/thread ~@body)))

(defn start-async-consumer
  [num-consumers]
  (async/thread num-consumers
    (while true
      (let [line (async/<!! in-chan)
          data (process line)]
            (async/>!! out-chan data)))))

(defn start-async-aggregator
  "Take items from the out-chan and print it."
  []
  (async/thread
    (while true
      (let [data (async/<!! out-chan)]
      (println data)))))

(defn -main
  "Starts the workers"
  [& args]
  (do
    (start-async-consumers 2)
    (start-async-aggregator)
    (doseq [line (line-seq stdin-reader)]
    (async/>!! in-chan line))))