eslick

(p/subscribe [:test] :printer)


(p/subscribe #{:test} :printer
        (async/map> #(select-keys % [:ts :value]) (async/chan)))

eslick

{
 :node "ip-10-226-117-71",
 :ns "switchboard.api.synchronize",
 :thread-id 397,
 :app 1
 :mtx 3292,
 :atx 2154,
 :dts 1395756593391,
 :date #inst "2014-03-25T14:09:53.390-00:00",
 :auth #uuid "532cb9c1-39cc-486e-9e34-62d154d0fb84",

eslick

16:57:27.147 ERROR com.amazonaws.http.AmazonHttpClient - Unable to unmarshall response (ParseError at [row,col]:[1,1]
Message: JAXP00010001: The parser has encountered more than "64000" entity expansions in this document; this is the limit imposed by the JDK.)
javax.xml.stream.XMLStreamException: ParseError at [row,col]:[1,1]
Message: JAXP00010001: The parser has encountered more than "64000" entity expansions in this document; this is the limit imposed by the JDK.
	at com.sun.org.apache.xerces.internal.impl.XMLStreamReaderImpl.setInputSource(XMLStreamReaderImpl.java:219) ~[na:1.6.0_27]
	at com.sun.org.apache.xerces.internal.impl.XMLStreamReaderImpl.<init>(XMLStreamReaderImpl.java:189) ~[na:1.6.0_27]
	at com.sun.xml.internal.stream.XMLInputFactoryImpl.getXMLStreamReaderImpl(XMLInputFactoryImpl.java:277) ~[na:1.6.0_27]
	at com.sun.xml.internal.stream.XMLInputFactoryImpl.createXMLStreamReader(XMLInputFactoryImpl.java:129) ~[na:1.6.0_27]
	at com.sun.xml.internal.stream.XMLInputFactoryImpl.createXMLEventReader(XM

eslick

 :profiles {;; PRODUCTION
            :prod {:jvm-opts ["-Xms512M" "-Xmx2G" "-server" "-Djdk.xml.entityExpansionLimit=0"
                              "-XX:+UseConcMarkSweepGC" "-XX:+CMSIncrementalMode"]}}

eslick

22:34:03,045 WARN  [org.hornetq.core.client] (MSC service thread 1-2) HQ212002: Timed out waiting for handler to complete processing
22:34:03,056 ERROR [stderr] (Thread-13638 (HornetQ-client-global-threads-396177756)) java.lang.NullPointerException
22:34:03,056 ERROR [stderr] (Thread-13638 (HornetQ-client-global-threads-396177756)) 	at org.projectodd.shimdandy.impl.ClojureRuntimeShimImpl.invoke(ClojureRuntimeShimImpl.java:91)
22:34:03,057 ERROR [stderr] (Thread-13638 (HornetQ-client-global-threads-396177756)) 	at org.immutant.messaging.MessageProcessor.onMessage(MessageProcessor.java:51)
22:34:03,058 ERROR [stderr] (Thread-13638 (HornetQ-client-global-threads-396177756)) 	at org.projectodd.polyglot.messaging.BaseMessageProcessor.onMessage(BaseMessageProcessor.java:132)
22:34:03,058 ERROR [stderr] (Thread-13638 (HornetQ-client-global-threads-396177756)) 	at org.hornetq.core.client.impl.ClientConsumerImpl.callOnMessage(ClientConsumerImpl.java:1085)
22:34:03,059 ERROR [stderr] (Thread-13638 (HornetQ-client-globa

eslick

(define-sink :null []
  :reduce-writer wrap/clojure-writer
  :output-format NullOutputFormat
  :output-key Text
  :output-value Text)

eslick

(define-source :hbase [table constraints]
  :input-format TableInputFormat
  :map-reader hbase-table-reader-latest
  :configure (fn [job]
	       (configure-hbase job table constraints)))

eslick

(define-shuffle :clojure []
  :map-writer wrap/clojure-writer
  :reduce-reader wrap/clojure-reduce-reader)

eslick

;; Pseudocode example for guards in flows

(def *guard-time* [:days 7])

(defmacro with-resource-time-guard 
  [[name timespec inhibit?] &body ]
  `(let [resource# (get-resource name)]
     (cond (and ~inhibit? (resource-value resource#))
           (resource-value resource#)
           (or (resource-expired? resource# ~timespec)

eslick

;; hadoop -jar compass.jar -step count-users
;; hadoop -jar compass.jar -flow count-user-flow

(defn map-count-users [k fmap]
  (ctx/increment-counter "Compass" "rows")
  (ctx/increment-counter "Compass" (format "src=%s" (:src (:userinfo fmap))))
  [])

(defn count-users-total [job]
  (job-counter-value job "Compass" "rows"))