-
-
Save tce/1754274 to your computer and use it in GitHub Desktop.
Using CLI in clojure
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
;; from http://twitch.nervestaple.com/2012/01/12/clojure-hbase/?utm_source=NoSQL%20Weekly%20Newsletter | |
(defn tool-run | |
"Provides the main function needed to bootstrap the Hadoop application." | |
[^Tool this args-in] | |
;; define our command line flags and parse out the provided | |
;; arguments | |
(let [[options args banner] | |
(cli args-in | |
["-h" "--help" | |
"Show usage information" :default false :flag true] | |
["-p" "--path" "HDFS path of data to consume"] | |
["-o" "--output" "HDFS path for the output report"])] | |
;; display the help message | |
(if (:help options) | |
(do (println banner) 0) | |
;; setup and run our job | |
(do | |
(doto (Job.) | |
(.setJarByClass (.getClass this)) | |
(.setJobName "crawl-log-load") | |
(.setOutputKeyClass Text) | |
(.setOutputValueClass LongWritable) | |
(.setMapperClass (Class/forName "crawl-log-loader.core_mapper")) | |
(.setReducerClass (Class/forName "crawl-log-loader.core_reducer")) | |
(.setInputFormatClass TextInputFormat) | |
(.setOutputFormatClass TextOutputFormat) | |
(FileInputFormat/setInputPaths (:path options)) | |
(FileOutputFormat/setOutputPath (Path. (:output options))) | |
(.waitForCompletion true)) | |
0)))) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment