Skip to content

Instantly share code, notes, and snippets.

@lgouger
Last active Jan 12, 2021
Embed
What would you like to do?
A method for paging through AWS results. *paging-invoke*
#!/usr/bin/env bb
(require '[babashka.pods :as pods])
(pods/load-pod 'org.babashka/aws "0.0.1")
(require '[pod.babashka.aws :as aws])
(require '[clojure.tools.cli :refer [parse-opts]])
(defn paging-invoke
([client paginator-map op-map]
(let [{:keys [op request]} op-map
{:keys [input-token limit-key output-token result-key]} paginator-map]
(printf ";; +++ paging-invoke %s :: NEXT INVOKE (w/ request)\n" op)
(let [response (aws/invoke client {:op op :request request})]
(if (:cognitect.anomalies/category response)
(let [op-name (-> client aws/ops op :name)]
(throw (ex-info (format "%s Failed" op-name) {:error (:message response)})))
(let [token (response output-token)
result (response result-key)]
(if (empty? result)
(printf ";; +++ paging-invoke %s :: empty %s in initial request\n" op result-key)
(do
(printf ";; -- paging-invoke %s :: initial request returned %d %s\n" op (count result) result-key)
(lazy-seq (cons (first result) (paging-invoke client paginator-map op-map {input-token token result-key (rest result)})))))))))
)
([client paginator-map op-map prev-response]
(let [{:keys [op request]} op-map
{:keys [input-token limit-key output-token result-key]} paginator-map
token (input-token prev-response)
result (result-key prev-response)]
(printf ";; +++ paging-invoke %s :: get next item (w/ request & prev-response)\n" op)
(if (empty? result)
(if (some? token)
(do
(printf ";; +++ paging-invoke %s :: NEXT INVOKE (w/ request & prev-response)\n" op)
(let [new-response (aws/invoke client {:op op :request (assoc request input-token token)})
token (new-response output-token)
result (new-response result-key)]
(if (empty? result)
(printf ";; +++ paging-invoke %s :: empty %s in new request\n" op result-key)
(do
(printf ";; -- paging-invoke %s :: %d %s in new request\n" op (count result) result-key)
(lazy-seq (cons (first result) (paging-invoke client paginator-map op-map {input-token token result-key (rest result)})))))))
(printf ";; -- paging-invoke %s :: empty %s, no %s\n" op result-key output-token))
(do
(printf ";; -- paging-invoke %s :: with %s\n" op result-key)
(lazy-seq (cons (first result) (paging-invoke client paginator-map op-map {input-token token result-key (rest result)}))))))
)
)
(defn paging-list-objects-v2
"Returns a lazy sequence of S3 Objects using the S3 ListObjectsV2 operation."
[client request]
(let [paginator {
:input-token :ContinuationToken
:limit-key :MaxKeys
:output-token :NextContinuationToken
:result-key :Contents}]
(paging-invoke client paginator {:op :ListObjectsV2 :request request})))
(def cli-options
[["-b" "--bucket BUCKET" "AWS S3 Bucket Name"]
["-p" "--prefix PREFIX" "Prefix string for limiting keys listed" :default ""]
["-r" "--region REGION" "AWS Region Name"
:default (or (System/getenv "AWS_REGION") "us-west-1")]
["-n" "--num num-items" "Number of items to retrieve per request"
:default 1000
:parse-fn #(Integer/parseInt %)]
;; A non-idempotent option
["-v" nil "Verbosity level"
:id :verbosity
:default 0
:update-fn inc]
["-h" "--help"]])
(defn usage [options-summary]
(->> ["find-in-bucket, retrieve the contents of a bucket until the target object is found."
""
"Usage: fib.clj [options] target-key"
""
"Options:"
options-summary
""]
(str/join \newline)))
(defn error-msg [errors]
(str "The following errors occurred while parsing your command:\n\n"
(str/join \newline errors)))
(defn exit [status msg]
(println msg)
(System/exit status))
(defn differ [key target-key verbose]
(when (> verbose 0)
(printf "Checking '%s' :: %s\n" key (if (= target-key key) "MATCHES" "doesn't match")))
(not= key target-key))
(defn -main [args]
(let [{:keys [options arguments errors summary]} (parse-opts args cli-options)
region (:region options)
verbosity (:verbosity options)
bucket (:bucket options)
prefix (:prefix options)
max-keys (:num options)]
(cond
(:help options) (exit 0 (usage summary))
(not= (count arguments) 1) (do (println "One argument is expected, target-key") (exit 1 (usage summary)))
errors (exit 1 (error-msg errors)))
(let [s3-client (aws/client {:api :s3 :region region})
target-key (first arguments)
objs (paging-list-objects-v2 s3-client {:Bucket bucket :Prefix prefix :MaxKeys max-keys})
first-matching-obj (first (drop-while #(differ (:Key %) target-key verbosity) objs))]
(println (:Key first-matching-obj)))))
(-main *command-line-args*)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment