A method for paging through AWS results. *paging-invoke*

find_in_bucket.clj

#!/usr/bin/env bb

(require '[babashka.pods :as pods])

(pods/load-pod 'org.babashka/aws "0.0.1")

(require '[pod.babashka.aws :as aws])
(require '[clojure.tools.cli :refer [parse-opts]])


(defn paging-invoke
  ([client paginator-map op-map]
   (let [{:keys [op request]} op-map
         {:keys [input-token limit-key output-token result-key]} paginator-map]
     (printf ";; +++ paging-invoke %s :: NEXT INVOKE (w/ request)\n" op)
     (let [response (aws/invoke client {:op op :request request})]
       (if (:cognitect.anomalies/category response)
         (let [op-name (-> client aws/ops op :name)]
           (throw (ex-info (format "%s Failed" op-name) {:error (:message response)})))

         (let [token (response output-token)
               result (response result-key)]
           (if (empty? result)
             (printf ";; +++ paging-invoke %s :: empty %s in initial request\n" op result-key)
             (do
               (printf ";; -- paging-invoke %s :: initial request returned %d %s\n" op (count result) result-key)
               (lazy-seq (cons (first result) (paging-invoke client paginator-map op-map {input-token token result-key (rest result)})))))))))
   )
  ([client paginator-map op-map prev-response]
   (let [{:keys [op request]} op-map
         {:keys [input-token limit-key output-token result-key]} paginator-map
         token (input-token prev-response)
         result (result-key prev-response)]
     (printf ";; +++ paging-invoke %s :: get next item (w/ request & prev-response)\n" op)
     (if (empty? result)
       (if (some? token)
         (do
           (printf ";; +++ paging-invoke %s :: NEXT INVOKE (w/ request & prev-response)\n" op)
           (let [new-response (aws/invoke client {:op op :request (assoc request input-token token)})
                 token (new-response output-token)
                 result (new-response result-key)]
             (if (empty? result)
               (printf ";; +++ paging-invoke %s :: empty %s in new request\n" op result-key)
               (do
                 (printf ";; -- paging-invoke %s :: %d %s in new request\n" op (count result) result-key)
                 (lazy-seq (cons (first result) (paging-invoke client paginator-map op-map {input-token token result-key (rest result)})))))))
         (printf ";; -- paging-invoke %s :: empty %s, no %s\n" op result-key output-token))
       (do
         (printf ";; -- paging-invoke %s :: with %s\n" op result-key)
         (lazy-seq (cons (first result) (paging-invoke client paginator-map op-map {input-token token result-key (rest result)}))))))
   )
  )

(defn paging-list-objects-v2
"Returns a lazy sequence of S3 Objects using the S3 ListObjectsV2 operation."
  [client request]
  (let [paginator {
                   :input-token  :ContinuationToken
                   :limit-key    :MaxKeys
                   :output-token :NextContinuationToken
                   :result-key   :Contents}]
    (paging-invoke client paginator {:op :ListObjectsV2 :request request})))

(def cli-options
  [["-b" "--bucket BUCKET" "AWS S3 Bucket Name"]
   ["-p" "--prefix PREFIX" "Prefix string for limiting keys listed" :default ""]
   ["-r" "--region REGION" "AWS Region Name"
    :default (or (System/getenv "AWS_REGION") "us-west-1")]
   ["-n" "--num num-items" "Number of items to retrieve per request"
    :default 1000
    :parse-fn #(Integer/parseInt %)]
   ;; A non-idempotent option
   ["-v" nil "Verbosity level"
    :id :verbosity
    :default 0
    :update-fn inc]
   ["-h" "--help"]])


(defn usage [options-summary]
  (->> ["find-in-bucket, retrieve the contents of a bucket until the target object is found."
        ""
        "Usage: fib.clj [options] target-key"
        ""
        "Options:"
        options-summary
        ""]
       (str/join \newline)))

(defn error-msg [errors]
  (str "The following errors occurred while parsing your command:\n\n"
       (str/join \newline errors)))

(defn exit [status msg]
  (println msg)
  (System/exit status))


(defn differ [key target-key verbose]
  (when (> verbose 0)
    (printf "Checking '%s' :: %s\n" key (if (= target-key key) "MATCHES" "doesn't match")))
  (not= key target-key))

(defn -main [args]

  (let [{:keys [options arguments errors summary]} (parse-opts args cli-options)
        region (:region options)
        verbosity (:verbosity options)
        bucket (:bucket options)
        prefix (:prefix options)
        max-keys (:num options)]

    (cond
      (:help options) (exit 0 (usage summary))
      (not= (count arguments) 1) (do (println "One argument is expected, target-key") (exit 1 (usage summary)))
      errors (exit 1 (error-msg errors)))

    (let [s3-client (aws/client {:api :s3 :region region})
          target-key (first arguments)
          objs (paging-list-objects-v2 s3-client {:Bucket bucket :Prefix prefix :MaxKeys max-keys})
          first-matching-obj (first (drop-while #(differ (:Key %) target-key verbosity) objs))]
      (println (:Key first-matching-obj)))))


(-main *command-line-args*)