A script to empty and delete buckets

s3_bucket_remover.clj

#!/usr/bin/env bb

(require '[babashka.pods :as pods])

(pods/load-pod 'org.babashka/aws "0.0.6")

(require '[pod.babashka.aws :as aws])
(require '[pod.babashka.aws.credentials :as credentials])

(def region "us-east-1")

(def s3-client
  (aws/client {:api :s3 :region region
               :credentials-provider (credentials/basic-credentials-provider
                                      {:access-key-id     ""
                                       :secret-access-key ""})}))

(defn list-objects [bucket]
  (aws/invoke s3-client {:op :ListObjects
                         :request {:Bucket bucket}}))

(defn objects->keys [objects]
  (map #(select-keys % [:Key]) (:Contents objects)))

(defn clear-s3-results
  [bucket]
  (loop [next-set (objects->keys (list-objects bucket))
         deleted-objects []]
    (if (pos? (count next-set))
      (let [deleted (aws/invoke s3-client  {:op :DeleteObjects
                                            :request {:Bucket bucket
                                                      :Delete {:Objects next-set}}})]
        (recur (objects->keys (list-objects bucket))
               (apply conj deleted-objects (:Deleted deleted))))
      deleted-objects)))

(defn delete-s3-bucket [bucket]
  (aws/invoke s3-client {:op :DeleteBucket
                         :request {:Bucket bucket}}))

(defn empty-and-delete-bucket [bucket]
  (clear-s3-results bucket)
  (delete-s3-bucket bucket))


(defn bucket-names [buckets]
  (map :Name buckets))

(defn empty-and-delete-buckets []
  (let [{:keys [Buckets]} (aws/invoke s3-client {:op :ListBuckets})
        bucket-names (bucket-names Buckets)]
    (pmap
     empty-and-delete-bucket
     bucket-names)))

(empty-and-delete-buckets)

When running this code I get an exception:

"ERROR: can't serialize to transit:" {:cognitect.anomalies/category :cognitect.anomalies/fault, :cognitect.anomalies/message "Invalid 'Location' header: null", :cognitect.http-client/throwable #error { :cause "Invalid 'Location' header: null" :via [{:type org.eclipse.jetty.client.HttpResponseException :message "Invalid 'Location' header: null" :at [org.eclipse.jetty.client.HttpRedirector redirect "HttpRedirector.java" 167]}] :trace [[org.eclipse.jetty.client.HttpRedirector redirect "HttpRedirector.java" 167] [org.eclipse.jetty.client.RedirectProtocolHandler onComplete "RedirectProtocolHandler.java" 73] [org.eclipse.jetty.client.ResponseNotifier notifyComplete "ResponseNotifier.java" 218] [org.eclipse.jetty.client.ResponseNotifier notifyComplete "ResponseNotifier.java" 210] [org.eclipse.jetty.client.HttpReceiver terminateResponse "HttpReceiver.java" 481] [org.eclipse.jetty.client.HttpReceiver terminateResponse "HttpReceiver.java" 461] [org.eclipse.jetty.client.HttpReceiver responseSuccess "HttpReceiver.java" 424] [org.eclipse.jetty.client.http.HttpReceiverOverHTTP messageComplete "HttpReceiverOverHTTP.java" 365] [org.eclipse.jetty.http.HttpParser parseFields "HttpParser.java" 1175] [org.eclipse.jetty.http.HttpParser parseNext "HttpParser.java" 1548] [org.eclipse.jetty.client.http.HttpReceiverOverHTTP parse "HttpReceiverOverHTTP.java" 204] [org.eclipse.jetty.client.http.HttpReceiverOverHTTP process "HttpReceiverOverHTTP.java" 144] [org.eclipse.jetty.client.http.HttpReceiverOverHTTP receive "HttpReceiverOverHTTP.java" 79] [org.eclipse.jetty.client.http.HttpChannelOverHTTP receive "HttpChannelOverHTTP.java" 131] [org.eclipse.jetty.client.http.HttpConnectionOverHTTP onFillable "HttpConnectionOverHTTP.java" 169] [org.eclipse.jetty.io.AbstractConnection$ReadCallback succeeded "AbstractConnection.java" 311] [org.eclipse.jetty.io.FillInterest fillable "FillInterest.java" 105] [org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint onFillable "SslConnection.java" 540] [org.eclipse.jetty.io.ssl.SslConnection onFillable "SslConnection.java" 395] [org.eclipse.jetty.io.ssl.SslConnection$2 succeeded "SslConnection.java" 161] [org.eclipse.jetty.io.FillInterest fillable "FillInterest.java" 105] [org.eclipse.jetty.io.ChannelEndPoint$1 run "ChannelEndPoint.java" 104] [org.eclipse.jetty.util.thread.strategy.EatWhatYouKill runTask "EatWhatYouKill.java" 336] [org.eclipse.jetty.util.thread.strategy.EatWhatYouKill doProduce "EatWhatYouKill.java" 313] [org.eclipse.jetty.util.thread.strategy.EatWhatYouKill tryProduce "EatWhatYouKill.java" 171] [org.eclipse.jetty.util.thread.strategy.EatWhatYouKill run "EatWhatYouKill.java" 129] [org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread run "ReservedThreadExecutor.java" 375] [org.eclipse.jetty.util.thread.QueuedThreadPool runJob "QueuedThreadPool.java" 773] [org.eclipse.jetty.util.thread.QueuedThreadPool$Runner run "QueuedThreadPool.java" 905] [java.lang.Thread run "Thread.java" 829] [com.oracle.svm.core.thread.JavaThreads threadStartRoutine "JavaThreads.java" 553] [com.oracle.svm.core.posix.thread.PosixJavaThreads pthreadStartRoutine "PosixJavaThreads.java" 192]]}}

Solution from Slack:

Yeah, in my gist I set the region in the client. But apparently, when getting the buckets with this client, it gives back buckets in other regions as well. If I then try to list object from a bucket outside that region this error occurs. The solution is to retrieve the region by calling GetBucketLocation and add that to the list objects request