multi-part-upload-aws-api.clj

(defn get-file-splits
  [size-in-bytes mb-per-split]
  (let [num-parts (Math/ceil (/ size-in-bytes mb-per-split))]
    (map
      (fn [part-num]
        {::part-number           (inc part-num)
         ::start-pos             (if (zero? part-num)
                                   0
                                   (inc (* part-num mb-per-split)))
         ::max-bytes-to-transfer mb-per-split})
      (range num-parts))))
​
(defn upload-parts!
  [s3-client ^File file bucket key upload-id]
  (let [content-length (.length file)
        ;; part size to 5mb
        part-size (* 1024 1024 5)]
    (with-open [ra-f (RandomAccessFile. file "r")]
      (reduce
        (fn [acc {::keys [part-number start-pos max-bytes-to-transfer] :as part}]
          (log/info {:tag     ::upload-part
                     :message (format "Uploading part %s for s3://%s/%s"
                                      part-number
                                      bucket key)})
          (.seek ra-f start-pos)
          (let [ba (byte-array max-bytes-to-transfer)
                _ (.read ra-f ba)
                resp (aws.api/invoke
                       s3-client
                       {:op      :UploadPart
                        :request {:Bucket     bucket
                                  :Key        key
                                  :UploadId   upload-id
                                  :PartNumber part-number
                                  :Body       ba}})]
            (if (anom/anomaly? resp)
              (reduced
                (assoc resp
                  ::failing-part part))
              (conj acc (assoc resp ::part-number part-number)))))
        [] (get-file-splits content-length part-size)))))
​
(defn multipart-upload-file!
  [s3-client ^File file {::keys [bucket key]}]
  ;; based off: https://docs.aws.amazon.com/AmazonS3/latest/dev/llJavaUploadFile.html
  (let [init-req (aws.api/invoke
                   s3-client
                   {:op      :CreateMultipartUpload
                    :request {:Bucket bucket
                              :Key    key}})
        upload-id (:UploadId init-req)]
    (if (anom/anomaly? init-req)
      init-req
      (let [parts (upload-parts! s3-client file bucket key upload-id)]
        (if (anom/anomaly? parts)
          parts
          (let [complete-resp (aws.api/invoke
                                s3-client
                                {:op      :CompleteMultipartUpload
                                 :request {:Bucket          bucket
                                           :Key             key
                                           :UploadId        upload-id
                                           :MultipartUpload {:Parts
                                                             (map (fn [{:keys [:ETag ::part-number]}]
                                                                    {:ETag       ETag
                                                                     :PartNumber part-number})
                                                                  parts)}}})]
            (assoc complete-resp ::parts parts)))))))