Storing and retrieving Clojure data structures as GZIP compressed JSON in Amazon S3

s3.clj

(ns aws.s3
  (:refer-clojure :exclude [get])
  (:use [clojure.walk :only (keywordize-keys stringify-keys)]
        [clojure.contrib.def :only (defonce-)]
        [clojure.contrib.json :only (read-json write-json)])
  (:import [java.io PrintWriter InputStreamReader ByteArrayInputStream ByteArrayOutputStream]
           [java.util.zip GZIPInputStream GZIPOutputStream]
           [com.google.common.base Charsets]
           [com.amazonaws.services.s3 AmazonS3Client]
           [com.amazonaws.services.s3.model Region CreateBucketRequest ObjectMetadata
            BucketVersioningConfiguration SetBucketVersioningConfigurationRequest]))

;;;; Setup

;;; May want to share this among all AWS clients
(defonce credentials (PropertiesCredentials.
                      (.getResourceAsStream (clojure.lang.RT/baseLoader) "aws.properties")))

(defonce- s3 (AmazonS3Client. credentials))

(defonce- bucket "some-bucket")
(when-not (.doesBucketExist s3 bucket)
  (.createBucket s3 (CreateBucketRequest. bucket Region/US_Standard)))
(let [enabled (BucketVersioningConfiguration. BucketVersioningConfiguration/ENABLED)]
  (when-not (= (.getBucketVersioningConfiguration s3 bucket) enabled)
    (.setBucketVersioningConfiguration s3 (SetBucketVersioningConfigurationRequest. bucket enabled))))


;;;; Storage

(defn- compressed-json [x]
  (with-open [out    (new ByteArrayOutputStream)
              gzip   (GZIPOutputStream. out)
              writer (PrintWriter. gzip)]
    (write-json x writer)
    (.flush writer)
    (.finish gzip)
    (.toByteArray out)))

(defn put! [k v]
  "Stores a Clojure data structure as compressed JSON."
  (let [bytes (compressed-json v)
        len   (count bytes)
        in    (ByteArrayInputStream. bytes)
        meta  (doto (new ObjectMetadata)
                (.setContentType "text/plain") ; show in browser, but should be "application/json" 
                (.setContentLength len)
                (.setContentEncoding "gzip")
                (.setUserMetadata (-> v meta stringify-keys)))]
    (.putObject s3 bucket k in meta)))


;;;; Retrieval

(defn- get-decompressed-json [o]
  (-> (.getObjectContent o)
      (GZIPInputStream.)
      (InputStreamReader. Charsets/UTF_8)
      read-json))

(defn- get-meta [o]
  (->> (.. o getObjectMetadata getUserMetadata)
       (into {})
       keywordize-keys))

(defn get [k]
  "Returns a Clojure data structure stored as compressed JSON."
  (let [o (.getObject s3 bucket k)
        v (get-decompressed-json o)
        m (get-meta o)]
    (with-meta v m)))

Thanks for this! I was trying to figure out how to gzip a string and send it to S3, and this helped immensely.