demo reading and writing json with clojure jsonista library in a streaming style

jsonista-streaming.clj

(ns streaming
  "read-write large json arrays with jsonista - techniques from 
  https://cassiomolin.com/2019/08/19/combining-jackson-streaming-api-with-objectmapper-for-parsing-json/
  
  comment at end shows running/testing code"
  (:require [jsonista.core :as j])
  (:import (com.fasterxml.jackson.databind ObjectMapper)
           (com.fasterxml.jackson.core JsonToken JsonGenerator JsonParser)
           (java.io OutputStream ByteArrayOutputStream ByteArrayInputStream InputStream)))

(defn write-array 
  "writes xs to out and closes"
  [^OutputStream out ^ObjectMapper mapper xs]
  (with-open [^JsonGenerator json-generator
        (-> (.getFactory mapper)
            (.createGenerator out))]
    (.writeStartArray json-generator)
    (doseq [x xs]
      (.writeValue mapper json-generator x))
    (.writeEndArray json-generator)))

(defn- streamed-seq [^ObjectMapper mapper ^JsonParser json-parser ^InputStream is]
  (lazy-seq
    (if-not (= JsonToken/END_ARRAY (.nextToken json-parser))
      (cons
        (.readValue mapper json-parser ^Class Object)
        (streamed-seq mapper json-parser is))
      (.close is))))

(defn read-array 
  "returns a seq which closes the input-stream when drained"
  [is mapper]
  (let [^JsonParser json-parser (-> (.getFactory mapper)
                                    (.createParser is))
        start (.nextToken json-parser)]
    (assert (= JsonToken/START_ARRAY start))
    (streamed-seq mapper json-parser is)))

(comment
  
  (defn temp-file []
    (let [tmp (java.io.File/createTempFile (str (rand-int 1000)) ".json")]
      (.deleteOnExit tmp)
      tmp))
  ;running the following and eyeballing max heap usage, the stream way is flat, even though 2x objs
  
  ;stream way
  (let [mapper j/default-object-mapper
        f (temp-file)]
    (write-array (java.io.FileOutputStream. f) mapper (map (fn [n] {:foo n}) (range 10000000)))
    (let [is (java.io.FileInputStream. f)]
      ;(String. (.toByteArray baos))
      (last (read-array is mapper))))
  
  ; using api
  (let [f (temp-file)
        mapper j/default-object-mapper]
    (j/write-value (java.io.FileOutputStream. f)
      (map (fn [n] {:foo n}) (range 5000000))
      mapper)
    (last
        (j/read-value (java.io.FileInputStream. f) mapper)))

  )