Babashka script to download contentful images

script.clj

(ns script
  (:require [babashka.pods :as pods]
            [babashka.fs :as fs]
            [babashka.http-client :as http]
            [clojure.data.csv :as csv]
            [clojure.java.io :as io]
            [clojure.string :as str]
            [babashka.cli :as cli]))

;; read the CSV line-by-line into a data structure
(defn csv-data*
  [path]
  (with-open [reader (io/reader path)]
    ;; Babashka aliases clojure.data.csv as csv
    (doall (csv/read-csv reader))))

(defn csv-data
  [path]
  (let [[header & data] (csv-data* path)]
    (map (partial zipmap (map keyword header)) data)))


(defn mkdirs
  [{:keys [out-dir] :or {out-dir "./cfassets"}} test-cases]
  (doseq [{:keys [test-name fmt]} test-cases]
    (fs/create-dirs (str/join "/" [out-dir test-name "full"]))
    (fs/create-dirs (str/join "/" [out-dir test-name "thumbs"]))))

(defn url->filename
  [url]
  (->> url
    (re-find #".*\/(.*\.\w+)" )
    last))


(defn download
  [{:keys [out-dir]} {:keys [fmt-params test-name]} {:keys [url filename thumbnail width-requested] :as asset}]
  (let [out-path (str/join "/" [out-dir
                                test-name
                                (if (= thumbnail "true") "thumbs" "full")
                                filename])]
    (if (.exists (io/file out-path))
      (println (format "\t%s .. exists, skipping" filename))
      (do
        (println (format "\t%s %s" filename (fmt-params asset)))
        (io/copy
          (:body (http/get (str "https:" url)
                   {:query-params
                    (merge {:w width-requested}
                      (fmt-params asset))
                    "Accept-Encoding" ["gzip" "deflate"]
                    :as :stream}))
          (io/file out-path))))))

(defn download-assets-for-test
  [{:keys [out-dir] :as cfg} {:keys [fmt-params test-name] :as test} assets]
  (doseq [asset assets]
    (download cfg test asset)))

(defn download-all
  [{:keys [out-dir] :as cfg} tests assets]
  (doseq [test tests]
    (println (format "%s - Downloading assets" (:test-name test)))
    (download-assets-for-test cfg test assets)))

(def test-cases
  "Thumbnails are always requested as thumbnails, images are always
  requested with the same width in the test file. Only the format and
  specialisation (like 8 bit png / progressive) is changed."
  [{:test-name "default"
    :fmt-params  (constantly {:fl "progressive"})}

   {:test-name "default-70"
    :fmt-params  (constantly {:fl "progressive"
                              :q "70"})}

   {:test-name "smart-logic"
    :fmt-params
    (fn [{:keys [original-size]}]
      (let [sz (Long/parseLong original-size)
            q  (cond
                 (> sz 2000000) "50"
                 (> sz 1000000) "50"
                 :else "70")]
        (cond-> {:fl "progressive"
                 :fm "jpg"}
          q (assoc :q q))))}

   {:test-name "progressive+jpgs"
    :fmt-params  (constantly {:fl "progressive" :fm "jpg"})}

   {:test-name "progressive+jpgs-70"
    :fmt-params  (constantly {:fl "progressive" :fm "jpg" :q "70"})}

   {:test-name "avif"
    :fmt-params (constantly {:fm "avif"})}

   {:test-name "png"
    :fmt-params (constantly {:fm "png"})}

   {:test-name "8bit-png"
    :fmt-params (constantly {:fm "png"
                             :fl "png8"})}])


(def DEFAULT-CONFIG {:out-dir "./cfassets"})

;; (defmulti command (fn [{:keys [dispatch]}] dispatch))

;; (defmethod command :default
;;   [{:keys [opts]}]
;;   (println "default " opts))

;; (def table
;;   [{:cmds ["mkdirs"]   :fn command   :args->opts [:out-dir]}
;;    {:cmds ["download"] :fn command :args->opts [:out-dir]}
;;    {:cmds ["report"]   :fn command   :args->opts [:out-dir]}
;;    {:cmds []   :fn command }])

;; (defn -main [& args]
;;   (cli/dispatch table args {:exec-args {:out-dir "./cfassets"}}))


;; (apply -main *command-line-args*)

(def assets
  (->> "./test-data.log"
    csv-data
    (map #(assoc % :filename (url->filename (:url %))))))

(mkdirs DEFAULT-CONFIG
  test-cases)

(download-all
  {:out-dir "./cfassets"}
  test-cases
  assets)