dataset-memory-usage.clj

user> (require '[tech.io :as io])
nil
user> (require '[tech.ml.dataset :as ds])
nil
user> (require '[clj-memory-meter.core :as mm])
nil
user> (def double-mapseq
        (vec (repeatedly 1000
                         #(->> (range 100)
                               (map (fn [idx]
                                      [idx (rand)]))
                               (into {})))))
#'user/double-mapseq
user> (mm/measure double-mapseq)
"4.9 MB"
user> (mm/measure (ds/->dataset double-mapseq))
"859.8 KB"
user> (def short-mapseq
        (vec (repeatedly 1000
                         #(->> (range 100)
                               (map (fn [idx]
                                      [idx (unchecked-short (rand-int 65535))]))
                               (into {})))))
#'user/short-mapseq
user> (mm/measure short-mapseq)
"4.1 MB"
user> (mm/measure (ds/->dataset short-mapseq))
"235.6 KB"
user> (def object-mapseq
        (vec (repeatedly 1000
                         #(->> (range 100)
                               (map (fn [idx]
                                      [idx (keyword (str idx))]))
                               (into {})))))
#'user/object-mapseq
user> (mm/measure object-mapseq)
"2.6 MB"
user> (mm/measure (ds/->dataset object-mapseq))
"457.3 KB"

user> (def ames-path "data/ames-house-prices/train.csv")
#'user/ames-path
user> (def raw-csv-load (vec (io/csv->mapseq ames-path)))
#'user/raw-csv-load
user> (mm/measure raw-csv-load)
"9.0 MB"
user> (mm/measure (ds/->dataset raw-csv-load))
"1.6 MB"
user> (mm/measure (ds/->dataset ames-path))
"270.9 KB"