script to download walpapers from windowsonearth.org

gallery.cljs

#!/usr/bin/env lumo
(ns spacedl.spacedl
  (:require [cljs.nodejs :as node]
            [clojure.pprint :refer [pprint]]
            [clojure.string :as string :refer [join split starts-with?]]
            [cljs.core :refer [*command-line-args*]]))

(node/enable-util-print!)
(.on js/process "uncaughtException" #(js/console.error %))

(defonce http (node/require "http"))
(defonce https (node/require "https"))
(defonce fs (node/require "fs"))

(defn page [gallery-url page-number]
  (string/replace gallery-url #"PageNumber=\d+&" (str "PageNumber=" page-number "&")))

(defn parse-image-url [{:keys [GalleryUrl Index UrlSignature URLFilename ImageKey PhotoBy Sizes]}]
  (let [size      (->> [:O :5K :4K :X5 :X4 :X3]
                       (filter #(Sizes %))
                       (first))
        ext       (-> size Sizes :ext)
        file-name (str URLFilename "-" (name size) "." ext)]
    (when size
      {:url       (join "/" [GalleryUrl Index UrlSignature (name size) file-name])
       :file-name file-name})))

(defn get-url [url cb]
  (.get (if (starts-with? url "https") https http) url cb))

(defn parse-page [url]
  (fn [resolve reject]
    (get-url url
             (fn [res]
               (let [body (atom "")]
                 (-> res
                     (.on "data" #(swap! body str (.toString %)))
                     (.on "end" #(-> @body
                                     js/JSON.parse
                                     (js->clj :keywordize-keys true)
                                     resolve))))))))

(defn follow-redirects [url cb]
  (get-url
    url
    (fn [res]
      (if-let [location (aget (.-headers res) "location")]
        (follow-redirects location cb)
        (cb res)))))

(defn save-file [{:keys [url file-name]} cb]
  (follow-redirects
    url
    (fn [res]
      (println "saving" url)
      (.pipe res (.createWriteStream fs file-name)
             (js/setTimeout cb 100)))))

(defn save-links [[link & links]]
  (when link
    (if (.existsSync fs (:file-name link))
      (do
        (println "skipping" (:file-name link))
        (save-links links))
      (save-file link #(save-links links)))))

(defn parse-image-links [pages]
  (-> (map #(js/Promise. (parse-page %)) pages)
      (js/Promise.all)
      (.then #(->> % (map :Images) (apply concat) (map parse-image-url) set save-links))
      (.catch js/console.error)))

(defn parse-gallery [url {:keys [TotalItems TotalPages]}]
  (println "gallery contains" TotalPages "pages with" TotalItems "images")
  (parse-image-links (map (partial page url) (range TotalPages))))

(defn parse-url [url]
  (-> (js/Promise. (parse-page url))
      (.then #(parse-gallery url (:Pagination %)))
      (.catch js/console.error)))

(if (empty? *command-line-args*)
  (println "please provide a gallery URL as the argument")
  (parse-url (first *command-line-args*)))

intructions.md

installation

1. install Node.js
2. install Lumo: npm install -g lumo-cljs
3. save script locally
4. run the script with the URL, e.g:

    ./gallery.cljs "http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=37434732&albumKey=TBQqg7&nodeId=FDP9N&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=16&imageSizes=L%2CXL&method=rpc.gallery.getalbum"

gallery list

• Night Lights - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=37344726&albumKey=6kQLh9&nodeId=HnNLw&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Glaciers - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=37403539&albumKey=S8TJBC&nodeId=HXf83&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Glorious Earth - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=52574604&albumKey=Sd8tWH&nodeId=Wd8XTq&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=14&imageSizes=M%2CL&method=rpc.gallery.getalbum
• Most Popular - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=47556713&albumKey=Ctf9F7&nodeId=2kf6x&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Cupola Views - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=57712022&albumKey=KR9cwp&nodeId=rt6SwR&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=14&imageSizes=XL%2CX2&method=rpc.gallery.getalbum
• North America - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089914&albumKey=3MKq54&nodeId=DP7bM&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• South America - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089923&albumKey=6mm3w3&nodeId=k5PL8&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Europe - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089906&albumKey=MGWwk6&nodeId=gS2cx&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Africa - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089935&albumKey=XRCm4Z&nodeId=6wDBX&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Asia - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089928&albumKey=wVcXTX&nodeId=ZpN6x&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Australia / New Zealand - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44096028&albumKey=ZBLvbJ&nodeId=M7fLK&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Art Gallery - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=43860666&albumKey=3bSfrB&nodeId=VsTsV&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Atlantic - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089952&albumKey=HFkcz8&nodeId=nChbw&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum
• Pacific - http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=44089964&albumKey=WKj2sc&nodeId=7fVVQ&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=30&imageSizes=X2%2CX3&method=rpc.gallery.getalbum

---

Steps to find a gallery URL:

• go to http://www.windowsonearth.org and select the gallery you wish to download
• open the network tab in the browser and look for the XHR request with the method=rpc.gallery.getalbum option, e.g: http://www.windowsonearth.org/services/api/json/1.4.0/?galleryType=album&albumId=37434732&albumKey=TBQqg7&nodeId=FDP9N&PageNumber=0&imageId=0&imageKey=&returnModelList=true&PageSize=16&imageSizes=L%2CXL&method=rpc.gallery.getalbum

Just wanted to drop a thanks for this sample.
It has a lot of good examples when it is your first dive into lumo :)

I think lines 1-9 can improve.

#!/usr/bin/env lumo
(require '[cljs.nodejs :as node]
         '[clojure.pprint :refer [pprint]]
         '[clojure.string :as string :refer [join split starts-with?]]
         'http
         'https
         'fs)

(node/enable-util-print!)
(js/process.on "uncaughtException" js/console.error)