Skip to content

Instantly share code, notes, and snippets.

@jackrusher jackrusher/data.txt
Last active Aug 29, 2015

Embed
What would you like to do?
TO GET THE DATA
===============
$ ftp ftp.ncdc.noaa.gov
[... login as user 'ftp', password is your email address ...]
ftp> cd pub/data/gsod
250 CWD command successful
ftp> get ish-history.txt
[... transfer info ...]
ftp> cd 2013
250 CWD command successful
ftp> get gsod_2013.tar
[... transfer info ...]
Put 'ish-history.txt' in the top directory of your clojure project, make a sub-directory called 'weather', then untar the archive of GSOD data into that directory.
(ns workbook.weather
(use quil.core))
(defn parse-fixed-width-fields [line cuts]
(loop [in-str line cuts cuts out []]
(if (seq cuts)
(recur (subs in-str (first cuts)) (rest cuts) (conj out (clojure.string/trim (subs in-str 0 (first cuts)))))
out)))
(defn parse-weather-station [line]
(zipmap [:usaf :wban :station-name :country :state :call-sign :lat :long :elevation :begin :end]
(parse-fixed-width-fields line [7 6 30 3 6 6 7 8 10 9 8])))
(defn load-weather-stations [filename]
(reduce #(assoc %1 (str (:usaf %2) "-" (:wban %2)) %2) {}
(map parse-weather-station (drop 22 (clojure.string/split-lines (slurp filename))))))
(def weather-stations (load-weather-stations "ish-history.txt"))
(defn load-weather-file [filename]
(rest (for [line (clojure.string/split-lines (slurp filename))]
;; slp-a slp-b stp-a stp-b visibility-min visibility-max windspeed-min windspeed-max maxspeed gust max min precipitation sndp frshtt
(let [[usaf wban ymd-date temp temp-time dew-point dew-time & junk] (clojure.string/split line #"[ ]+")]
{:usaf usaf :wban wban :ymd-date ymd-date :temp temp :dew-point dew-point}))))
(def weather-files
(filter #(.contains % "op") (map str (file-seq (clojure.java.io/file "weather")))))
(defn get-weather-station [records]
(let [sample (first records)]
(weather-stations (str (sample :usaf) "-" (sample :wban)))))
(def all-records
(remove (comp nil? get-weather-station) (pmap load-weather-file weather-files)))
(defn f-to-c
"I can't believe weather stations use the units of perfidy!"
[f]
(float (* (- f 32) 5/9)))
(defn string-to-lat-or-long [s]
(let [len (count s)]
(if (> 3 len)
-999999
(read-string (str (subs s 0 (- len 3)) "." (subs s (- len 3)))))))
(def all-temps
(for [records all-records]
(let [station (get-weather-station records)
temps (map (comp f-to-c read-string :temp) records)
dews (map (comp #(if (> % 100) 0 %)
f-to-c read-string :dew-point) records)]
(list (apply min temps) (apply max temps) (apply max dews)
(station :station-name) (station :country)
(string-to-lat-or-long (station :lat)) (string-to-lat-or-long (station :long))))))
;;(count all-temps)
;; => 12089
;;(apply min (map first all-temps))
;; => -79.27778
;;(apply max (map second all-temps))
;; => 43.22222
(def best-weather
(filter #(let [[low high dew station-name country lat long] %]
(and (> low 0)
(< high 29)
(> high 20)
(< dew 23)
(> dew 9)
(not= country "")
(not= lat -999999))) all-temps))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; whole planet
(def min-long -180)
(def max-long 180)
(def max-lat -90)
(def min-lat 90)
;; just western eurasia
;;(def min-long -40)
;;(def max-long 40)
;;(def max-lat 24)
;;(def min-lat 64)
(defn setup []
(frame-rate 10))
(defn draw []
(smooth)
(background 0x181818)
(stroke-weight 3)
(no-fill)
(stroke 60)
(doseq [[_ _ _ _ _ lat long] all-temps]
(point (map-range long min-long max-long 0 (width))
(map-range lat min-lat max-lat 0 (height))))
(stroke 255)
(doseq [[_ _ _ _ _ lat long] best-weather]
(point (map-range long min-long max-long 0 (width))
(map-range lat min-lat max-lat 0 (height)))))
;; this will start the sketch
(defsketch example
:title "Mapping the good weather"
:setup setup
:draw draw
:size [800 400])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.