Ungisting Files and Tidying up Metadata

README.md

This gist covers

• some of the snippets of code that was used to remove the gists from generated pages.
• The tidying up of metadata and project strcuture

files.clj

(ns briefcase.files
  (require [stasis.core :as stasis]
           [clygments.core :as pygments]
           [cheshire.core :refer :all]
           [briefcase.content.utils :as util]
           [net.cgrand.enlive-html :as html]
           [clj-yaml.core :as yaml]))

(def articles
  (stasis/slurp-directory "resources/content/articles" #".*\.md$"))

(defn has-gists [[path content]]
  (not (nil? (re-find #"://gist\.github\.com/" content))))

(defn load-gist [path]
  [path (gist-content path)])

(defn extract-gists [content]
  (->> content
       (re-seq #"[\"|\'](http[s]?://gist\.github\.com/\d*\.js(\?[^\'\"]*)?)[\"|\']")
       (map #(nth % 1))
       (map load-gist)
       (into {})))

(defn gist-content
  "given a path to a gists embed script this function will grab the contents of
   the gist.  If a file name is also specified for a multi file gist it will
   extract the correct one"
  [path]
  (let [id       (gist-id path)
        data     (fetch-gist id)
        files    (:files data)
        filename (last (re-find #"\?file=([^&]*)" path))]
    (:content (if (nil? filename)
                  (first (vals files))
                  ((keyword filename) files)))))

(def gist-source
  (let [gisted (filter has-gists articles)]
    (zipmap (keys gisted)
            (map extract-gists (vals gisted)))))

(spit "data.edn" (with-out-str (clojure.pprint/pprint gist-source)))

(def gist-content (load-string (slurp "data.edn")))

(def gist-content-types (load-string (slurp "types.edn")))


(spit "merged.edn" (with-out-str (clojure.pprint/pprint (into {}
      (map (fn [[path gists]]
        (let [types (get gist-content-types path)]
          [path  (merge-with (fn [a b] { :content a
                                         :type (get pygments-shortnames b "text") })
                             gists types)])) gist-content)) )) )

(def gist-data (load-string (slurp "merged.edn")))

(->> gist-content-types
     (vals)
     (map vals)
     (apply concat)
     (distinct))

(def pygments-shortnames
  { "C#"         "csharp"
    "CSS"        "css"
    "XML"        "xml"
    "JavaScript" "javascript"
    "HTML"       "html"
    "ASP"        "html"
    "PostScript" "powershell" ; this was supposed to be powershell not postscript
    "PowerShell" "powershell"
    "Ruby"       "ruby"
    "Cucumber"   "cucumber"
    "SQL"        "sql" })

(def gist-types
  (let [gisted (filter has-gists articles)]
    (zipmap (keys gisted)
            (map extract-gist-types (vals gisted)))))

(defn gist-content-type
  [path]
  (let [id       (gist-id path)
        data     (fetch-gist id)
        files    (:files data)
        filename (last (re-find #"\?file=([^&]*)" path))]
    (:language (if (nil? filename)
                  (first (vals files))
                  ((keyword filename) files)))))

(defn load-gist-type [path]
  [path (gist-content-type path)])

(defn extract-gist-types [content]
  (->> content
       (re-seq #"[\"|\'](http[s]?://gist\.github\.com/\d*\.js(\?[^\'\"]*)?)[\"|\']")
       (map #(nth % 1))
       (map load-gist-type)
       (into {})))

(spit "types.edn" (with-out-str (clojure.pprint/pprint gist-types)))
;
(= loaded gist-source) ; proves that original data is the same as the slurped data

(defn gist-id [path]
  (last (re-find #"/(\d*)\.js" path)))

(defn get-one [path]
  (get gist-source path))

(get-one "/2011-06-21-microorms-for-dotnet-inserts-updates-deletes.md")


(defn fetch-url[address]
  (with-open [stream (.openStream (java.net.URL. address))]
    (let  [reader (java.io.InputStreamReader. stream)
           buf    (java.io.BufferedReader. reader)]
      (apply str (line-seq buf)))))

(defn fetch-json [address]
  (-> address
      (fetch-url)
      (parse-string true)))


(defn fetch-gist [gist-id]
  (fetch-json (str "https://api.github.com/gists/" gist-id "?client_id=" client-id "&client_secret=" client-secret)))

(defn highlight-codes [gist-map]
  (let [gists (vals gist-map)]
    (zipmap (keys gist-map)
            (map #(pygments/highlight (:content %)
                                      (:type %)
                                      :html) gists))))


(defn wrap-codes [gist-map]
  (let [gists (vals gist-map)]
    (zipmap (keys gist-map)
            (map #(str "<pre><code>" (:content %) "</code></pre>") gists))))

(def highlighted-gist-code (zipmap (keys gist-data) (map highlight-codes (vals gist-data))))


(spit "formatted.edn" (with-out-str (clojure.pprint/pprint highlighted-gist-code)))

(def snippets (read-string (slurp "wrapped.edn")))

(defn wrap-block [snippet]
  (html/sniptest "<div></div>" [:div] (html/substitute (html/at (html/html-resource (java.io.StringReader. snippet))
                                         [#{:html :body}]
                                         html/unwrap
                                         [:div.highlight :pre]
                                         (html/do->
                                           (html/wrap :code)
                                           (html/wrap :pre))
                                         [:div.highlight :pre :code :pre]
                                         html/unwrap))))

(spit "wrapped.edn" (with-out-str (clojure.pprint/pprint (zipmap (keys snippets)
                                                                 (map wrap-blocks (vals snippets))))))

(defn wrap-blocks [blocks]
  (into {} (map (fn [[path code]] [path (wrap-block code)]) blocks)))




(defn replace-gist [source [path snippet]]
  (html/sniptest source
                 [[:script (html/attr-starts :src path)]]
                 (html/substitute (html/at (html/html-resource (java.io.StringReader. snippet))
                                           [#{:html :body}] html/unwrap))))


(defn replace-gists [content snippets]
  (let [un-no-script (html/sniptest content [:noscript] (fn [_] ""))]
    (reduce replace-gist un-no-script snippets)))

(defn transmogrify-article
  "take the original gisted article and replaces its gists with real code. it
   expects the original content (yep yaml frontmatter as well) and the map of
   gists generated including content, url and type"
  [original snippets]
  (let [fm (util/frontmatter original)
        bd (util/content-body original)]
    (println (:title fm))
    (str "---\n"
         (yaml/generate-string fm)
         "\n---\n"
         (replace-gists bd snippets))))

(defn run! []
  (doall
    (for [[path content] articles
          :when (contains? snippets path)
          :let [snippets (get snippets path)
                updated (transmogrify-article content snippets)]]
      (spit (str "./temp" path) updated))))

(defn test! []
  (into {} (for [[path content] articles
                  :when (contains? snippets path)
                  :let [snippets (get snippets path)
                        updated (transmogrify-article content snippets)]]
              [path  updated])))

(first (test!))


(get (test!) "/2009-04-15-updatepanels-alternatives.md")

(run!)

(keys (filter has-gists articles))


(html/at (html/html-resource (java.io.StringReader. "<div class='highlight'>asdasdas</div>")) [#{:html :body}] html/unwrap)

(html/sniptest "<a></a><script src='test.html'></script><h1>pants</h1>"
               [[:script (html/attr-starts :src "test.html")]]
               (html/substitute (html/at (html/html-resource (java.io.StringReader. "<div class='highlight'><pre><span>1</span><span>2</span></pre></div>"))
                                         [#{:html :body}]
                                         html/unwrap
                                         [:div.highlight :pre]
                                         (html/do->
                                           (html/wrap :code)
                                           (html/wrap :pre))
                                         [:div.highlight :pre :code :pre]
                                         html/unwrap)))

tidy_up.clj

(ns briefcase.tidy-up
  (require [briefcase.content.utils :refer [slurp-content]]
           [clj-yaml.core :as yaml]))

; # Work to be done
;
; - rename :name in projects to :title
; - remove :layout :slug :location :sequence :subtitle :comments :thumb :events :screencast_thumb
; - add :type to each
; - talk url logic should be hardcoded
; - use vimeo urls for videos
(defn drop-keys
  [content]
  (dissoc content :layout
                  :slug
                  :location
                  :sequence
                  :subtitle
                  :comments
                  :thumb
                  :events
                  :screencast_thumb
                  :video
                  :presentation
                  :html
                  :description
                  :video_url
                  :name
                  :uri))

(def whitelisted-frontmatter
  "identifies frontmatter keys that should be kept."
  [:date        ; *
   :title       ; *
   :published   ; *
   :url         ; projects, screencasts
   :categories  ; *
   :external])  ; screencasts

(defn set-published
  [content]
  (assoc content :published (get content :published true)))

(defn set-screencast-url [screencast]
  (if (not (get screencast :external false))
    (assoc screencast
           :url (str "http://player.vimeo.com/video/" (:video_url screencast) "?title=0&byline=0&portrait=0")
           :external false)
    screencast))


(defn rename-screencast-path
  [screencast]
  (let [path      (:path screencast)
        date      (:date screencast)
        formatter (java.text.SimpleDateFormat. "yyyy-MM-dd")
        date-part (.format formatter date)
        new-path  (clojure.string/replace path #"^/\d\d\d-" (str "/" date-part "-"))]
    (assoc screencast :path new-path)))

(defn set-externality [entry]
  (assoc entry :external (get entry :external false)))

(defn set-article-date
  [article]
  (let [formatter (java.text.SimpleDateFormat. "yyyy-MM-dd")
        date-part (subs (:path article) 1 11)
        date      (.parse formatter date-part)]
    (assoc article :date date)))

(def projects-redux
  (->> (slurp-content "resources/content/projects")
       (map #(assoc % :type "project"))
       (map set-published)
       (map #(assoc % :title (str (:name %) " - " (:description %))))
       (map #(assoc % :external true))
       (map drop-keys)))

(def talks-redux
  (->> (slurp-content "resources/content/talks")
       (map #(assoc % :url (or (:video %) (:presentation %))))
       (map set-published)
       (map #(assoc % :type "talk"))
       (map #(assoc % :external true))
       (map drop-keys)))

(def screencasts-redux
  (->> (slurp-content "resources/content/screencasts")
       (map set-screencast-url)
       (map set-published)
       (map #(assoc % :type "screencast"))
       (map rename-screencast-path)
       (map set-externality)
       (map drop-keys)))

(def article-redux
  (->> (slurp-content "resources/content/articles")
       (map #(assoc % :type "article"))
       (map set-published)
       (map set-article-date)
       (map #(assoc % :external false))
       (map drop-keys)))

(defn write [items]
  (doall
    (for [item items
           :let [path    (:path item)
                 content (clojure.string/trim (:content item))
                 source  (dissoc item :path :content)
                 yaml    (yaml/generate-string source)]]
      (spit (str "output" path) (str "---\n" yaml "---\n" content)))))

(write (concat article-redux talks-redux projects-redux screencasts-redux))