JSoup clojure snippet

README.md

https://clojurians.slack.com/archives/C053PTJE6/p1598996773028100

Prestance 11:46 PM Hi the team! For learning web scraping with Clojure and Jsoup, I transpose this “Python BeautifulSoup" video tutorial https://www.youtube.com/watch?v=XQgXKtPSzUI: My Clojure version: https://github.com/PrestanceDesign/clj-jsoup-scraping The Python source code: https://code.datasciencedojo.com/datasciencedojo/tutorials/blob/master/Web%20Scraping%20with%20Python%20and%20BeautifulSoup/Web%20Scraping%20with%20Python%20and%20Beautiful%20Soup.py

core.clj

(ns jsoup-scraping.core
  (:gen-class)
  (:require [clojure.string :as str]
            [clojure.data.csv :as csv]
            [clojure.java.io :as io])
  (:import org.jsoup.Jsoup))


(def newegg-url "http://www.newegg.com/Product/ProductList.aspx?Submit=ENE&N=-1&IsNodeId=1&Description=GTX&bop=And&Page=1&PageSize=36&order=BESTMATCH")


(defn fetch-page [url]
  (.get (Jsoup/connect url)))


(defn dom-elements [root selector]
  (.select root selector))


(defn dom-text [root selector]
  (-> (dom-elements root selector)
      (.text)))


(defn dom-attr [root selector attribute]
  (-> (dom-elements root selector)
      (.attr attribute)))


(defn parse-product [item]
  (let [product-name (-> (dom-text item ".item-title")
                         (str/replace "," "|"))
        brand        (-> (dom-attr item "div.item-branding > a > img" "title"))
        shipping     (-> (dom-text item "div.item-action ul.price > li.price-ship")
                         (str/replace "$" "")
                         (str/replace "Shipping" ""))]
    [brand product-name shipping]))


(def csv-headers
  ["brand" "product_name" "shipping"])


(defn generate-csv [filename url]
  (let [html     (fetch-page url) 
        csv-rows (map parse-product (dom-elements html "div.item-container"))]
    (with-open [writer (io/writer filename)]
      (csv/write-csv writer (cons csv-headers csv-rows)))))


(defn -main []
  (generate-csv "test-graphics.csv" newegg-url))


(comment

  (def html (fetch-page newegg-url))


  (def items (dom-elements html "div.item-container"))


  (dom-text (first items) ".item-title")


  (dom-attr (first items) ".item-branding > a > img" "title")


  (parse-product (first items))


  (generate-csv "graphics_cards.csv" newegg-url)


  )