public
Created

Snarf historical prices from ebay... via common lisp

  • Download Gist
gistfile1.txt
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
(in-package "CL-USER")
 
(let ((*print-case* :upcase))
(ql:quickload '(#:fare-memoization #:parse-number
#:optima #:optima.ppcre #:fare-quasiquote-optima
#:drakma #:closure-html)))
 
(defpackage #:eb
(:use #:common-lisp #:fare-memoization #:parse-number
#:optima #:optima.ppcre
#:drakma #:closure-html))
 
(in-package "EB")
 
(defun fetch-price-page (query page-number)
;; for example: (fetch-price-page "macbook air 13" 3)
(let* ((url-template
"http://www.ebay.com/csc/i.html?_from=R40&_sacat=0&LH_Complete=1&_nkw=~a&_pgn=~d&_skc=50&rt=nc")
(url (format nil url-template (url-encode query *drakma-default-external-format*) page-number)))
(parse (http-request url) (closure-html:make-lhtml-builder))))
 
(memoize 'fetch-price-page)
 
(defun snarf-prices (page-sxpr)
;; for example: (snarf-prices (fetch-price-page "macbook air 13" 3))
;;
;; They look like this:
;; (:div ((:class "g-b bidsold") (:itemprop "price")) "
;; $18.99")
(let ((result ()))
(labels ((recure (x)
(match x
(`(:div ((:class "g-b bidsold") (:itemprop "price"))
,(ppcre "\\w*\\$([\\d.]+)" price))
(push (parse-number price) result))
((list* (satisfies keywordp) * children)
(map nil #'recure children)))))
(recure page-sxpr)
(nreverse result))))

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.