scraper.clj

#!/usr/bin/env hython
; not actually clojure ...

(import "urllib2")
(import-from "lxml" "html")


(defn form-pts-url [developer]
  "Return the PTS URL for a given developer."
  (+ "http://qa.debian.org/developer.php?login=" developer))


(defn digest-page [url]
  "Take a URL and return an lxml.html page ready for scraping."
  (.fromstring html (.read (.urlopen urllib2 url))))


(defn scrape-page [page]
  "Scrape the PTS for bugs"
  (do
    (def xpath-base "//table[contains(@class, 'packagetable sortable')]")
    (zip
      (.xpath page (+ xpath-base "//a[@name]/text()"))
      (.xpath page (+ xpath-base "//td[2]//a[contains(@href, 'bugs.debian.org')]/text()")))))


(print (scrape-page (digest-page (form-pts-url "paultag"))))