script to dump hackernews in Guile (requires guile-json)

hn.scm

(define-module (hn))

(use-modules (rnrs io ports))
(use-modules (ice-9 receive))
(use-modules (http))
(use-modules (json))
(use-modules (msgpack))


(define (max-id)
  (receive (response body) (http-get "https://hacker-news.firebaseio.com/v0/maxitem.json")
    (string->number body)))


(define (assocify ht)
  (hash-map->list cons ht))

(define (dump)
  (call-with-output-file "hn.msgpack"
    (lambda (port)
      (let loop ((uid (max-id)))
        (format #t "~a\n" uid)
        (unless (eq? uid 0)
          (catch #true
            (lambda ()
              (let* ((url "https://hacker-news.firebaseio.com/v0/item/~a.json")
                     (url (format #f url uid)))
                (receive (response body) (http-get url)
                  (let ((bv (pack (assocify (json-string->scm body)))))
                    (put-bytevector port bv))))
              (loop (1- uid)))
            (lambda _ (loop (1- uid)))))))))

http.scm

(define-module (http))


(use-modules (curl)
             (web response)
             (ice-9 pretty-print)
             (rnrs bytevectors))


(define-public (http-get url)
  ;; Create a Curl handle
  (let ((handle (curl-easy-init)))
    ;; Set the URL from which to get the data
    (curl-easy-setopt handle 'url url)
    ;; Request that the HTTP headers be included in the response
    (curl-easy-setopt handle 'header #t)
    ;; Get the result as a Latin-1 string
    (let* ((response-string (curl-easy-perform handle))
           ;; Create a string port from the response
           (response-port (open-input-string response-string))
           ;; Have the (web response) module to parse the response
           (response (read-response response-port))
           (body (utf8->string (read-response-body response))))
      (close response-port)
      ;; Have the (web response) module extract the body from the
      ;; response
      (values response body))))