An Archive.org metadata miner written in Go.

ia-mine.go

package main

import (
    "os"
    "bufio"
    "crypto/tls"
    "net/http"
    "io/ioutil"
    "fmt"
    "time"
)

type Resp struct {
    *http.Response
    err error
}

func fetch(uri string, queue chan string) {
    tr := &http.Transport{
        TLSClientConfig:    &tls.Config{},
    }

    client := http.Client{Transport: tr}
    resp, err := client.Get(uri)
    defer resp.Body.Close()

    if err != nil {
        // re-queue
        queue <- uri
        return
    }

    r := Resp{resp, err}
    body, _ := ioutil.ReadAll(r.Body)

    fmt.Println(string(body))
}

func q(itemlist string, queue chan string) {
    file, _ := os.Open(itemlist)
    defer file.Close()
    scanner := bufio.NewScanner(file)

    for scanner.Scan() {
        uri := "http://archive.org/metadata/" + scanner.Text()
        queue <- uri
    }
    return

}

func main() {
    // rate-limiter
    rate := time.Second / 500
    throttle := time.Tick(rate)

    queue := make(chan string)

    go q("itemlist.txt", queue)

    for uri := range queue {
        <-throttle
        go fetch(uri, queue)
    }

}