Script to scrape links (or imgs, scripts, etc.) from a web page -- could be incorporated into a simple crawler or mass downloader

links.go

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_url "net/url"

	"code.google.com/p/go.net/html"
)

var url = flag.String("url", "http://www.google.com", "URL to fetch")
var tag = flag.String("tag", "a", "tag names to look into")
var attr = flag.String("attr", "href", "attribute to print")

func main() {
	flag.Parse()
	r, err := http.Get(*url)
	if err != nil {
		log.Fatal(err)
	}
	defer r.Body.Close()

	u, err := _url.Parse(*url)
	if err != nil {
		log.Fatal(err)
	}

	z := html.NewTokenizer(r.Body)
	for {
		tt := z.Next()
		if tt == html.ErrorToken {
			break
		} else if tt == html.StartTagToken {
			if t, more := z.TagName(); string(t) == *tag && more {
				k, v, more := z.TagAttr()
				if string(k) == *attr {
					printURL(u, v)
				}
				for more {
					k, v, more = z.TagAttr()
					if string(k) == *attr {
						printURL(u, v)
						break
					}
				}
			}
		}
	}
}

func printURL(u *_url.URL, ref []byte) {
	u2, err := u.Parse(string(ref))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(u2)
}