dedup.go

package main

import (
        "bufio"
        "log"
        "fmt"
        "os"
        "regexp"
)

var scanRe = regexp.MustCompile(`^SHA1\((.+)\)= ([a-z0-9]+)$`)

func main() {
        seen := make(map[string]bool)
        var dupes []string

        s := bufio.NewScanner(os.Stdin)
        for s.Scan() {
                m := scanRe.FindStringSubmatch(s.Text())
                if m == nil {
                        log.Println("bad match: %v", s.Text())
                        continue
                }
                fn, hash := m[1], m[2]
                if seen[hash] {
                        dupes = append(dupes, fn)
                } else {
                        seen[hash] = true
                }
        }

        for _, fn := range dupes {
                fmt.Println(fn)
        }
}

scrape.go

package main

import (
  "fmt"
	"log"
	"net/http"
	"strings"

	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
)

func main() {
	base := "http://www.muffwiggler.com/forum/"
	u := base + "viewtopic.php?t=1135"
	for u != "" {
		resp, err := http.Get(u)
		if err != nil {
			log.Fatal(err)
		}
		root, err := html.Parse(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Fatal(err)
		}
		for _, post := range findAll(root, hasClass("postbody")) {
			for _, img := range findAll(post, isTag(atom.Img)) {
				fmt.Println(attr(img, "src"))
			}
		}
		u = ""
		for _, nav := range findAll(root, hasClass("nav")) {
			for _, a := range findAll(nav, isTag(atom.A)) {
				for c := a.FirstChild; c != nil; c = c.NextSibling {
					if c.Type == html.TextNode && c.Data == "Next" {
						u = base + attr(a, "href")
						log.Println(u)
					}
				}

			}
		}
	}
}

type selector func(*html.Node) bool

func isTag(a atom.Atom) selector {
	return func(n *html.Node) bool {
		return n.DataAtom == a
	}
}

func hasClass(name string) selector {
	return func(n *html.Node) bool {
		for _, a := range n.Attr {
			if a.Key == "class" {
				for _, c := range strings.Fields(a.Val) {
					if c == name {
						return true
					}
				}
			}
		}
		return false
	}
}

func hasAttr(key, val string) selector {
	return func(n *html.Node) bool {
		for _, a := range n.Attr {
			if a.Key == key && a.Val == val {
				return true
			}
		}
		return false
	}
}

func attr(node *html.Node, key string) (value string) {
	for _, attr := range node.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func findAll(node *html.Node, fn selector) (nodes []*html.Node) {
	walk(node, func(n *html.Node) bool {
		if fn(n) {
			nodes = append(nodes, n)
		}
		return true
	})
	return
}

func find(n *html.Node, fn selector) *html.Node {
	var result *html.Node
	walk(n, func(n *html.Node) bool {
		if result != nil {
			return false
		}
		if fn(n) {
			result = n
			return false
		}
		return true
	})
	return result
}

func walk(n *html.Node, fn selector) {
	if fn(n) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c, fn)
		}
	}
}