Crawls golang.org/pkg for interfaces and writes them to a file.

interface_spider.go

package main

import (
	"log"
	"os"
	"strings"

	gq "github.com/PuerkitoBio/goquery"
)

func main() {
	var interfaces []Interface

	log.Println("Loading standard library packages...")
	packages := standardPackages(BASE_URL)

	log.Println("Looking for interface definitions in each package...")
	for _, pkg := range packages {
		log.Println(pkg.name)
		interfaces = append(interfaces, packageInterfaces(pkg)...)
	}

	log.Println("Found", len(interfaces), "interfaces.")
	log.Println("Dumping to " + OUTPUT_FILE + "...")

	file, err := os.Create(OUTPUT_FILE)
	checkErr(err)

	for _, iface := range interfaces {
		write(file, "// "+iface.url+"\r\n")
		write(file, iface.def+"\r\n")
		write(file, "/*"+iface.doc+"*/\r\n\r\n\r\n")
	}

	// Also write out list of interfaces and their packages
	write(file, "\r\n\r\n\r\n// ===============================================\r\n\r\n\r\n")
	for _, iface := range interfaces {
		write(file, iface.pkg.shortName+"."+iface.name+"\r\n")
	}

	log.Println("All done!")
}

func standardPackages(basePath string) []Package {
	var packages []Package

	doc, err := gq.NewDocument(basePath)
	checkErr(err)

	doc.Find(".dir td.name a").Each(func(i int, s *gq.Selection) {
		if relativePath, ok := s.Attr("href"); ok {
			packages = append(packages, Package{
				shortName: s.Text(),
				name:      strings.TrimSuffix(relativePath, "/"),
				url:       basePath + relativePath,
			})
		}
	})

	return packages
}

func packageInterfaces(pkg Package) []Interface {
	var ifaces []Interface

	doc, err := gq.NewDocument(pkg.url)
	checkErr(err)

	// Iterate all code blocks immediately after h2
	doc.Find("h2 + pre").Each(func(i int, pre *gq.Selection) {
		def := pre.Text()

		// Must be an interface definition
		if !strings.Contains(def, " interface {") {
			return
		}

		// Get description from p tags following
		var pText []string
		pre.NextFilteredUntil("p", ":not(p)").Each(func(i int, p *gq.Selection) {
			pText = append(pText, p.Text())
		})

		h2 := pre.PrevFiltered("h2")
		ifaceName := strings.TrimPrefix(h2.Text(), "type ")

		// Save interface along with a direct link to its definition
		if headerId, ok := h2.Attr("id"); ok {
			ifaces = append(ifaces, Interface{
				pkg:  pkg,
				name: ifaceName,
				url:  pkg.url + "#" + headerId,
				def:  def,
				doc:  strings.Join(pText, "\r\n"),
			})
		} else {
			return
		}
	})

	return ifaces
}

func write(file *os.File, s string) {
	_, err := file.WriteString(s)
	checkErr(err)
	file.Sync()
}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type Package struct {
	shortName, name, url string
}

type Interface struct {
	pkg                 Package
	name, url, def, doc string
}

const (
	BASE_URL    = "http://golang.org/pkg/"
	OUTPUT_FILE = "interfaces.go"
)