Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Crawls golang.org/pkg for interfaces and writes them to a file.
package main
import (
"log"
"os"
"strings"
gq "github.com/PuerkitoBio/goquery"
)
func main() {
var interfaces []Interface
log.Println("Loading standard library packages...")
packages := standardPackages(BASE_URL)
log.Println("Looking for interface definitions in each package...")
for _, pkg := range packages {
log.Println(pkg.name)
interfaces = append(interfaces, packageInterfaces(pkg)...)
}
log.Println("Found", len(interfaces), "interfaces.")
log.Println("Dumping to " + OUTPUT_FILE + "...")
file, err := os.Create(OUTPUT_FILE)
checkErr(err)
for _, iface := range interfaces {
write(file, "// "+iface.url+"\r\n")
write(file, iface.def+"\r\n")
write(file, "/*"+iface.doc+"*/\r\n\r\n\r\n")
}
// Also write out list of interfaces and their packages
write(file, "\r\n\r\n\r\n// ===============================================\r\n\r\n\r\n")
for _, iface := range interfaces {
write(file, iface.pkg.shortName+"."+iface.name+"\r\n")
}
log.Println("All done!")
}
func standardPackages(basePath string) []Package {
var packages []Package
doc, err := gq.NewDocument(basePath)
checkErr(err)
doc.Find(".dir td.name a").Each(func(i int, s *gq.Selection) {
if relativePath, ok := s.Attr("href"); ok {
packages = append(packages, Package{
shortName: s.Text(),
name: strings.TrimSuffix(relativePath, "/"),
url: basePath + relativePath,
})
}
})
return packages
}
func packageInterfaces(pkg Package) []Interface {
var ifaces []Interface
doc, err := gq.NewDocument(pkg.url)
checkErr(err)
// Iterate all code blocks immediately after h2
doc.Find("h2 + pre").Each(func(i int, pre *gq.Selection) {
def := pre.Text()
// Must be an interface definition
if !strings.Contains(def, " interface {") {
return
}
// Get description from p tags following
var pText []string
pre.NextFilteredUntil("p", ":not(p)").Each(func(i int, p *gq.Selection) {
pText = append(pText, p.Text())
})
h2 := pre.PrevFiltered("h2")
ifaceName := strings.TrimPrefix(h2.Text(), "type ")
// Save interface along with a direct link to its definition
if headerId, ok := h2.Attr("id"); ok {
ifaces = append(ifaces, Interface{
pkg: pkg,
name: ifaceName,
url: pkg.url + "#" + headerId,
def: def,
doc: strings.Join(pText, "\r\n"),
})
} else {
return
}
})
return ifaces
}
func write(file *os.File, s string) {
_, err := file.WriteString(s)
checkErr(err)
file.Sync()
}
func checkErr(err error) {
if err != nil {
log.Fatal(err)
}
}
type Package struct {
shortName, name, url string
}
type Interface struct {
pkg Package
name, url, def, doc string
}
const (
BASE_URL = "http://golang.org/pkg/"
OUTPUT_FILE = "interfaces.go"
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment