Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Crawls golang.org/pkg for interfaces and writes them to a file.
package main
import (
"log"
"os"
"strings"
gq "github.com/PuerkitoBio/goquery"
)
func main() {
var interfaces []Interface
log.Println("Loading standard library packages...")
packages := standardPackages(BASE_URL)
log.Println("Looking for interface definitions in each package...")
for _, pkg := range packages {
log.Println(pkg.name)
interfaces = append(interfaces, packageInterfaces(pkg)...)
}
log.Println("Found", len(interfaces), "interfaces.")
log.Println("Dumping to " + OUTPUT_FILE + "...")
file, err := os.Create(OUTPUT_FILE)
checkErr(err)
for _, iface := range interfaces {
write(file, "// "+iface.url+"\r\n")
write(file, iface.def+"\r\n")
write(file, "/*"+iface.doc+"*/\r\n\r\n\r\n")
}
// Also write out list of interfaces and their packages
write(file, "\r\n\r\n\r\n// ===============================================\r\n\r\n\r\n")
for _, iface := range interfaces {
write(file, iface.pkg.shortName+"."+iface.name+"\r\n")
}
log.Println("All done!")
}
func standardPackages(basePath string) []Package {
var packages []Package
doc, err := gq.NewDocument(basePath)
checkErr(err)
doc.Find(".dir td.name a").Each(func(i int, s *gq.Selection) {
if relativePath, ok := s.Attr("href"); ok {
packages = append(packages, Package{
shortName: s.Text(),
name: strings.TrimSuffix(relativePath, "/"),
url: basePath + relativePath,
})
}
})
return packages
}
func packageInterfaces(pkg Package) []Interface {
var ifaces []Interface
doc, err := gq.NewDocument(pkg.url)
checkErr(err)
// Iterate all code blocks immediately after h2
doc.Find("h2 + pre").Each(func(i int, pre *gq.Selection) {
def := pre.Text()
// Must be an interface definition
if !strings.Contains(def, " interface {") {
return
}
// Get description from p tags following
var pText []string
pre.NextFilteredUntil("p", ":not(p)").Each(func(i int, p *gq.Selection) {
pText = append(pText, p.Text())
})
h2 := pre.PrevFiltered("h2")
ifaceName := strings.TrimPrefix(h2.Text(), "type ")
// Save interface along with a direct link to its definition
if headerId, ok := h2.Attr("id"); ok {
ifaces = append(ifaces, Interface{
pkg: pkg,
name: ifaceName,
url: pkg.url + "#" + headerId,
def: def,
doc: strings.Join(pText, "\r\n"),
})
} else {
return
}
})
return ifaces
}
func write(file *os.File, s string) {
_, err := file.WriteString(s)
checkErr(err)
file.Sync()
}
func checkErr(err error) {
if err != nil {
log.Fatal(err)
}
}
type Package struct {
shortName, name, url string
}
type Interface struct {
pkg Package
name, url, def, doc string
}
const (
BASE_URL = "http://golang.org/pkg/"
OUTPUT_FILE = "interfaces.go"
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.