Wiki-links - Gets JSON API endpoints of hyperlinks present in a given wiki page [go get github.com/opesun/goquery]

wikilinks.go

package main

import (
	"fmt"
	"github.com/opesun/goquery"
	"strings"
)

const wikiBase string = "http://en.wikipedia.org"
const wikiAPIURL string = "http://en.wikipedia.org/w/api.php?format=json&action=query&titles=$title&prop=revisions&rvprop=content"

func getTitle(node *goquery.Node) *string {
	var title string
	for _, attr := range node.Attr {
		if attr.Key == "title" {
			title = attr.Val		
		}
	}
	return &title
}

func getLinks(baseURL string) *map[string]string {
	linksMap := make(map[string]string)
	x, err := goquery.ParseUrl(baseURL)
	if err != nil {
		panic(err)
	}
	for _, v := range x.Find("a") {
		attributes := v.Attr
		
			for _, attr := range attributes {
				if attr.Key == "href" {
					if strings.HasPrefix(attr.Val, "/wiki/") && !strings.Contains(attr.Val, "/wiki/Wikipedia") {					
						linkTitle := *getTitle(v)
						linksMap[linkTitle] = strings.Replace(wikiAPIURL,"$title",linkTitle,-1)
					}
				}
			}					
	}
	return &linksMap
}



func main() {
	
	baseURL := "http://en.wikipedia.org/wiki/Tf%E2%80%93idf"	
	links := getLinks(baseURL)
	for key,value := range *links {
		fmt.Println(key," -> ",value)	
	}
	
}