A Tour Of Go: Web Crawler

notreallya_crawler.go

package main

import (
    "fmt"
    "sync"
)

type StringArray []string

var urlmap StringArray= make([]string,1)
var wg sync.WaitGroup

func (sa *StringArray) search(w string) int{
    for i,v := range *sa {
        if v==w {
            return i
        }
    }
    return -1
}


type Fetcher interface {
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {

    defer wg.Done()

    // This implementation doesn't do either:
    if depth <= 0 {
        return
    }
    
    if urlmap.search(url) == -1 {
    	fmt.Printf("Crawl: %s\n",url)
        urlmap=append(urlmap,url)
    	body, urls, err := fetcher.Fetch(url)
    	if err != nil {
       	 fmt.Println(err)
       	 return
    	}
    	fmt.Printf("found: %s %q\n", url, body)
    	for _, u := range urls {
       	    wg.Add(1)
            go Crawl(u, depth-1, fetcher)
    	}
    }else{
       fmt.Printf("Already Crawled: %s\n",url) 
    }
    return
}

func main() {
    wg.Add(1)
    go Crawl("http://golang.org/", 4, fetcher)
    wg.Wait()
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f *fakeFetcher) Fetch(url string) (string, []string, error) {
    if res, ok := (*f)[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}


// fetcher is a populated fakeFetcher.
var fetcher = &fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

Note: the package "sort" is designed for ordered arrays and performs binary searches. Using sort.Search, sort.SearchString or StringSlices's methods will return the index at which ordered insertion should be performed when an element is not found... likely it's not what you need here, better go with a handmade search on an unordered array.