Count words on webpages. Usage: echo http://golang.org | go run search.go

search.go

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
)

// getLinks takes arguments from command line
func getLinks(out chan<- string) {

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		link, err := url.ParseRequestURI(strings.TrimSpace(scanner.Text())) // not better way, but...
		if err != nil {
			fmt.Printf("Bad url trown away: %v\n", err)
			continue // skip bad urls
		}
		out <- link.String()
	}

	close(out)
}

// countWords counts world on the page
// search word is CASE SENSITIVE
func countWords(out chan<- int, in <-chan string) {

	var tokens = make(chan struct{}, 5) // max concurrent requests
	var wg sync.WaitGroup

	for link := range in {
		tokens <- struct{}{}
		wg.Add(1)
		go countWordsRequest(out, tokens, &wg, link)
	}

	wg.Wait()
	close(out)
}

// countWordsRequest make get requests, count words and release semaphore
func countWordsRequest(out chan<- int, tokens <-chan struct{}, wg *sync.WaitGroup, link string) {

	defer wg.Done()

	var count int
	var body bytes.Buffer

	res, err := http.Get(link)
	<-tokens
	if err != nil {
		fmt.Printf("Http error: %v\n", err)
		return
	}
	defer res.Body.Close()

	io.Copy(&body, res.Body)
	count = strings.Count(body.String(), "Go")

	fmt.Printf("Count for %s: %d\n", link, count)

	out <- count
}

// showResults show results and prints total
func showResults(in <-chan int) {

	var total int
	for c := range in {
		total += c
	}

	fmt.Printf("Total: %d\n", total)
}

func main() {

	var (
		links = make(chan string, 10) // limit reading from stdin too...
		count = make(chan int)
	)

	go getLinks(links)
	go countWords(count, links)

	showResults(count)

}