downloader

gistfile1.go

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
)

var linkRegexp = regexp.MustCompile(`<A HREF="(\w+.txt)">`)

func main() {
	resp, err := http.Get("http://www.textfiles.com/programming/")

	if err != nil {
		fmt.Print(err)
	}
	defer resp.Body.Close()

	downloadFile := downloadFile()
	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		link := extractLink(scanner.Text())

		if len(link) != 0 {
			select {
			case downloadFile <- link:
			}
		}
	}
}

func extractLink(text string) string {
	result := linkRegexp.FindStringSubmatch(text)

	if len(result) > 0 {
		return result[1]
	} else {
		return ""
	}
}

func downloadFile() chan<- string {
	files := make(chan string, 5)
	go func() {
		for filename := range files {
			resp, err := http.Get("http://www.textfiles.com/programming/" + filename)
			if err != nil {
				fmt.Print(err)
			}
			defer resp.Body.Close()
			contents, err := ioutil.ReadAll(resp.Body)
			writeFile(filename, contents)
		}
	}()

	return files
}

func writeFile(filename string, contents []byte) {
	file, err := os.Create("out/" + filename)

	if err != nil {
		fmt.Print(err)
	}

	defer file.Close()

	file.Write(contents)
}