Grabber quotationspage.com

grabber.go

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"time"
	"strings"
	"os"
	
	"github.com/PuerkitoBio/goquery"
)

var (
	WORKERS int = 2
	COUNT int = 150
	QUOTES_FILE string = "quotes.txt"
)

func grab() <-chan string {
	c := make(chan string)

	for i:=0; i <= WORKERS; i++ {
		go func(){
			for {
				res, err := http.Get("http://www.quotationspage.com/random.php")
				if err != nil {
					log.Fatal(err)
				}
				defer res.Body.Close()
				
				if res.StatusCode != 200 {
					log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
				}

				doc, err := goquery.NewDocumentFromReader(res.Body)
				if err != nil {
					log.Fatal(err)
				}


				if s:= strings.TrimSpace(doc.Find("dt.quote a").First().Text()); s!= "" {
					c <- s
				}

				time.Sleep(100 * time.Millisecond)
			}			
		}()
	}

	fmt.Println("Workers: ", WORKERS)

	return c
}

func init() {
	flag.IntVar(&WORKERS, "w", WORKERS, "Workers count")
	flag.IntVar(&COUNT, "c", COUNT, "Count quoutes")
	flag.StringVar(&QUOTES_FILE, "qf", QUOTES_FILE, "Name of quotes file")
}

func main() {
	flag.Parse()

	// File to quotes
	quotes_file, err := os.OpenFile(QUOTES_FILE, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer quotes_file.Close()

	quote_chan := grab()
	for i:=0; i < COUNT; i++ {
		quote := <-quote_chan
		quotes_file.WriteString(quote + "\n\n\n")

		fmt.Println(quote, "\n\n\n")
	}
}