Async Http Download

example.go

package main

import ("flag"
       "net/http"
       "net/url"
       "io"
       "log"
       "io/ioutil"
       "sync"
       )


// Resource represents an HTTP URL to be polled by this program.
type Resource struct {
     url      string
     errCount int
}


func (r * Resource) Download()  {

     _, err := url.Parse(r.url)

     if err != nil {
        log.Panic(err, "Invalid url", r.url)
     }

     resp, err := http.Get(r.url) 
     if err != nil {
        log.Panic(err)
     }
     
     defer resp.Body.Close()
     
     fd, err := ioutil.TempFile("/tmp", "download.tmp.")
     if err != nil {    
        log.Panic(err)
     }
     defer fd.Close()

     log.Print(r.url, " WRITING ", fd.Name())

     defer log.Print(r.url, " DONE " )

     io.Copy(fd, resp.Body)
}


func Poller(in <-chan *Resource, out chan <- *Resource, error  chan <- *Resource, workers *sync.WaitGroup) {
     defer func(){
        err := recover()
        log.Println("Error !", err)
     }()

     defer workers.Done()

     for r := range in {
          r.Download()
          log.Println("Message about to be sent", r)
          out <- r
          log.Println("Message sent", r)
     }

     log.Println("No more input data")

}


func main(){

     flag.Parse()

     pending := make(chan *Resource)
     completed := make(chan *Resource)
     error := make(chan *Resource)

     numPollers := 1

     workers := &sync.WaitGroup{}

     for i := 0; i < numPollers; i++ {
          workers.Add(1)
          go Poller(pending, completed, error, workers)
     }
     
     for _, url := range flag.Args() {
          log.Print("Scheduling ", url)
          pending <- &Resource{url: url}
      }

     close(pending)

     log.Print("Here")

     go func(){
          defer log.Print("goroutine 1 terminated ")
          workers.Wait()
          close(completed)
          close(error)
     }()

     log.Print("Here2 ")
     go func() {
     defer log.Print("goroutine 2 terminated ")

     log.Print("Starting XXX")

     for resource_complete := range completed {
         log.Print("Complete: ", resource_complete)
     }

     for resource_complete := range error {
         log.Print("Error: ", resource_complete)
     }
     }()

     log.Print("Here3 ")
     log.Print("Finish")
}