DownloadFiles func

http.go

func DownloadFiles(destDir string, urls []string) (err error) {
	if _, err := os.Stat(destDir); os.IsNotExist(err) {
		os.MkdirAll(destDir, 0755)
		log.Infof("Created directory %s.", destDir)
	}

	client := grab.NewClient()
	client.UserAgent = "pogos"
	requests := make([]*grab.Request, 0)
	for _, url := range urls {
		request, err := grab.NewRequest(destDir, url)
		if err != nil {
			return err
		}
		request.NoResume = true
		requests = append(requests, request)
	}
	log.Infof("Downloading %d files ... ", len(urls))
	responseChannel := client.DoBatch(3, requests...)
	ticker := time.NewTicker(200 * time.Millisecond)
	completed := 0
	inProgress := 0
	responses := make([]*grab.Response, 0)
	for completed < len(requests) {
		select {
		case response := <-responseChannel:
			if response != nil {
				responses = append(responses, response)
			}
		case <-ticker.C:
			if inProgress > 0 {
				fmt.Printf("\033[%dA\033[K", inProgress)
			}
			// update completed downloads
			for count, response := range responses {
				if response != nil && response.IsComplete() {
					// print final result
					if response.Err() != nil {
						log.Errorf("Error downloading %s: %v\n", response.Request.URL(), response.Err())
					} else {
						log.Infof(
							"Finished %s %d / %d bytes (%d%%)\n",
							response.Filename, response.BytesComplete(), response.Size, int(100*response.Progress()),
						)
					}

					// mark completed
					responses[count] = nil
					completed++
				}
			}
			// update downloads in progress
			inProgress = 0
			for _, response := range responses {
				if response != nil {
					inProgress++
					fmt.Printf(
						"\tDownloading %s %d / %d bytes (%d%%)\033[K\n",
						response.Filename, response.BytesComplete(), response.Size, int(100*response.Progress()),
					)
				}
			}
		}
	}
	ticker.Stop()
	log.Infof("%d files successfully downloaded.", len(requests))
	return nil
}