conc_downloader.go

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"strings"
	"path/filepath"
)

func feeder(file *os.File) chan string {

	comm := make(chan string)
	// create a new scanner and read the file line by line
	scanner := bufio.NewScanner(file)

	go func() {
		// check for errors
		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		for scanner.Scan() {
			comm <- scanner.Text()
		}
		close(comm)
	}()
	return comm
}

func downloader(full_url string, wg *sync.WaitGroup, conc_chan chan bool) {
	// Parse the URL and ensure there are no errors.
	defer wg.Done()

	u, err := url.Parse(full_url)
	if err != nil {
		panic(err)
	}

	s := []string{}
	s = append(s, "/Users/pablo/Desktop")
	s = append(s, u.Path)
	filename := strings.Join(s, "")
    
    fmt.Println(filename)
	if os.MkdirAll(filepath.Dir(filename), 0777) != nil {
		panic("Unable to create directory for tagfile!")
	} 

	output, err := os.Create(filename)
	if err != nil {
		fmt.Println("Error while creating", filename, "-", err)
		return
	}
	defer output.Close()

	response, err := http.Get(full_url)
	if err != nil {
		fmt.Println("Error while downloading", full_url, "-", err)
		return
	}
	defer response.Body.Close()

	_, err = io.Copy(output, response.Body)
	if err != nil {
		fmt.Println("Error copying", full_url, "-", err)
		return
	}

	<- conc_chan

}

func main() {

	if len(os.Args) != 3 {
		fmt.Println("Pass the file to read and the level of concurrency!")
		os.Exit(0)
	}

	// open the file
	if file, err := os.Open(os.Args[1]); err == nil {

		defer file.Close()
		comm := feeder(file)

		var wg sync.WaitGroup
		conc_level, _ := strconv.Atoi(os.Args[2])
		conc_chan := make(chan bool, conc_level)
		for line := range comm {
			conc_chan <- true
			wg.Add(1)
			go downloader(line, &wg, conc_chan)
		}
		wg.Wait()	
	} else {
		log.Fatal(err)
	}

}