File Downloader in Golang

godownload.go

package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

var (
	textFile       = "files.txt"
	downloadFolder = "files"
	singleFolder   = false
)

func init() {
	// Verbose logging with file name and line number
	log.SetFlags(log.Lshortfile)
}

func main() {

	files := readFile(textFile)

	for _, url := range files {
		download(url)
		//log.Println(url)
	}
}

func readFile(filename string) []string {
	lines := []string{}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Println("Error reading file:", err)
	} else {
		stringContent := string(content)

		lineEnding := "\n"

		if windows := strings.Index(stringContent, "\r\n"); windows > -1 {
			lineEnding = "\r\n"
		}

		lines = strings.Split(stringContent, lineEnding)
	}

	return lines
}

func download(url string) {
	cleanURL := strings.TrimSpace(url)
	newUrl := cleanURL

	// Skip blank lines
	if newUrl == "" {
		return
	}

	sep := `\`
	if strings.Index(newUrl, "/") > -1 {
		sep = "/"
	}

	if newUrl[len(newUrl)-1:] == sep {
		return
	}

	newUrl = strings.Replace(newUrl, "http://", "", -1)
	newUrl = strings.Replace(newUrl, "https://", "", -1)

	urlParts := strings.Split(newUrl, sep)
	filename := urlParts[len(urlParts)-1]

	// Just a simple GET request to the image URL
	// We get back a *Response, and an error
	res, err := http.Get(cleanURL)

	if err != nil {
		log.Printf("http.Get -> %v", err)
		return
	}

	// We read all the bytes of the image
	// Types: data []byte
	data, err := ioutil.ReadAll(res.Body)

	if err != nil {
		log.Println("ioutil.ReadAll -> %v", err)
		return
	}

	// You have to manually close the body, check docs
	// This is required if you want to use things like
	// Keep-Alive and other HTTP sorcery.
	res.Body.Close()

	folder := downloadFolder

	if !singleFolder {
		if len(urlParts) > 1 {
			folder = downloadFolder + string(filepath.Separator) + strings.Join(urlParts[0:len(urlParts)-1], string(filepath.Separator))
		}
	}

	os.MkdirAll(folder, 0777)

	// You can now save it to disk or whatever...
	if err = ioutil.WriteFile(folder+string(filepath.Separator)+filename, data, 0777); err != nil {
		log.Println("Error Saving:", filename, err)
	} else {
		log.Println("Saved:", filename)
	}
}