Created
October 29, 2016 15:33
-
-
Save Gitart/e5336bcd72d1f0c98e10662d36698956 to your computer and use it in GitHub Desktop.
Download
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"io/ioutil" | |
"log" | |
"net/http" | |
"os" | |
"path/filepath" | |
"strings" | |
) | |
var ( | |
textFile = "files.txt" | |
downloadFolder = "files" | |
singleFolder = false | |
) | |
func init() { | |
// Verbose logging with file name and line number | |
log.SetFlags(log.Lshortfile) | |
} | |
func main() { | |
files := readFile(textFile) | |
for _, url := range files { | |
download(url) | |
//log.Println(url) | |
} | |
} | |
func readFile(filename string) []string { | |
lines := []string{} | |
content, err := ioutil.ReadFile(filename) | |
if err != nil { | |
log.Println("Error reading file:", err) | |
} else { | |
stringContent := string(content) | |
lineEnding := "\n" | |
if windows := strings.Index(stringContent, "\r\n"); windows > -1 { | |
lineEnding = "\r\n" | |
} | |
lines = strings.Split(stringContent, lineEnding) | |
} | |
return lines | |
} | |
func download(url string) { | |
cleanURL := strings.TrimSpace(url) | |
newUrl := cleanURL | |
// Skip blank lines | |
if newUrl == "" { | |
return | |
} | |
sep := `\` | |
if strings.Index(newUrl, "/") > -1 { | |
sep = "/" | |
} | |
if newUrl[len(newUrl)-1:] == sep { | |
return | |
} | |
newUrl = strings.Replace(newUrl, "http://", "", -1) | |
newUrl = strings.Replace(newUrl, "https://", "", -1) | |
urlParts := strings.Split(newUrl, sep) | |
filename := urlParts[len(urlParts)-1] | |
// Just a simple GET request to the image URL | |
// We get back a *Response, and an error | |
res, err := http.Get(cleanURL) | |
if err != nil { | |
log.Printf("http.Get -> %v", err) | |
return | |
} | |
// We read all the bytes of the image | |
// Types: data []byte | |
data, err := ioutil.ReadAll(res.Body) | |
if err != nil { | |
log.Println("ioutil.ReadAll -> %v", err) | |
return | |
} | |
// You have to manually close the body, check docs | |
// This is required if you want to use things like | |
// Keep-Alive and other HTTP sorcery. | |
res.Body.Close() | |
folder := downloadFolder | |
if !singleFolder { | |
if len(urlParts) > 1 { | |
folder = downloadFolder + string(filepath.Separator) + strings.Join(urlParts[0:len(urlParts)-1], string(filepath.Separator)) | |
} | |
} | |
os.MkdirAll(folder, 0777) | |
// You can now save it to disk or whatever... | |
if err = ioutil.WriteFile(folder+string(filepath.Separator)+filename, data, 0777); err != nil { | |
log.Println("Error Saving:", filename, err) | |
} else { | |
log.Println("Saved:", filename) | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment