Skip to content

Instantly share code, notes, and snippets.

@cnu
Created December 2, 2018 19:23
Show Gist options
  • Star 10 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save cnu/026744b1e86c6d9e22313d06cba4c2e9 to your computer and use it in GitHub Desktop.
Save cnu/026744b1e86c6d9e22313d06cba4c2e9 to your computer and use it in GitHub Desktop.
Download files in Go
package main
import (
"fmt"
"io"
"net/http"
"os"
)
func main() {
if len(os.Args) != 3 {
fmt.Println("usage: download url filename")
os.Exit(1)
}
url := os.Args[1]
filename := os.Args[2]
err := DownloadFile(url, filename)
if err != nil {
panic(err)
}
}
// DownloadFile will download a url and store it in local filepath.
// It writes to the destination file as it downloads it, without
// loading the entire file into memory.
func DownloadFile(url string, filepath string) error {
// Create the file
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()
// Get the data
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
return nil
}
package main
import (
"fmt"
"io"
"net/http"
"os"
"strings"
"github.com/dustin/go-humanize"
)
// WriteCounter counts the number of bytes written to it. By implementing the Write method,
// it is of the io.Writer interface and we can pass this into io.TeeReader()
// Every write to this writer, will print the progress of the file write.
type WriteCounter struct {
Total uint64
}
func (wc *WriteCounter) Write(p []byte) (int, error) {
n := len(p)
wc.Total += uint64(n)
wc.PrintProgress()
return n, nil
}
// PrintProgress prints the progress of a file write
func (wc WriteCounter) PrintProgress() {
// Clear the line by using a character return to go back to the start and remove
// the remaining characters by filling it with spaces
fmt.Printf("\r%s", strings.Repeat(" ", 50))
// Return again and print current status of download
// We use the humanize package to print the bytes in a meaningful way (e.g. 10 MB)
fmt.Printf("\rDownloading... %s complete", humanize.Bytes(wc.Total))
}
func main() {
if len(os.Args) != 3 {
fmt.Println("usage: download url filename")
os.Exit(1)
}
fmt.Println("Download Started")
url := os.Args[1]
filename := os.Args[2]
err := DownloadFile(url, filename)
if err != nil {
panic(err)
}
fmt.Println("Download Finished")
}
// DownloadFile will download a url and store it in local filepath.
// It writes to the destination file as it downloads it, without
// loading the entire file into memory.
// We pass an io.TeeReader into Copy() to report progress on the download.
func DownloadFile(url string, filepath string) error {
// Create the file with .tmp extension, so that we won't overwrite a
// file until it's downloaded fully
out, err := os.Create(filepath + ".tmp")
if err != nil {
return err
}
defer out.Close()
// Get the data
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
// Create our bytes counter and pass it to be used alongside our writer
counter := &WriteCounter{}
_, err = io.Copy(out, io.TeeReader(resp.Body, counter))
if err != nil {
return err
}
// The progress use the same line so print a new line once it's finished downloading
fmt.Println()
// Rename the tmp file back to the original file
err = os.Rename(filepath+".tmp", filepath)
if err != nil {
return err
}
return nil
}
@quangnguyenbh
Copy link

do you know Unexpected EOF in the o.Copy(out, resp.Body) ? I'm using your first download snippet

@msterzhang
Copy link

func DownloadFile(url string, filepath string) error {

	// Create the file with .tmp extension, so that we won't overwrite a
	// file until it's downloaded fully
	out, err := os.Create(filepath + ".tmp")
	if err != nil {
		return err
	}

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Create our bytes counter and pass it to be used alongside our writer
	counter := &WriteCounter{}
	_, err = io.Copy(out, io.TeeReader(resp.Body, counter))
	if err != nil {
		return err
	}

	// The progress use the same line so print a new line once it's finished downloading
	fmt.Println()
        out.Close()
	// Rename the tmp file back to the original file
	err = os.Rename(filepath+".tmp", filepath)
	if err != nil {
		return err
	}

	return nil
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment