Skip to content

Instantly share code, notes, and snippets.

@sglyon
Last active May 12, 2017 18:00
Show Gist options
  • Save sglyon/47a6b4c490fa1726461d5458e736b802 to your computer and use it in GitHub Desktop.
Save sglyon/47a6b4c490fa1726461d5458e736b802 to your computer and use it in GitHub Desktop.
async file downloads
package main
import (
"fmt"
"github.com/mitchellh/go-homedir"
"io"
"net/http"
"os"
"path"
"sync"
)
var urls = []string{
"https://www.irs.gov/pub/irs-soi/1990to1991countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1991to1992countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1992to1993countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1993to1994countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1994to1995countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1995to1996countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1996to1997countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1997to1998countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1998to1999countymigration.zip",
"https://www.irs.gov/pub/irs-soi/1999to2000countymigration.zip",
"https://www.irs.gov/pub/irs-soi/2000to2001countymigration.zip",
"https://www.irs.gov/pub/irs-soi/2001to2002countymigration.zip",
"https://www.irs.gov/pub/irs-soi/2002to2003countymigration.zip",
"https://www.irs.gov/pub/irs-soi/2003to2004countymigration.zip",
"https://www.irs.gov/pub/irs-soi/county0405.zip",
"https://www.irs.gov/pub/irs-soi/county0506.zip",
"https://www.irs.gov/pub/irs-soi/county0607.zip",
"https://www.irs.gov/pub/irs-soi/county0708.zip",
"https://www.irs.gov/pub/irs-soi/county0809.zip",
"https://www.irs.gov/pub/irs-soi/county0910.zip",
"https://www.irs.gov/pub/irs-soi/county1011.zip",
"https://www.irs.gov/pub/irs-soi/countyinflow1112.csv",
"https://www.irs.gov/pub/irs-soi/countyoutflow1112.csv",
"https://www.irs.gov/pub/irs-soi/countyinflow1213.csv",
"https://www.irs.gov/pub/irs-soi/countyoutflow1213.csv",
"https://www.irs.gov/pub/irs-soi/countyinflow1314.csv",
"https://www.irs.gov/pub/irs-soi/countyoutflow1314.csv",
"https://www.irs.gov/pub/irs-soi/countyinflow1415.csv",
"https://www.irs.gov/pub/irs-soi/countyoutflow1415.csv",
}
func downloadFromURL(url string, fileName string) {
fmt.Println("Downloading", url, "to", fileName)
output, err := os.Create(fileName)
if err != nil {
fmt.Println("Error while creating", fileName, "-", err)
return
}
defer output.Close()
response, err := http.Get(url)
if err != nil {
fmt.Println("Error while downloading", url, "-", err)
return
}
defer response.Body.Close()
n, err := io.Copy(output, response.Body)
if err != nil {
fmt.Println("Error while downloading", url, "-", err)
return
}
fmt.Println(n, "bytes downloaded.")
}
func main() {
fmt.Println("About to run for", len(urls), "urls")
var wg sync.WaitGroup
userHome, err := homedir.Dir()
if err != nil {
fmt.Println("Couldn't obtain user home directory -", err)
return
}
rawPath := path.Join(userHome, "data", "irs_migration", "raw")
if _, err := os.Stat(rawPath); os.IsNotExist(err) {
os.MkdirAll(rawPath, 0777)
}
for _, url := range urls {
wg.Add(1)
go func(url string) {
defer wg.Done()
fileName := path.Join(rawPath, path.Base(url))
downloadFromURL(url, fileName)
}(url)
}
wg.Wait()
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment