Fast parallel downloads using HTTP Accept-Ranges and Go-routines

gistfile1.go

package main

import (
	"io/ioutil"
	"net/http"
	"strconv"
	"sync"
)

var wg sync.WaitGroup

func main() {
	res, _ := http.Head("http://localhost/rand.txt"); // 187 MB file of random numbers per line
	maps := res.Header
	length, _ := strconv.Atoi(maps["Content-Length"][0]) // Get the content length from the header request
	limit := 10 // 10 Go-routines for the process so each downloads 18.7MB
	len_sub := length / limit // Bytes for each Go-routine
	diff := length % limit // Get the remaining for the last request
	body := make([]string, 11) // Make up a temporary array to hold the data to be written to the file
	for i := 0; i < limit ; i++ {
		wg.Add(1)
	
		min := len_sub * i // Min range
		max := len_sub * (i + 1) // Max range

		if (i == limit - 1) {
			max += diff // Add the remaining bytes in the last request
		}

		go func(min int, max int, i int) {
			client := &http.Client {}
			req, _ := http.NewRequest("GET", "http://localhost/rand.txt", nil)	
			range_header := "bytes=" + strconv.Itoa(min) +"-" + strconv.Itoa(max-1) // Add the data for the Range header of the form "bytes=0-100"
			req.Header.Add("Range", range_header)
			resp,_ := client.Do(req)
			defer resp.Body.Close()
			reader, _ := ioutil.ReadAll(resp.Body)
			body[i] = string(reader)
			ioutil.WriteFile(strconv.Itoa(i), []byte(string(body[i])), 0x777) // Write to the file i as a byte array
			wg.Done()
			//			ioutil.WriteFile("new_oct.png", []byte(string(body)), 0x777)
		}(min, max, i)
	}
	wg.Wait()
}

/*

alias combine="perl -E 'say for 0..10' | xargs cat > output.txt"
alias clean-dir="ls -1 | perl -ne 'print if /^\d+$/' | xargs rm"
alias verify="diff /var/www/rand.txt output.txt"

Combine - read the files and append them to the text file
Clean - Remove all the files in the folder which are numbers. (Temp files)
Verify - Verify the diff of the files from the current directory to the original file
*/

/*

Results : 

Parallel : 

10 Go routines :

real	0m4.349s
user	0m0.484s
sys	0m0.356s

60 Go routines : 

real	0m0.891s
user	0m0.484s
sys	0m0.432s

Wget :

real	0m19.536s
user	0m4.652s
sys	0m0.580s

Combine files : perl -E 'say for 0..59' | xargs cat > output.txt

real	0m1.532s
user	0m0.000s
sys	0m0.244s


*/