Load Bitstamp Historical Data Into Redis

import_bitstamp_data.go

package main

import (
	"bytes"
	"compress/gzip"
	"encoding/csv"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/garyburd/redigo/redis"
	"io"
	"net/http"
	"strconv"
)

const (
	DATA_URL          = "http://api.bitcoincharts.com/v1/csv/bitstampUSD.csv.gz"
	buffer_size int64 = 256
)

func main() {
	resp, err := http.Get(DATA_URL)

	if err != nil {
		fmt.Println("HEAD Error:", err)
		return
	}

	dlSize, err := strconv.Atoi(resp.Header.Get("Content-Length"))

	if err != nil {
		fmt.Println("Download Size Error:", err)
		return
	}

	count := (dlSize / 245)
	bar := pb.StartNew(count)

	file := make([]byte, 0)

	for {
		data := make([]byte, buffer_size)
		i, err := resp.Body.Read(data)

		bar.Increment()

		if err == io.EOF {
			bar.FinishPrint("")
			break
		}

		if err != nil {
			fmt.Println("Download Err:", err)
		}

		file = append(file, data[:i]...)
	}

	b := bytes.NewBuffer(file)

	fmt.Println("Unzipping...")
	r, err := gzip.NewReader(b)
	defer r.Close()

	c := csv.NewReader(r)

	rDB, err := redis.Dial("tcp", ":6379")

	if err != nil {
		fmt.Println("Redis Connection Error:", err)
	}

	fmt.Println("Parsing and writing data to redis...")
	for {
		fields, err := c.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}

		rDB.Do("SET", fields[0], fields[1])
	}
}