total.go

package main

import (
	"fmt"
	"log"
	"runtime"
	"strconv"
	"time"

	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Fatal(err)
	}

	s := s3.New(auth, aws.USEast)

	merged := s.Bucket("twitter-merged-count")

	total := uint64(0)
	start := time.Date(2006, time.March, 21, 12, 0, 0, 0, time.UTC)
	end := time.Date(2011, time.January, 1, 12, 0, 0, 0, time.UTC)
	for current := start; current.Before(end); current = current.AddDate(0, 0, 1) {
		for h := 0; h < 24; h++ {
			hour := current.Format("2006/01/02") + fmt.Sprintf("/%02d/", h)
			name := hour + "tweet.gz"
		readCount:
			body, err := merged.Get(name)
			if err != nil {
				log.Printf("error getting reader for %s: %s\n", name, err)
				time.Sleep(10 * time.Millisecond)
				goto readCount
			}
			count, err := strconv.Atoi(string(body))
			if err != nil {
				log.Printf("name: %s ERROR: %s\n", name, err)
			}
			log.Printf("name: %s count: %d", name, count)
			total = total + uint64(count)
		}
	}
	log.Println("total count:", total)
}