code.google.com web hook -> fetch archive -> upload to GCS

app.go

package foo

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"appengine"
	"appengine/delay"
	"appengine/urlfetch"

	"code.google.com/p/google-api-go-client/googleapi"
	storage "code.google.com/p/google-api-go-client/storage/v1beta2"
	value "github.com/ImJasonH/appengine-value"
)

const storageScope = "https://www.googleapis.com/auth/devstorage.read_write"

var (
	rootBucket string
)

func init() {
	http.HandleFunc("/webhook", hookHandler)
}

type hook struct {
	ProjectName string `json:"project_name"`
	Revisions   []struct {
		Revision string `json:"revision"`
	} `json:"revisions"`
}

// RoundTripper that uses urlfetch and adds the service account access token for all requests.
type tokenRT struct {
	c      appengine.Context
	client *http.Client
}

func (t tokenRT) RoundTrip(r *http.Request) (*http.Response, error) {
	tok, _, err := appengine.AccessToken(t.c, storageScope)
	if err != nil {
		return nil, err
	}
	r.Header.Set("Authorization", "Bearer "+tok)
	return t.client.Do(r)
}

var fetchAsync = delay.Func("fetch", fetch)

func fetch(c appengine.Context, projectName, revID string) error {
	// TODO: Support sub-repos.

	// Set up API client
	gcs, err := storage.New(&http.Client{
		Transport: tokenRT{c, urlfetch.Client(c)},
	})
	if err != nil {
		return nil
	}

	// TODO: Ensure the bucket is world-readable
	// This method requires a client for v1, not v1beta2
	//if _, err := gcs.Buckets.Update(bucketName).PredefinedAcl("public-read").Do(); err != nil {
	//	handleAPIError(err)
	//	return err
	//}

	// Fetch repo archive
	url := fmt.Sprintf("https://%s.googlecode.com/archive/%s.tar.gz", projectName, revID)
	resp, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		all, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return fmt.Errorf("http status %d\n%s", resp.StatusCode, string(all))
	}

	// Add all files from the tar to the bucket
	gr, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	tr := tar.NewReader(gr)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		filename := hdr.Name
		if strings.HasPrefix(filename, fmt.Sprintf("%s-%s", projectName, revID[:12])) {
			filename = filename[len(projectName)+13:]
			if filename == "" {
				c.Infof("skipping %s", hdr.Name)
				continue
			}
		}
		// filename should be /path/to/stuff
		c.Infof("inserting %s%s", revID, filename)
		if _, err := gcs.Objects.Insert(rootBucket, &storage.Object{
			Name: revID + filename,
		}).Media(tr).Do(); err != nil {
			handleAPIError(c, err)
			continue
		}
	}
	return nil
}

func handleAPIError(c appengine.Context, err error) {
	if gerr, ok := err.(*googleapi.Error); ok {
		c.Errorf("api error: %s\n%s", gerr.Message, gerr.Body)
		return
	}
	c.Errorf("unknown error: %v", err)
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	// TODO: Check hook secret.
	c := appengine.NewContext(r)
	defer r.Body.Close()
	rootBucket = value.Get(c, "root_bucket")

	var h hook
	if err := json.NewDecoder(r.Body).Decode(&h); err != nil {
		c.Errorf("%v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	for _, r := range h.Revisions {
		fetchAsync.Call(c, h.ProjectName, r)
	}
}

app.yaml

application: nth-setup-586
version: 1
runtime: go
api_version: go1

handlers:
- url: /.*
  script: _go_app

queue.yaml

queue:
- name: default
  rate: 1/s