Skip to content

Instantly share code, notes, and snippets.

@kyledinh

kyledinh/0.md

Forked from disolovyov/0.md
Created October 13, 2013 01:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save kyledinh/6957026 to your computer and use it in GitHub Desktop.
Save kyledinh/6957026 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
0.md 
func error2(err error, c appengine.Context) bool {
    if err != nil {
        c.Errorf("%v", err.Error())
        return true
    }
    return false
}

func error3(err error, c appengine.Context, w http.ResponseWriter) bool {
    if err != nil {
        msg := err.Error()
        c.Errorf("%v", msg)
        http.Error(w, msg, http.StatusInternalServerError)
        return true
    }
    return false
}

/*
 * Use it like this:
 */
func index(w http.ResponseWriter, r *http.Request) {
    // ...
    
    c := appengine.NewContext(r)
    token, err := channel.Create(c, clientId)
    if error3(err, c, w) { return }
    // ...
}
Raw
1a.md

1. Create project structure

app_name
├── app.yaml
├── src
│   └── module_name
│       └── module_name.go
├── static
│   └── start.html
└── templates
    └── gallery.html

2. Setup routing

  • / → HTTP redirect to /static/start.html
  • /gallery?name=Gopher → render templates/gallery.html: "Hello Gopher!"

3. Verify with local dev app server

$ cd app_name && dev_appserver.py --use_sqlite ./

4. Deploy to App Engine

$ appcfg.py update --email your_account@gmail.com --oauth2 ./
Raw
1b.md

app.yaml

application: app-name
version: 1
runtime: go
api_version: go1

handlers:
- url: /static
  static_dir: static

- url: /.*
  script: _go_app

gallery.html

<!DOCTYPE html>
<html>
<body>
    Hello {{.}}!
</body>
</html>

module_name.go

package module_name

import (
    "strings"
    "time"
    "io"
    "fmt"
    "regexp"
    "crypto/rand"
    "encoding/hex"
    "net/http"
    "net/url"
    "html/template"
    "appengine"
    "appengine/user"
    "appengine/urlfetch"
    "appengine/channel"
    "appengine/datastore"
    "appengine/taskqueue"
    "appengine/blobstore"
    "appengine/image"
)

func init() {
    http.HandleFunc("/", index)
    http.HandleFunc("/gallery", gallery)
}

func index(w http.ResponseWriter, r *http.Request) {
    http.Redirect(w, r, "/static/start.html", http.StatusTemporaryRedirect)
}

var templates *template.Template = nil

func gallery(w http.ResponseWriter, r *http.Request) {
    name := r.FormValue("name")
    if templates == nil {
        var err error
        templates, err = template.ParseFiles("templates/gallery.html")
        if error3(err, c, w) { return }
    }
    w.Header().Set("Content-Type", "text/html")
    templates.ExecuteTemplate(w, "gallery.html", name)
}
Raw
2a.md
  1. Create a HTML page with a single text field, to POST URLs to App Engine's server.
  2. On the server, create a handler that fetches the HTML file using the URL Fetch service.
  3. Parse the HTML to extract image addresses.
Raw
2b.md

start.html

<!DOCTYPE html>
<html>
<head>
    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
    <title>My App</title>
    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
    <script type="text/javascript">
        $(document).ready(function() {
            $('#target').focus();
        });
    </script>
</head>
<body>
    <form action="/start">
        <input id="target" name="target" type="text" style="width: 30em;" />
        &nbsp;
        <button id="go">Go!</button>
    </form>
</body>
</html>

module_name.go

func init() {
    http.HandleFunc("/", index)
    http.HandleFunc("/start", start)
}

func start(w http.ResponseWriter, r *http.Request) {
    target := r.FormValue("target")
    c := appengine.NewContext(r)

    client := urlfetch.Client(c)
    resp, err := client.Get("http://" + target)
    if error3(err, c, w) { return }

    len := int(resp.ContentLength)
    buf := make([]byte, len)
    read, err := resp.Body.Read(buf)
    if error3(err, c, w) { return }
    if read != len {
        http.Error(w, fmt.Sprintf("Target page Content-Length is %v but read %v bytes", len, read), http.StatusInternalServerError)
        return
    }

    rx, _ := regexp.Compile("<img .*? src=\"(.*?)\"")
    images := rx.FindAllSubmatch(buf, len)
    if images == nil {
        w.Header().Set("Content-Type", "text/plain")
        fmt.Fprintf(w, "HTTP GET returned status %v\nNo images found\n\n", resp.Status)
        w.Write(buf)
        return
    }

    for _, image := range images {
        addr := string(image[1])
        if strings.Index(addr, "http") == 0 {
            fmt.Fprintf(w, "image found: %v\n", addr)
        }
    }
}
Raw
3a.md
  1. Enqueue image addresses as separate tasks via Task Queue.
  2. The task should fetch the image and
  3. Store in Blobstore;
  4. Create thumbnail URL using the Images service.
Raw
3b.md

app.yaml

handlers:
- url: /fetch
  script: _go_app
  login: admin

module_name.go

func init() {
    // ...
    http.HandleFunc("/fetch", fetch)
}

func start(w http.ResponseWriter, r *http.Request) {
    // ...

    for _, image := range images {
        addr := string(image[1])
        if strings.Index(addr, "http") == 0 {
            task := taskqueue.NewPOSTTask("/fetch", url.Values{"image": {addr}})
            _, err := taskqueue.Add(c, task, "default")
            if error3(err, c, w) { return }
        }
    }
}


func fetch(w http.ResponseWriter, r *http.Request) {
    c := appengine.NewContext(r)

    imageUrl := r.FormValue("image")

    client := urlfetch.Client(c)
    resp, err := client.Get(imageUrl)
    if error3(err, c, w) { return }

    blob, err := blobstore.Create(c, resp.Header.Get("Content-Type"))
    if error3(err, c, w) { return }
    written, err := io.Copy(blob, resp.Body)
    if error3(err, c, w) { return }
    if written < 100 {
        c.Infof("image is too small %v", written)
        return
    }
    err = blob.Close()
    if error3(err, c, w) { return }

    blobkey, err := blob.Key()
    if error3(err, c, w) { return }

    thumbnailUrl, err := image.ServingURL(c, blobkey, &image.ServingURLOptions{Size: 100})
    if error3(err, c, w) { return }
    thumbnail := thumbnailUrl.String()

    // ...
}
Raw
4a.md
  1. For client's web browser, generate a new page backed by the Channel service and let the client listen for updates.
  2. Fetch task should send thumbnail URLs to client via channel obtained by client id.
Raw
4b.md

module_name.go

const cookieName = "image-scrap-clientid"

func start(w http.ResponseWriter, r *http.Request) {
    // ...

    var clientId string
    u := user.Current(c)
    if u != nil {
        clientId = u.Email
    } else {
        len := 16
        r := make([]byte, len)
        read, err := rand.Read(r)
        if error3(err, c, w) && read < 8 { return }
        clientId = hex.EncodeToString(r)
    }

    // ...
    for _, image := range images {
        addr := string(image[1])
        if strings.Index(addr, "http") == 0 {
            task := taskqueue.NewPOSTTask("/fetch", url.Values{"clientId": {clientId}, "image": {addr}})
            _, err := taskqueue.Add(c, task, "default")
            if error3(err, c, w) { return }
        }
    }

    token, err := channel.Create(c, clientId)
    if error3(err, c, w) { return }

    c.Infof("setting cookie to '%v'", clientId)
    http.SetCookie(w, &http.Cookie{Name: cookieName, Value: clientId})
    gallery(token, c, w)
}

func gallery(token string, c appengine.Context, w http.ResponseWriter) {
    if templates == nil {
        var err error
        templates, err = template.ParseFiles("templates/gallery.html")
        if error3(err, c, w) { return }
    }
    w.Header().Set("Content-Type", "text/html")
    templates.ExecuteTemplate(w, "gallery.html", token)
}

func fetch(w http.ResponseWriter, r *http.Request) {
    // ...

    thumbnail := thumbnailUrl.String()

    c.Infof("pushing just fetched '%v'", thumbnail)
    clientId := r.FormValue("clientId")
    channel.Send(c, clientId, thumbnail)
}

gallery.html

<!DOCTYPE html>
<html>
<head>
    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
    <title>My App</title>
    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
    <script type="text/javascript" src="/_ah/channel/jsapi"></script>
    <script type="text/javascript">
        $(document).ready(function() {
            window.channel = new goog.appengine.Channel({{.}});
            window.socket = channel.open();
            var images = $('#images');
            socket.onmessage = function(msg) {
                images.append('<img style="display: none;" onload="$(this).fadeIn()" src="' + msg.data + '"/>')
            };
        });
    </script>
</head>
<body>
    <a href="/reset">Start over</a>
    <div id="images"></div>
</body>
</html>
Raw
5a.md

Some images are lost!?

... because there is no one to receive thumbnail URLs until the client connects, and Channel messages are dropped.

  1. start should save clientId into Datastore.
  2. fetch should write thumbnail URLs into Datastore as clienId's entity ancestors.
  3. When the client connects, push thumbnail URLs (already retrieved from Datastore).
Raw
5b.md

app.yaml

inbound_services:
- channel_presence

handlers:
- url: /_ah/channel/connected/
  script: _go_app

module_name.go

func init() {
    // ...

    http.HandleFunc("/_ah/channel/connected/", connected)
}

func connected(w http.ResponseWriter, r *http.Request) {
    clientId := r.FormValue("from")
    c := appengine.NewContext(r)
    c.Infof("connected '%v'", clientId)
    images(clientId, c)
}

type Request struct {
    ClientId string
}

type Thumbnail struct {
    ThumbnailURL string
    Blob appengine.BlobKey // BlobKey is a string
}

const rootNode = "image-scrap-request"
const thumbnailLeaf = "image-scrap-thumbnail"
const rubbish = "rubbish"

func start(w http.ResponseWriter, r *http.Request) {
    // ...

    isr := Request{clientId}
    key, err := datastore.Put(c, datastore.NewIncompleteKey(c, rootNode, nil), &isr)
    if error3(err, c, w) { return }

    for _, image := range images {
        addr := string(image[1])
        if strings.Index(addr, "http") == 0 {
            task := taskqueue.NewPOSTTask("/fetch", url.Values{"clientId": {clientId}, "image": {addr}, "key": {key.Encode()}})
            _, err := taskqueue.Add(c, task, "default")
            if error3(err, c, w) { return }
        }
    }

    // ...
}

func fetch(w http.ResponseWriter, r *http.Request) {
    // ...

    key, err := datastore.DecodeKey(r.FormValue("key"))
    if error3(err, c, w) { return }

    // ...
    thumbnail := thumbnailUrl.String()

    // ...
    ist := Thumbnail{thumbnail, blobkey}
    _, err = datastore.Put(c, datastore.NewIncompleteKey(c, thumbnailLeaf, key), &ist)
    if error3(err, c, w) { return }
}

func images(clientId string, c appengine.Context) {
    c.Infof("images for '%v'", clientId)
    iterate(clientId, c)
}

func iterate(clientId string, c appengine.Context) {
    root := datastore.NewQuery(rootNode).Filter("ClientId = ", clientId).Run(c)
    for {
        var parent Request
        rootKey, err := root.Next(&parent)
        if err == datastore.Done {
            break
        }
        if error2(err, c) { return }
        thumbnails := datastore.NewQuery(thumbnailLeaf).Ancestor(rootKey).Run(c)
        for {
            var thumbnail Thumbnail
            key, err := thumbnails.Next(&thumbnail)
            if err == datastore.Done {
                break
            }
            if error2(err, c) { return }
            c.Infof("pushing from db '%v'", thumbnail.ThumbnailURL)
            channel.Send(c, clientId, thumbnail.ThumbnailURL)
        }
    }
}
Raw
6a.md

On repeated visits with a cookie, retrieve the entity and its ancestors (thumbnail URLs), display the images, and re-open the listening channel.

Raw
6b.md

module_name.go

func index(w http.ResponseWriter, r *http.Request) {
    cookie, err := r.Cookie(cookieName)
    if err == nil && cookie != nil {
        c := appengine.NewContext(r)
        clientId := cookie.Value
        token, err := channel.Create(c, clientId)
        if error3(err, c, w) { return }
        gallery(token, c, w)
    } else {
        http.Redirect(w, r, "/static/start.html", http.StatusTemporaryRedirect)
    }
}
Raw
7a.md
  1. Implement “Start over” functionality: /reset.
  2. BONUS! perform cleanup in background via goroutines: use go func().
Raw
7b.md

module_name.go

func init() {
    // ...
    http.HandleFunc("/reset", reset)
}

func reset(w http.ResponseWriter, r *http.Request) {
    cookie, err := r.Cookie(cookieName)
    if err == nil && cookie != nil {
        c := appengine.NewContext(r)
        clientId := cookie.Value
        c.Infof("removing cookie")
        http.SetCookie(w, &http.Cookie{Name: cookieName, Value: rubbish, Expires: time.Unix(1, 0)}) // do not use (0, 0)!
        // go delete(clientId, c) -- ?
        delete(clientId, c)
    }
    http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

func delete(clientId string, c appengine.Context) {
    c.Infof("deleting '%v'", clientId)
    iterate(clientId, c, "delete")
}

func iterate(clientId string, c appengine.Context, op string) {
    root := datastore.NewQuery(rootNode).Filter("ClientId = ", clientId).Run(c)
    var keys []*datastore.Key
    for {
        var parent Request
        rootKey, err := root.Next(&parent)
        if err == datastore.Done {
            break
        }
        if error2(err, c) { return }
        thumbnails := datastore.NewQuery(thumbnailLeaf).Ancestor(rootKey).Run(c)
        for {
            var thumbnail Thumbnail
            key, err := thumbnails.Next(&thumbnail)
            if err == datastore.Done {
                break
            }
            if error2(err, c) { return }
            if op == "send" {
                c.Infof("pushing from db '%v'", thumbnail.ThumbnailURL)
                channel.Send(c, clientId, thumbnail.ThumbnailURL)
            } else if op == "delete" {
                c.Infof("deleting thumbnail from db '%v'", key)
                keys = append(keys, key)
                err := image.DeleteServingURL(c, thumbnail.Blob)
                error2(err, c)
            }
        }
        if op == "delete" {
            c.Infof("deleting request from db '%v'", rootKey)
            keys = append(keys, rootKey)
        }
    }
    if op == "delete" {
        err := datastore.DeleteMulti(c, keys) // Deleting parent deletes ancestors? No
        error2(err, c)
    }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment