Varnish VCL for Apr 9, 2013 SFPHP Meetup

sfphp.vcl

# VCL for Varnish - How to cache your dynamic pages 
# SFPHP.org Apr 9, 2013

# See: https://github.com/varnish/libvmod-header
# This varnish mod allows us to easily manipulate the Set-Cookie responses
import header;

probe varnish_probe {
  .url = "/ping";
  .timeout = 2s;
  .interval = 5m;
  .window = 3;
  .threshold = 1;
}

backend localhost {
  .host = "127.0.0.1";
  .port = "10088";
}

backend widgets {
  .host = "127.0.0.1";
  .port = "80";
}

backend google {
  .host = "74.125.28.99";
}

backend server1 {
  .host = "sfphp.org";
  .probe = varnish_probe;
}

backend server2 {
  .host = "sfmysql.org";
  .probe = varnish_probe;
}

director default round-robin {
  { .backend = server1; }
  { .backend = server2; }
}

# We only want to allow purge requests to come from the localhost
acl purge {
    "localhost";
}


sub vcl_recv {

    # Health check for varnish itself
    if (req.url == "/ping") {
        # Varnish doesn't let us use "synthetic" in vcl_recv, so throw a bogus error
        error 600 "OK";
    }

    # Set a header with the Varnish request ID so that we can add it to our logging
    set req.http.X-Varnish-XID = req.xid;

    # defaults to the "default" director, but let's override if the host matches these patterns
    if ( req.http.host ~ "^local.widgets" ) {
        set req.backend = widgets;
    # proxy a search endpoint to google
    } elseif ( req.url ~ "/google" ) {
        set req.backend = google;
        set req.http.host = "www.google.com";
        set req.url = regsub(req.url, "^/google", "/search");
        return (pass);
    } elseif ( req.http.host ~ "^localhost" ) {
        set req.backend = localhost;
    }

    # call a custom method so we can normalize the user agent from the browser
    call normalize_user_agent;

    # If not one of the sites we support Varnish for we should pass.
    if ( req.http.host !~ "^(local\.widgets|localhost)" ) {
        return (pass);
    }

    # Set a header announcing Surrogate Capability to the origin servers
    set req.http.Surrogate-Capability = "varnish=ESI/1.0";

    if (req.http.Cookie ~ "regioncookie=\d") {
        set req.http.X-regioncookie  = regsub(req.http.Cookie, "^.*?regioncookie=(\d)$", "\1");
        set req.http.X-regioncookie-value = regsub(req.http.X-regioncookie, "^(\d)$", "\1");

        set req.http.X-regioncookie-found = "true";
        unset req.http.X-regioncookie;
        unset req.http.X-regioncookie-value;
    } else {
        set req.http.X-regioncookie-found = "false";
        return (pass);
    }

    # pass if it is a "private" page
    if ( req.url ~ "/private/" ) {
        return (pass);
    } elseif (req.http.Cookie ~ "(myauth)=") {
        # only return a pass if the backend is healthy
        # Otherwise, the code below will strip the auth cookie and treat the user as logged out
        if (req.backend.healthy) {
            return (pass);
        }
    }

    if (req.http.Cookie) {
        # strip all cookies except for "regioncookie"
        set req.http.Cookie = ";" + req.http.Cookie;
        set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
        set req.http.Cookie = regsuball(req.http.Cookie, ";(regioncookie|otheracceptablecookie)=", "; \1=");
        set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
        set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");

        if (req.http.Cookie == "") {
            # If there are no remaining cookies, remove the cookie header. If there
            # aren't any cookie headers, Varnish's default behavior will be to cache
            # the page.
            unset req.http.Cookie;
        }
    }

    # Normalize Accept-Encoding header
    #
    # Browsers send this in many different ways; Varnish will cache multiple
    # versions of the same page
    if (req.http.Accept-Encoding) {
        if (req.http.Accept-Encoding ~ "gzip") {
            # If the browser supports it, we'll use gzip.
            set req.http.Accept-Encoding = "gzip";
        } else if (req.http.Accept-Encoding ~ "deflate") {
            # Next, try deflate if it is supported.
            set req.http.Accept-Encoding = "deflate";
        } else {
            # Unknown algorithm. Remove it and send unencoded.
            unset req.http.Accept-Encoding;
        }
    }

    # Allow serving stale objects from cache if origin servers are down
    #
    # This will give us 3 hours to fix the origin servers if they go down.
    # This setting must be <= beresp.grace in vcl_fetch.
    set req.grace = 3h;

    if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
                req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            error 405 "Not allowed.";
        }
        return (lookup);
    }

    if (req.request != "GET" &&
        req.request != "HEAD" &&
        req.request != "PUT" &&
        req.request != "POST" &&
        req.request != "TRACE" &&
        req.request != "OPTIONS" &&
        req.request != "DELETE") {
            /* Non-RFC2616 or CONNECT which is weird. */
            return (pipe);
    }
    if (req.request != "GET" && req.request != "HEAD") {
        /* We only deal with GET and HEAD by default */
        return (pass);
    }

    # only pass for authorization not the default cookie too since we always pass the i18n cookie
    if (req.http.Authorization) {
        /* Not cacheable by default */
        return (pass);
    }

    # If client requests no-cache, pass to origin
    if (req.http.Cache-Control ~ "no-cache") {
        return (pass);
    }

    # Otherwise, lookup in cache
    # we don't fall back to the default varnish vcl_recv or else the page
    # won't cache because cookies are set
    return (lookup);
}

sub vcl_hash {
    # If there are any cookies still set, we need to include them in the hash
    # this is because there are different cache values based on the regioncookie
    if (req.http.Cookie != "") {
        hash_data(req.http.Cookie);
    }

}

sub normalize_user_agent {
    if (req.http.user-agent ~ "Mobile") {
        set req.http.X-UA = "mobile";
    } else if (req.http.user-agent ~ "Android") {
        set req.http.X-UA = "androind";
    } else if (req.http.user-agent ~ "Opera Mini/") {
        set req.http.X-UA = "mobile";
    } else if (req.http.user-agent ~ "Opera Mobi/") {
        set req.http.X-UA = "mobile";
    } else if (req.http.user-agent ~ "iP(od|ad|hone)") {
        set req.http.X-UA = "iOS";
    } else if (req.http.user-agent ~ "MSIE") {
        set req.http.X-UA = "desktop";
    } else if (req.http.user-agent ~ "Chrome") {
        set req.http.X-UA = "desktop";
    } else if (req.http.user-agent ~ "Firefox") {
        set req.http.X-UA = "desktop";
    } else if (req.http.user-agent ~ "Safari") {
        set req.http.X-UA = "desktop";
    } else if (req.http.user-agent ~ "Opera") {
        set req.http.X-UA = "desktop";
    # I like to treat curl as desktop
    } else if (req.http.user-agent ~ "curl") {
        set req.http.X-UA = "desktop";
    } else {
        set req.http.X-UA = req.http.user-agent;
    }
}

sub vcl_fetch {

    # Keep object in cache up to 3 hours after expiration
    #
    # This will give us 3 hours to fix the origin servers if they go down.
    # This setting must be >= req.grace in vcl_recv.
    set beresp.grace = 3h;

    # Unset the Surrogate Control header and do ESI
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # If X-Varnish-Ttl is set, use this header's value as the TTL for the varnish cache.
    # Expires, cache-control, etc. will be passed directly through to the client.
    # Stolen from http://open.blogs.nytimes.com/2010/09/15/using-varnish-so-news-doesnt-break-your-server/
    if (beresp.http.X-Varnish-Ttl) {
        C{
            char *ttl;
            /* first char in third param is length of header plus colon in octal */
            ttl = VRT_GetHdr(sp, HDR_BERESP, "\016X-Varnish-Ttl:");
            VRT_l_beresp_ttl(sp, atoi(ttl));
        }C
        remove beresp.http.X-Varnish-Ttl;
    } elseif (req.url ~ "\.(gif|jpg|jpeg|swf|css|js|flv|mp3|mp4|pdf|ico|png)(\?.*|)$") {
        remove beresp.http.Set-Cookie;
        set beresp.ttl = 24h;
    }

    # Cache errors for 5s, ignoring what the origin server may have
    # said about the cacheability of the response. If the origin said the
    # error is cacheable for several minutes, that is too long and should
    # be overridden. If the origin said the error is not cacheable at all,
    # a bunch of clients making the same request will result in a flood of
    # requests hitting the origin. Caching the error for one second means
    # the backend will get at most one request per 5 seconds for that URL.
    if (beresp.status >= 400) {
        set beresp.ttl = 5s;
        set beresp.grace = 0s;
    }

    # You don't wish to cache content for logged in users
    if (req.http.Cookie ~ "myauth") {
        set beresp.http.X-Cacheable = "NO:Got Session";
        return (hit_for_pass);
    } else if (beresp.ttl <= 0s) {
        set beresp.http.X-Cacheable = "NO:Not Cacheable <= 0s ttl";
    } else if ( ( req.url ~ "/private/" ) ) {
        set beresp.http.X-Cacheable = "NO:Private URL";
        return (hit_for_pass);
    # You are respecting the Cache-Control=private header from the backend
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.http.X-Cacheable = "NO:Cache-Control=private";
        return (hit_for_pass);
    } else {
        # Remove any cookie that isn't the ignauth or i18n cookie
        header.remove(beresp.http.Set-Cookie,"^(?!((myauth|regioncookie)=))");

        if (beresp.http.Set-Cookie == "") {
            set beresp.http.X-Cacheable = "YES";

            # If there are no remaining cookies, remove the cookie header. If there
            # aren't any cookie headers, Varnish's default behavior will be to cache
            # the page.
            remove beresp.http.Set-Cookie;
        }
    }

    set beresp.http.X-UA = req.http.X-UA;
    set beresp.http.Vary = "Accept-Encoding,X-UA";

    # Unconditional return intentionally omitted, fall through to default vcl_fetch
}

sub vcl_deliver {
    set resp.http.X-Served-By = server.hostname;
    if (obj.hits > 0) {
        set resp.http.X-Cache-Result = "HIT";
        set resp.http.X-Cache-Hits = obj.hits;
    } else {
        set resp.http.X-Cache-Result = "MISS";
    }
}

sub vcl_error {
    # Catch bogus error and return synthetic health check page
    if (obj.status == 600) {
        set obj.status = 200;
        synthetic "pong";
        return (deliver);
    }

    # Unconditional return intentionally omitted, fall through to default vcl_error

}

# Called if the cache has a copy of the page.
sub vcl_hit {
    if (req.request == "PURGE") {
        purge;
        error 200 "Purged";
    }
}

# Called if the cache does not have a copy of the page.
sub vcl_miss {
    if (req.request == "PURGE") {
        purge;
        error 200 "Not in cache";
    }
}