VCL snippets

cmdline.sh

mkfs.ext4 -b 4k -C 128k -E num_backup_sb=0,packed_meta_blocks=1,root_owner=111:118 -O bigalloc,extent,flex_bg,^has_journal,mmp,sparse_super2 -L varnish-cache -N 64 -v /dev/sdc

crawler.vcl

vcl 4.0;

import purge;
import std;

acl local {
    "localhost";
    "127.0.0.1"/8;
    "::1"/128;
}

sub vcl_recv {
    if (req.restarts > 0 && req.hash_always_miss) {
        // No need to redo URL normalization, conditional request passing, etc.
        return(hash);
    }

    // ..

    if (req.http.User-Agent ~ "\AWordPress Crawler" && client.ip ~ local) {
        set req.http.X-Crawled-At = "" + now;
    }
    else if (req.http.X-Crawled-At) {
        unset req.http.X-Crawled-At;
    }
}

sub vcl_hit {
    if (req.http.X-Crawled-At) {
        if ((obj.ttl + obj.grace) > 4h) {
            purge.soft(2m, 4h);
        }

        set req.hash_always_miss = true;

        return(restart);
    }
}

fstab

/dev/sdc /var/lib/varnish/extvol ext4 defaults,discard,dirsync,errors=remount-ro,noatime,nobarrier,nosuid,nodev,noexec 0 0

override.conf

-s static=file,/var/lib/varnish/extvol/static.bin,9G,128K,random

wordpress.vcl

sub prune_cookies {
    set req.http.X-Cookie-Primary = regsuball(
        regsuball(
            ";" + req.http.Cookie,
            ";\s*((?:wordpress_(?!test)|wp-settings-).+?)\s*=\s*",
            ";__KEEP__\1="
        ),
        ";(?!__KEEP__)[^;]*",
        ""
    );

    if (req.http.X-Cookie-Primary ~ "__KEEP__") {
        // Only include these cookie(s) if the primary cookie(s) are present.
        set req.http.X-Cookie-Secondary = regsuball(
            regsuball(
                ";" + req.http.Cookie,
                ";\s*((?:[Mm][Yy])?PHPSESSID|wordpress_test_cookie)\s*=\s*",
                ";__KEEP__\1="
            ),
            ";(?!__KEEP__)[^;]*",
            ""
        );

        set req.http.Cookie = regsuball(req.http.X-Cookie-Primary + req.http.X-Cookie-Secondary, "__KEEP__", " ");
        set req.http.Cookie = regsub(regsub(req.http.Cookie, "\A;\s*", ""), "\s*\z", "");

        unset req.http.X-Cookie-Secondary;
    }
    else {
        unset req.http.Cookie;
    }

    unset req.http.X-Cookie-Primary;
}

sub vcl_recv {
    if (req.http.Cookie) { 
        if (req.http.Cookie ~ "\b(?:wordpress_(?:logged_in|sec)_[[:xdigit:]]+|wp-settings-\d+)\b") {
            return(pass);
        }

        unset req.http.Cookie;
    }

    if (req.http.Cache-Control ~ "\bno-?cache\b" || req.http.Pragma ~ "\bno-?cache\b") {
        return(pass);
    }

    // TODO: Do we need to whitelist any query parameters?
    //set req.url = regsub(req.url, "(?:(?<=.)/+)?(?:\?.*)?\z", "");
    set req.url = regsub(req.url, "(?:(?<=.)/+)?(?:index\.php)?(?:\?.*)?\z", "");
}

sub vcl_backend_response {
    if (!(bereq.uncacheable || beresp.http.Cache-Control || beresp.http.Expires)) {
       // The backend didn't set a TTL, but it also didn't explicitly not set a
       // TTL, so we'll set a default here.
       set beresp.ttl = 86400s;

       set beresp.http.Cache-Control =
         "s-maxage=" + std.integer(beresp.ttl) + ", " +
         "max-age=" + 30 * std.integer(beresp.ttl);
    }
}

working.vcl

vcl 4.0;

import std;

acl local {
    "localhost";
    "127.0.0.0"/8;
    "::1"/128;
    "fe80::"/10;
}

backend default {
    .host = "127.0.0.1";
    .port = "2020";
    .proxy_header = 2;
}

sub vcl_recv {
    if (client.ip !~ local) {
        return(synth(403, "Forbidden"));
    }

    // Try to format the URL in such a way to minimize WordPress's canonical
    // redirection subroutine and maximize cache hit rate. And do it early so
    // PURGE works the way it should (and to make subsequent regexps terser).
    set req.url = regsuball(req.url, "/{2,}", "/");
    set req.url = regsuball(req.url, "/index\.php/?", "/");
    set req.url = regsub(req.url, "(?!\A)/(\?.*)?\z", "\1");
    set req.url = std.querysort(req.url);

    if (req.method == "PURGE") {
        if (std.ip(req.http.X-Real-IP, "0.0.0.0") !~ local) {
            return(synth(405, "Forbidden"));
        }

        return(purge);
    }

    if (req.url ~ "\A(?:/wp)?/wp-(?:admin|login\.php)\b" || req.url ~ "[?&]preview=true\b") {
        return(pass);
    }

    if (req.http.Cookie) {
        if (req.http.Cookie ~ "\bwordpress_(?:logged_in|sec)_[[:xdigit:]]") {
            return(pass);
        }

        unset req.http.Cookie;
    }
}

sub vcl_backend_fetch {
    // The WordPress Dashboard gets all wonky-donkey without a trailing slash,
    // *and* it seems to prefer the index filename in the actual request URI.
    set bereq.url = regsub(bereq.url, "(?<=/wp-admin)(\?.*)?\z", "/index.php\1");
}

sub vcl_backend_response {
    // https://varnish-cache.org/docs/6.3/users-guide/vcl-grace.html#misbehaving-servers
    if (bereq.is_bgfetch && beresp.status >= 500 && beresp.status < 600) {
        return(abandon);
    }

    set beresp.do_gzip = true
        && (!beresp.http.Content-Length || std.bytes(beresp.http.Content-Length, 0B) > 400B)
        && (!beresp.http.Content-Type || (
            beresp.http.Content-Type ~ "text|javascript|json|svg\+xml|icon|font"
            && beresp.http.Content-Type !~ "woff"
        ));

    if (beresp.ttl <= 0s && (beresp.http.ETag || beresp.http.Last-Modified)) {
        // We can't cache this object, but it *can* be revalidated by the
        // backend, so automatically `(pass)' subsequent matching requests for
        // some time (hit-for-pass).
        //
        // https://varnish-cache.org/docs/6.3/users-guide/increasing-your-hitrate.html#hit-for-pass
        //
        // N.B. Putting this statement above `(deliver)' will cause the
        // transient hit-for-pass object to be extended indefinitely on
        // successive matching requests within the TTL -- which is fine, as
        // long as the backend is well-behaved. I think.
        return(pass(2m));
    }

    // The next two blocks are lifted (mosty) as-is from the built-in VCL.
    // They've been inserted here to prevent unnecessary work being done on
    // uncacheable requests.
    //
    // https://github.com/varnishcache/varnish-cache/blob/6.3/bin/varnishd/builtin.vcl#L147-L161
    if (bereq.uncacheable) {
        // Pass, or previously hit-for-pass.
        return(deliver);
    }

    if (beresp.ttl <= 0s
        || beresp.http.Set-Cookie
        || std.tolower(beresp.http.Surrogate-Control) ~ "\bno-?store\b"
        || (!beresp.http.Surrogate-Control && std.tolower(beresp.http.Cache-Control) ~ "\b(?:private|no-?(?:cache|store))\b")
        || beresp.http.Vary == "*"
    ) {
        // Hit-for-miss.
        //
        // https://varnish-cache.org/docs/6.3/users-guide/increasing-your-hitrate.html#hit-for-miss
        set beresp.ttl = 2m;
        set beresp.uncacheable = true;

        return(deliver);
    }

    if (!beresp.http.Cache-Control) {
        if (!beresp.http.Expires) {
            set beresp.ttl = 5m;
        }

        // https://varnish-cache.org/docs/6.3/users-guide/vcl-grace.html#grace-mode
        set beresp.grace = 1h;

        // The backend didn't set a Cache-Control header, so we'll set it based
        // on TTL and Grace. This doesn't affect Varnish per se, but it advises
        // and informs the user-agent (and your humble webmaster). We only want
        // to do this once and cache the result for the happy path, so we'll do
        // it here instead of in `vcl_deliver', where it might seem more
        // logical.
        set beresp.http.Cache-Control = "public"
            + ", max-age=" + 32 * std.integer(beresp.ttl)
            + ", s-maxage=" + std.integer(beresp.ttl)
            + ", stale-while-revalidate=" + std.integer(beresp.grace);
    }

    // We'll (re-)set this dynamically in `vcl_deliver'. (Just don't cache it.
    // That'd be silly.)
    unset beresp.http.Expires;

    if (beresp.http.ETag || beresp.http.Last-Modified) {
        // Keep stale and out-of-grace objects in cache for future backend
        // revalidations.
        //
        // https://varnish-cache.org/docs/6.3/users-guide/vcl-grace.html#keep
        // https://varnish-cache.org/docs/6.3/users-guide/increasing-your-hitrate.html#cache-misses
        set beresp.keep = 4h;
    }

    return(deliver);
}

sub vcl_deliver {
    if (obj.uncacheable) {
        // Tell the user-agent not to cache this request -- which was a pass,
        // hit-for-pass, or hit-for-miss -- regardless of the headers resolved
        // in `vcl_backend_response'.
        set resp.http.Cache-Control = "private, max-age=0, must-revalidate";
        set resp.http.Expires = "Thu, 01 Jan 1970 00:00:00 GMT";
    }
    else {
        set resp.http.Expires = "" + (now + obj.ttl);
    }
}

// vim: et:ts=4:sw=4:tw=80