grace with sick backends

server s1 {     # Part 1 requests
     rxreq
     txresp -body "1"
     expect req.url == "/1"
     rxreq
     txresp -body "2"
     expect req.url == "/1"

     # Part 2 requests
     rxreq
     txresp -status 500 -body "3"
     expect req.url == "/500"

     rxreq
     txresp -status 500 -body "4"
     expect req.url == "/500"

     # Finalize
     rxreq
     txresp -status 204
     expect req.url == "/empty"
} -start

varnish v1 -vcl+backend {
     import std;
     import vsthrottle;

     sub vcl_recv {
          # Here we create a special header which identifies
          # which "5xx key" this request corresponds to. Probably
          # you want to calculate something more canonical here,
          # maybe from the URL sans query parameters, and maybe
          # you want to collapse all request to certain kinds of
          # images.
          set req.http.X-5xx-throttle-key = std.tolower(req.http.host) + req.url;

          # First check if this URL is in trouble,
          if (vsthrottle.remaining(req.http.X-5xx-throttle-key, 2, 10s) == 0) {
               std.log("This resource is in trouble");
               set req.http.X-5xx = "1";
          } else {
               unset req.http.X-5xx;
          }
          if (!req.http.X-5xx && std.healthy(s1)) {
               set req.grace = 1ms;
          }
     }
     sub vcl_backend_fetch {
          if (bereq.http.X-5xx) {
               # We know that this resource has caused problems recently, so we just give up
               # before we run to the backend.
               return (abandon);
               # Note: If we are in a bgfetch, processing stops with the abandon. If there is
               # a client waiting for this, we end up synth, where we need to give the client
               # something sensible. This is tested below.
          }
     }
     sub vcl_backend_response {
          if (beresp.status > 499 && beresp.status < 600) {
               # We remember that this resource went
               if (vsthrottle.is_denied(bereq.http.X-5xx-throttle-key, 2, 10s)) {
                    # Here we want the vsthrottle to remember that something
                    # happened, so that code in vcl_recv can notice that
                    # something is wrong with this URL. The return value should
                    # always be false, unless there are different requests to
                    # the same URL fetched in parallell (which can happen for
                    # pass or vary missmatch with predictive vary in place.
                    std.log(bereq.http.X-5xx-throttle-key + " was already denied, so this request was in parallell with a nother request for the same URL");
               } else {
                    std.log(bereq.http.X-5xx-throttle-key + " is now closer to being denied.");
               }
               if (bereq.is_bgfetch) {
                    std.log("Abandoning the bgfetch");
                    return(abandon);
               }
               # We cache 5xx by default for a while (knowing that this is not a bgfetch). This
               # is probably much better than immediately going to the backend to ask for this
               # resource.
               set beresp.ttl = 1s;
               set beresp.grace = 0s;
          } else {
               set beresp.ttl = 1ms;
               set beresp.grace = 1h;
          }
     }
     sub vcl_synth {
          if (req.http.X-5xx) {
               synthetic("throttled");
               set resp.status = 503;
               return (deliver);
          }
     }
} -start

# Part 1, check that req.grace for healty/unhealth backends work as expected with the 

client c1 {
     txreq -url "/1"
     rxresp
     expect resp.status == 200
     expect resp.body == "1"
     delay .1
     # No grace since the backend is healthy
     txreq -url "/1"
     rxresp
     expect resp.status == 200
     expect resp.body == "2"
} -run

# Set the backend unhealthy by using varnishadm

varnish v1 -cliok "backend.set_health s1 sick"
delay .1

client c2 {
     # The backend is sick, req.grace should not be set. Also, the background fetch should error out
     txreq -url "/1"
     rxresp
     expect resp.status == 200
     expect resp.body == "2"
     delay .1
     # Since we abandoned the background fetch, we should get the same result again
     txreq -url "/1"
     rxresp
     expect resp.status == 200
     expect resp.body == "2"
} -run

# Part 2, set the backend back to healthy and mix failing and non-failing requests

varnish v1 -cliok "backend.set_health s1 healthy"
delay .1

client c3 {
     txreq -url "/500"
     rxresp
     expect resp.status == 500
     expect resp.body == "3"
     # The TTL of the newly inserted object should give us a cache hit here.
     txreq -url "/500"
     rxresp
     expect resp.status == 500
     expect resp.body == "3"
     # Wait for the TTL to expire
     delay 1.1
     # Provoke the second 5xx from the backend
     txreq -url "/500"
     rxresp
     expect resp.status == 500
     expect resp.body == "4"
     # Wait for that to expire, too
     delay 1.1
} -run

# Try to get it the resource many more times, vsthrottle should see that it is not allowed and return a 503 in synth

client c4 -repeat 3 {
     txreq -url "/500"
     rxresp
     expect resp.status == 503
     expect resp.body == "throttled"
} -run

# Finally kick of a request to a totally different resource, just to see that we are in sync with the server

client c5 {
     txreq -url "/empty"
     rxresp
     expect resp.status == 204
} -run