Cache primer from a Redis queue

pledge.lua

local redis = require "redis"
local socket = require "socket"

local queue = "REVALIDATE"

local redis_client = redis.connect("127.0.0.1", 6379)

function log(msg)
    io.stdout:write(os.date('%c', os.time()) .. ' ' .. msg .. "\n")
    io.stdout:flush()
end

while true do
    socket.sleep(1)
    local expired = redis_client:zrangebyscore('ledge:uris_by_expiry', 0, os.time())
    for i,uri in ipairs(expired) do
        redis_client:zadd('ledge:uris_by_expiry', -1, uri) 
        redis_client:rpush(queue, uri)
    end 
end

pledge_worker.lua

local redis = require "redis"
local http = require "socket.http"
local url = require "socket.url"

local queue = "REVALIDATE"

local redis_client = redis.connect("127.0.0.1", 6379)

-- Block waiting for jobs on the queue, yield for each job.
function producer()
    return coroutine.create(function ()
        while true do
            local res = redis_client:blpop(queue, 0)
            coroutine.yield(res[2])
        end 
    end)
end

-- Resume produer, and fetch the job URI.
function consumer(prod)
    while true do
        local status, msg = coroutine.resume(prod)
        log("Priming " .. msg)

        -- Change the host to localhost.. we'll manually add the Host header.
        local parsed_url = url.parse(msg)
        local origin_host = parsed_url.host
        parsed_url.host = '127.0.0.1'

        -- Fetch
        local s, c, h = http.request({
            url = url.build(parsed_url),
            redirect = false,
            headers = { 
                ['Cache-Control'] = 'no-cache',
                ['Host'] = origin_host
            },  
        })  

        log("Done ("..c..")")
    end 
end

function log(msg)
    io.stdout:write(os.date('%c', os.time()) .. ' ' .. msg .. "\n")
    io.stdout:flush()
end

consumer(producer())

This works using the BLPOP redis command, which blocks the client until a value appears on the list. So rather than polling, simply run as many of these as you like, and add work to the queue with:

RPUSH REVALIDATE http://example.com http://example.com/2 http://example.com/3

The worker which has blocked the longest will get the next job. Magic.