logflare_cloudflare_worker_with_batching

logflare_cloudflare_worker_with_batching.js

const sleep = ms => {
    return new Promise(resolve => {
        setTimeout(resolve, ms)
    })
}

const makeid = length => {
    let text = ""
    const possible = "ABCDEFGHIJKLMNPQRSTUVWXYZ0123456789"
    for (let i = 0; i < length; i += 1) {
        text += possible.charAt(Math.floor(Math.random() * possible.length))
    }
    return text
}


const buildMetadataFromHeaders = headers => {
    const responseMetadata = {}
    Array.from(headers).forEach(([key, value]) => {
        responseMetadata[key.replace(/-/g, "_")] = value
    })
    return responseMetadata
}

// IpInfo
let ipInfoBackoff = 0

// Batching
const BATCH_INTERVAL_MS = 500
const MAX_REQUESTS_PER_BATCH = 100
const WORKER_ID = makeid(6)


let batchTimeoutReached = true
let logEventsBatch = []

// Backoff

const BACKOFF_INTERVAL = 10000
let backoff = 0

// IpInfo
const ipInfoToken = "XXXXXXXXXXXXXX" // Change This
const ipInfoMaxAge = 86400

// Logflare API
const sourceKey = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" // Change This
const apiKey = "XXXXXXXXXXXX" // Change This
const logflareApiURL = "https://api.logflare.app/logs/cloudflare"

const fetchIpDataWithCache = async ip => {
    const cache = caches.default

    // Do not switch to HTTPS until this is fixed:
    // deployed Cloudflare workers throw SSL handshake error
    // * this doesn't happen neither in cloudflareworkers.com environment
    // * this also doesn't happen in test console of preview console for deployed workers within Cloudflare dashboard
    const url = new URL(`http://ipinfo.io/${ip}/json?token=${ipInfoToken}`)

    const cacheKey = new Request(url, {
        method: "GET",
        headers: { "Content-Type": "application/json" },
    })

    let cachedResponse = await cache.match(cacheKey)

    if (!cachedResponse) {
        const resp = await fetch(cacheKey)
        if (resp.status === 200) {
            cachedResponse = new Response(resp.body, resp)
            cachedResponse.headers.set("Cache-Control", `max-age=${ipInfoMaxAge}`)
            await cache.put(cacheKey, cachedResponse.clone())
            return cachedResponse.json()
        }
        ipInfoBackoff = Date.now() + 10000
        return {
            error: await resp.text(),
        }
    }
    return cachedResponse.json()
}

async function addToBatch(body, connectingIp, event) {
    try {
        if (ipInfoToken && ipInfoBackoff < Date.now()) {
            body.metadata.request.ipData = await fetchIpDataWithCache(connectingIp)
        }
    } catch (e) { }
    logEventsBatch.push(body)

    if (logEventsBatch.length >= MAX_REQUESTS_PER_BATCH) {
        event.waitUntil(postBatch(event))
    }

    return true
}

async function handleRequest(event) {
    const { request } = event

    const requestMetadata = buildMetadataFromHeaders(request.headers)

    const t1 = Date.now()
    const response = await fetch(request)
    const originTimeMs = Date.now() - t1

    const responseMetadata = buildMetadataFromHeaders(response.headers)

    const rMeth = request.method
    const rUrl = request.url
    const uAgent = request.headers.get("user-agent")
    const cfRay = request.headers.get("cf-ray")
    const cIP = request.headers.get("cf-connecting-ip")
    const rCf = request.cf
    const statusCode = response.status

    const buildLogMessage = `${rMeth} | ${statusCode} | ${cIP} | ${cfRay} | ${rUrl} | ${uAgent}`

    const logflareEventBody = {
        source: sourceKey,
        message: buildLogMessage,
        timestamp: new Date().toISOString(),
        metadata: {
            response: {
                headers: responseMetadata,
                origin_time: originTimeMs,
                status_code: response.status,
            },
            request: {
                url: rUrl,
                method: rMeth,
                headers: requestMetadata,
                cf: rCf,
            },
            logflare_worker: {
                worker_id: WORKER_ID,
            },
        },
    }
    event.waitUntil(
        addToBatch(logflareEventBody, requestMetadata.cf_connecting_ip, event),
    )

    return response
}

const fetchAndSetBackOff = async (lfRequest, event) => {
    if (backoff <= Date.now()) {
        const resp = await fetch(logflareApiURL, lfRequest)
        if (resp.status === 403 || resp.status === 429) {
            backoff = Date.now() + BACKOFF_INTERVAL
        }
    }

    event.waitUntil(scheduleBatch(event))

    return true
}

const postBatch = async event => {
    const batchInFlight = [...logEventsBatch]
    logEventsBatch = []
    const rHost = batchInFlight[0].metadata.request.headers.host
    const body = JSON.stringify({ batch: batchInFlight, source: sourceKey })
    const request = {
        method: "POST",
        headers: {
            "X-API-KEY": apiKey,
            "Content-Type": "application/json",
            "User-Agent": `Cloudflare Worker via ${rHost}`,
        },
        body,
    }
    event.waitUntil(fetchAndSetBackOff(request, event))
}

const scheduleBatch = async event => {
    if (batchTimeoutReached) {
        batchTimeoutReached = false
        await sleep(BATCH_INTERVAL_MS)
        if (logEventsBatch.length > 0) {
            event.waitUntil(postBatch(event))
        }
        batchTimeoutReached = true
    }
    return true
}

addEventListener("fetch", event => {
    event.passThroughOnException()

    event.waitUntil(scheduleBatch(event))
    event.respondWith(handleRequest(event))
})

I'm using your code and put it in cf worker after modifying the needed values, and i can see log going in when doing GET request to worker URL, but not when visiting my site. What else i need to configure?