Webflow API rate limiter workaround (ES6 / Nodejs)

README.md

Webflow API rate limit workaround

This is a naive work-around to the glitchy rate limiter of the current Webflow API. Any suggestions to optimizations is much appreciated.

How to use

It's a wrapper to your existing webflow-api functions, where you define two parameters:

1. a collection ID and
2. the function you want to call

If you have reached the rate limit this wrapper will handle the error and try to reset the rate limiter count by basically spamming until a new pool of requests has been granted.

Pro: easy to setup and integrate into existing solution Con: your script might time-out if the Webflow API does not provide a new pool fast enough

Example

This was your function before:

const getCollection = async ({ collectionId }) =>
    await webflow.items({ collectionId });

... and this is your function with this wrapper:

const getCollection = async ({ collectionId }) =>
    await webflowRateLimiterWorkaround({
        collectionId,
        callback: async () => await webflow.items({ collectionId }),
    });

example.js

import { webflowRateLimiterWorkaround } from "webflowAPI-ratelimit-workaround.js";
/* don't forget to also import and configure webflow-api */

const getCollection = async ({collectionId}) => {
  return await webflowRateLimiterWorkaround({
      collectionId,
      callback: async () =>
          await webflow.items(
              { collectionId },
              {
                  limit: 100,
                  offset: 0,
              }
          ),
  });  
}


const _debugFill = async ({collectionId}) => {
    const hitIt = async () => {
        return await webflow.items(
            { collectionId },
            {
                limit: 1,
                offset: 0,
            }
        );
    };

    for (let i = 0; i < 100; i++) {
        const res = await webflowRateLimiterWorkaround({
            collectionId,
            callback: async () => await hitIt(),
        });
    }
};

webflowAPI-ratelimit-workaround.js

const sleep = async ms => await new Promise(r => setTimeout(r, ms));


export const webflowRateLimiterWorkaround = async ({ collectionId, callback }) => {
    while (true) {
        const hitIt = async () => {
            const x = await webflow.items(
                { collectionId },
                {
                    limit: 1,
                    offset: 0,
                }
            );

            return x["_meta"].rateLimit.remaining;
        };

        try {
            const response = await callback();
            let webflowRateLimitRemaining = response["_meta"].rateLimit.remaining;
            console.log({ webflowRateLimitRemaining });

            if (webflowRateLimitRemaining === 0) {
                console.log( "webflow rate limit. hit it again until reset *sigh*");
                let hasReset = false;
                while (!hasReset) {
                    const result = await hitIt();
                    if (result > webflowRateLimitRemaining) {
                        hasReset = true;
                        return await callback();
                    }
                }
            } else {
                return response;
            }
        } catch (e) {
            console.log(e.code || e);
            await sleep(1000);
            continue;
        }
    }
};