Wrapper code for making calls to Shopify using `shopify-api-node` that get automatically retried when the rate limit is exceeded

retryShopifyCall.ts

import pRetry from "p-retry";
import { isArray, isNil, isObject, isString } from "lodash";

const responseFromError = (error: any): Response | undefined => {
  if ("response" in error) {
    const response = error.response;
    if (response && "body" in response) {
      return response as Response;
    }
  }
};

const sleep = (ms: number): Promise<void> => {
  return new Promise((resolve) => setTimeout(resolve, ms));
};

export const RetryableErrorCodes = new Set([
  "ETIMEDOUT",
  "ECONNRESET",
  "EADDRINUSE",
  "ECONNREFUSED",
  "EPIPE",
  "ENOTFOUND",
  "ENETUNREACH",
  "EAI_AGAIN",
]);

/**
 *  Report if an error connecting to a remote system can be retried
 **/
export const isRetryableConnectionError = (error: any): boolean => {
  return isObject(error) && "code" in error && RetryableErrorCodes.has((error as any).code);
};



/**
 * When retrying requests to Shopify, we introduce a random amount of jitter to the retry time to avoid all making requests in lockstep in a thundering herd.
 * See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ for more information
 * See https://github.com/tim-kos/node-retry#retryoperationoptions for an explanation of the math
 */
const addJitter = (baseRetrySeconds: number, attemptNumber: number) => {
  return baseRetrySeconds + Math.min(Math.random() * Math.pow(2, attemptNumber), 6);
};

/**
 * When a given error should be retried, if it should be retried at all.
 *
 * @return The number of seconds from now when the operation should be retried. A value of 0 means it can be retried
 *   immediately, whereas a negative value means it should not be retried.
 */
export const shouldRetryError = (error: any): number | null => {
  if (isRetryableConnectionError(error)) {
    return 0;
  }

  const response = responseFromError(error);
  if (!response) {
    return null;
  }

  if (response.statusCode >= 500 && response.statusCode < 600) {
    return 0;
  }

  if (response.headers["retry-after"]) {
    const value = parseFloat(response.headers["retry-after"]);
    if (isNaN(value)) {
      const when = new Date(value).valueOf();
      return when - Date.now().valueOf();
    } else if (isFinite(value)) {
      return value;
    } else {
      return null;
    }
  }

  if (response.statusCode == 429) {
    // Arbitrary 2 seconds, incase we get a 429 without a Retry-After response header
    return 2;
  }

  // detect graphql request throttling
  if (response.body && isObject(response.body)) {
    const body = response.body as {
      errors?: {
        message: string;
        extensions?: {
          code: string;
        };
      }[];
      extensions?: {
        cost?: {
          requestedQueryCost: number;
          throttleStatus: {
            maximumAvailable: number;
            currentlyAvailable: number;
            restoreRate: number;
          };
        };
      };
    };

    if (body.errors && isArray(body.errors) && isObject(body.errors[0]) && body.errors[0].extensions?.code == "THROTTLED") {
      const costData = body.extensions?.cost;
      if (costData) {
        return (costData.requestedQueryCost - costData.throttleStatus.currentlyAvailable) / costData.throttleStatus.restoreRate;
      } else {
        return 2;
      }
    }
  }

  return null;
};

/**
 * Retry making an API call with `shopify-api-node` until it succeeds or we've tried too many times.
 */
export const retryShopifyCall = async <T>(run: () => Promise<T>): Promise<T> => {
  return await pRetry(run, {
    // we don't use p-retry's exponential backoff and instead rely on shopify's retry-after header to tell us when to retry
    // we implement the delay between attempts by sleeping in the `onFailedAttempt` function, which blocks the next attempt.
    minTimeout: 1,
    maxTimeout: 1,
    onFailedAttempt: async (error) => {
      if (error.retriesLeft > 0) {
        const maybeRetryAfterSeconds = shouldRetryError(error);
        if (maybeRetryAfterSeconds != null) {
          const retryAfterSeconds = addJitter(maybeRetryAfterSeconds, error.attemptNumber);
          console.debug({ error, retryAfterSeconds }, "error communicating with the shopify API, retrying");
          await sleep(retryAfterSeconds * 1000);
        } else {
          throw error;
        }
      }
    },
  });
};