Example how to handle OpenAI rate limiting

safe-rate-limit-chatgpt.js

import openai from '/path/to/openai';

const MAX_RETRIES_IN_CASE_UNKNOWN_ERROR = 5;

export default async function promptChatGPT(
  data,
  retriesLeft = MAX_RETRIES_IN_CASE_UNKNOWN_ERROR
) {

  const completionRequest = buildCompletionRequest(data); // <-- insert your own logic

  try {
    const completion = await openai.createChatCompletion(completionRequest, {
      timeout: 60000,
    });

    const { content } = completion.data?.choices?.[0].message ?? {};

    return content;
  } catch (error) {

    // if the error is an openai rate limit error
    if (error.status === 429) {
      return retryBasedOnRateLimit(promptChatGPT, data, error.headers, retriesLeft);
    }

    // if the error is unknown, we still try up to MAX_RETRIES_IN_CASE_UNKNOWN_ERROR times,
    // after which we give up and potentially notify error service
    return retryOrGiveUp(promptChatGPT, data, retriesLeft - 1);
  }
}

function retryBasedOnRateLimit(
  promptChatGPT,
  data,
  headers,
  retriesLeft
) {
  const remainingRequests =
    headers?.['x-ratelimit-remaining-requests'] ?? 0;
    
  const remainingTokens =
    headers?.['x-ratelimit-remaining-tokens'] ?? 0;
    
  const requestsResetInMS =
    timeToMillisecondDelay(
      headers?.['x-ratelimit-reset-requests']
    ) ?? 1000;
    
  const tokensResetInMS =
    timeToMillisecondDelay(
      headers?.['x-ratelimit-reset-tokens']
    ) ?? 1000;

  if (remainingRequests < 1) {
    // OpenAI rate limit reached. Waiting for requests limit to reset

    return new Promise((resolve) => {
      setTimeout(() => {
        resolve(promptChatGPT(data)); 
      }, requestsResetInMS);
    });
  }

  if (remainingTokens < data.maxTokens) {
    // OpenAI rate limit reached. Waiting for tokens limit to reset

    return new Promise((resolve) => {
      setTimeout(() => {
        resolve(promptChatGPT(data));
      }, tokensResetInMS);
    });
  }

  // OpenAI rate limit reached but their response is malformed or invalid. Waiting to manually retry
  const retryInMS = (1 / retriesLeft) * 10000; // <-- basic backoff, but can obv be more fancy

  return new Promise((resolve) => {
    setTimeout(() => {
      resolve(promptChatGPT(data, retriesLeft - 1));
    }, retryInMS);
  });
}


// '2.874s' -> 2874
// '300ms' -> 300
function timeToMillisecondDelay(t) {
  if (t.endsWith('s')) {
    return parseFloat(t) * 1000;
  } 
  if (t.endsWith('ms')) {
    return parseFloat(t);
  } 
  
  // can throw or log and fallback
  return 2000;
}