From 2c7e5b41b6ccf3c6cf898968f92c9307ef01ccfb Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Wed, 31 Dec 2025 14:19:42 +0000 Subject: [PATCH] Add provider-specific retry delay extraction for rate limits Implement extractRetryDelay methods in Claude, Gemini, and OpenAI classes to parse provider-specific retry delay headers/responses. Update StreamingService to use these delays when available, falling back to exponential backoff. Enhance ApiError to include response headers and body for retry delay extraction. --- AIClasses/Claude/Claude.ts | 27 +++++++++++++++++++++++++- AIClasses/Gemini/Gemini.ts | 33 +++++++++++++++++++++++++++++++- AIClasses/OpenAI/OpenAI.ts | 37 ++++++++++++++++++++++++++++++++++-- Services/StreamingService.ts | 24 ++++++++++++++++++++--- Types/ApiError.ts | 9 ++++++--- 5 files changed, 120 insertions(+), 10 deletions(-) diff --git a/AIClasses/Claude/Claude.ts b/AIClasses/Claude/Claude.ts index df01d40..b402a40 100644 --- a/AIClasses/Claude/Claude.ts +++ b/AIClasses/Claude/Claude.ts @@ -13,6 +13,7 @@ import { Exception } from "Helpers/Exception"; import { MimeType, toMimeType } from "Enums/MimeType"; import { isTextFile } from "Enums/FileType"; import { MimeTypeToFileTypes } from "Enums/FileTypeMimeTypeMapping"; +import { ApiError, ApiErrorType } from "Types/ApiError"; export class Claude extends BaseAIClass { @@ -89,7 +90,8 @@ export class Claude extends BaseAIClass { AIProviderURL.Claude, requestBody, (chunk: string) => this.parseStreamChunk(chunk), - headers + headers, + (error) => this.extractRetryDelay(error) ); } @@ -309,6 +311,29 @@ export class Claude extends BaseAIClass { return JSON.stringify(contentBlocks); } + private extractRetryDelay(error: ApiError): number | undefined { + if (error.info.type !== ApiErrorType.RATE_LIMIT || !error.info.responseHeaders) { + return undefined; + } + + const retryAfter = error.info.responseHeaders.get('Retry-After'); + if (!retryAfter) return undefined; + + // Try parsing as seconds (number) + const seconds = parseInt(retryAfter, 10); + if (!isNaN(seconds)) return seconds; + + // Try parsing as HTTP date + const date = new Date(retryAfter); + if (!isNaN(date.getTime())) { + const now = Date.now(); + const delayMs = date.getTime() - now; + return Math.max(0, Math.ceil(delayMs / 1000)); + } + + return undefined; + } + private isSupportedMimeType(mimeType: MimeType): boolean { return this.SUPPORTED_MIMETYPES.includes(mimeType); } diff --git a/AIClasses/Gemini/Gemini.ts b/AIClasses/Gemini/Gemini.ts index a158253..0f8b05e 100644 --- a/AIClasses/Gemini/Gemini.ts +++ b/AIClasses/Gemini/Gemini.ts @@ -14,6 +14,7 @@ import { MimeType, toMimeType } from "Enums/MimeType"; import { isTextFile } from "Enums/FileType"; import { MimeTypeToFileTypes } from "Enums/FileTypeMimeTypeMapping"; import { Exception } from "Helpers/Exception"; +import { ApiError, ApiErrorType } from "Types/ApiError"; export class Gemini extends BaseAIClass { @@ -137,7 +138,9 @@ export class Gemini extends BaseAIClass { yield* this.streamingService.streamRequest( `${AIProviderURL.Gemini}/${this.settingsService.settings.model}:streamGenerateContent?key=${this.apiKey}&alt=sse`, requestBody, - (chunk: string) => this.parseStreamChunk(chunk) + (chunk: string) => this.parseStreamChunk(chunk), + undefined, // No additional headers + (error) => this.extractRetryDelay(error) ); } @@ -342,6 +345,34 @@ export class Gemini extends BaseAIClass { return JSON.stringify(parts); } + private extractRetryDelay(error: ApiError): number | undefined { + if (error.info.type !== ApiErrorType.RATE_LIMIT || !error.info.responseBody) { + return undefined; + } + + try { + const parsed = JSON.parse(error.info.responseBody) as { + error?: { + details?: Array<{ retryDelay?: string }> + } + }; + + const retryDelay = parsed.error?.details?.[0]?.retryDelay; + if (!retryDelay) return undefined; + + // Parse duration string (e.g., "60s", "1.5s") + const match = retryDelay.match(/^(\d+\.?\d*)s$/); + if (match) { + const seconds = parseFloat(match[1]); + return Math.ceil(seconds); + } + + return undefined; + } catch { + return undefined; + } + } + private isSupportedMimeType(mimeType: MimeType): boolean { return this.SUPPORTED_MIMETYPES.includes(mimeType); } diff --git a/AIClasses/OpenAI/OpenAI.ts b/AIClasses/OpenAI/OpenAI.ts index 574e4e5..8ff28de 100644 --- a/AIClasses/OpenAI/OpenAI.ts +++ b/AIClasses/OpenAI/OpenAI.ts @@ -9,7 +9,7 @@ import { fromString as aiFunctionFromString } from "Enums/AIFunction"; import type { IAIFunctionDefinition } from "AIClasses/FunctionDefinitions/IAIFunctionDefinition"; import type { ResponseEvent, ResponseOutputTextDelta, ResponseOutputItemDone, ResponseErrorEvent, ResponseFailedEvent, OpenAIFunctionTool, ResponsesAPIInput } from "./OpenAITypes"; import { Exception } from "Helpers/Exception"; -import { ApiErrorType } from "Types/ApiError"; +import { ApiError, ApiErrorType } from "Types/ApiError"; import { MimeType, toMimeType } from "Enums/MimeType"; import { isTextFile } from "Enums/FileType"; import { MimeTypeToFileTypes } from "Enums/FileTypeMimeTypeMapping"; @@ -60,7 +60,8 @@ export class OpenAI extends BaseAIClass { AIProviderURL.OpenAI, requestBody, (chunk: string) => this.parseStreamChunk(chunk), - headers + headers, + (error) => this.extractRetryDelay(error) ); } @@ -348,6 +349,38 @@ export class OpenAI extends BaseAIClass { }]); } + private extractRetryDelay(error: ApiError): number | undefined { + if (error.info.type !== ApiErrorType.RATE_LIMIT || !error.info.responseHeaders) { + return undefined; + } + + const headers = error.info.responseHeaders; + + // Try x-ratelimit-reset-requests first (most common) + const resetRequests = headers.get('x-ratelimit-reset-requests'); + if (resetRequests) { + const resetTimestamp = parseInt(resetRequests, 10); + if (!isNaN(resetTimestamp)) { + const now = Math.floor(Date.now() / 1000); + const delaySeconds = Math.max(0, resetTimestamp - now); + return delaySeconds; + } + } + + // Fallback to x-ratelimit-reset-tokens + const resetTokens = headers.get('x-ratelimit-reset-tokens'); + if (resetTokens) { + const resetTimestamp = parseInt(resetTokens, 10); + if (!isNaN(resetTimestamp)) { + const now = Math.floor(Date.now() / 1000); + const delaySeconds = Math.max(0, resetTimestamp - now); + return delaySeconds; + } + } + + return undefined; + } + private isSupportedMimeType(mimeType: MimeType): boolean { return this.SUPPORTED_MIMETYPES.includes(mimeType); } diff --git a/Services/StreamingService.ts b/Services/StreamingService.ts index c9d9af8..05054a7 100644 --- a/Services/StreamingService.ts +++ b/Services/StreamingService.ts @@ -27,7 +27,7 @@ export class StreamingService { } public async* streamRequest(url: string, requestBody: unknown, parseStreamChunk: (chunk: string) => IStreamChunk, - additionalHeaders?: Record): AsyncGenerator { + additionalHeaders?: Record, extractRetryDelay?: (error: ApiError) => number | undefined): AsyncGenerator { let lastError: Error | null = null; @@ -61,7 +61,9 @@ export class StreamingService { return; } - await sleep(StreamingService.RETRY_DELAYS[attempt]); + const delayMs = this.calculateRetryDelay(error, attempt, extractRetryDelay); + Exception.warn(`Rate limit exceeded, waiting for ${delayMs}ms...`); + await sleep(delayMs); } } @@ -86,7 +88,7 @@ export class StreamingService { if (!response.ok) { const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + throw ApiError.fromResponse(response.status, response.statusText, responseBody, response.headers); } return response; @@ -178,4 +180,20 @@ export class StreamingService { return attempt < StreamingService.MAX_RETRIES; } + private calculateRetryDelay(error: unknown, attempt: number, + extractRetryDelay?: (error: ApiError) => number | undefined + ): number { + // Only use provider-specific delay for 429 rate limits + if (error instanceof ApiError && error.info.type === ApiErrorType.RATE_LIMIT && extractRetryDelay) { + const providerDelaySeconds = extractRetryDelay(error); + if (providerDelaySeconds !== undefined) { + // Convert to milliseconds + return providerDelaySeconds * 1000; + } + } + + // Fall back to exponential backoff + return StreamingService.RETRY_DELAYS[attempt]; + } + } \ No newline at end of file diff --git a/Types/ApiError.ts b/Types/ApiError.ts index 4044d5c..bdc14df 100644 --- a/Types/ApiError.ts +++ b/Types/ApiError.ts @@ -14,7 +14,8 @@ export interface ApiErrorInfo { message: string; userMessage: string; isRetryable: boolean; - retryAfter?: number; // seconds + responseHeaders?: Headers; + responseBody?: string; } export class ApiError extends Error { @@ -27,7 +28,7 @@ export class ApiError extends Error { return error instanceof ApiError; } - static fromResponse(status: number, statusText: string, responseBody: string): ApiError { + static fromResponse(status: number, statusText: string, responseBody: string, headers?: Headers): ApiError { let type: ApiErrorType; let userMessage: string; let isRetryable: boolean; @@ -83,7 +84,9 @@ export class ApiError extends Error { statusCode: status, message, userMessage, - isRetryable + isRetryable, + responseHeaders: headers, + responseBody: responseBody }); }