feat: Add resilience features - retry, rate limiting, and timeouts

src/Lighthouse/ipns/getAllKeys.ts

@@ -1,4 +1,6 @@
 import { lighthouseConfig } from '../../lighthouse.config'
+import { resilientFetch } from '../utils/resilientFetch'
+import { quickApiConfig } from '../utils/apiConfig'
 
 type ipnsObject = {
   ipnsName: string
@@ -14,18 +16,19 @@ export type keyDataResponse = {
 
 export default async (apiKey: string): Promise<keyDataResponse> => {
   try {
-    const response = await fetch(
+    const response = await resilientFetch(
       lighthouseConfig.lighthouseAPI + `/api/ipns/get_ipns_records`,
       {
         headers: {
           Authorization: `Bearer ${apiKey}`,
           'Content-Type': 'application/json',
         },
+        retryOptions: quickApiConfig.retryOptions,
+        rateLimiter: quickApiConfig.rateLimiter,
+        timeout: quickApiConfig.timeout,
       }
     )
-    if (!response.ok) {
-      throw new Error(`Request failed with status code ${response.status}`)
-    }
+
     const data = (await response.json()) as ipnsObject[]
     /*
       return:

src/Lighthouse/podsi/index.ts

@@ -1,4 +1,6 @@
 import { defaultConfig } from '../../lighthouse.config'
+import { resilientFetch } from '../utils/resilientFetch'
+import { defaultApiConfig } from '../utils/apiConfig'
 
 type Proof = {
   verifierData: {
@@ -36,19 +38,27 @@ type PODSIData = {
 
 export default async (cid: string): Promise<{ data: PODSIData }> => {
   try {
-    const response = await fetch(
-      defaultConfig.lighthouseAPI + `/api/lighthouse/get_proof?cid=${cid}`
-    )
-    if (!response.ok) {
-      if (response.status === 400) {
-        throw new Error("Proof Doesn't exist yet")
+    const response = await resilientFetch(
+      defaultConfig.lighthouseAPI + `/api/lighthouse/get_proof?cid=${cid}`,
+      {
+        retryOptions: {
+          ...defaultApiConfig.retryOptions,
+          retryCondition: (error: any) => {
+            // Don't retry on 400 errors for PODSI - proof might not exist yet
+            if (error.status === 400) return false
+            // Use default retry logic for other errors
+            return error.status === 429 || error.status >= 502
+          }
+        },
+        rateLimiter: defaultApiConfig.rateLimiter,
+        timeout: defaultApiConfig.timeout,
       }
-      throw new Error(`Request failed with status code ${response.status}`)
-    }
+    )
+
     const data = (await response.json()) as PODSIData
     return { data }
   } catch (error: any) {
-    if (error?.response?.status === 400) {
+    if (error.status === 400) {
       throw new Error("Proof Doesn't exist yet")
     }
     throw new Error(error.message)

src/Lighthouse/upload/files/browser.ts

@@ -4,7 +4,7 @@ import {
   IUploadProgressCallback,
   IFileUploadedResponse
 } from '../../../types'
-import { fetchWithTimeout } from '../../utils/util'
+import { resilientUpload } from '../../utils/resilientUpload'
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
 export default async (
@@ -32,24 +32,16 @@ export default async (
       Authorization: token
     })
 
-    const response = uploadProgressCallback
-      ? await fetchWithTimeout(endpoint, {
-          method: 'POST',
-          body: formData,
-          headers: headers,
-          timeout: 7200000,
-          onProgress: (progress) => {
-            uploadProgressCallback({
-              progress: progress,
-            })
-          },
-        })
-      : await fetchWithTimeout(endpoint, {
-          method: 'POST',
-          body: formData,
-          headers: headers,
-          timeout: 7200000,
+    const response = await resilientUpload(endpoint, {
+      method: 'POST',
+      body: formData,
+      headers: headers,
+      onProgress: uploadProgressCallback ? (progress) => {
+        uploadProgressCallback({
+          progress: progress,
         })
+      } : undefined,
+    })
 
     if (!response.ok) {
       const res = (await response.json())

src/Lighthouse/utils/apiConfig.ts

@@ -0,0 +1,39 @@
+import { RetryOptions } from './retry'
+import { RateLimiter } from './rateLimiter'
+
+export interface ApiConfig {
+  retryOptions: RetryOptions
+  rateLimiter: RateLimiter
+  timeout: number
+}
+
+// Default configuration for different operation types
+export const defaultApiConfig: ApiConfig = {
+  retryOptions: {
+    maxRetries: 3,
+    baseDelay: 1000,
+    maxDelay: 10000
+  },
+  rateLimiter: new RateLimiter(10, 20), // 10 requests per second, burst of 20
+  timeout: 30000 // 30 seconds
+}
+
+export const uploadApiConfig: ApiConfig = {
+  retryOptions: {
+    maxRetries: 5,
+    baseDelay: 2000,
+    maxDelay: 30000
+  },
+  rateLimiter: new RateLimiter(5, 10), // 5 requests per second for uploads
+  timeout: 7200000 // 2 hours for large uploads
+}
+
+export const quickApiConfig: ApiConfig = {
+  retryOptions: {
+    maxRetries: 2,
+    baseDelay: 500,
+    maxDelay: 5000
+  },
+  rateLimiter: new RateLimiter(20, 50), // 20 requests per second for quick operations
+  timeout: 10000 // 10 seconds
+}

src/Lighthouse/utils/index.ts

@@ -0,0 +1,23 @@
+// Core resilience utilities
+export { withRetry, type RetryOptions, type RetryableError } from './retry'
+export { RateLimiter } from './rateLimiter'
+export { resilientFetch, FetchError, type ResilientFetchOptions } from './resilientFetch'
+export { resilientUpload } from './resilientUpload'
+
+// Configuration presets
+export {
+  defaultApiConfig,
+  uploadApiConfig,
+  quickApiConfig,
+  type ApiConfig
+} from './apiConfig'
+
+// Existing utilities
+export {
+  isCID,
+  isPrivateKey,
+  addressValidator,
+  checkDuplicateFileNames,
+  fetchWithTimeout,
+  fetchWithDirectStream,
+} from './util'

src/Lighthouse/utils/rateLimiter.ts

@@ -0,0 +1,46 @@
+export class RateLimiter {
+  private tokens: number
+  private lastRefill: number
+  private readonly refillRate: number // tokens per second
+  private readonly capacity: number
+
+  constructor(
+    private requestsPerSecond: number,
+    private burstSize: number = requestsPerSecond
+  ) {
+    this.capacity = burstSize
+    this.tokens = burstSize
+    this.lastRefill = Date.now()
+    this.refillRate = requestsPerSecond
+  }
+
+  async waitForToken(): Promise<void> {
+    this.refillTokens()
+
+    if (this.tokens >= 1) {
+      this.tokens -= 1
+      return
+    }
+
+    // Calculate wait time for next token
+    const waitTime = (1 / this.refillRate) * 1000
+    await new Promise(resolve => setTimeout(resolve, waitTime))
+
+    // Recursively wait if still no tokens available
+    return this.waitForToken()
+  }
+
+  private refillTokens(): void {
+    const now = Date.now()
+    const timePassed = (now - this.lastRefill) / 1000
+    const tokensToAdd = timePassed * this.refillRate
+
+    this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd)
+    this.lastRefill = now
+  }
+
+  getAvailableTokens(): number {
+    this.refillTokens()
+    return Math.floor(this.tokens)
+  }
+}

src/Lighthouse/utils/resilientFetch.ts

@@ -0,0 +1,74 @@
+import { withRetry, RetryOptions, RetryableError } from './retry'
+import { RateLimiter } from './rateLimiter'
+
+export interface ResilientFetchOptions extends RequestInit {
+  timeout?: number
+  retryOptions?: RetryOptions
+  rateLimiter?: RateLimiter
+}
+
+export class FetchError extends Error implements RetryableError {
+  constructor(
+    message: string,
+    public status?: number,
+    public code?: string
+  ) {
+    super(message)
+    this.name = 'FetchError'
+  }
+}
+
+export async function resilientFetch(
+  url: string,
+  options: ResilientFetchOptions = {}
+): Promise<Response> {
+  const {
+    timeout = 30000,
+    retryOptions,
+    rateLimiter,
+    ...fetchOptions
+  } = options
+
+  const operation = async (): Promise<Response> => {
+    if (rateLimiter) {
+      await rateLimiter.waitForToken()
+    }
+
+    // Create abort controller for timeout
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), timeout)
+
+    try {
+      const response = await fetch(url, {
+        ...fetchOptions,
+        signal: controller.signal
+      })
+
+      clearTimeout(timeoutId)
+
+      if (!response.ok) {
+        throw new FetchError(
+          `Request failed with status code ${response.status}`,
+          response.status
+        )
+      }
+
+      return response
+    } catch (error: any) {
+      clearTimeout(timeoutId)
+
+      if (error.name === 'AbortError') {
+        throw new FetchError('Request timeout', undefined, 'ETIMEDOUT')
+      }
+
+      // Network errors
+      if (error.code) {
+        throw new FetchError(error.message, undefined, error.code)
+      }
+
+      throw error
+    }
+  }
+
+  return withRetry(operation, retryOptions)
+}