1
export interface RetryConfig {
2
  maxAttempts: number;     // Tổng attempts (mặc định: 3)
3
  baseDelayMs: number;     // Delay ban đầu (mặc định: 1000)
4
  maxDelayMs: number;      // Delay tối đa (mặc định: 30000)
5
  jitterFactor: number;    // Nhiễu ngẫu nhiên 0-1 (mặc định: 0.2)
6
  retryableErrors: string[];  // Lỗi nào thì retry
7
}
8

9
export const DEFAULT_RETRY: RetryConfig = {
10
  maxAttempts: 3,
11
  baseDelayMs: 1000,
12
  maxDelayMs: 30_000,
13
  jitterFactor: 0.2,
14
  retryableErrors: ["429","503","timeout","rate limit","ECONNRESET"],
15
};
16

17
interface RetryResult<T> {
18
  value: T;
19
  attempts: number;
20
  totalDelayMs: number;
21
}
22

23
export async function withRetry<T>(
24
  fn: () => Promise<T>,
25
  config: RetryConfig = DEFAULT_RETRY
26
): Promise<RetryResult<T>> {
27
  let lastError: Error | null = null;
28
  let totalDelayMs = 0;
29

30
  for (let attempt = 1; attempt <= config.maxAttempts; attempt++) {
31
    try {
32
      const value = await fn();
33
      return { value, attempts: attempt, totalDelayMs };
34
    } catch (error) {
35
      lastError = error instanceof Error ? error : new Error(String(error));
36
      if (attempt >= config.maxAttempts) break;
37
      if (!isRetryable(lastError.message, config.retryableErrors)) break;
38

39
      const delay = calculateDelay(attempt, config);
40
      totalDelayMs += delay;
41
      await new Promise(r => setTimeout(r, delay));
42
    }
43
  }
44
  throw lastError!;
45
}
46

47
function isRetryable(msg: string, patterns: string[]): boolean {
48
  return patterns.some(p => msg.toLowerCase().includes(p.toLowerCase()));
49
}
50

51
function calculateDelay(attempt: number, config: RetryConfig): number {
52
  const exp = Math.min(config.baseDelayMs * Math.pow(2, attempt - 1), config.maxDelayMs);
53
  return Math.floor(exp + exp * config.jitterFactor * Math.random());
54
}

1
type CircuitState = "closed" | "open" | "half-open";
2

3
export class CircuitBreaker {
4
  private state = new Map<string, {
5
    status: CircuitState; failures: number;
6
    lastFailure: number; halfOpenSuccesses: number;
7
  }>();
8
  private threshold: number;
9
  private cooldownMs: number;
10

11
  constructor(threshold = 5, cooldownMs = 30_000) {
12
    this.threshold = threshold;
13
    this.cooldownMs = cooldownMs;
14
  }
15

16
  async call<T>(tool: string, fn: () => Promise<T>): Promise<T> {
17
    const s = this.getState(tool);
18

19
    if (s.status === "open") {
20
      const elapsed = Date.now() - s.lastFailure;
21
      if (elapsed >= this.cooldownMs) {
22
        this.transition(tool, "half-open");
23
      } else {
24
        throw new CircuitOpenError(`"${tool}" circuit open. Retry in ${this.cooldownMs - elapsed}ms`);
25
      }
26
    }
27

28
    try {
29
      const result = await fn();
30
      this.onSuccess(tool);
31
      return result;
32
    } catch (error) {
33
      this.onFailure(tool);
34
      throw error;
35
    }
36
  }
37

38
  private onSuccess(tool: string) {
39
    const s = this.getState(tool);
40
    if (s.status === "half-open") {
41
      s.halfOpenSuccesses++;
42
      if (s.halfOpenSuccesses >= 1) this.transition(tool, "closed");
43
    } else if (s.status === "closed") {
44
      s.failures = 0;
45
    }
46
  }
47

48
  private onFailure(tool: string) {
49
    const s = this.getState(tool);
50
    s.failures++; s.lastFailure = Date.now();
51
    if (s.status === "half-open") this.transition(tool, "open");
52
    else if (s.status === "closed" && s.failures >= this.threshold)
53
      this.transition(tool, "open");
54
  }
55

56
  private transition(tool: string, newStatus: CircuitState) {
57
    const s = this.getState(tool);
58
    console.warn(`[Breaker] ${tool}: ${s.status} → ${newStatus}`);
59
    s.status = newStatus;
60
    if (newStatus === "closed") { s.failures = 0; s.halfOpenSuccesses = 0; }
61
  }
62

63
  private getState(tool: string) {
64
    if (!this.state.has(tool))
65
      this.state.set(tool, { status: "closed", failures: 0, lastFailure: 0, halfOpenSuccesses: 0 });
66
    return this.state.get(tool)!;
67
  }
68

69
  getStatus(tool: string) { const s = this.getState(tool); return { status: s.status, failures: s.failures }; }
70
  reset(tool: string) { this.transition(tool, "closed"); }
71
}
72

73
export class CircuitOpenError extends Error {
74
  public retryAfterMs: number;
75
  constructor(msg: string, retryAfterMs = 0) { super(msg); this.name = "CircuitOpenError"; this.retryAfterMs = retryAfterMs; }
76
}

1
CLOSED ──(fail ≥ threshold)──▶ OPEN ──(cooldown)──▶ HALF-OPEN
2
  ▲                                                  │
3
  └────(success ≥ limit)◄────────────────────────────┘
4
                                    │
5
                                    └──(fail)──▶ OPEN

1
export interface FallbackStep<T> {
2
  name: string;
3
  execute: () => Promise<T>;
4
  isAvailable?: () => boolean; // Bỏ qua step này nếu unavailable
5
}
6

7
export class FallbackChain<T> {
8
  private steps: FallbackStep<T>[];
9

10
  constructor(steps: FallbackStep<T>[]) {
11
    if (steps.length === 0) throw new Error("Need ≥ 1 step");
12
    this.steps = steps;
13
  }
14

15
  async execute(): Promise<{ value: T; step: string; warnings: string[] }> {
16
    const warnings: string[] = [];
17

18
    for (const step of this.steps) {
19
      if (step.isAvailable && !step.isAvailable()) {
20
        warnings.push(`${step.name}: skip (unavailable)`);
21
        continue;
22
      }
23
      try {
24
        const value = await step.execute();
25
        return { value, step: step.name, warnings };
26
      } catch (e) {
27
        warnings.push(`${step.name}: ${e instanceof Error ? e.message : String(e)}`);
28
      }
29
    }
30
    throw new Error(`All fallback exhausted:\n${warnings.join("\n")}`);
31
  }
32
}

1
export class ResilientToolCaller {
2
  private breaker: CircuitBreaker;
3

4
  constructor(options?: { failureThreshold?: number; cooldownMs?: number }) {
5
    this.breaker = new CircuitBreaker(options?.failureThreshold || 5, options?.cooldownMs || 30_000);
6
  }
7

8
  async call<T>(
9
    tool: string,
10
    fn: () => Promise<T>,
11
    fallbackSteps?: FallbackStep<T>[]
12
  ): Promise<T | { value: T; warnings: string[] }> {
13
    try {
14
      return await withRetry(() => this.breaker.call(tool, fn));
15
    } catch (error) {
16
      if (fallbackSteps && fallbackSteps.length > 0) {
17
        const chain = new FallbackChain(fallbackSteps);
18
        try {
19
          return await chain.execute();
20
        } catch {
21
          // Both primary + fallback failed — graceful degradation
22
          return {
23
            value: (`⚠️ "${tool}" unavailable.\nAutomatic retry in 30s.\nTry again or use different tool.`) as unknown as T,
24
            warnings: [`${tool}: all paths failed`],
25
          };
26
        }
27
      }
28
      return {
29
        value: (`⚠️ "${tool}" temporarily unavailable.`) as unknown as T,
30
        warnings: [`${tool}: ${error}`],
31
      };
32
    }
33
  }
34

35
  getBreakerStatus(tool: string) { return this.breaker.getStatus(tool); }
36
  resetBreaker(tool: string) { this.breaker.reset(tool); }
37
}