1
// src/resilience/retry.ts — Configurable retry with exponential backoff and jitter
2

3
export interface RetryConfig {
4
  maxAttempts: number;        // Total attempts including first (default: 3)
5
  baseDelayMs: number;        // Initial delay (default: 1000)
6
  maxDelayMs: number;         // Cap on delay (default: 30000)
7
  jitterFactor: number;       // Random jitter 0-1 (default: 0.2)
8
  retryableErrors: string[];  // Error messages that trigger retry
9
}
10

11
export const DEFAULT_RETRY: RetryConfig = {
12
  maxAttempts: 3,
13
  baseDelayMs: 1000,
14
  maxDelayMs: 30_000,
15
  jitterFactor: 0.2,
16
  retryableErrors: [
17
    "429", "503", "502", "504",  // HTTP status codes
18
    "timeout", "timed out",
19
    "rate limit", "rate_limit",
20
    "too many requests",
21
    "ECONNRESET", "ETIMEDOUT",
22
    "network error", "network failure",
23
    "internal server error",
24
    "service unavailable",
25
  ],
26
};
27

28
interface RetryResult<T> {
29
  value: T;
30
  attempts: number;
31
  totalDelayMs: number;
32
}
33

34
export async function withRetry<T>(
35
  fn: () => Promise<T>,
36
  config: RetryConfig = DEFAULT_RETRY
37
): Promise<RetryResult<T>> {
38
  let lastError: Error | null = null;
39
  let totalDelayMs = 0;
40

41
  for (let attempt = 1; attempt <= config.maxAttempts; attempt++) {
42
    try {
43
      const value = await fn();
44
      return { value, attempts: attempt, totalDelayMs };
45
    } catch (error) {
46
      lastError = error instanceof Error ? error : new Error(String(error));
47

48
      // Don't retry on last attempt
49
      if (attempt >= config.maxAttempts) break;
50

51
      // Check if error is retryable
52
      if (!isRetryable(lastError.message, config.retryableErrors)) break;
53

54
      // Calculate delay: exponential backoff + jitter
55
      const delay = calculateDelay(attempt, config);
56
      totalDelayMs += delay;
57

58
      console.warn(
59
        `[Retry] Attempt ${attempt}/${config.maxAttempts} failed. ` +
60
        `Retrying in ${delay}ms. Error: ${lastError.message}`
61
      );
62

63
      await sleep(delay);
64
    }
65
  }
66

67
  throw lastError!;
68
}
69

70
function isRetryable(message: string, patterns: string[]): boolean {
71
  return patterns.some(pattern =>
72
    message.toLowerCase().includes(pattern.toLowerCase())
73
  );
74
}
75

76
function calculateDelay(attempt: number, config: RetryConfig): number {
77
  const exponential = config.baseDelayMs * Math.pow(2, attempt - 1);
78
  const capped = Math.min(exponential, config.maxDelayMs);
79
  const jitter = capped * config.jitterFactor * Math.random();
80
  return Math.floor(capped + jitter);
81
}
82

83
function sleep(ms: number): Promise<void> {
84
  return new Promise(resolve => setTimeout(resolve, ms));
85
}

1
// Simulate a flaky API
2
let callCount = 0;
3
const result = await withRetry(async () => {
4
  callCount++;
5
  if (callCount < 3) throw new Error("429 rate limit exceeded");
6
  return "success!";
7
});
8

9
console.log(result); // { value: "success!", attempts: 3, totalDelayMs: 3400 }

1
// src/resilience/circuit-breaker.ts — Per-tool circuit breaker (state machine)
2

3
type CircuitState = "closed" | "open" | "half-open";
4

5
export interface BreakerConfig {
6
  failureThreshold: number;   // Failures before opening (default: 5)
7
  cooldownMs: number;         // Time before half-open (default: 30_000)
8
  halfOpenMaxRequests: number;// Successes needed to close (default: 1)
9
  onStateChange?: (tool: string, from: CircuitState, to: CircuitState) => void;
10
}
11

12
const DEFAULT_BREAKER: BreakerConfig = {
13
  failureThreshold: 5,
14
  cooldownMs: 30_000,
15
  halfOpenMaxRequests: 1,
16
};
17

18
export class CircuitBreaker {
19
  private state: Map<string, {
20
    status: CircuitState;
21
    failures: number;
22
    lastFailure: number;
23
    halfOpenSuccesses: number;
24
  }> = new Map();
25

26
  private config: BreakerConfig;
27

28
  constructor(config: Partial<BreakerConfig> = {}) {
29
    this.config = { ...DEFAULT_BREAKER, ...config };
30
  }
31

32
  // ──── Public API ────
33

34
  /**
35
   * Call a function through the circuit breaker.
36
   * Throws immediately if circuit is open.
37
   */
38
  async call<T>(tool: string, fn: () => Promise<T>): Promise<T> {
39
    const state = this.getState(tool);
40

41
    if (state.status === "open") {
42
      const timeSinceFailure = Date.now() - state.lastFailure;
43
      if (timeSinceFailure >= this.config.cooldownMs) {
44
        // Transition to half-open — allow one probe request
45
        this.transition(tool, state, "half-open");
46
      } else {
47
        const retryAfter = this.config.cooldownMs - timeSinceFailure;
48
        throw new CircuitOpenError(
49
          `Circuit breaker open for "${tool}". Retry in ${retryAfter}ms.`,
50
          retryAfter
51
        );
52
      }
53
    }
54

55
    try {
56
      const result = await fn();
57
      this.onSuccess(tool);
58
      return result;
59
    } catch (error) {
60
      this.onFailure(tool);
61
      throw error; // Re-throw — let caller decide if retry is appropriate
62
    }
63
  }
64

65
  /**
66
   * Manually reset a tripped circuit.
67
   */
68
  reset(tool: string) {
69
    const state = this.getState(tool);
70
    this.transition(tool, state, "closed");
71
  }
72

73
  /**
74
   * Get current status for monitoring.
75
   */
76
  getStatus(tool: string): { status: CircuitState; failures: number; lastFailure: number | null } {
77
    const state = this.getState(tool);
78
    return {
79
      status: state.status,
80
      failures: state.failures,
81
      lastFailure: state.lastFailure > 0 ? state.lastFailure : null,
82
    };
83
  }
84

85
  /**
86
   * Get status for all tracked tools.
87
   */
88
  getAllStatus(): Record<string, CircuitState> {
89
    const result: Record<string, CircuitState> = {};
90
    for (const [tool, state] of this.state) {
91
      result[tool] = state.status;
92
    }
93
    return result;
94
  }
95

96
  // ──── Internal ────
97

98
  private getState(tool: string) {
99
    if (!this.state.has(tool)) {
100
      this.state.set(tool, {
101
        status: "closed",
102
        failures: 0,
103
        lastFailure: 0,
104
        halfOpenSuccesses: 0,
105
      });
106
    }
107
    return this.state.get(tool)!;
108
  }
109

110
  private onSuccess(tool: string) {
111
    const state = this.getState(tool);
112
    if (state.status === "half-open") {
113
      state.halfOpenSuccesses++;
114
      if (state.halfOpenSuccesses >= this.config.halfOpenMaxRequests) {
115
        this.transition(tool, state, "closed");
116
      }
117
    } else if (state.status === "closed") {
118
      // Reset failure counter on success
119
      state.failures = 0;
120
    }
121
  }
122

123
  private onFailure(tool: string) {
124
    const state = this.getState(tool);
125
    state.failures++;
126
    state.lastFailure = Date.now();
127

128
    if (state.status === "half-open") {
129
      // Single failure in half-open → back to open
130
      this.transition(tool, state, "open");
131
    } else if (state.status === "closed" && state.failures >= this.config.failureThreshold) {
132
      this.transition(tool, state, "open");
133
    }
134
  }
135

136
  private transition(tool: string, state: any, newStatus: CircuitState) {
137
    const oldStatus = state.status;
138
    state.status = newStatus;
139
    if (newStatus === "closed") {
140
      state.failures = 0;
141
      state.halfOpenSuccesses = 0;
142
    }
143
    if (this.config.onStateChange) {
144
      this.config.onStateChange(tool, oldStatus, newStatus);
145
    }
146
    console.warn(`[CircuitBreaker] ${tool}: ${oldStatus} → ${newStatus}`);
147
  }
148
}
149

150
export class CircuitOpenError extends Error {
151
  public retryAfterMs: number;
152

153
  constructor(message: string, retryAfterMs: number) {
154
    super(message);
155
    this.name = "CircuitOpenError";
156
    this.retryAfterMs = retryAfterMs;
157
  }
158
}

1
         failures ≥ threshold
2
  ┌──────────────────────────────┐
3
  │                              ▼
4
┌──────┐   cooldown elapsed   ┌──────┐
5
│ CLOSED│────────────────────▶│ OPEN │
6
│       │◀────────────────────│      │
7
└──────┘   successes ≥ limit  └──────┘
8
    ▲                              │
9
    │     half-open failure         │
10
    └──────────────────────────────┘
11
            ┌──────────┐
12
            │HALF-OPEN │
13
            └──────────┘

1
// src/resilience/fallback.ts — Graceful degradation fallback chain
2

3
import { CircuitBreaker, CircuitOpenError } from "./circuit-breaker.js";
4
import { withRetry, RetryConfig } from "./retry.js";
5

6
export interface FallbackStep<T> {
7
  name: string;
8
  execute: () => Promise<T>;
9
  isAvailable?: () => boolean; // Optional — skip this step if unavailable
10
}
11

12
export interface FallbackResult<T> {
13
  value: T;
14
  finalStep: string;
15
  attempts: number;
16
  totalDurationMs: number;
17
  warnings: string[];
18
}
19

20
export class FallbackChain<T> {
21
  private steps: FallbackStep<T>[];
22
  private circuitBreaker: CircuitBreaker;
23
  private retryConfig: RetryConfig;
24

25
  constructor(
26
    steps: FallbackStep<T>[],
27
    circuitBreaker?: CircuitBreaker,
28
    retryConfig?: RetryConfig
29
  ) {
30
    if (steps.length === 0) throw new Error("Fallback chain must have at least one step");
31
    this.steps = steps;
32
    this.circuitBreaker = circuitBreaker || new CircuitBreaker();
33
    this.retryConfig = retryConfig || { maxAttempts: 2, baseDelayMs: 500, maxDelayMs: 5000, jitterFactor: 0.2, retryableErrors: [] };
34
  }
35

36
  async execute(): Promise<FallbackResult<T>> {
37
    const startTime = Date.now();
38
    const warnings: string[] = [];
39
    let totalAttempts = 0;
40

41
    for (const step of this.steps) {
42
      // Check availability
43
      if (step.isAvailable && !step.isAvailable()) {
44
        warnings.push(`${step.name}: unavailable, skipping`);
45
        continue;
46
      }
47

48
      try {
49
        const { value, attempts } = await withRetry(
50
          () => this.circuitBreaker.call(step.name, step.execute),
51
          this.retryConfig
52
        );
53
        totalAttempts += attempts;
54

55
        return {
56
          value,
57
          finalStep: step.name,
58
          attempts: totalAttempts,
59
          totalDurationMs: Date.now() - startTime,
60
          warnings,
61
        };
62
      } catch (error) {
63
        totalAttempts++;
64
        if (error instanceof CircuitOpenError) {
65
          warnings.push(`${step.name}: circuit open (retry in ${error.retryAfterMs}ms)`);
66
        } else {
67
          warnings.push(`${step.name}: ${error instanceof Error ? error.message : String(error)}`);
68
        }
69
        // Continue to next fallback step
70
      }
71
    }
72

73
    // All steps failed
74
    throw new FallbackExhaustedError(
75
      `All fallback steps exhausted. Warnings:\n${warnings.join("\n")}`,
76
      warnings,
77
      Date.now() - startTime
78
    );
79
  }
80
}
81

82
export class FallbackExhaustedError extends Error {
83
  public warnings: string[];
84
  public totalDurationMs: number;
85

86
  constructor(message: string, warnings: string[], totalDurationMs: number) {
87
    super(message);
88
    this.name = "FallbackExhaustedError";
89
    this.warnings = warnings;
90
    this.totalDurationMs = totalDurationMs;
91
  }
92
}

1
// src/resilience/index.ts — Unified resilience wrapper for MCP tools
2

3
import { CircuitBreaker, CircuitOpenError } from "./circuit-breaker.js";
4
import { withRetry, RetryConfig } from "./retry.js";
5
import { FallbackChain, FallbackStep, FallbackResult, FallbackExhaustedError } from "./fallback.js";
6
import { AgentLogger } from "../telemetry/logger.js";
7
import { AgentMetrics } from "../telemetry/metrics.js";
8

9
export interface ResilienceOptions {
10
  retryConfig?: Partial<RetryConfig>;
11
  breakerConfig?: Partial<{
12
    failureThreshold: number;
13
    cooldownMs: number;
14
  }>;
15
  logger?: AgentLogger;
16
  metrics?: AgentMetrics;
17
}
18

19
export class ResilientToolCaller {
20
  private circuitBreaker: CircuitBreaker;
21
  private logger?: AgentLogger;
22
  private metrics?: AgentMetrics;
23

24
  constructor(options: ResilienceOptions = {}) {
25
    this.circuitBreaker = new CircuitBreaker({
26
      failureThreshold: options.breakerConfig?.failureThreshold || 5,
27
      cooldownMs: options.breakerConfig?.cooldownMs || 30_000,
28
      onStateChange: (tool, from, to) => {
29
        console.warn(`[Breaker] ${tool}: ${from} → ${to}`);
30
        this.metrics?.increment(`breaker:${tool}:${to}`);
31
      },
32
    });
33
    this.logger = options.logger;
34
    this.metrics = options.metrics;
35
  }
36

37
  /**
38
   * Call a tool with full resilience: retry → circuit breaker → fallback.
39
   */
40
  async call<T>(
41
    toolName: string,
42
    primaryFn: () => Promise<T>,
43
    fallbackSteps?: FallbackStep<T>[],
44
    options?: {
45
      logResult?: (result: T) => void;
46
      retryConfig?: Partial<RetryConfig>;
47
    }
48
  ): Promise<FallbackResult<T> | T> {
49
    const retryConfig: RetryConfig = {
50
      maxAttempts: options?.retryConfig?.maxAttempts || 3,
51
      baseDelayMs: options?.retryConfig?.baseDelayMs || 1000,
52
      maxDelayMs: options?.retryConfig?.maxDelayMs || 30_000,
53
      jitterFactor: options?.retryConfig?.jitterFactor || 0.2,
54
      retryableErrors: options?.retryConfig?.retryableErrors || [
55
        "429", "503", "timeout", "rate limit", "ECONNRESET",
56
      ],
57
    };
58

59
    if (fallbackSteps && fallbackSteps.length > 0) {
60
      // Use fallback chain
61
      const firstStep: FallbackStep<T> = {
62
        name: toolName,
63
        execute: () => withRetry(primaryFn, retryConfig).then(r => {
64
          this.logger?.info("tool_success", { tool: toolName, attempts: r.attempts });
65
          this.metrics?.increment(`tool:${toolName}:success`);
66
          return r.value;
67
        }),
68
      };
69

70
      const chain = new FallbackChain<T>(
71
        [firstStep, ...fallbackSteps],
72
        this.circuitBreaker
73
      );
74

75
      return await chain.execute();
76
    } else {
77
      // No fallback — just retry + circuit breaker
78
      try {
79
        const result = await withRetry(
80
          () => this.circuitBreaker.call(toolName, primaryFn),
81
          retryConfig
82
        );
83
        this.metrics?.increment(`tool:${toolName}:success`);
84
        this.logger?.info("tool_success", { tool: toolName, attempts: result.attempts });
85
        return result.value;
86
      } catch (error) {
87
        this.metrics?.increment(`tool:${toolName}:failure`);
88
        this.logger?.error("tool_failure", { tool: toolName });
89

90
        if (error instanceof CircuitOpenError) {
91
          return this.gracefulDegradation(toolName, error.retryAfterMs) as T;
92
        }
93

94
        throw error;
95
      }
96
    }
97
  }
98

99
  /**
100
   * Last resort: return a user-friendly message instead of crashing.
101
   */
102
  private gracefulDegradation(tool: string, retryAfterMs: number): FallbackResult<string> {
103
    const message = [
104
      `⚠️ The "${tool}" tool is temporarily unavailable due to repeated failures.`,
105
      ``,
106
      `This usually means:`,
107
      `- The external API (GitHub, database, etc.) is down or rate limiting`,
108
      `- There's a network issue between the agent and the service`,
109
      ``,
110
      `⏱️ The system will automatically retry in ${Math.ceil(retryAfterMs / 1000)} seconds.`,
111
      `Try again in a moment, or rephrase your request to use a different tool.`,
112
      ``,
113
      `_If this persists, contact your administrator._`,
114
    ].join("\n");
115

116
    return {
117
      value: message as unknown as T,
118
      finalStep: "graceful-degradation",
119
      attempts: 0,
120
      totalDurationMs: 0,
121
      warnings: [`${tool}: circuit open for ${retryAfterMs}ms`],
122
    };
123
  }
124

125
  getBreakerStatus(tool: string) {
126
    return this.circuitBreaker.getStatus(tool);
127
  }
128

129
  getAllBreakerStatus() {
130
    return this.circuitBreaker.getAllStatus();
131
  }
132

133
  resetBreaker(tool: string) {
134
    this.circuitBreaker.reset(tool);
135
  }
136
}

1
import { ResilientToolCaller } from "./resilience/index.js";
2
import { FallbackStep } from "./resilience/fallback.js";
3
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
4
import { AgentLogger } from "./telemetry/logger.js";
5
import { AgentMetrics } from "./telemetry/metrics.js";
6

7
const server = new McpServer({ name: "github-issue-manager", version: "1.0.4" });
8
const metrics = new AgentMetrics();
9
const resilient = new ResilientToolCaller({
10
  breakerConfig: { failureThreshold: 5, cooldownMs: 30_000 },
11
  metrics,
12
});
13

14
function resilientTool(
15
  name: string,
16
  description: string,
17
  schema: any,
18
  handler: (args: any) => Promise<any>
19
) {
20
  server.tool(name, description, schema, async (args) => {
21
    const logger = new AgentLogger();
22

23
    try {
24
      // Define fallbacks for critical read tools
25
      const fallbacks: FallbackStep<any>[] = [];
26
      if (name === "get_issue" || name === "list_issues") {
27
        fallbacks.push({
28
          name: `${name}-cached`,
29
          execute: async () => {
30
            // Return cached/limited data or a reasonable default
31
            return { content: [{ type: "text", text: "⚠️ Live data unavailable. Showing cached information may be stale." }], isError: false };
32
          },
33
          isAvailable: () => true,
34
        });
35
      }
36

37
      const result = await resilient.call(
38
        name,
39
        () => handler(args),
40
        fallbacks.length > 0 ? fallbacks : undefined,
41
        { logResult: (r) => logger.info("result", { tool: name }) }
42
      );
43

44
      // FallbackResult has `value` — unwrap it
45
      return result && typeof result === "object" && "value" in result
46
        ? (result as any).value
47
        : result;
48

49
    } catch (error) {
50
      return {
51
        content: [{
52
          type: "text",
53
          text: `❌ Failed after multiple attempts.\n\nError: ${error}`,
54
        }],
55
        isError: true,
56
      };
57
    }
58
  });
59
}
60

61
// Register all tools through resilient wrapper
62
resilientTool("list_issues", "List issues", { /* schema */ }, async (args) => { /* handler */ });
63
resilientTool("get_issue", "Get issue", { /* schema */ }, async (args) => { /* handler */ });
64
resilientTool("create_issue", "Create issue", { /* schema */ }, async (args) => { /* handler */ });
65
// ... remaining tools

1
app.get("/resilience/breakers", (req, res) => {
2
  res.json({
3
    breakers: resilient.getAllBreakerStatus(),
4
    metrics: {
5
      // From Prometheus metrics
6
    },
7
  });
8
});
9

10
// Manual breaker reset (admin only)
11
app.post("/resilience/breakers/:tool/reset", (req, res) => {
12
  resilient.resetBreaker(req.params.tool);
13
  res.json({ status: "reset", tool: req.params.tool });
14
});

1
let count = 0;
2
const result = await resilient.call(
3
  "test-tool",
4
  async () => {
5
    count++;
6
    if (count < 3) throw new Error("429 rate limit");
7
    return "ok";
8
  }
9
);
10
// result.value === "ok"
11
// result.attempts === 3

1
const breaker = new CircuitBreaker({ failureThreshold: 5, cooldownMs: 1000 });
2

3
for (let i = 0; i < 5; i++) {
4
  await expect(
5
    breaker.call("flaky", async () => { throw new Error("fail"); })
6
  ).rejects.toThrow();
7
}
8

9
// 6th call should fail fast
10
await expect(
11
  breaker.call("flaky", async () => "should not reach")
12
).rejects.toThrow(CircuitOpenError);

1
const chain = new FallbackChain<string>([
2
  { name: "primary", execute: async () => { throw new Error("fail"); } },
3
  { name: "backup", execute: async () => { throw new Error("also fail"); } },
4
]);
5

6
await expect(chain.execute()).rejects.toThrow(FallbackExhaustedError);