1
// src/multi-region/region-router.ts — Resolve which region handles a request
2

3
export interface RegionConfig {
4
  name: string;           // "us-east-1" | "eu-west-1" | "ap-southeast-1"
5
  endpoint: string;       // Agent endpoint URL
6
  weight: number;         // Traffic weight (0-100)
7
  priority: number;       // Lower = preferred (0 = primary)
8
  status: "active" | "degraded" | "down";
9
}
10

11
export class RegionRouter {
12
  private regions: RegionConfig[] = [];
13

14
  constructor(regions: RegionConfig[]) {
15
    this.regions = regions.sort((a, b) => a.priority - b.priority);
16
  }
17

18
  /**
19
   * Get the optimal region for a request.
20
   * Prefers same-continent regions and healthy ones.
21
   */
22
  resolve(clientRegion: string): RegionConfig {
23
    const healthy = this.regions.filter(r => r.status === "active");
24
    if (healthy.length === 0) {
25
      // Everything is down — return the closest region anyway
26
      return this.findClosest(clientRegion);
27
    }
28

29
    // Try closest active region first
30
    const closest = this.findClosest(clientRegion, healthy);
31
    if (closest) return closest;
32

33
    // Fallback to any healthy region
34
    return healthy[0];
35
  }
36

37
  /**
38
   * Detect region failure and mark as down.
39
   */
40
  markDown(regionName: string): void {
41
    const region = this.regions.find(r => r.name === regionName);
42
    if (region) {
43
      region.status = "down";
44
      console.warn(`[Region] ${regionName}: marked DOWN`);
45
    }
46
  }
47

48
  /**
49
   * Health check all regions.
50
   */
51
  async healthCheck(): Promise<Record<string, "active" | "degraded" | "down">> {
52
    const results: Record<string, "active" | "degraded" | "down"> = {};
53

54
    for (const region of this.regions) {
55
      try {
56
        const controller = new AbortController();
57
        const timeout = setTimeout(() => controller.abort(), 5000);
58

59
        const response = await fetch(`${region.endpoint}/health`, {
60
          signal: controller.signal,
61
        });
62
        clearTimeout(timeout);
63

64
        if (response.ok) {
65
          const body = await response.json();
66
          region.status = body.status || "active";
67
        } else {
68
          region.status = "degraded";
69
        }
70
      } catch {
71
        region.status = "down";
72
      }
73

74
      results[region.name] = region.status;
75
    }
76

77
    return results;
78
  }
79

80
  private findClosest(clientRegion: string, pool?: RegionConfig[]): RegionConfig | undefined {
81
    const candidates = pool || this.regions;
82
    // Simple continent mapping — in production, use geo-IP database
83
    const continent = this.mapToContinent(clientRegion);
84
    const sameContinent = candidates.filter(r => this.mapToContinent(r.name) === continent);
85

86
    if (sameContinent.length > 0) {
87
      return sameContinent.reduce((best, r) =>
88
        r.priority < best.priority ? r : best
89
      );
90
    }
91

92
    return undefined;
93
  }
94

95
  private mapToContinent(region: string): string {
96
    const prefix = region.startsWith("us-") || region.startsWith("ca-") ? "americas"
97
      : region.startsWith("eu-") ? "europe"
98
      : region.startsWith("ap-") || region.startsWith("sa-") ? "asia-pacific"
99
      : "other";
100
    return prefix;
101
  }
102

103
  getStatus(): { regions: RegionConfig[]; healthy: number; total: number } {
104
    return {
105
      regions: this.regions,
106
      healthy: this.regions.filter(r => r.status === "active").length,
107
      total: this.regions.length,
108
    };
109
  }
110
}

1
// src/multi-region/global-state.ts — Cross-region session state replication
2

3
export interface SessionSnapshot {
4
  sessionId: string;
5
  conversation: Array<{
6
    role: "user" | "assistant" | "tool";
7
    content: string;
8
    timestamp: number;
9
  }>;
10
  state: Record<string, unknown>;   // Agent internal state (paused tools, etc.)
11
  version: number;                    // Monotonic version for conflict resolution
12
  lastActiveRegion: string;
13
  lastActiveAt: number;
14
}
15

16
export interface GlobalStateConfig {
17
  checkpointIntervalMs: number;   // How often to checkpoint (default: 5000)
18
  ttlMs: number;                  // Session TTL after last activity (default: 3600_000)
19
  conflictStrategy: "last-write-wins" | "version-wins" | "merge";
20
}
21

22
const DEFAULT_GLOBAL_CONFIG: GlobalStateConfig = {
23
  checkpointIntervalMs: 5_000,
24
  ttlMs: 3_600_000,  // 1 hour
25
  conflictStrategy: "version-wins",
26
};
27

28
export class GlobalStateStore {
29
  private store: Map<string, SessionSnapshot> = new Map();
30
  private currentRegion: string;
31
  private config: GlobalStateConfig;
32

33
  constructor(currentRegion: string, config: Partial<GlobalStateConfig> = {}) {
34
    this.currentRegion = currentRegion;
35
    this.config = { ...DEFAULT_GLOBAL_CONFIG, ...config };
36
  }
37

38
  /**
39
   * Save a session checkpoint to the global store.
40
   * In production, this writes to DynamoDB / CockroachDB / Redis Enterprise.
41
   * For this example, we use an in-memory map (simulating a global KV store).
42
   */
43
  async checkpoint(sessionId: string, snapshot: Omit<SessionSnapshot, "lastActiveRegion" | "lastActiveAt">): Promise<void> {
44
    const existing = this.store.get(sessionId);
45

46
    const updated: SessionSnapshot = {
47
      ...snapshot,
48
      lastActiveRegion: this.currentRegion,
49
      lastActiveAt: Date.now(),
50
    };
51

52
    // Conflict resolution
53
    if (existing && this.config.conflictStrategy === "version-wins") {
54
      if (snapshot.version <= existing.version) {
55
        console.warn(`[GlobalState] Conflict: version ${snapshot.version} ≤ ${existing.version}, rejecting`);
56
        return;
57
      }
58
    }
59

60
    this.store.set(sessionId, updated);
61

62
    // Log activity for monitoring
63
    console.debug(`[GlobalState] Checkpoint: ${sessionId} v${snapshot.version} @ ${this.currentRegion}`);
64
  }
65

66
  /**
67
   * Restore a session from the global store.
68
   * Used during failover — new region picks up where old region left off.
69
   */
70
  async restore(sessionId: string): Promise<SessionSnapshot | null> {
71
    const snapshot = this.store.get(sessionId);
72
    if (!snapshot) return null;
73

74
    // Check TTL
75
    if (Date.now() - snapshot.lastActiveAt > this.config.ttlMs) {
76
      this.store.delete(sessionId);
77
      return null;
78
    }
79

80
    // Update the active region on restore
81
    snapshot.lastActiveRegion = this.currentRegion;
82
    snapshot.lastActiveAt = Date.now();
83

84
    console.log(`[GlobalState] Restored: ${sessionId} v${snapshot.version} (was in ${snapshot.lastActiveRegion})`);
85
    return snapshot;
86
  }
87

88
  /**
89
   * Checkpoint periodically during a session.
90
   */
91
  startPeriodicCheckpoint(
92
    sessionId: string,
93
    getSnapshot: () => Omit<SessionSnapshot, "lastActiveRegion" | "lastActiveAt">
94
  ): () => void {
95
    const interval = setInterval(() => {
96
      this.checkpoint(sessionId, getSnapshot());
97
    }, this.config.checkpointIntervalMs);
98

99
    return () => clearInterval(interval);
100
  }
101

102
  /**
103
   * List all active sessions globally.
104
   */
105
  listActiveSessions(): SessionSnapshot[] {
106
    const now = Date.now();
107
    return Array.from(this.store.values())
108
      .filter(s => now - s.lastActiveAt <= this.config.ttlMs)
109
      .sort((a, b) => b.lastActiveAt - a.lastActiveAt);
110
  }
111

112
  async stats(): Promise<{
113
    totalSessions: number;
114
    activeSessions: number;
115
    byRegion: Record<string, number>;
116
  }> {
117
    const allSessions = Array.from(this.store.values());
118
    const now = Date.now();
119
    const active = allSessions.filter(s => now - s.lastActiveAt <= this.config.ttlMs);
120
    const byRegion: Record<string, number> = {};
121

122
    for (const s of active) {
123
      byRegion[s.lastActiveRegion] = (byRegion[s.lastActiveRegion] || 0) + 1;
124
    }
125

126
    return {
127
      totalSessions: allSessions.length,
128
      activeSessions: active.length,
129
      byRegion,
130
    };
131
  }
132
}

1
// In production, replace the in-memory map with DynamoDB:
2
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
3
import { DynamoDBDocumentClient, PutCommand, GetCommand } from "@aws-sdk/lib-dynamodb";
4

5
export class DynamoGlobalState extends GlobalStateStore {
6
  private docClient: DynamoDBDocumentClient;
7
  private tableName: string;
8

9
  constructor(region: string, tableName: string) {
10
    super(region);
11
    this.tableName = tableName;
12
    this.docClient = DynamoDBDocumentClient.from(
13
      new DynamoDBClient({ region })
14
    );
15
  }
16

17
  async checkpoint(sessionId: string, snapshot: Omit<SessionSnapshot, "lastActiveRegion" | "lastActiveAt">): Promise<void> {
18
    await this.docClient.send(new PutCommand({
19
      TableName: this.tableName,
20
      Item: {
21
        pk: `session#${sessionId}`,
22
        ...snapshot,
23
        lastActiveRegion: this.currentRegion,
24
        lastActiveAt: Date.now(),
25
        ttl: Math.floor(Date.now() / 1000) + 3600, // DynamoDB TTL
26
      },
27
      // Conditional write for version-wins consistency
28
      ConditionExpression: "attribute_not_exists(version) OR version < :newVersion",
29
      ExpressionAttributeValues: { ":newVersion": snapshot.version },
30
    }));
31
  }
32

33
  async restore(sessionId: string): Promise<SessionSnapshot | null> {
34
    const result = await this.docClient.send(new GetCommand({
35
      TableName: this.tableName,
36
      Key: { pk: `session#${sessionId}` },
37
    }));
38
    return result.Item as SessionSnapshot ?? null;
39
  }
40
}

1
// src/multi-region/cache-replication.ts — Cross-region cache warming
2

3
import { SemanticCache } from "../cache/semantic-cache.js";
4
import { ExactCache } from "../cache/exact-cache.js";
5

6
interface CacheWarmupEntry {
7
  key: string;
8
  value: string;
9
  ttlMs: number;
10
  sourceRegion: string;
11
  timestamp: number;
12
}
13

14
export class CacheReplicator {
15
  private sourceCache: SemanticCache | ExactCache;
16
  private peerEndpoints: string[];
17
  private regionName: string;
18

19
  constructor(
20
    cache: SemanticCache | ExactCache,
21
    regionName: string,
22
    peerEndpoints: string[]
23
  ) {
24
    this.sourceCache = cache;
25
    this.regionName = regionName;
26
    this.peerEndpoints = peerEndpoints;
27
  }
28

29
  /**
30
   * Broadcast a cache entry to peer regions.
31
   */
32
  async broadcast(key: string, value: string, ttlMs: number): Promise<void> {
33
    const entry: CacheWarmupEntry = {
34
      key,
35
      value,
36
      ttlMs,
37
      sourceRegion: this.regionName,
38
      timestamp: Date.now(),
39
    };
40

41
    const promises = this.peerEndpoints.map(endpoint =>
42
      fetch(`${endpoint}/cache/warmup`, {
43
        method: "POST",
44
        headers: { "Content-Type": "application/json" },
45
        body: JSON.stringify(entry),
46
        signal: AbortSignal.timeout(2000),
47
      }).catch(() => {
48
        console.warn(`[CacheReplicator] Failed to warm ${endpoint}`);
49
      })
50
    );
51

52
    await Promise.allSettled(promises);
53
  }
54

55
  /**
56
   * Warmup endpoint — receives entries from peer regions.
57
   */
58
  async receiveWarmup(entry: CacheWarmupEntry): Promise<void> {
59
    const stalenessMs = Date.now() - entry.timestamp;
60

61
    // Only accept entries less than 30 seconds old
62
    if (stalenessMs > 30_000) {
63
      console.debug(`[CacheReplicator] Skipping stale warmup from ${entry.sourceRegion} (${stalenessMs}ms old)`);
64
      return;
65
    }
66

67
    const remainingTtl = entry.ttlMs - stalenessMs;
68
    if (remainingTtl <= 0) return;
69

70
    await this.sourceCache.set(entry.key, JSON.parse(entry.value), remainingTtl);
71
    console.debug(`[CacheReplicator] Warmed: ${entry.key.slice(0, 40)}... from ${entry.sourceRegion}`);
72
  }
73
}

1
app.post("/cache/warmup", async (req, res) => {
2
  await cacheReplicator.receiveWarmup(req.body);
3
  res.json({ status: "warmed" });
4
});

1
// src/multi-region/failover-manager.ts — Automatic region failover
2

3
import { RegionRouter } from "./region-router.js";
4
import { GlobalStateStore } from "./global-state.js";
5

6
export interface FailoverEvent {
7
  fromRegion: string;
8
  toRegion: string;
9
  sessionCount: number;
10
  reason: string;
11
  timestamp: number;
12
}
13

14
type FailoverListener = (event: FailoverEvent) => void;
15

16
export class FailoverManager {
17
  private router: RegionRouter;
18
  private globalState: GlobalStateStore;
19
  private currentRegion: string;
20
  private listeners: FailoverListener[] = [];
21
  private failoverHistory: FailoverEvent[] = [];
22
  private healthCheckInterval: ReturnType<typeof setInterval> | null = null;
23

24
  constructor(
25
    router: RegionRouter,
26
    globalState: GlobalStateStore,
27
    currentRegion: string
28
  ) {
29
    this.router = router;
30
    this.globalState = globalState;
31
    this.currentRegion = currentRegion;
32
  }
33

34
  /**
35
   * Start periodic health checks.
36
   */
37
  startHealthChecks(intervalMs = 30_000): void {
38
    this.healthCheckInterval = setInterval(async () => {
39
      const status = await this.router.healthCheck();
40

41
      if (status[this.currentRegion] === "down") {
42
        const failover = this.initiateFailover("health-check-failure");
43
        console.warn(`[Failover] Region ${this.currentRegion} is DOWN. Failover triggered.`);
44
        this.notifyListeners(failover);
45
      }
46
    }, intervalMs);
47
  }
48

49
  /**
50
   * Initiate failover to the next available region.
51
   */
52
  initiateFailover(reason: string): FailoverEvent {
53
    const healthyRegions = this.router.getStatus().regions.filter(r =>
54
      r.status === "active" && r.name !== this.currentRegion
55
    );
56

57
    if (healthyRegions.length === 0) {
58
      throw new Error("No healthy regions available for failover");
59
    }
60

61
    // Pick the highest priority healthy region
62
    const target = healthyRegions.sort((a, b) => a.priority - b.priority)[0];
63
    const activeSessions = this.globalState.listActiveSessions();
64
    const activeSessionCount = activeSessions.length;
65

66
    // Final checkpoint all active sessions
67
    for (const session of activeSessions) {
68
      this.globalState.checkpoint(session.sessionId, {
69
        conversation: session.conversation,
70
        state: session.state,
71
        version: session.version + 1,
72
      });
73
    }
74

75
    const event: FailoverEvent = {
76
      fromRegion: this.currentRegion,
77
      toRegion: target.name,
78
      sessionCount: activeSessionCount,
79
      reason,
80
      timestamp: Date.now(),
81
    };
82

83
    this.failoverHistory.push(event);
84
    this.currentRegion = target.name;
85

86
    return event;
87
  }
88

89
  /**
90
   * Test failover without actually switching.
91
   */
92
  dryRunFailover(): { targetRegion: string; sessionCount: number } {
93
    const healthyRegions = this.router.getStatus().regions.filter(r =>
94
      r.status === "active" && r.name !== this.currentRegion
95
    );
96
    const target = healthyRegions.sort((a, b) => a.priority - b.priority)[0];
97
    const activeSessions = this.globalState.listActiveSessions();
98

99
    return {
100
      targetRegion: target?.name || "none",
101
      sessionCount: activeSessions.length,
102
    };
103
  }
104

105
  /**
106
   * Restore a session — called by the new region after failover.
107
   */
108
  async restoreSession(sessionId: string): Promise<boolean> {
109
    const snapshot = await this.globalState.restore(sessionId);
110
    if (!snapshot) return false;
111

112
    console.log(`[Failover] Session ${sessionId} restored in ${this.currentRegion}`);
113
    return true;
114
  }
115

116
  onFailover(listener: FailoverListener): void {
117
    this.listeners.push(listener);
118
  }
119

120
  getHistory(): FailoverEvent[] {
121
    return this.failoverHistory;
122
  }
123

124
  stopHealthChecks(): void {
125
    if (this.healthCheckInterval) {
126
      clearInterval(this.healthCheckInterval);
127
      this.healthCheckInterval = null;
128
    }
129
  }
130

131
  private notifyListeners(event: FailoverEvent): void {
132
    for (const listener of this.listeners) {
133
      try { listener(event); } catch (e) {
134
        console.error("[Failover] Listener error:", e);
135
      }
136
    }
137
  }
138
}

1
import { GlobalStateStore } from "./global-state.js";
2
import { AgentLogger } from "../telemetry/logger.js";
3

4
export class CheckpointAwareAgent {
5
  private globalState: GlobalStateStore;
6
  private logger: AgentLogger;
7
  private stopCheckpointing: (() => void) | null = null;
8

9
  constructor(globalState: GlobalStateStore) {
10
    this.globalState = globalState;
11
    this.logger = new AgentLogger();
12
  }
13

14
  /**
15
   * Start a session with periodic checkpointing.
16
   */
17
  startSession(sessionId: string): CheckpointContext {
18
    const context: CheckpointContext = {
19
      sessionId,
20
      conversation: [],
21
      state: {},
22
      version: 1,
23
    };
24

25
    // Periodic checkpoint
26
    this.stopCheckpointing = this.globalState.startPeriodicCheckpoint(
27
      sessionId,
28
      () => ({
29
        sessionId,
30
        conversation: context.conversation,
31
        state: context.state,
32
        version: context.version,
33
      })
34
    );
35

36
    this.logger.info("session_started", {
37
      sessionId,
38
      region: (this.globalState as any).currentRegion,
39
    });
40

41
    return context;
42
  }
43

44
  /**
45
   * Handle a tool call — checkpoint after each mutation.
46
   */
47
  async handleToolCall(context: CheckpointContext, tool: string, result: unknown): Promise<void> {
48
    context.conversation.push({
49
      role: "tool",
50
      content: JSON.stringify(result).slice(0, 1000),
51
      timestamp: Date.now(),
52
    });
53
    context.version++;
54

55
    // Checkpoint after every mutation
56
    if (["create_issue", "update_issue", "delete_issue"].includes(tool)) {
57
      await this.globalState.checkpoint(context.sessionId, {
58
        sessionId: context.sessionId,
59
        conversation: context.conversation,
60
        state: context.state,
61
        version: context.version,
62
      });
63

64
      this.logger.info("session_checkpoint", {
65
        sessionId: context.sessionId,
66
        version: context.version,
67
        tool,
68
      });
69
    }
70
  }
71

72
  /**
73
   * End session — final checkpoint and cleanup.
74
   */
75
  async endSession(context: CheckpointContext): Promise<void> {
76
    if (this.stopCheckpointing) {
77
      this.stopCheckpointing();
78
    }
79

80
    await this.globalState.checkpoint(context.sessionId, {
81
      sessionId: context.sessionId,
82
      conversation: context.conversation,
83
      state: context.state,
84
      version: context.version,
85
    });
86

87
    this.logger.info("session_ended", {
88
      sessionId: context.sessionId,
89
      version: context.version,
90
      messageCount: context.conversation.length,
91
    });
92
  }
93
}
94

95
interface CheckpointContext {
96
  sessionId: string;
97
  conversation: Array<{ role: string; content: string; timestamp: number }>;
98
  state: Record<string, unknown>;
99
  version: number;
100
}

1
agent.minixium.com
2
  ├── us-east-1: weight 50 (primary)
3
  ├── eu-west-1: weight 30
4
  └── ap-southeast-1: weight 20
5
  Health check: /health → failover on consecutive failures

1
us-east-1 cache hit → broadcast to eu-west-1 and ap-southeast-1
2
                                       ↓
3
Other regions warm their cache → 502 → 200 on failover