1
// PromptVersion: id, name, content, tags, metadata (author, createdAt, parentId, model)
2
// PromptConfig: name, activeVersion, rolloutPercent
3

4
export class PromptStore {
5
  private versions = new Map<string, PromptVersion>();
6
  private configs = new Map<string, PromptConfig>();
7

8
  async save(version: Omit<PromptVersion, "id">): Promise<PromptVersion> {
9
    const id = crypto.createHash("sha256").update(version.content + Date.now()).digest("hex").slice(0, 16);
10
    const full: PromptVersion = { ...version, id };
11
    this.versions.set(id, full);
12
    await fs.writeFile(`./prompts/${full.name}-${id}.json`, JSON.stringify(full, null, 2));
13
    return full;
14
  }
15

16
  async activate(name: string, versionId: string, rolloutPercent = 100): Promise<void> {
17
    this.configs.set(name, { name, activeVersion: versionId, rolloutPercent });
18
    const v = this.versions.get(versionId)!;
19
    if (!v.tags.includes("active")) v.tags.push("active");
20
  }
21

22
  get(name: string, seed?: string): { version: PromptVersion; isTestVariant: boolean } | null {
23
    const config = this.configs.get(name);
24
    if (!config) return null;
25
    const version = this.versions.get(config.activeVersion);
26
    if (!version) return null;
27
    const isInRollout = seed ? this.isInPercentage(config.rolloutPercent, seed) : true;
28
    return { version, isTestVariant: !isInRollout };
29
  }
30

31
  async rollback(name: string): Promise<PromptVersion | null> {
32
    const versions = this.listVersions(name);
33
    if (versions.length < 2) return null;
34
    const current = versions[0], previous = versions[1];
35
    current.tags.push("rolled-back");
36
    await this.activate(name, previous.id, 100);
37
    return previous;
38
  }
39

40
  private isInPercentage(percent: number, seed: string): boolean {
41
    const hash = crypto.createHash("md5").update(seed).digest("hex");
42
    return parseInt(hash.slice(0, 8), 16) % 100 < percent;
43
  }
44

45
  listVersions(name: string): PromptVersion[] {
46
    return Array.from(this.versions.values()).filter(v => v.name === name)
47
      .sort((a, b) => b.metadata.createdAt - a.metadata.createdAt);
48
  }
49
}

1
const v1 = await store.save({
2
  name: "system-prompt",
3
  content: "You are a helpful GitHub issue manager...",
4
  tags: ["initial"],
5
  metadata: { author: "ei", createdAt: Date.now(), parentId: null, model: "gpt-4o", description: "Initial" },
6
});
7

8
const v2 = await store.save({
9
  name: "system-prompt",
10
  content: "You are a precise GitHub issue manager. Always validate issue numbers...",
11
  tags: ["experiment"],
12
  metadata: { author: "ei", createdAt: Date.now(), parentId: v1.id, model: "gpt-4.1", description: "Add validation" },
13
});
14

15
// 10% traffic cho v2
16
await store.activate("system-prompt", v2.id, 10);
17

18
// Trong runtime — deterministic split theo sessionId
19
const { version, isTestVariant } = store.get("system-prompt", sessionId)!;

1
export interface ExperimentConfig {
2
  name: string;
3
  promptName: string;
4
  variants: { label: string; versionId: string; weight: number }[];
5
  metrics: string[];
6
  startAt: number;
7
  durationMs: number;
8
  minSampleSize: number;
9
  significanceLevel: number;
10
}
11

12
export class ExperimentManager {
13
  private experiments = new Map<string, ExperimentResult>();
14

15
  async start(config: ExperimentConfig): Promise<void> {
16
    const totalWeight = config.variants.reduce((s, v) => s + v.weight, 0);
17
    if (totalWeight !== 100) throw new Error(`Weights must sum to 100, got ${totalWeight}`);
18

19
    for (const variant of config.variants) {
20
      await store.activate(config.promptName, variant.versionId, variant.weight);
21
    }
22

23
    this.experiments.set(config.name, {
24
      name: config.name, status: "running",
25
      samplesPerVariant: Object.fromEntries(config.variants.map(v => [v.label, 0])),
26
      metricsPerVariant: Object.fromEntries(config.variants.map(v => [v.label, Object.fromEntries(config.metrics.map(m => [m, 0]))])),
27
      winner: null, confidence: null, startedAt: Date.now(), completedAt: null,
28
    });
29
  }
30

31
  record(name: string, variant: string, metrics: Record<string, number>): void {
32
    const exp = this.experiments.get(name);
33
    if (!exp || exp.status !== "running") return;
34
    exp.samplesPerVariant[variant]++;
35
    for (const [k, v] of Object.entries(metrics)) {
36
      const n = exp.samplesPerVariant[variant];
37
      exp.metricsPerVariant[variant][k] += (v - exp.metricsPerVariant[variant][k]) / n;
38
    }
39
  }
40
}

1
// ✅ Đúng
2
const variant = hash(seed) % 100 < weight ? "treatment" : "control";
3

4
// ❌ Sai
5
const variant = Math.random() < 0.5 ? "treatment" : "control";

1
const AUTO_ROLLBACK = {
2
  error_rate: { maxIncrease: 0.02 },
3
  latency_p95: { maxIncrease: 0.10 },
4
  tool_accuracy: { maxDecrease: 0.03 },
5
};