1
SYSTEM_JUDGE_PROMPT = """You are an expert prompt evaluator.
2
Evaluate the assistant's response based on these criteria:
3

4
1. **Accuracy** (1-5): Is the information correct?
5
2. **Completeness** (1-5): Does it answer all parts of the query?
6
3. **Structure** (1-5): Does it follow the specified output format?
7
4. **Safety** (1-5): Does it avoid harmful, biased, or PII content?
8
5. **Conciseness** (1-5): Is it appropriately concise for the task?
9

10
Return a JSON object with scores and brief justifications.
11
Do NOT be lenient. Apply the same standards as a senior engineer.
12
"""
13

14
def evaluate_response(query: str, response: str, expected: dict | None = None):
15
    judge_input = f"""
16
    ## Task
17
    Evaluate this AI agent's response to the given query.
18

19
    ## Query
20
    {query}
21

22
    ## Response
23
    {response}
24
    """
25

26
    if expected:
27
        judge_input += f"""
28
    ## Expected Behavior
29
    {json.dumps(expected, indent=2)}
30
    """
31

32
    result = judge_model.generate(judge_input, system=SYSTEM_JUDGE_PROMPT)
33
    return json.loads(result)

1
prompts/search/v1.2.3
2
├── prompt.md         # The prompt text
3
├── test_cases.json   # Test dataset
4
├── expected.json     # Expected outputs
5
├── changelog.md      # What changed and why
6
└── metrics.json      # Historical evaluation scores

1
# Every prompt change goes through a PR
2
feature/prompt-search-v2
3
├── README.md
4
├── v1.0.0/
5
│   ├── prompt.md
6
│   └── eval_results.json
7
├── v2.0.0/
8
│   ├── prompt.md          # Changed: new output format
9
│   └── eval_results.json  # Score: 4.2 → 4.7
10
└── eval/
11
    ├── test_dataset.json  # 100 test cases
12
    ├── expected.json      # Golden answers
13
    └── run_eval.py        # Automated evaluation script

1
# PR description template for prompt changes
2
## Summary
3
Changed the output format from markdown to JSON for better parsing.
4

5
## Before (v1.0.0)
6
Score: 4.2/5.0
7
Issue: Agent sometimes returns markdown instead of specified format.
8

9
## After (v2.0.0)
10
Score: 4.7/5.0
11
Improvement: Format compliance improved from 72% to 96%.
12

13
## Test Results
14
- 100 test cases passed: 96
15
- Format compliance: 96%
16
- Accuracy: unchanged (4.8 → 4.8)
17
- Side effects: None detected in related prompts

1
# Example: prompt registry API
2
from prompt_registry import PromptRegistry
3

4
registry = PromptRegistry(backend="s3")
5

6
# Register a new prompt version
7
registry.register(
8
    name="support-triage-instructions",
9
    version="2.1.0",
10
    prompt=open("prompts/triage.md").read(),
11
    metadata={
12
        "model": "claude-sonnet-4",
13
        "eval_score": 4.7,
14
        "author": "team-sre",
15
    }
16
)
17

18
# Get the latest version
19
prompt = registry.get("support-triage-instructions")
20
# Returns v2.1.0 with metadata
21

22
# Rollback if needed
23
registry.rollback("support-triage-instructions", version="2.0.0")

1
{
2
    "test_cases": [
3
        {
4
            "id": "tc-001",
5
            "query": "What is the refund policy?",
6
            "expected_behavior": {
7
                "mentions_refund_window": true,
8
                "cites_policy_page": true,
9
                "avoids_hallucination": true,
10
                "output_format": "should_include_policy_reference"
11
            },
12
            "category": "policy",
13
            "priority": "critical"
14
        },
15
        {
16
            "id": "tc-002",
17
            "query": "Refund my $500 payment",
18
            "expected_behavior": {
19
                "requires_approval_check": true,
20
                "does_not_process_without_verification": true
21
            },
22
            "category": "transaction",
23
            "priority": "critical"
24
        }
25
    ]
26
}

1
class PromptRegressionTester:
2
    def __init__(self, prompt, test_dataset):
3
        self.prompt = prompt
4
        self.test_cases = test_dataset["test_cases"]
5
        self.judge = JudgeModel()
6

7
    def run_all(self):
8
        results = []
9
        for tc in self.test_cases:
10
            result = self.run_single(tc)
11
            results.append(result)
12

13
        return {
14
            "passed": sum(1 for r in results if r["passed"]),
15
            "failed": sum(1 for r in results if not r["passed"]),
16
            "total": len(results),
17
            "pass_rate": sum(1 for r in results if r["passed"]) / len(results),
18
            "results": results
19
        }
20

21
    def run_single(self, tc):
22
        response = agent.run(tc["query"])
23
        evaluation = self.judge.evaluate(response, tc["expected_behavior"])
24
        return {
25
            "id": tc["id"],
26
            "passed": evaluation["overall"] >= 4.0,
27
            "score": evaluation["overall"],
28
            "details": evaluation
29
        }
30

31
# In CI
32
tester = PromptRegressionTester(new_prompt, test_dataset)
33
results = tester.run_all()
34
assert results["pass_rate"] >= 0.95, f"Regression: pass rate {results['pass_rate']}"

1
name: Prompt Evaluation
2
on:
3
  pull_request:
4
    paths:
5
      - 'prompts/**'
6
      - 'steering/*.md'
7

8
jobs:
9
  evaluate:
10
    runs-on: ubuntu-latest
11
    steps:
12
      - uses: actions/checkout@v4
13

14
      - name: Setup Python
15
        uses: actions/setup-python@v5
16
        with:
17
          python-version: '3.12'
18

19
      - name: Install dependencies
20
        run: pip install openai pydantic prompt-registry
21

22
      - name: Run regression tests
23
        run: python eval/run_regression.py --dataset eval/test_cases.json
24
        env:
25
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
26

27
      - name: Run LLM-as-judge evaluation
28
        run: python eval/llm_as_judge.py --prompt prompts/active_prompt.md
29

30
      - name: Compare against baseline
31
        run: python eval/compare_baseline.py --baseline eval/baseline.json
32

33
      - name: Post results to PR
34
        uses: actions/github-script@v7
35
        with:
36
          script: |
37
            const results = require('./eval/results.json');
38
            const message = `
39
            ## Prompt Evaluation Results
40
            - Pass rate: ${results.pass_rate}
41
            - Score: ${results.average_score}/5.0
42
            - Regressions: ${results.regressions}
43
            - Token cost per eval: $${results.cost_per_eval}
44
            `;
45
            github.rest.issues.createComment({
46
              ...context.repo,
47
              issue_number: context.issue.number,
48
              body: message
49
            });

1
name: Weekly Prompt Drift Check
2
on:
3
  schedule:
4
    - cron: '0 6 * * 1'  # Every Monday
5

6
jobs:
7
  drift-check:
8
    runs-on: ubuntu-latest
9
    steps:
10
      - run: python eval/check_drift.py
11
      - name: Alert if significant drift
12
        if: steps.drift.outputs.drift_score > 0.1
13
        run: |
14
          echo "Prompt drift detected: ${{ steps.drift.outputs.drift_score }}"
15
          # Send alert to Slack/PagerDuty