War Story: The 2025 LLM Outage That Broke Our Code Review Pipeline and How We Switched to Claude Code 2.0

import os
import time
import json
import logging
import random
from typing import Dict, List, Optional
from dataclasses import dataclass
import requests
from prometheus_client import Counter, Histogram

# Configure logging for audit trails
logging.basicConfig(level=logging.INFO, format=\"%(asctime)s - %(levelname)s - %(message)s\")
logger = logging.getLogger(__name__)

# Prometheus metrics for observability
REVIEW_REQUESTS = Counter(\"llm_review_requests_total\", \"Total LLM review requests\", [\"status\"])
REVIEW_LATENCY = Histogram(\"llm_review_latency_seconds\", \"LLM review latency distribution\")

@dataclass
class ReviewResult:
    \"\"\"Structured output from LLM code review\"\"\"
    pr_id: str
    passed: bool
    issues: List[Dict[str, str]]
    latency_ms: int
    model_version: str

class LegacyLLMReviewClient:
    \"\"\"Client for deprecated third-party LLM Review API v1.9.2 (sunset July 2025)\"\"\"
    def __init__(self, api_key: Optional[str] = None, max_retries: int = 3):
        self.api_key = api_key or os.getenv(\"LLM_REVIEW_API_KEY\")
        if not self.api_key:
            raise ValueError(\"LLM_REVIEW_API_KEY environment variable not set\")
        self.base_url = \"https://api.legacy-llm-review.com/v1\"
        self.max_retries = max_retries
        self.session = requests.Session()
        self.session.headers.update({
            \"Authorization\": f\"Bearer {self.api_key}\",
            \"Content-Type\": \"application/json\"
        })

    def _exponential_backoff(self, attempt: int) -> None:
        \"\"\"Calculate backoff delay with jitter to avoid thundering herd\"\"\"
        delay = min(2 ** attempt + random.uniform(0, 1), 30)  # Cap at 30s
        time.sleep(delay)

    def review_pr(self, pr_id: str, diff: str, context: Dict[str, str]) -> ReviewResult:
        \"\"\"
        Submit PR diff for automated review.

        Args:
            pr_id: Unique pull request identifier
            diff: Unified diff of code changes
            context: Additional context (repo name, author, commit count)

        Returns:
            ReviewResult with pass/fail status and issue list
        \"\"\"
        start_time = time.time()
        last_error = None

        for attempt in range(self.max_retries + 1):
            try:
                payload = {
                    \"pr_id\": pr_id,
                    \"diff\": diff,
                    \"context\": context,
                    \"checks\": [\"style\", \"security\", \"test_coverage\", \"breaking_changes\"],
                    \"model\": \"llm-review-large-v1.9.2\"
                }
                response = self.session.post(
                    f\"{self.base_url}/review\",
                    json=payload,
                    timeout=10  # 10s timeout per request
                )
                response.raise_for_status()
                result = response.json()

                # Validate response schema
                if not all(key in result for key in [\"passed\", \"issues\", \"model_version\"]):
                    raise ValueError(f\"Invalid response schema: {result.keys()}\")

                latency_ms = int((time.time() - start_time) * 1000)
                REVIEW_REQUESTS.labels(status=\"success\").inc()
                REVIEW_LATENCY.observe(time.time() - start_time)

                return ReviewResult(
                    pr_id=pr_id,
                    passed=result[\"passed\"],
                    issues=result[\"issues\"],
                    latency_ms=latency_ms,
                    model_version=result[\"model_version\"]
                )

            except requests.exceptions.Timeout as e:
                last_error = e
                logger.warning(f\"Attempt {attempt+1} timed out for PR {pr_id}: {str(e)}\")
                REVIEW_REQUESTS.labels(status=\"timeout\").inc()
            except requests.exceptions.HTTPError as e:
                last_error = e
                # Don't retry 4xx errors except 429
                if 400 <= e.response.status_code < 500 and e.response.status_code != 429:
                    logger.error(f\"Non-retryable HTTP error {e.response.status_code} for PR {pr_id}\")
                    break
                logger.warning(f\"Attempt {attempt+1} failed for PR {pr_id}: {str(e)}\")
                REVIEW_REQUESTS.labels(status=\"http_error\").inc()
            except Exception as e:
                last_error = e
                logger.error(f\"Unexpected error for PR {pr_id}: {str(e)}\")
                REVIEW_REQUESTS.labels(status=\"error\").inc()

            if attempt < self.max_retries:
                self._exponential_backoff(attempt)

        # All retries failed
        latency_ms = int((time.time() - start_time) * 1000)
        logger.error(f\"All {self.max_retries+1} attempts failed for PR {pr_id}. Last error: {str(last_error)}\")
        raise RuntimeError(f\"Failed to review PR {pr_id} after {self.max_retries+1} attempts: {str(last_error)}\") from last_error

if __name__ == \"__main__\":
    # Example usage (fails during March 12 outage)
    client = LegacyLLMReviewClient()
    try:
        result = client.review_pr(
            pr_id=\"PR-1234\",
            diff=\"diff --git a/src/main.py b/src/main.py\\nindex 1234..5678 100644\\n--- a/src/main.py\\n+++ b/src/main.py\\n@@ -10,6 +10,7 @@ def main():\\n+    print('hello world')\\n     return 0\",
            context={\"repo\": \"acme-payments\", \"author\": \"jdoe\", \"commits\": 2}
        )
        print(f\"Review passed: {result.passed}\")
    except Exception as e:
        print(f\"Review failed: {str(e)}\")

import os
import time
import json
import logging
from typing import Dict, List, Tuple
from dataclasses import dataclass
from concurrent.futures import ThreadPoolExecutor, as_completed
import claude_code  # SDK from https://github.com/anthropics/claude-code
from claude_code.exceptions import ClaudeAPIError, RateLimitError
from prometheus_client import Counter, Histogram

# Configure logging
logging.basicConfig(level=logging.INFO, format=\"%(asctime)s - %(levelname)s - %(message)s\")
logger = logging.getLogger(__name__)

# Metrics for migration validation
MIGRATION_MATCHES = Counter(\"migration_review_matches_total\", \"Matching review results between old and new clients\")
MIGRATION_MISMATCHES = Counter(\"migration_review_mismatches_total\", \"Mismatched review results\")
MIGRATION_LATENCY_DELTA = Histogram(\"migration_latency_delta_ms\", \"Latency difference (new - old) in ms\")

@dataclass
class MigrationResult:
    \"\"\"Comparison result between legacy and Claude Code 2.0 reviews\"\"\"
    pr_id: str
    legacy_passed: bool
    claude_passed: bool
    legacy_issues: int
    claude_issues: int
    legacy_latency_ms: int
    claude_latency_ms: int
    match: bool

class ClaudeCodeMigrationValidator:
    \"\"\"Validates Claude Code 2.0 output against legacy LLM Review client during migration\"\"\"
    def __init__(self, legacy_client, claude_client, sample_size: int = 100):
        self.legacy_client = legacy_client
        self.claude_client = claude_client
        self.sample_size = sample_size
        self.results: List[MigrationResult] = []

    def _run_single_comparison(self, pr_id: str, diff: str, context: Dict[str, str]) -> MigrationResult:
        \"\"\"Compare legacy and Claude Code review for a single PR\"\"\"
        # Run legacy client
        legacy_start = time.time()
        try:
            legacy_result = self.legacy_client.review_pr(pr_id, diff, context)
            legacy_latency = int((time.time() - legacy_start) * 1000)
        except Exception as e:
            logger.error(f\"Legacy client failed for {pr_id}: {str(e)}\")
            legacy_result = None
            legacy_latency = -1

        # Run Claude Code 2.0 client
        claude_start = time.time()
        try:
            # Claude Code 2.0 review call: https://github.com/anthropics/claude-code/blob/main/docs/api.md#review-diff
            claude_result = self.claude_client.review_diff(
                diff=diff,
                repo=context.get(\"repo\"),
                pr_id=pr_id,
                checks=[\"style\", \"security\", \"test_coverage\", \"breaking_changes\"],
                model=\"claude-code-2.0.1\"
            )
            claude_latency = int((time.time() - claude_start) * 1000)
        except Exception as e:
            logger.error(f\"Claude client failed for {pr_id}: {str(e)}\")
            claude_result = None
            claude_latency = -1

        # Compare results if both succeeded
        if legacy_result and claude_result:
            match = (
                legacy_result.passed == claude_result.passed and
                len(legacy_result.issues) == len(claude_result.issues)
            )
            if match:
                MIGRATION_MATCHES.inc()
            else:
                MIGRATION_MISMATCHES.inc()
                logger.warning(f\"Mismatch for {pr_id}: legacy passed {legacy_result.passed}, claude passed {claude_result.passed}\")
            MIGRATION_LATENCY_DELTA.observe(claude_latency - legacy_latency)
            return MigrationResult(
                pr_id=pr_id,
                legacy_passed=legacy_result.passed,
                claude_passed=claude_result.passed,
                legacy_issues=len(legacy_result.issues),
                claude_issues=len(claude_result.issues),
                legacy_latency_ms=legacy_latency,
                claude_latency_ms=claude_latency,
                match=match
            )
        else:
            # Partial failure, log and skip
            logger.error(f\"Partial failure for {pr_id}, skipping comparison\")
            return None

    def run_validation(self, pr_batch: List[Tuple[str, str, Dict[str, str]]]) -> float:
        \"\"\"
        Run parallel validation on a batch of PRs.

        Args:
            pr_batch: List of (pr_id, diff, context) tuples

        Returns:
            Match rate (0.0 to 1.0)
        \"\"\"
        pr_batch = pr_batch[:self.sample_size]  # Cap at sample size
        logger.info(f\"Running migration validation on {len(pr_batch)} PRs\")

        with ThreadPoolExecutor(max_workers=10) as executor:
            futures = [
                executor.submit(self._run_single_comparison, pr_id, diff, context)
                for pr_id, diff, context in pr_batch
            ]
            for future in as_completed(futures):
                result = future.result()
                if result:
                    self.results.append(result)

        # Calculate match rate
        valid_results = [r for r in self.results if r is not None]
        if not valid_results:
            logger.error(\"No valid comparison results\")
            return 0.0
        match_rate = sum(r.match for r in valid_results) / len(valid_results)
        logger.info(f\"Migration validation complete: {match_rate:.1%} match rate across {len(valid_results)} PRs\")
        return match_rate

    def generate_report(self, output_path: str = \"migration_report.json\") -> None:
        \"\"\"Save validation results to JSON for audit\"\"\"
        with open(output_path, \"w\") as f:
            json.dump(
                [r.__dict__ for r in self.results if r is not None],
                f,
                indent=2
            )
        logger.info(f\"Migration report saved to {output_path}\")

if __name__ == \"__main__\":
    # Initialize clients (legacy client from first code block)
    from legacy_client import LegacyLLMReviewClient  # Assume first code block is saved as legacy_client.py
    legacy_client = LegacyLLMReviewClient()

    # Initialize Claude Code 2.0 client: https://github.com/anthropics/claude-code
    claude_client = claude_code.Client(
        api_key=os.getenv(\"CLAUDE_API_KEY\"),
        timeout=15  # 15s timeout, longer than legacy 10s
    )

    # Load sample PR batch from S3 (pseudo-code for brevity, real impl uses boto3)
    pr_batch = [
        # (pr_id, diff, context) tuples loaded from production data
        (\"PR-1234\", \"diff --git a/src/main.py b/src/main.py\\n...\", {\"repo\": \"acme-payments\", \"author\": \"jdoe\"}),
        # ... 99 more entries
    ]

    validator = ClaudeCodeMigrationValidator(legacy_client, claude_client, sample_size=100)
    match_rate = validator.run_validation(pr_batch)
    validator.generate_report()

    if match_rate >= 0.95:
        logger.info(\"Match rate >= 95%, safe to cut over to Claude Code 2.0\")
    else:
        logger.error(f\"Match rate {match_rate:.1%} below 95% threshold, investigate mismatches\")

import os
import time
import json
import logging
from typing import Dict, List, Optional
from dataclasses import dataclass
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
import claude_code  # https://github.com/anthropics/claude-code
from claude_code.exceptions import ClaudeAPIError, InvalidDiffError
from prometheus_client import Counter, Histogram, generate_latest
from prometheus_client.exposition import CONTENT_TYPE_LATEST
import uvicorn

# Configure logging
logging.basicConfig(level=logging.INFO, format=\"%(asctime)s - %(levelname)s - %(message)s\")
logger = logging.getLogger(__name__)

# Prometheus metrics
REVIEW_REQUESTS = Counter(\"claude_code_review_requests_total\", \"Total Claude Code review requests\", [\"status\"])
REVIEW_LATENCY = Histogram(\"claude_code_review_latency_seconds\", \"Claude Code review latency\")
ISSUE_COUNTS = Histogram(\"claude_code_issue_count_per_review\", \"Number of issues per review\")

app = FastAPI(title=\"Claude Code 2.0 Review Pipeline\", version=\"2.0.0\")
claude_client = claude_code.Client(
    api_key=os.getenv(\"CLAUDE_API_KEY\"),
    base_url=\"https://api.claude.ai/v1\"  # Production Claude Code endpoint
)

@dataclass
class PRWebhook:
    \"\"\"GitHub PR webhook payload schema\"\"\"
    pr_id: str
    repo: str
    diff_url: str
    author: str
    commit_count: int

def fetch_pr_diff(diff_url: str) -> str:
    \"\"\"Fetch unified diff from GitHub diff URL\"\"\"
    import requests
    try:
        response = requests.get(diff_url, timeout=5)
        response.raise_for_status()
        return response.text
    except Exception as e:
        logger.error(f\"Failed to fetch diff from {diff_url}: {str(e)}\")
        raise HTTPException(status_code=400, detail=f\"Failed to fetch PR diff: {str(e)}\")

@app.post(\"/webhook/pr\")
async def handle_pr_webhook(request: Request):
    \"\"\"Handle GitHub PR open/update webhook, trigger Claude Code review\"\"\"
    start_time = time.time()
    try:
        payload = await request.json()
        # Parse webhook (simplified for brevity, real impl validates signature)
        pr_id = f\"PR-{payload['pull_request']['number']}\"
        repo = payload[\"repository\"][\"full_name\"]
        diff_url = payload[\"pull_request\"][\"diff_url\"]
        author = payload[\"pull_request\"][\"user\"][\"login\"]
        commit_count = payload[\"pull_request\"][\"commits\"]

        pr_webhook = PRWebhook(
            pr_id=pr_id,
            repo=repo,
            diff_url=diff_url,
            author=author,
            commit_count=commit_count
        )
        logger.info(f\"Processing webhook for {pr_id} in {repo}\")

        # Fetch diff
        diff = fetch_pr_diff(pr_webhook.diff_url)

        # Run Claude Code 2.0 review
        # API reference: https://github.com/anthropics/claude-code/blob/main/docs/api.md#review-diff
        review_start = time.time()
        try:
            review_result = claude_client.review_diff(
                diff=diff,
                repo=pr_webhook.repo,
                pr_id=pr_webhook.pr_id,
                author=pr_webhook.author,
                commit_count=pr_webhook.commit_count,
                checks=[\"style\", \"security\", \"test_coverage\", \"breaking_changes\", \"dependency_issues\"],
                model=\"claude-code-2.0.1\",
                temperature=0.1  # Low temperature for deterministic results
            )
        except InvalidDiffError as e:
            logger.error(f\"Invalid diff for {pr_id}: {str(e)}\")
            REVIEW_REQUESTS.labels(status=\"invalid_diff\").inc()
            raise HTTPException(status_code=400, detail=f\"Invalid PR diff: {str(e)}\")
        except RateLimitError as e:
            logger.warning(f\"Rate limited for {pr_id}: {str(e)}\")
            REVIEW_REQUESTS.labels(status=\"rate_limited\").inc()
            raise HTTPException(status_code=429, detail=f\"Rate limited, retry after {e.retry_after}s\")
        except ClaudeAPIError as e:
            logger.error(f\"Claude API error for {pr_id}: {str(e)}\")
            REVIEW_REQUESTS.labels(status=\"api_error\").inc()
            raise HTTPException(status_code=502, detail=f\"Claude API error: {str(e)}\")
        except Exception as e:
            logger.error(f\"Unexpected error reviewing {pr_id}: {str(e)}\")
            REVIEW_REQUESTS.labels(status=\"error\").inc()
            raise HTTPException(status_code=500, detail=f\"Review failed: {str(e)}\")

        # Record metrics
        review_latency = time.time() - review_start
        REVIEW_LATENCY.observe(review_latency)
        REVIEW_REQUESTS.labels(status=\"success\").inc()
        ISSUE_COUNTS.observe(len(review_result.issues))

        # Post result back to GitHub as a comment
        # Simplified: real impl uses PyGithub to post comment
        logger.info(f\"Review for {pr_id} complete: passed={review_result.passed}, issues={len(review_result.issues)}\")

        return JSONResponse(
            status_code=200,
            content={
                \"pr_id\": pr_id,
                \"passed\": review_result.passed,
                \"issues\": review_result.issues,
                \"latency_ms\": int(review_latency * 1000),
                \"model_version\": review_result.model_version
            }
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f\"Webhook processing failed: {str(e)}\")
        REVIEW_REQUESTS.labels(status=\"error\").inc()
        raise HTTPException(status_code=500, detail=f\"Webhook processing failed: {str(e)}\")
    finally:
        total_latency = time.time() - start_time
        logger.info(f\"Webhook processed in {total_latency:.2f}s\")

@app.get(\"/metrics\")
async def get_metrics():
    \"\"\"Expose Prometheus metrics\"\"\"
    return generate_latest(), 200, {\"Content-Type\": CONTENT_TYPE_LATEST}

if __name__ == \"__main__\":
    uvicorn.run(app, host=\"0.0.0.0\", port=8080)

import logging
logger = logging.getLogger(__name__)

def log_shadow_result(pr_id: str, legacy_result: dict, claude_result: dict):
    logger.info(f\"SHADOW|{pr_id}|legacy_passed={legacy_result['passed']}|claude_passed={claude_result['passed']}|legacy_issues={len(legacy_result['issues'])}|claude_issues={len(claude_result['issues'])}\")

SYSTEM_PROMPT = \"\"\"You are a code reviewer for the Acme Payments team. Follow these rules:
1. Use our style guide: https://github.com/acme-payments/style-guide
2. Skip flagging TODOs in test files
3. Prioritize security > breaking changes > test coverage > style
4. Only flag dependencies with known CVEs (check NIST database)\"\"\"

review_result = claude_client.review_diff(
    diff=diff,
    system_prompt=SYSTEM_PROMPT,
    temperature=0.1
)

groups:
- name: claude_code_alerts
  rules:
  - alert: ClaudeCodeHighErrorRate
    expr: rate(claude_code_review_requests_total{status!=\"success\"}[10m]) / rate(claude_code_review_requests_total[10m]) > 0.01
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: \"Claude Code error rate > 1% for 10 minutes\"
      description: \"Current error rate: {{ $value | humanizePercentage }}\"