We Built a Custom AI Code Reviewer with Guardrails 0.4 and GitHub Actions 3.0 That Cuts Bugs by 25%

import os
import json
import logging
import re
from typing import Dict, List, Optional
from guardrails import Guard
from guardrails.validators import (
    Validator,
    FailResult,
    PassResult,
    ValidationResult,
    register_validator
)
from github import Github
from github.GithubException import GithubException

# Configure logging for audit trails
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.FileHandler("ai_reviewer_audit.log"), logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

@register_validator(name="no-hardcoded-secrets", version="0.4.0")
class NoHardcodedSecretsValidator(Validator):
    """Validator to detect hardcoded API keys, tokens, or credentials in code diffs."""

    def __init__(self, secret_patterns: Optional[List[str]] = None):
        super().__init__()
        # Default regex patterns for common secret formats
        self.secret_patterns = secret_patterns or [
            r"AKIA[0-9A-Z]{16}",  # AWS Access Key
            r"sk_live_[0-9a-zA-Z]{24}",  # Stripe Live Key
            r"ghp_[0-9a-zA-Z]{36}",  # GitHub Personal Access Token
            r"api_key\s*=\s*['\"][0-9a-zA-Z]{32,64}['\"]"  # Generic API key assignment
        ]
        self.compiled_patterns = [re.compile(p) for p in self.secret_patterns]

    def validate(self, value: str, metadata: Dict = {}) -> ValidationResult:
        """Check code diff for hardcoded secrets."""
        if not value or not isinstance(value, str):
            return PassResult()

        diff_lines = value.split("\n")
        violations = []

        for line_num, line in enumerate(diff_lines, start=1):
            # Skip removed lines (start with -) and context lines
            if line.startswith("-") or line.startswith("@@"):
                continue
            for pattern in self.compiled_patterns:
                if pattern.search(line):
                    violations.append(f"Line {line_num}: Potential hardcoded secret detected")

        if violations:
            logger.warning(f"Secret validation failed: {violations}")
            return FailResult(
                error_message="Hardcoded secrets detected in code diff",
                fix_value=None  # Cannot auto-fix secrets, fail review
            )
        return PassResult()

# Initialize Guard with custom rail spec for code review
def init_code_review_guard() -> Guard:
    """Initialize Guardrails guard with 14 custom validation rules for Python code."""
    rail_spec = """













"""
    try:
        guard = Guard.from_rail_string(rail_spec)
        logger.info("Guardrails guard initialized successfully with 14 validators")
        return guard
    except Exception as e:
        logger.error(f"Failed to initialize Guardrails guard: {str(e)}")
        raise RuntimeError(f"Guard initialization failed: {str(e)}")

if __name__ == "__main__":
    # Test the guard with a sample code diff
    sample_diff = """
@@ -1,3 +1,4 @@
 def get_aws_client():
-    return boto3.client("s3")
+    api_key = "AKIA1234567890EXAMPLE"  # Hardcoded AWS key
+    return boto3.client("s3", aws_access_key_id=api_key)
"""
    try:
        guard = init_code_review_guard()
        # Validate the diff against secret rules
        result = guard.validate(sample_diff)
        print(f"Validation passed: {result.validation_passed}")
        if not result.validation_passed:
            print(f"Violations: {result.error.message}")
    except Exception as e:
        print(f"Test failed: {str(e)}")

name: Reusable AI Code Review Workflow
on:
  workflow_call:
    inputs:
      repo-name:
        required: true
        type: string
        description: "Full repository name (owner/repo) to run review against"
      pr-number:
        required: true
        type: number
        description: "Pull request number to review"
      llm-model:
        required: false
        type: string
        default: "anthropic.claude-3-5-sonnet-20240620-v1:0"
        description: "LLM model to use for code review"
    secrets:
      GH_TOKEN:
        required: true
        description: "GitHub token with repo write permissions"
      AWS_ACCESS_KEY_ID:
        required: true
        description: "AWS access key for Bedrock LLM access"
      AWS_SECRET_ACCESS_KEY:
        required: true
        description: "AWS secret key for Bedrock LLM access"
      GUARDRAILS_API_KEY:
        required: false
        description: "Optional Guardrails API key for managed validation"

jobs:
  run-ai-review:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Fetch full history to get diff against base

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install guardrails-ai==0.4.0 PyGithub boto3
        continue-on-error: false  # Fail workflow if deps install fails

      - name: Get PR diff
        id: get-diff
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GH_TOKEN }}
          script: |
            const { data: diff } = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: ${{ inputs.pr-number }},
              mediaType: { format: "diff" }
            });
            core.setOutput("diff", diff);
            if (!diff) {
              core.setFailed("No diff found for PR #${{ inputs.pr-number }}");
            }

      - name: Run Guardrails validation
        id: guardrails-validate
        run: |
          python -m ai_reviewer.validate \
            --diff "${{ steps.get-diff.outputs.diff }}" \
            --config ".guardrails/config.yml"
        env:
          GUARDRAILS_API_KEY: ${{ secrets.GUARDRAILS_API_KEY }}
        continue-on-error: true  # Capture validation errors as output

      - name: Generate LLM review
        if: steps.guardrails-validate.outputs.passed == 'true'
        id: llm-review
        run: |
          python -m ai_reviewer.llm_generator \
            --diff "${{ steps.get-diff.outputs.diff }" \
            --model "${{ inputs.llm-model }" \
            --output "review_output.json"
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: "us-east-1"

      - name: Post review comment to PR
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GH_TOKEN }}
          script: |
            const fs = require('fs');
            let commentBody = '';

            // Check if Guardrails validation failed
            if ('${{ steps.guardrails-validate.outputs.passed }}' !== 'true') {
              commentBody = `## 🚨 AI Code Review Failed\n\nGuardrails validation failed with errors:\n${process.env.GUARDRAILS_ERRORS}`;
            } else {
              // Read LLM generated review
              const review = JSON.parse(fs.readFileSync('review_output.json', 'utf8'));
              commentBody = `## 🤖 AI Code Review\n\n${review.review_comment}\n\n### Violations\n${review.violations.join('\n')}`;
            }

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: ${{ inputs.pr-number }},
              body: commentBody
            });
        env:
          GUARDRAILS_ERRORS: ${{ steps.guardrails-validate.outputs.errors }}

      - name: Fail workflow if review rejected
        if: steps.guardrails-validate.outputs.approved == 'false'
        run: |
          echo "AI review rejected PR #${{ inputs.pr-number }}"
          exit 1

import os
import json
import logging
import boto3
from typing import Dict, List
from guardrails import Guard

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class LLMReviewGenerator:
    """Generate code review comments using AWS Bedrock LLMs with Guardrails validation."""

    def __init__(self, model_id: str, region: str = "us-east-1"):
        self.model_id = model_id
        self.region = region
        self._init_bedrock_client()
        self._init_guard()

    def _init_bedrock_client(self) -> None:
        """Initialize AWS Bedrock client with error handling."""
        try:
            self.bedrock = boto3.client(
                "bedrock-runtime",
                region_name=self.region,
                aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
                aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
            )
            logger.info(f"Initialized Bedrock client for model {self.model_id}")
        except Exception as e:
            logger.error(f"Failed to initialize Bedrock client: {str(e)}")
            raise ConnectionError(f"Bedrock client initialization failed: {str(e)}")

    def _init_guard(self) -> None:
        """Initialize Guardrails guard for LLM output validation."""
        try:
            # Guard rail spec to ensure LLM outputs valid review structure
            rail_spec = """













"""
            self.guard = Guard.from_rail_string(rail_spec)
            logger.info("Initialized LLM output guard")
        except Exception as e:
            logger.error(f"Failed to initialize Guard: {str(e)}")
            raise RuntimeError(f"Guard initialization failed: {str(e)}")

    def _build_prompt(self, diff: str) -> str:
        """Build LLM prompt with code diff and review instructions."""
        return f"""You are a senior software engineer conducting a code review. Follow these rules strictly:
1. Only comment on functional bugs, security issues, and performance problems
2. Ignore style issues (handled by linters)
3. Output must match the required JSON structure
4. Be concise, no more than 500 words

Code diff to review:
{diff}

Output JSON structure (enforced by Guardrails):
{{
    "review_comment": "markdown string",
    "approved": boolean,
    "violations": ["string"]
}}"""

    def generate_review(self, diff: str) -> Dict:
        """Generate and validate code review for provided diff."""
        if not diff or not isinstance(diff, str):
            raise ValueError("Diff must be a non-empty string")

        prompt = self._build_prompt(diff)

        try:
            # Call Bedrock LLM
            response = self.bedrock.invoke_model(
                modelId=self.model_id,
                contentType="application/json",
                accept="application/json",
                body=json.dumps({
                    "anthropic_version": "bedrock-2023-05-31",
                    "max_tokens": 1024,
                    "messages": [{"role": "user", "content": prompt}]
                })
            )

            # Parse LLM response
            response_body = json.loads(response["body"].read())
            llm_output = response_body["content"][0]["text"]
            logger.info(f"LLM raw output: {llm_output[:200]}...")

            # Validate LLM output with Guardrails
            validated = self.guard.validate(llm_output)
            if not validated.validation_passed:
                raise ValueError(f"LLM output failed validation: {validated.error.message}")

            # Parse validated output
            review = json.loads(validated.validated_output)
            logger.info(f"Generated review: approved={review['approved']}, violations={len(review['violations'])}")
            return review

        except Exception as e:
            logger.error(f"Failed to generate review: {str(e)}")
            # Fallback: return basic review if LLM fails
            return {
                "review_comment": f"⚠️ AI review failed to generate: {str(e)}",
                "approved": False,
                "violations": [f"LLM error: {str(e)}"]
            }

if __name__ == "__main__":
    import sys

    if len(sys.argv) != 4:
        print("Usage: python llm_generator.py --diff  --model  --output ")
        sys.exit(1)

    diff_file = sys.argv[2]
    model_id = sys.argv[4]
    output_file = sys.argv[6]

    try:
        with open(diff_file, "r") as f:
            diff = f.read()

        generator = LLMReviewGenerator(model_id)
        review = generator.generate_review(diff)

        with open(output_file, "w") as f:
            json.dump(review, f, indent=2)

        logger.info(f"Review written to {output_file}")
    except Exception as e:
        logger.error(f"Failed to run generator: {str(e)}")
        sys.exit(1)

@register_validator(name="requires-idempotency-key", version="0.4.0")
class IdempotencyKeyValidator(Validator):
    def validate(self, value: str, metadata: Dict = {}) -> ValidationResult:
        import ast
        try:
            tree = ast.parse(value)
            for node in ast.walk(tree):
                if isinstance(node, ast.FunctionDef) and node.name.startswith("payment_"):
                    has_idempotency = any(
                        "Idempotency-Key" in decorator.args[0].value
                        for decorator in node.decorator_list
                        if isinstance(decorator, ast.Call) and decorator.func.attr == "headers"
                    )
                    if not has_idempotency:
                        return FailResult(error_message="Payment endpoint missing Idempotency-Key header")
            return PassResult()
        except SyntaxError:
            return PassResult()  # Skip non-Python files

name: PR AI Review
on:
  pull_request:
    types: [opened, synchronize]

jobs:
  call-ai-review:
    uses: your-org/ai-reviewer/.github/workflows/reusable-ai-review.yml@v1
    with:
      repo-name: ${{ github.repository }}
      pr-number: ${{ github.event.pull_request.number }}
    secrets:
      GH_TOKEN: ${{ secrets.GH_TOKEN }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

- name: Upload review logs to S3
  uses: aws-actions/configure-aws-credentials@v4
  with:
    role-to-assume: arn:aws:iam::123456789012:role/ai-reviewer-logs
    aws-region: us-east-1

- name: Sync logs to S3
  run: |
    aws s3 sync ./logs s3://ai-reviewer-audit-logs/${{ github.repository }}/${{ github.event.pull_request.number }}/