The Performance Battle compliance with Gitleaks and Snyk: Results

#!/usr/bin/env python3
"""
Gitleaks vs Snyk Batch Benchmark Tool
Scans a directory of Git repositories and compares scan performance, accuracy, and cost.
Requires: gitleaks (v8.18.0+), snyk (v1.1200.0+), git
"""

import argparse
import json
import logging
import os
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Tuple

# Configure logging for audit trails
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

class SecretScannerBenchmark:
    """Benchmark runner for Gitleaks and Snyk secret detection scans."""

    def __init__(self, repo_dir: Path, gitleaks_path: str = "gitleaks", snyk_path: str = "snyk"):
        self.repo_dir = repo_dir.resolve()
        self.gitleaks_path = gitleaks_path
        self.snyk_path = snyk_path
        self.results: List[Dict] = []

        if not self.repo_dir.exists() or not self.repo_dir.is_dir():
            raise FileNotFoundError(f"Repository directory {self.repo_dir} does not exist")

        # Verify tools are installed
        self._verify_tool(self.gitleaks_path, "gitleaks")
        self._verify_tool(self.snyk_path, "snyk")

    def _verify_tool(self, tool_path: str, tool_name: str) -> None:
        """Check if a required tool is installed and accessible."""
        try:
            subprocess.run(
                [tool_path, "--version"],
                capture_output=True,
                text=True,
                check=True,
                timeout=10
            )
            logger.info(f"Verified {tool_name} is installed at {tool_path}")
        except (subprocess.CalledProcessError, FileNotFoundError) as e:
            logger.error(f"Failed to verify {tool_name}: {e}")
            sys.exit(1)

    def _run_gitleaks_scan(self, repo_path: Path) -> Tuple[float, int, int]:
        """
        Run Gitleaks scan on a single repository.
        Returns: (scan_duration_sec, secrets_found, exit_code)
        """
        start_time = time.perf_counter()
        try:
            result = subprocess.run(
                [
                    self.gitleaks_path,
                    "detect",
                    "--source", str(repo_path),
                    "--report-format", "json",
                    "--report-path", f"/tmp/gitleaks_{repo_path.name}.json",
                    "--verbose"
                ],
                capture_output=True,
                text=True,
                timeout=300  # 5 minute timeout per repo
            )
            duration = time.perf_counter() - start_time

            # Gitleaks exits 1 if secrets found, 0 if none, >1 on error
            secrets_found = 0
            if result.returncode == 1:
                # Parse report to get actual count
                report_path = Path(f"/tmp/gitleaks_{repo_path.name}.json")
                if report_path.exists():
                    with open(report_path, "r") as f:
                        report = json.load(f)
                        secrets_found = len(report)
                    report_path.unlink()  # Clean up
            elif result.returncode > 1:
                logger.warning(f"Gitleaks scan failed for {repo_path.name}: {result.stderr}")

            return duration, secrets_found, result.returncode

        except subprocess.TimeoutExpired:
            logger.error(f"Gitleaks scan timed out for {repo_path.name}")
            return 300.0, 0, -1

    def _run_snyk_scan(self, repo_path: Path) -> Tuple[float, int, int]:
        """
        Run Snyk secret scan on a single repository.
        Returns: (scan_duration_sec, secrets_found, exit_code)
        """
        start_time = time.perf_counter()
        try:
            result = subprocess.run(
                [
                    self.snyk_path,
                    "code",
                    "test",
                    str(repo_path),
                    "--json",
                    "--output", f"/tmp/snyk_{repo_path.name}.json"
                ],
                capture_output=True,
                text=True,
                timeout=300
            )
            duration = time.perf_counter() - start_time

            secrets_found = 0
            # Snyk exits 1 if issues found, 0 if none, >1 on error
            if result.returncode == 1:
                report_path = Path(f"/tmp/snyk_{repo_path.name}.json")
                if report_path.exists():
                    with open(report_path, "r") as f:
                        report = json.load(f)
                        # Snyk secret findings are under 'vulnerabilities' with type 'secret'
                        secrets_found = len([
                            v for v in report.get("vulnerabilities", [])
                            if v.get("type") == "secret"
                        ])
                    report_path.unlink()
            elif result.returncode > 1:
                logger.warning(f"Snyk scan failed for {repo_path.name}: {result.stderr}")

            return duration, secrets_found, result.returncode

        except subprocess.TimeoutExpired:
            logger.error(f"Snyk scan timed out for {repo_path.name}")
            return 300.0, 0, -1

    def run_benchmark(self) -> None:
        """Iterate over all repos in repo_dir and run scans for both tools."""
        repos = [p for p in self.repo_dir.iterdir() if p.is_dir() and (p / ".git").exists()]
        logger.info(f"Found {len(repos)} Git repositories to scan")

        for repo in repos:
            logger.info(f"Scanning {repo.name}...")
            g_duration, g_secrets, g_code = self._run_gitleaks_scan(repo)
            s_duration, s_secrets, s_code = self._run_snyk_scan(repo)

            self.results.append({
                "repo_name": repo.name,
                "gitleaks_duration_sec": round(g_duration, 2),
                "gitleaks_secrets_found": g_secrets,
                "gitleaks_exit_code": g_code,
                "snyk_duration_sec": round(s_duration, 2),
                "snyk_secrets_found": s_secrets,
                "snyk_exit_code": s_code,
                "scan_timestamp": datetime.utcnow().isoformat()
            })

            # Log interim results
            logger.info(f"Completed {repo.name}: Gitleaks {g_duration:.2f}s ({g_secrets} secrets), Snyk {s_duration:.2f}s ({s_secrets} secrets)")

    def save_results(self, output_path: Path) -> None:
        """Save benchmark results to JSON file."""
        with open(output_path, "w") as f:
            json.dump({
                "benchmark_metadata": {
                    "tool": "Gitleaks vs Snyk Benchmark",
                    "gitleaks_version": subprocess.run([self.gitleaks_path, "--version"], capture_output=True, text=True).stdout.strip(),
                    "snyk_version": subprocess.run([self.snyk_path, "--version"], capture_output=True, text=True).stdout.strip(),
                    "repo_count": len(self.results),
                    "run_date": datetime.utcnow().isoformat()
                },
                "results": self.results
            }, f, indent=2)
        logger.info(f"Saved results to {output_path}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Benchmark Gitleaks and Snyk secret scanning performance")
    parser.add_argument("--repo-dir", type=Path, required=True, help="Directory containing Git repositories to scan")
    parser.add_argument("--output", type=Path, default=Path("benchmark_results.json"), help="Output path for results JSON")
    parser.add_argument("--gitleaks-path", type=str, default="gitleaks", help="Path to Gitleaks binary")
    parser.add_argument("--snyk-path", type=str, default="snyk", help="Path to Snyk CLI binary")

    args = parser.parse_args()

    try:
        benchmark = SecretScannerBenchmark(
            repo_dir=args.repo_dir,
            gitleaks_path=args.gitleaks_path,
            snyk_path=args.snyk_path
        )
        benchmark.run_benchmark()
        benchmark.save_results(args.output)
    except Exception as e:
        logger.error(f"Benchmark failed: {e}")
        sys.exit(1)

name: Secret Compliance Scan
on:
  push:
    branches: [ main, release/* ]
  pull_request:
    branches: [ main ]

permissions:
  contents: read
  security-events: write
  pull-requests: write

jobs:
  secret-scan:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Fetch all history for full commit scan

      - name: Install Gitleaks v8.18.0
        run: |
          curl -sSfL https://raw.githubusercontent.com/gitleaks/gitleaks/master/scripts/install.sh | sh -s -- v8.18.0
          sudo install gitleaks /usr/local/bin/gitleaks
          gitleaks --version  # Verify install
        shell: bash

      - name: Install Snyk v1.1200.0
        run: |
          npm install -g snyk@1.1200.0
          snyk --version  # Verify install
        shell: bash

      - name: Run Gitleaks scan
        id: gitleaks
        continue-on-error: true  # We'll handle failure manually to compare results
        run: |
          gitleaks detect \
            --source . \
            --report-format sarif \
            --report-path gitleaks.sarif \
            --exit-code 0  # Don't fail here, we'll check report
          echo "gitleaks_secrets=$(cat gitleaks.sarif | jq '.runs[0].results | length')" >> $GITHUB_OUTPUT
        shell: bash

      - name: Run Snyk secret scan
        id: snyk
        continue-on-error: true
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        run: |
          snyk code test \
            --json \
            --output snyk_report.json \
            --skip-unreachable \
            --include-ignored-paths="node_modules/,vendor/" \
            2>&1 | tee snyk_output.log

          # Extract secret count from Snyk report
          SECRET_COUNT=$(cat snyk_report.json | jq '[.vulnerabilities[] | select(.type == "secret")] | length')
          echo "snyk_secrets=$SECRET_COUNT" >> $GITHUB_OUTPUT

          # Check for scan errors
          if grep -q "Authentication failed" snyk_output.log; then
            echo "snyk_error=auth_failed" >> $GITHUB_OUTPUT
          elif grep -q "rate limit" snyk_output.log; then
            echo "snyk_error=rate_limited" >> $GITHUB_OUTPUT
          else
            echo "snyk_error=none" >> $GITHUB_OUTPUT
          fi
        shell: bash

      - name: Compare scan results
        id: compare
        run: |
          GITLEAKS_COUNT="${{ steps.gitleaks.outputs.gitleaks_secrets }}"
          SNYK_COUNT="${{ steps.snyk.outputs.snyk_secrets }}"
          SNYK_ERROR="${{ steps.snyk.outputs.snyk_error }}"

          echo "Gitleaks found $GITLEAKS_COUNT secrets"
          echo "Snyk found $SNYK_COUNT secrets (error: $SNYK_ERROR)"

          # Fail if either tool found secrets, unless Snyk had an auth error
          if [ "$GITLEAKS_COUNT" -gt 0 ] || ([ "$SNYK_COUNT" -gt 0 ] && [ "$SNYK_ERROR" = "none" ]); then
            echo "SECRETS_FOUND=true" >> $GITHUB_OUTPUT
            # Generate human-readable report
            echo "## Secret Compliance Failure" >> $GITHUB_STEP_SUMMARY
            echo "- Gitleaks detected $GITLEAKS_COUNT potential secrets" >> $GITHUB_STEP_SUMMARY
            echo "- Snyk detected $SNYK_COUNT potential secrets" >> $GITHUB_STEP_SUMMARY
            echo "Review the attached SARIF and JSON reports for details." >> $GITHUB_STEP_SUMMARY
          else
            echo "SECRETS_FOUND=false" >> $GITHUB_OUTPUT
            echo "## Secret Compliance Passed" >> $GITHUB_STEP_SUMMARY
            echo "No secrets detected by Gitleaks or Snyk." >> $GITHUB_STEP_SUMMARY
          fi
        shell: bash

      - name: Upload Gitleaks SARIF
        if: always()
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: gitleaks.sarif
          category: gitleaks-secret-scan

      - name: Upload Snyk report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: snyk-report
          path: snyk_report.json

      - name: Fail build if secrets found
        if: steps.compare.outputs.SECRETS_FOUND == 'true'
        run: |
          echo "::error::Secrets detected in commit. Remove secrets and re-push."
          exit 1
        shell: bash

      - name: Alert on Snyk errors
        if: steps.snyk.outputs.snyk_error != 'none'
        run: |
          echo "::warning::Snyk scan encountered error: ${{ steps.snyk.outputs.snyk_error }}"
        shell: bash

package main

import (
    "encoding/json"
    "fmt"
    "log"
    "os"
    "path/filepath"
    "sort"
    "time"
)

// BenchmarkResult represents a single repo scan result from the Python benchmark tool
type BenchmarkResult struct {
    RepoName            string  `json:"repo_name"`
    GitleaksDurationSec float64 `json:"gitleaks_duration_sec"`
    GitleaksSecrets     int     `json:"gitleaks_secrets_found"`
    GitleaksExitCode    int     `json:"gitleaks_exit_code"`
    SnykDurationSec     float64 `json:"snyk_duration_sec"`
    SnykSecrets         int     `json:"snyk_secrets_found"`
    SnykExitCode        int     `json:"snyk_exit_code"`
    ScanTimestamp       string  `json:"scan_timestamp"`
}

// BenchmarkMetadata contains metadata about the benchmark run
type BenchmarkMetadata struct {
    Tool            string `json:"tool"`
    GitleaksVersion string `json:"gitleaks_version"`
    SnykVersion     string `json:"snyk_version"`
    RepoCount       int    `json:"repo_count"`
    RunDate         string `json:"run_date"`
}

// FullBenchmarkReport is the top-level structure of the benchmark JSON
type FullBenchmarkReport struct {
    Metadata BenchmarkMetadata  `json:"benchmark_metadata"`
    Results  []BenchmarkResult `json:"results"`
}

// ToolMetrics holds aggregated performance metrics for a single tool
type ToolMetrics struct {
    TotalScans          int
    TotalSecretsFound   int
    TotalDurationSec    float64
    AvgDurationPerRepo  float64
    AvgSecretsPerRepo   float64
    FalsePositiveCount  int
    FalsePositiveRate  float64
    CostPer10kCommits   float64
}

func main() {
    if len(os.Args) < 2 {
        log.Fatal("Usage: go run compliance_report.go ")
    }

    inputPath := os.Args[1]
    report, err := loadBenchmarkReport(inputPath)
    if err != nil {
        log.Fatalf("Failed to load benchmark report: %v", err)
    }

    // Calculate metrics for Gitleaks and Snyk
    gitleaksMetrics := calculateMetrics(report, "gitleaks")
    snykMetrics := calculateMetrics(report, "snyk")

    // Generate compliance report
    outputPath := filepath.Join("compliance_report_"+time.Now().Format("20060102_150405")+".md")
    err = generateMarkdownReport(report.Metadata, gitleaksMetrics, snykMetrics, outputPath)
    if err != nil {
        log.Fatalf("Failed to generate report: %v", err)
    }

    log.Printf("Compliance report generated at %s", outputPath)
}

func loadBenchmarkReport(path string) (FullBenchmarkReport, error) {
    var report FullBenchmarkReport
    data, err := os.ReadFile(path)
    if err != nil {
        return report, fmt.Errorf("read file: %w", err)
    }

    err = json.Unmarshal(data, &report)
    if err != nil {
        return report, fmt.Errorf("unmarshal json: %w", err)
    }

    // Validate report has results
    if len(report.Results) == 0 {
        return report, fmt.Errorf("no benchmark results found in file")
    }

    return report, nil
}

func calculateMetrics(report FullBenchmarkReport, tool string) ToolMetrics {
    var metrics ToolMetrics
    metrics.TotalScans = len(report.Results)

    for _, res := range report.Results {
        if tool == "gitleaks" {
            metrics.TotalDurationSec += res.GitleaksDurationSec
            metrics.TotalSecretsFound += res.GitleaksSecrets
            // Assume exit code >1 is tool error
            if res.GitleaksExitCode > 1 {
                metrics.FalsePositiveCount++
            }
        } else if tool == "snyk" {
            metrics.TotalDurationSec += res.SnykDurationSec
            metrics.TotalSecretsFound += res.SnykSecrets
            if res.SnykExitCode > 1 {
                metrics.FalsePositiveCount++
            }
        }
    }

    // Calculate averages
    metrics.AvgDurationPerRepo = metrics.TotalDurationSec / float64(metrics.TotalScans)
    metrics.AvgSecretsPerRepo = float64(metrics.TotalSecretsFound) / float64(metrics.TotalScans)
    metrics.FalsePositiveRate = float64(metrics.FalsePositiveCount) / float64(metrics.TotalScans) * 100

    // Cost calculation: assume 100 commits per repo, 10k commits total
    // Gitleaks: $0.12 per 10k commits, Snyk: $4.80 per 10k commits
    if tool == "gitleaks" {
        metrics.CostPer10kCommits = 0.12
    } else {
        metrics.CostPer10kCommits = 4.80
    }

    return metrics
}

func generateMarkdownReport(meta BenchmarkMetadata, gitleaks, snyk ToolMetrics, outputPath string) error {
    // Sort repos by Gitleaks duration to find slowest scans
    sort.SliceStable(report.Results, func(i, j int) bool {
        return report.Results[i].GitleaksDurationSec > report.Results[j].GitleaksDurationSec
    })

    reportContent := fmt.Sprintf(`# Secret Compliance Benchmark Report
Generated: %s
Tools Tested: Gitleaks %s, Snyk %s
Repositories Scanned: %d

## Performance Comparison
| Metric | Gitleaks v%s | Snyk v%s |
|--------|--------------|----------|
| Avg Scan Time per Repo | %.2f sec | %.2f sec |
| Total Secrets Found | %d | %d |
| False Positive Rate | %.2f%% | %.2f%% |
| Cost per 10k Commits | $%.2f | $%.2f |

## Top 5 Slowest Repos (Gitleaks)
`, time.Now().Format(time.RFC3339), meta.GitleaksVersion, meta.SnykVersion, meta.RepoCount, meta.GitleaksVersion, meta.SnykVersion, gitleaks.AvgDurationPerRepo, snyk.AvgDurationPerRepo, gitleaks.TotalSecretsFound, snyk.TotalSecretsFound, gitleaks.FalsePositiveRate, snyk.FalsePositiveRate, gitleaks.CostPer10kCommits, snyk.CostPer10kCommits)

    // Add top 5 slowest repos
    count := 5
    if len(report.Results) < 5 {
        count = len(report.Results)
    }
    for i := 0; i < count; i++ {
        res := report.Results[i]
        reportContent += fmt.Sprintf("%d. %s: %.2f sec (Gitleaks), %.2f sec (Snyk)\n", i+1, res.RepoName, res.GitleaksDurationSec, res.SnykDurationSec)
    }

    // Write to file
    err := os.WriteFile(outputPath, []byte(reportContent), 0644)
    if err != nil {
        return fmt.Errorf("write report: %w", err)
    }

    return nil
}

# .pre-commit-config.yaml
repos:
  - repo: https://github.com/gitleaks/gitleaks
    rev: v8.18.0
    hooks:
      - id: gitleaks
        args: ["protect", "--verbose", "--staged", "--no-color"]
        timeout: 5

# .gitleaksignore
# Ignore test fixtures with fake secrets
tests/**/*

# Ignore documentation examples
docs/**/*.md

# Custom rule to exclude placeholder secrets
# Add to gitleaks.toml:
# [[rules]]
# id = "ignore-placeholder-secrets"
# description = "Ignore placeholder API keys"
# regex = "API_KEY=replace_me|SECRET_KEY=changeme"
# paths = [{regex = ".*"}]

# Dockerfile for Gitleaks scan worker
FROM golang:1.21-alpine AS builder
WORKDIR /app
COPY benchmark_tool.go .
RUN go build -o gitleaks-benchmark .

FROM alpine:latest
RUN apk add --no-cache git gitleaks
COPY --from=builder /app/gitleaks-benchmark /usr/local/bin/
ENTRYPOINT ["gitleaks-benchmark"]