Migrate automation Gitleaks checklist SOPS: A Step-by-Step Guide

import argparse
import json
import os
import subprocess
import sys
from typing import Dict, List, Any

# Default Gitleaks version to install if not present
GITLEAKS_VERSION = "8.18.0"
# Supported secret types to enable by default
DEFAULT_RULES = ["aws", "gcp", "azure", "github", "gitlab", "slack", "stripe", "twilio"]

def check_gitleaks_installed() -> bool:
    """Verify Gitleaks is installed and matches target version."""
    try:
        result = subprocess.run(
            ["gitleaks", "version"],
            capture_output=True,
            text=True,
            check=False
        )
        if result.returncode != 0:
            return False
        # Parse version string (e.g., "gitleaks 8.18.0")
        version_str = result.stdout.strip().split()[-1]
        return version_str == GITLEAKS_VERSION
    except FileNotFoundError:
        return False

def install_gitleaks(target_version: str = GITLEAKS_VERSION) -> None:
    """Install Gitleaks via go install if not present."""
    print(f"Installing Gitleaks {target_version}...")
    try:
        # Use go install to get specific version
        subprocess.run(
            [
                "go", "install",
                f"github.com/gitleaks/gitleaks/v8@v{target_version}"
            ],
            check=True,
            capture_output=True
        )
        print(f"Successfully installed Gitleaks {target_version}")
    except subprocess.CalledProcessError as e:
        print(f"Failed to install Gitleaks: {e.stderr.decode()}", file=sys.stderr)
        sys.exit(1)
    except FileNotFoundError:
        print("Go is not installed. Please install Go 1.21+ first.", file=sys.stderr)
        sys.exit(1)

def generate_gitleaks_config(output_path: str, extra_rules: List[str] = None) -> Dict[str, Any]:
    """Generate a custom Gitleaks config with project-specific rules."""
    config = {
        "version": "2.1.0",
        "rules": []
    }
    # Load default rules from Gitleaks built-in config
    default_config_path = os.path.join(
        os.path.expanduser("~"), ".gitleaks", "config", "gitleaks.toml"
    )
    if not os.path.exists(default_config_path):
        print(f"Default Gitleaks config not found at {default_config_path}", file=sys.stderr)
        sys.exit(1)

    # Add user-specified rules
    rules_to_add = DEFAULT_RULES + (extra_rules or [])
    for rule in rules_to_add:
        config["rules"].append({
            "id": rule,
            "enabled": True
        })

    # Write config to output path
    with open(output_path, "w") as f:
        json.dump(config, f, indent=2)
    print(f"Generated Gitleaks config at {output_path}")
    return config

def main():
    parser = argparse.ArgumentParser(description="Generate Gitleaks config and install dependencies")
    parser.add_argument(
        "--output", "-o",
        default=".gitleaks.json",
        help="Path to write Gitleaks config (default: .gitleaks.json)"
    )
    parser.add_argument(
        "--extra-rules", "-e",
        nargs="+",
        help="Additional Gitleaks rule IDs to enable"
    )
    args = parser.parse_args()

    # Check and install Gitleaks if needed
    if not check_gitleaks_installed():
        print("Gitleaks not found or version mismatch. Installing...")
        install_gitleaks()
    else:
        print(f"Gitleaks {GITLEAKS_VERSION} already installed")

    # Generate config
    generate_gitleaks_config(args.output, args.extra_rules)

if __name__ == "__main__":
    main()

import argparse
import base64
import json
import os
import subprocess
import sys
from typing import Dict, Any

# SOPS version to use
SOPS_VERSION = "3.8.1"
# Age public key for encryption (replace with your own)
AGE_PUBLIC_KEY = "age1ql3z7w907l7q6u3z9u3y5z2q4z0z3z8z5z1z4z7z2z9z6z3z0z8z5z2z1"

def check_sops_installed() -> bool:
    """Verify SOPS is installed and matches target version."""
    try:
        result = subprocess.run(
            ["sops", "--version"],
            capture_output=True,
            text=True,
            check=False
        )
        if result.returncode != 0:
            return False
        # Parse version (e.g., "sops 3.8.1")
        version_str = result.stdout.strip().split()[-1]
        return version_str == SOPS_VERSION
    except FileNotFoundError:
        return False

def install_sops(target_version: str = SOPS_VERSION) -> None:
    """Install SOPS via go install."""
    print(f"Installing SOPS {target_version}...")
    try:
        subprocess.run(
            [
                "go", "install",
                f"github.com/getsops/sops/v3/cmd/sops@v{target_version}"
            ],
            check=True,
            capture_output=True
        )
        print(f"Successfully installed SOPS {target_version}")
    except subprocess.CalledProcessError as e:
        print(f"Failed to install SOPS: {e.stderr.decode()}", file=sys.stderr)
        sys.exit(1)
    except FileNotFoundError:
        print("Go is not installed. Please install Go 1.21+ first.", file=sys.stderr)
        sys.exit(1)

def encrypt_secret(secret_path: str, key: str, value: str) -> None:
    """Encrypt a single secret and write to SOPS file."""
    # Create initial plaintext file
    plaintext = {key: value}
    temp_path = f"{secret_path}.tmp"
    with open(temp_path, "w") as f:
        json.dump(plaintext, f, indent=2)

    try:
        # Encrypt with age public key
        subprocess.run(
            [
                "sops",
                "--age", AGE_PUBLIC_KEY,
                "--encrypt", temp_path
            ],
            stdout=open(secret_path, "w"),
            check=True
        )
        print(f"Encrypted secret written to {secret_path}")
    except subprocess.CalledProcessError as e:
        print(f"Failed to encrypt secret: {e.stderr.decode()}", file=sys.stderr)
        sys.exit(1)
    finally:
        # Clean up temp file
        if os.path.exists(temp_path):
            os.remove(temp_path)

def decrypt_secret(secret_path: str, key: str) -> str:
    """Decrypt a SOPS file and return the value for a given key."""
    try:
        result = subprocess.run(
            ["sops", "--decrypt", secret_path],
            capture_output=True,
            text=True,
            check=True
        )
        plaintext = json.loads(result.stdout)
        return plaintext.get(key, "")
    except subprocess.CalledProcessError as e:
        print(f"Failed to decrypt secret: {e.stderr.decode()}", file=sys.stderr)
        sys.exit(1)
    except json.JSONDecodeError:
        print(f"Decrypted content is not valid JSON", file=sys.stderr)
        sys.exit(1)

def main():
    parser = argparse.ArgumentParser(description="Manage SOPS-encrypted secrets")
    subparsers = parser.add_subparsers(dest="command", required=True)

    # Encrypt subcommand
    encrypt_parser = subparsers.add_parser("encrypt", help="Encrypt a secret")
    encrypt_parser.add_argument("--path", "-p", required=True, help="Path to SOPS file")
    encrypt_parser.add_argument("--key", "-k", required=True, help="Secret key")
    encrypt_parser.add_argument("--value", "-v", required=True, help="Secret value")

    # Decrypt subcommand
    decrypt_parser = subparsers.add_parser("decrypt", help="Decrypt a secret")
    decrypt_parser.add_argument("--path", "-p", required=True, help="Path to SOPS file")
    decrypt_parser.add_argument("--key", "-k", required=True, help="Secret key to retrieve")

    args = parser.parse_args()

    # Check and install SOPS if needed
    if not check_sops_installed():
        print("SOPS not found or version mismatch. Installing...")
        install_sops()
    else:
        print(f"SOPS {SOPS_VERSION} already installed")

    if args.command == "encrypt":
        encrypt_secret(args.path, args.key, args.value)
    elif args.command == "decrypt":
        value = decrypt_secret(args.path, args.key)
        print(f"Decrypted value: {value}")

if __name__ == "__main__":
    main()

import argparse
import json
import os
import subprocess
import sys
from typing import List, Dict, Any

# Paths to config files
GITLEAKS_CONFIG = ".gitleaks.json"
SOPS_CONFIG = ".sops.yaml"
# Required environment variables for CI
REQUIRED_ENV_VARS = ["GITHUB_SHA", "GITHUB_REF", "SOPS_AGE_KEY"]

def check_env_vars() -> List[str]:
    """Verify all required CI environment variables are set."""
    missing = []
    for var in REQUIRED_ENV_VARS:
        if var not in os.environ:
            missing.append(var)
    return missing

def run_gitleaks_scan(target_path: str = ".") -> bool:
    """Run Gitleaks scan on target path and return pass/fail."""
    print(f"Running Gitleaks scan on {target_path}...")
    try:
        result = subprocess.run(
            [
                "gitleaks", "detect",
                "--source", target_path,
                "--config", GITLEAKS_CONFIG,
                "--verbose",
                "--report-format", "json",
                "--report-path", "gitleaks-report.json"
            ],
            capture_output=True,
            text=True,
            check=False
        )
        if result.returncode != 0:
            print(f"Gitleaks detected leaks:\n{result.stdout}")
            # Parse report for details
            if os.path.exists("gitleaks-report.json"):
                with open("gitleaks-report.json") as f:
                    report = json.load(f)
                    print(f"Found {len(report)} leaks")
            return False
        print("Gitleaks scan passed: no leaks detected")
        return True
    except FileNotFoundError:
        print("Gitleaks not installed. Run gitleaks_config_generator.py first.", file=sys.stderr)
        return False

def validate_sops_files(sops_dir: str = "secrets") -> bool:
    """Validate all SOPS files in directory are properly encrypted."""
    print(f"Validating SOPS files in {sops_dir}...")
    if not os.path.isdir(sops_dir):
        print(f"SOPS directory {sops_dir} not found", file=sys.stderr)
        return False

    all_valid = True
    for filename in os.listdir(sops_dir):
        if not filename.endswith(".json"):
            continue
        file_path = os.path.join(sops_dir, filename)
        try:
            # Attempt to decrypt file to verify integrity
            result = subprocess.run(
                ["sops", "--decrypt", file_path],
                capture_output=True,
                text=True,
                check=False
            )
            if result.returncode != 0:
                print(f"Invalid SOPS file: {file_path} - {result.stderr.decode()}")
                all_valid = False
            else:
                print(f"Valid SOPS file: {file_path}")
        except FileNotFoundError:
            print("SOPS not installed. Run sops_setup.py first.", file=sys.stderr)
            return False
    return all_valid

def generate_ci_report(gitleaks_pass: bool, sops_pass: bool) -> Dict[str, Any]:
    """Generate a CI validation report."""
    return {
        "gitleaks": {
            "passed": gitleaks_pass,
            "config": GITLEAKS_CONFIG
        },
        "sops": {
            "passed": sops_pass,
            "config": SOPS_CONFIG
        },
        "timestamp": subprocess.run(
            ["date", "-u", "+%Y-%m-%dT%H:%M:%SZ"],
            capture_output=True,
            text=True
        ).stdout.strip()
    }

def main():
    parser = argparse.ArgumentParser(description="Run CI checks for Gitleaks and SOPS")
    parser.add_argument(
        "--target", "-t",
        default=".",
        help="Path to scan with Gitleaks (default: current directory)"
    )
    parser.add_argument(
        "--sops-dir", "-s",
        default="secrets",
        help="Directory containing SOPS files (default: secrets)"
    )
    parser.add_argument(
        "--report", "-r",
        default="ci-report.json",
        help="Path to write CI report (default: ci-report.json)"
    )
    args = parser.parse_args()

    # Check environment variables
    missing_vars = check_env_vars()
    if missing_vars:
        print(f"Missing required environment variables: {missing_vars}", file=sys.stderr)
        sys.exit(1)

    # Run checks
    gitleaks_pass = run_gitleaks_scan(args.target)
    sops_pass = validate_sops_files(args.sops_dir)

    # Generate report
    report = generate_ci_report(gitleaks_pass, sops_pass)
    with open(args.report, "w") as f:
        json.dump(report, f, indent=2)
    print(f"CI report written to {args.report}")

    # Exit with error if any check failed
    if not (gitleaks_pass and sops_pass):
        sys.exit(1)
    print("All CI checks passed!")

if __name__ == "__main__":
    main()

repos:
  - repo: https://github.com/gitleaks/gitleaks
    rev: v8.18.0
    hooks:
      - id: gitleaks
        args: ["detect", "--diff-only", "--config", ".gitleaks.json"]

#!/bin/bash
# Generate new age key pair
age-keygen -o age-key.txt
NEW_PUBLIC_KEY=$(age-keygen -y age-key.txt)
# Update all SOPS files
for file in secrets/*.json; do
  sops updatekeys $file $NEW_PUBLIC_KEY
done
# Update GitHub Secret (requires gh CLI)
gh secret set SOPS_AGE_KEY < age-key.txt

curl -X POST "https://api.datadoghq.com/api/v2/logs" \
  -H "Content-Type: application/json" \
  -H "DD-API-KEY: ${DATADOG_API_KEY}" \
  -d @gitleaks-report.json

gitleaks-sops-migration/
├── .gitleaks.json          # Gitleaks config
├── .pre-commit-config.yaml # Pre-commit hook config
├── sops_manager.py         # SOPS encrypt/decrypt script
├── gitleaks_config_generator.py # Gitleaks install/config script
├── ci_checker.py           # CI pipeline script
├── secrets/                # SOPS-encrypted secrets
│   ├── db.json             # Encrypted database credentials
│   └── api.json            # Encrypted API keys
├── .github/
│   └── workflows/
│       └── ci.yml          # GitHub Actions workflow
└── checklist.md            # 27-point migration checklist