War Story: A Snyk 1.140 Scan Timeout That Delayed 10+ PR Merges for 1 Day

#!/usr/bin/env python3
"""
Snyk 1.140 Scan Timeout Reproduction Script
Benchmarks scan time across Snyk versions, manifests, and timeout configurations.
"""

import argparse
import json
import logging
import os
import subprocess
import time
from datetime import datetime
from typing import Dict, List, Optional, Tuple

# Configure logging for audit trail
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

# Default Snyk CLI path (override with --snyk-path)
DEFAULT_SNYK_PATH = "/usr/local/bin/snyk"
# Default scan timeout in seconds (matches Snyk 1.140 default)
DEFAULT_SNYK_TIMEOUT = 300
# Supported manifest file extensions
SUPPORTED_MANIFESTS = {".json", ".lock", ".yaml", ".yml"}

def validate_manifest(manifest_path: str) -> bool:
    """Check if manifest exists and is supported."""
    if not os.path.isfile(manifest_path):
        logger.error(f"Manifest not found: {manifest_path}")
        return False
    ext = os.path.splitext(manifest_path)[1]
    if ext not in SUPPORTED_MANIFESTS:
        logger.error(f"Unsupported manifest extension: {ext}")
        return False
    return True

def run_snyk_scan(
    snyk_path: str,
    manifest_path: str,
    timeout: int,
    org_id: Optional[str] = None
) -> Tuple[int, float, str, str]:
    """
    Execute Snyk test scan and return (exit_code, duration_sec, stdout, stderr).
    Handles timeout via subprocess timeout parameter.
    """
    cmd = [snyk_path, "test", "--json"]
    if org_id:
        cmd.extend(["--org", org_id])
    # Add manifest path as positional arg
    cmd.append(manifest_path)

    start_time = time.perf_counter()
    try:
        result = subprocess.run(
            cmd,
            timeout=timeout,
            capture_output=True,
            text=True,
            env={**os.environ, "SNYK_CFG_DISABLE_ANALYTICS": "1"}  # Disable analytics to avoid noise
        )
        duration = time.perf_counter() - start_time
        return result.returncode, duration, result.stdout, result.stderr
    except subprocess.TimeoutExpired:
        duration = time.perf_counter() - start_time
        logger.warning(f"Scan timed out after {duration:.2f}s (timeout: {timeout}s)")
        return 124, duration, "", f"Scan timed out after {timeout} seconds"
    except Exception as e:
        duration = time.perf_counter() - start_time
        logger.error(f"Scan failed with exception: {str(e)}")
        return 1, duration, "", str(e)

def benchmark_scans(
    snyk_path: str,
    manifest_path: str,
    timeout: int,
    iterations: int = 3
) -> List[Dict]:
    """Run multiple scan iterations and collect benchmark data."""
    if not validate_manifest(manifest_path):
        return []

    results = []
    for i in range(iterations):
        logger.info(f"Running iteration {i+1}/{iterations}")
        exit_code, duration, stdout, stderr = run_snyk_scan(snyk_path, manifest_path, timeout)
        # Parse JSON output if available
        vulns = []
        if stdout:
            try:
                scan_data = json.loads(stdout)
                vulns = scan_data.get("vulnerabilities", [])
            except json.JSONDecodeError:
                logger.warning("Failed to parse Snyk JSON output")

        results.append({
            "iteration": i+1,
            "exit_code": exit_code,
            "duration_sec": round(duration, 2),
            "vuln_count": len(vulns),
            "stdout": stdout[:200],  # Truncate for storage
            "stderr": stderr[:200]
        })
    return results

def main():
    parser = argparse.ArgumentParser(description="Reproduce Snyk 1.140 scan timeout issues")
    parser.add_argument("--snyk-path", default=DEFAULT_SNYK_PATH, help="Path to Snyk CLI binary")
    parser.add_argument("--manifest", required=True, help="Path to package manifest (package-lock.json, etc.)")
    parser.add_argument("--timeout", type=int, default=DEFAULT_SNYK_TIMEOUT, help="Scan timeout in seconds")
    parser.add_argument("--iterations", type=int, default=3, help="Number of benchmark iterations")
    parser.add_argument("--output", default="snyk_benchmark.json", help="Output JSON file for results")
    parser.add_argument("--org-id", help="Snyk organization ID (optional)")

    args = parser.parse_args()

    # Check Snyk binary exists
    if not os.path.isfile(args.snyk_path):
        logger.error(f"Snyk binary not found at {args.snyk_path}")
        return

    # Get Snyk version
    try:
        version_result = subprocess.run([args.snyk_path, "--version"], capture_output=True, text=True)
        snyk_version = version_result.stdout.strip()
        logger.info(f"Using Snyk version: {snyk_version}")
    except Exception as e:
        logger.error(f"Failed to get Snyk version: {str(e)}")
        return

    # Run benchmarks
    benchmark_results = benchmark_scans(
        snyk_path=args.snyk_path,
        manifest_path=args.manifest,
        timeout=args.timeout,
        iterations=args.iterations
    )

    # Save results
    output_data = {
        "snyk_version": snyk_version,
        "manifest_path": args.manifest,
        "manifest_size_mb": round(os.path.getsize(args.manifest) / (1024 * 1024), 2),
        "timeout_sec": args.timeout,
        "benchmark_timestamp": datetime.utcnow().isoformat(),
        "iterations": benchmark_results
    }

    with open(args.output, "w") as f:
        json.dump(output_data, f, indent=2)
    logger.info(f"Benchmark results saved to {args.output}")

if __name__ == "__main__":
    main()

name: Secure CI Pipeline with Snyk Timeout Mitigation
on:
  pull_request:
    branches: [main, release/*]
  push:
    branches: [main]

# Global timeout for entire workflow
timeout-minutes: 60

env:
  SNYK_ORG_ID: ${{ secrets.SNYK_ORG_ID }}
  SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
  CACHE_VERSION: v2  # Bump to invalidate cache

jobs:
  snyk-security-scan:
    runs-on: ubuntu-24.04
    timeout-minutes: 15  # Per-job timeout to prevent hung runners
    strategy:
      matrix:
        node-version: [20.x]
      fail-fast: false  # Run all matrix jobs even if one fails

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Needed for Snyk to detect changed dependencies

      - name: Setup Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          cache: "npm"
          cache-dependency-path: package-lock.json

      # Tier 1: Cache Snyk CLI binary to avoid download delays
      - name: Cache Snyk CLI
        id: cache-snyk
        uses: actions/cache@v4
        with:
          path: /usr/local/bin/snyk
          key: ${{ runner.os }}-snyk-${{ env.CACHE_VERSION }}-1.141.0
          restore-keys: |
            ${{ runner.os }}-snyk-${{ env.CACHE_VERSION }}-

      - name: Install Snyk CLI 1.141.0 (fixed version)
        if: steps.cache-snyk.outputs.cache-hit != 'true'
        run: |
          curl -fsSL https://github.com/snyk/snyk/releases/download/v1.141.0/snyk-linux -o /usr/local/bin/snyk
          chmod +x /usr/local/bin/snyk
          snyk --version  # Verify install

      # Tier 2: Cache Snyk vulnerability database
      - name: Cache Snyk DB
        id: cache-snyk-db
        uses: actions/cache@v4
        with:
          path: ~/.snyk/vulnerability-db
          key: ${{ runner.os }}-snyk-db-${{ env.CACHE_VERSION }}-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-snyk-db-${{ env.CACHE_VERSION }}-

      - name: Authenticate Snyk
        run: snyk auth ${{ env.SNYK_TOKEN }}

      # Tier 3: Cache node_modules to speed up manifest parsing
      - name: Cache node_modules
        uses: actions/cache@v4
        with:
          path: node_modules
          key: ${{ runner.os }}-node-${{ matrix.node-version }}-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ matrix.node-version }}-

      - name: Install dependencies
        run: npm ci --prefer-offline  # Use cached packages

      # Retry logic for Snyk scan (max 3 attempts)
      - name: Run Snyk Scan with Retry
        id: snyk-scan
        run: |
          MAX_RETRIES=3
          RETRY_DELAY=10
          for i in $(seq 1 $MAX_RETRIES); do
            echo "Attempt $i/$MAX_RETRIES"
            # Run scan with 240s timeout (override default 300s to fail faster)
            if snyk test --json --timeout=240000 > snyk-results.json 2> snyk-errors.log; then
              echo "Scan succeeded on attempt $i"
              exit 0
            else
              EXIT_CODE=$?
              echo "Scan failed with exit code $EXIT_CODE on attempt $i"
              # Exit immediately if not a timeout (e.g., auth error)
              if [ $EXIT_CODE -ne 124 ]; then
                echo "Non-timeout error, exiting"
                exit $EXIT_CODE
              fi
              # Wait before retrying
              if [ $i -lt $MAX_RETRIES ]; then
                echo "Waiting $RETRY_DELAY seconds before retry"
                sleep $RETRY_DELAY
              fi
            fi
          done
          echo "All $MAX_RETRIES scan attempts failed"
          exit 1

      # Fallback: Run offline Snyk scan if online fails
      - name: Fallback to Offline Snyk Scan
        if: failure() && steps.snyk-scan.outcome == 'failure'
        run: |
          echo "Online scan failed, running offline scan with cached DB"
          snyk test --json --offline --timeout=120000 > snyk-offline-results.json 2> snyk-offline-errors.log
          # Upload offline results as artifact
          echo "Offline scan completed"

      - name: Upload Snyk Results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: snyk-scan-results-${{ matrix.node-version }}
          path: |
            snyk-results.json
            snyk-errors.log
            snyk-offline-results.json
            snyk-offline-errors.log
          retention-days: 7

      - name: Break build on critical vulnerabilities
        if: success()
        run: |
          # Parse Snyk results and fail on high/critical vulns
          CRITICAL=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' snyk-results.json)
          HIGH=$(jq '[.vulnerabilities[] | select(.severity == "high")] | length' snyk-results.json)
          if [ $CRITICAL -gt 0 ] || [ $HIGH -gt 5 ]; then
            echo "Found $CRITICAL critical, $HIGH high vulnerabilities. Failing build."
            exit 1
          fi

#!/usr/bin/env bash
"""
Snyk Cache Cleanup and Performance Monitor
Cleans stale Snyk caches, monitors scan performance, and alerts on anomalies.
"""

set -euo pipefail  # Exit on error, undefined var, pipe failure

# Configuration
SNYK_CACHE_DIR="${HOME}/.snyk"
CI_RUNNER_TEMP="${RUNNER_TEMP:-/tmp/ci-runner}"
MAX_CACHE_AGE_DAYS=7
SCAN_LOG_PATH="${CI_RUNNER_TEMP}/snyk-scan-metrics.log"
ALERT_THRESHOLD_SEC=300  # Alert if scan takes longer than 5 minutes
SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL:-}"

# Logging setup
log() {
    local level="$1"
    shift
    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] [${level}] $*"
}

# Error handling
trap 'log ERROR "Script failed at line $LINENO"; exit 1' ERR

# Check if Snyk is installed
check_snyk_installed() {
    if ! command -v snyk &> /dev/null; then
        log ERROR "Snyk CLI not found in PATH"
        exit 1
    fi
    log INFO "Snyk version: $(snyk --version)"
}

# Clean stale Snyk caches
clean_stale_caches() {
    log INFO "Cleaning stale Snyk caches older than ${MAX_CACHE_AGE_DAYS} days"

    # Clean vulnerability DB cache
    local vuln_db_dir="${SNYK_CACHE_DIR}/vulnerability-db"
    if [ -d "$vuln_db_dir" ]; then
        log INFO "Cleaning vulnerability DB cache: ${vuln_db_dir}"
        find "$vuln_db_dir" -type f -mtime +"${MAX_CACHE_AGE_DAYS}" -delete -print | while read -r file; do
            log DEBUG "Deleted stale DB file: ${file}"
        done
    fi

    # Clean Snyk CLI update cache
    local update_cache_dir="${SNYK_CACHE_DIR}/updates"
    if [ -d "$update_cache_dir" ]; then
        log INFO "Cleaning Snyk update cache: ${update_cache_dir}"
        find "$update_cache_dir" -type f -mtime +"${MAX_CACHE_AGE_DAYS}" -delete -print | while read -r file; do
            log DEBUG "Deleted stale update file: ${file}"
        done
    fi

    # Clean temp CI files related to Snyk
    if [ -d "$CI_RUNNER_TEMP" ]; then
        log INFO "Cleaning temp Snyk files in ${CI_RUNNER_TEMP}"
        find "$CI_RUNNER_TEMP" -name "snyk-*" -type f -mtime +1 -delete -print | while read -r file; do
            log DEBUG "Deleted stale temp file: ${file}"
        done
    fi
}

# Monitor scan performance
monitor_scan_performance() {
    log INFO "Monitoring Snyk scan performance"

    # Create metrics log if it doesn't exist
    touch "$SCAN_LOG_PATH"

    # Get recent scan metrics from Snyk's debug log (if available)
    local snyk_debug_log="${SNYK_CACHE_DIR}/debug.log"
    if [ -f "$snyk_debug_log" ]; then
        log INFO "Parsing Snyk debug log: ${snyk_debug_log}"
        # Extract scan duration lines (format: "Scan completed in Xms")
        grep -i "scan completed in" "$snyk_debug_log" | tail -10 | while read -r line; do
            local duration_ms=$(echo "$line" | grep -oP '\d+(?=ms)')
            if [ -n "$duration_ms" ]; then
                local duration_sec=$(echo "scale=2; $duration_ms / 1000" | bc)
                local timestamp=$(date +'%Y-%m-%dT%H:%M:%S%z')
                echo "${timestamp},${duration_sec}" >> "$SCAN_LOG_PATH"

                # Alert if duration exceeds threshold
                if (( $(echo "$duration_sec > $ALERT_THRESHOLD_SEC" | bc -l) )); then
                    log WARN "Scan duration ${duration_sec}s exceeds threshold ${ALERT_THRESHOLD_SEC}s"
                    send_slack_alert "Snyk scan took ${duration_sec}s (threshold: ${ALERT_THRESHOLD_SEC}s)"
                fi
            fi
        done
    else
        log WARN "Snyk debug log not found: ${snyk_debug_log}"
    fi

    # Print recent metrics
    log INFO "Recent scan metrics (last 5):"
    tail -5 "$SCAN_LOG_PATH" | while read -r line; do
        log INFO "  ${line}"
    done
}

# Send Slack alert
send_slack_alert() {
    local message="$1"
    if [ -n "$SLACK_WEBHOOK_URL" ]; then
        log INFO "Sending Slack alert: ${message}"
        curl -fsSL -X POST -H 'Content-type: application/json' \
            --data "{\"text\":\"Snyk Performance Alert: ${message}\"}" \
            "$SLACK_WEBHOOK_URL" || log WARN "Failed to send Slack alert"
    else
        log WARN "SLACK_WEBHOOK_URL not set, skipping alert"
    fi
}

# Generate performance report
generate_report() {
    log INFO "Generating Snyk performance report"
    local report_path="${CI_RUNNER_TEMP}/snyk-performance-report.txt"

    echo "Snyk Performance Report - $(date)" > "$report_path"
    echo "=========================================" >> "$report_path"
    echo "Snyk Version: $(snyk --version)" >> "$report_path"
    echo "Cache Directory: ${SNYK_CACHE_DIR}" >> "$report_path"
    echo "Max Cache Age: ${MAX_CACHE_AGE_DAYS} days" >> "$report_path"
    echo "" >> "$report_path"

    # Calculate average scan time from metrics log
    if [ -f "$SCAN_LOG_PATH" ]; then
        local avg_duration=$(awk -F',' '{sum+=$2; count++} END {if(count>0) print sum/count; else print 0}' "$SCAN_LOG_PATH")
        local max_duration=$(awk -F',' '{if($2>max) max=$2} END {print max}' "$SCAN_LOG_PATH")
        local min_duration=$(awk -F',' '{if($2> "$report_path"
        echo "  Average Duration: ${avg_duration}s" >> "$report_path"
        echo "  Max Duration: ${max_duration}s" >> "$report_path"
        echo "  Min Duration: ${min_duration}s" >> "$report_path"
    else
        echo "No scan metrics available" >> "$report_path"
    fi

    log INFO "Report generated at ${report_path}"
    cat "$report_path"
}

# Main execution
main() {
    log INFO "Starting Snyk cache cleanup and performance monitor"
    check_snyk_installed
    clean_stale_caches
    monitor_scan_performance
    generate_report
    log INFO "Script completed successfully"
}

main

# Install pinned Snyk CLI with checksum validation
SNYK_VERSION="1.141.0"
SNYK_CHECKSUM="a1b2c3d4e5f678901234567890abcdef1234567890abcdef1234567890abcdef"
curl -fsSL "https://github.com/snyk/snyk/releases/download/v${SNYK_VERSION}/snyk-linux" -o /usr/local/bin/snyk
echo "${SNYK_CHECKSUM} /usr/local/bin/snyk" | sha256sum -c
chmod +x /usr/local/bin/snyk

# Tier 2: Cache Snyk vulnerability database
- name: Cache Snyk DB
  uses: actions/cache@v4
  with:
    path: ~/.snyk/vulnerability-db
    key: ${{ runner.os }}-snyk-db-v2-${{ hashFiles('package-lock.json') }}
    restore-keys: |
      ${{ runner.os }}-snyk-db-v2-

# Retry loop for Snyk scan with fallback
for i in {1..3}; do
  if snyk test --timeout=240000; then exit 0; fi
  if [ $i -eq 3 ]; then snyk test --offline --timeout=120000; fi
  sleep $((i * 10))
done