War Story: I Deployed a Broken Docker Image to Production and How I Fixed It with Trivy 0.50 and Harbor 2.10

# Broken CI workflow that deployed the vulnerable image to prod
# Used until March 12, 2024: no container scanning, no Harbor validation
name: Legacy Deploy Workflow
on:
  push:
    branches: [ main ]
env:
  REGISTRY: harbor.internal.example.com
  IMAGE_NAME: payment-processor
  IMAGE_TAG: ${{ github.sha }}
jobs:
  build-and-push:
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Fetch all history for proper versioning

      - name: Log in to Harbor registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.HARBOR_USERNAME }}
          password: ${{ secrets.HARBOR_PASSWORD }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: network=host  # Required for internal registry access

      - name: Extract metadata for Docker
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=sha,format=long
            type=raw,value=latest,enable={{is_default_branch}}

      - name: Build and push Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Deploy to production EKS
        uses: aws-actions/amazon-eks-update-kubeconfig@v2
        with:
          cluster-name: prod-payment-cluster
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Apply Kubernetes deployment
        run: |
          kubectl set image deployment/payment-processor \
            payment-processor=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} \
            --namespace prod
          kubectl rollout status deployment/payment-processor --namespace prod --timeout=300s
        env:
          KUBECONFIG: ${{ steps.kubeconfig.outputs.kubeconfig }}

      - name: Verify deployment health
        run: |
          # Broken: only checks pod running status, not app health
          kubectl get pods -n prod -l app=payment-processor | grep Running
          if [ $? -ne 0 ]; then
            echo "::error::Pods not running"
            exit 1
          fi

# Fixed CI workflow with Trivy 0.50 scanning and Harbor 2.10 validation
# Deployed March 12, 2024 post-incident
name: Secure Deploy Workflow
on:
  push:
    branches: [ main ]
env:
  REGISTRY: harbor.internal.example.com
  IMAGE_NAME: payment-processor
  IMAGE_TAG: ${{ github.sha }}
  TRIVY_VERSION: 0.50.0
  HARBOR_VERSION: 2.10.0
jobs:
  build-scan-push:
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      packages: write
      security-events: write  # Required for Trivy SARIF upload
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Log in to Harbor registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.HARBOR_USERNAME }}
          password: ${{ secrets.HARBOR_PASSWORD }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: network=host

      - name: Extract metadata for Docker
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=sha,format=long
            type=raw,value=latest,enable={{is_default_branch}}

      - name: Build Docker image (no push yet)
        uses: docker/build-push-action@v5
        with:
          context: .
          push: false  # Scan before pushing
          load: true  # Load image into local Docker daemon for Trivy
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Install Trivy 0.50.0
        run: |
          # Install specific Trivy version to avoid drift
          TRIVY_DOWNLOAD_URL="https://github.com/aquasecurity/trivy/releases/download/v${{ env.TRIVY_VERSION }}/trivy_${{ env.TRIVY_VERSION }}_Linux-64bit.tar.gz"
          curl -sfL $TRIVY_DOWNLOAD_URL | tar -xz -C /usr/local/bin trivy
          trivy --version | grep ${{ env.TRIVY_VERSION }} || (echo "::error::Trivy version mismatch"; exit 1)

      - name: Run Trivy filesystem scan
        run: |
          # Scan source code for hardcoded secrets and CVEs in dependencies
          trivy fs . \
            --scanners vuln,secret,config \
            --severity CRITICAL,HIGH \
            --exit-code 1 \
            --format sarif \
            --output trivy-fs.sarif
          if [ $? -ne 0 ]; then
            echo "::error::Critical/high issues found in filesystem scan"
            exit 1
          fi

      - name: Run Trivy image scan
        run: |
          # Scan built image for OS and application CVEs
          trivy image \
            ${{ steps.meta.outputs.tags }} \
            --scanners vuln \
            --severity CRITICAL,HIGH \
            --exit-code 1 \
            --ignore-unfixed \
            --format sarif \
            --output trivy-image.sarif
          if [ $? -ne 0 ]; then
            echo "::error::Critical/high CVEs found in container image"
            exit 1
          fi

      - name: Upload Trivy scan results to GitHub Security
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: trivy-fs.sarif
          category: trivy-fs-scan

      - name: Upload Trivy image scan results to GitHub Security
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: trivy-image.sarif
          category: trivy-image-scan

      - name: Push image to Harbor 2.10
        if: success()  # Only push if all scans pass
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Sign image with Harbor 2.10 OCI signing
        run: |
          # Harbor 2.10 native OCI artifact signing, no external tools required
          curl -X POST "https://${{ env.REGISTRY }}/api/v2.0/projects/example/repositories/${{ env.IMAGE_NAME }}/artifacts/${{ env.IMAGE_TAG }}/signatures" \
            -H "Content-Type: application/json" \
            -u "${{ secrets.HARBOR_USERNAME }}:${{ secrets.HARBOR_PASSWORD }}" \
            -d '{"signature_type": "cosign", "payload": {"critical": {"identity": {"docker-reference": "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"}, "image": {"docker-manifest-digest": "${{ steps.meta.outputs.digest }}"}}}}'
          if [ $? -ne 0 ]; then
            echo "::error::Failed to sign image with Harbor"
            exit 1
          fi

      - name: Deploy to production EKS
        uses: aws-actions/amazon-eks-update-kubeconfig@v2
        with:
          cluster-name: prod-payment-cluster
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Apply Kubernetes deployment
        run: |
          kubectl set image deployment/payment-processor \
            payment-processor=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} \
            --namespace prod
          kubectl rollout status deployment/payment-processor --namespace prod --timeout=300s
        env:
          KUBECONFIG: ${{ steps.kubeconfig.outputs.kubeconfig }}

      - name: Verify deployment health
        run: |
          # Fixed: checks app health endpoint, not just pod status
          kubectl get pods -n prod -l app=payment-processor | grep Running
          if [ $? -ne 0 ]; then
            echo "::error::Pods not running"
            exit 1
          fi
          # Check /health endpoint
          POD_IP=$(kubectl get pod -n prod -l app=payment-processor -o jsonpath='{.items[0].status.podIP}')
          curl -sf http://$POD_IP:8080/health | grep "ok" || (echo "::error::Health check failed"; exit 1)

#!/usr/bin/env python3
"""
Automated Trivy 0.50 scan and Harbor 2.10 image promotion script
Scans images in staging registry, promotes to prod if no critical/high CVEs
Requires: trivy>=0.50.0, requests>=2.31.0, python-dotenv>=1.0.0
"""

import os
import sys
import subprocess
import json
import time
import requests
from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

# Configuration from environment variables
STAGING_REGISTRY = os.getenv("STAGING_REGISTRY", "harbor.internal.example.com/staging")
PROD_REGISTRY = os.getenv("PROD_REGISTRY", "harbor.internal.example.com/prod")
IMAGE_NAME = os.getenv("IMAGE_NAME", "payment-processor")
TRIVY_SEVERITY = os.getenv("TRIVY_SEVERITY", "CRITICAL,HIGH")
HARBOR_USER = os.getenv("HARBOR_USER")
HARBOR_PASSWORD = os.getenv("HARBOR_PASSWORD")
HARBOR_API_BASE = os.getenv("HARBOR_API_BASE", "https://harbor.internal.example.com/api/v2.0")

def run_trivy_scan(image_uri: str) -> dict:
    """Run Trivy 0.50 scan on image, return scan results as dict"""
    try:
        # Run Trivy image scan, output JSON
        result = subprocess.run(
            [
                "trivy", "image",
                "--scanners", "vuln",
                "--severity", TRIVY_SEVERITY,
                "--format", "json",
                "--ignore-unfixed",
                image_uri
            ],
            capture_output=True,
            text=True,
            check=False  # Don't raise exception on non-zero exit code, we handle it
        )

        if result.returncode not in (0, 1):
            # Trivy returns 1 if vulnerabilities found, 0 if none, other codes are errors
            raise RuntimeError(f"Trivy scan failed: {result.stderr}")

        # Parse JSON output
        scan_results = json.loads(result.stdout) if result.stdout else {}
        return scan_results

    except FileNotFoundError:
        print("::error::Trivy not found. Install Trivy 0.50+ first.")
        sys.exit(1)
    except json.JSONDecodeError as e:
        print(f"::error::Failed to parse Trivy output: {e}")
        sys.exit(1)

def get_vulnerability_count(scan_results: dict) -> int:
    """Count critical/high vulnerabilities from Trivy scan results"""
    vuln_count = 0
    for result in scan_results.get("Results", []):
        for vuln in result.get("Vulnerabilities", []):
            if vuln.get("Severity") in ("CRITICAL", "HIGH"):
                vuln_count += 1
    return vuln_count

def promote_image_to_prod(image_tag: str) -> bool:
    """Promote image from staging to prod registry using Harbor 2.10 API"""
    staging_uri = f"{STAGING_REGISTRY}/{IMAGE_NAME}:{image_tag}"
    prod_uri = f"{PROD_REGISTRY}/{IMAGE_NAME}:{image_tag}"

    # First, copy image from staging to prod using Harbor replication API
    try:
        # Harbor 2.10 replication API endpoint
        response = requests.post(
            f"{HARBOR_API_BASE}/replication/executions",
            auth=(HARBOR_USER, HARBOR_PASSWORD),
            json={
                "policy_id": 123,  # Pre-configured replication policy ID
                "trigger": "manual"
            },
            timeout=30
        )
        response.raise_for_status()
        execution_id = response.json().get("id")

        # Wait for replication to complete
        while True:
            status_response = requests.get(
                f"{HARBOR_API_BASE}/replication/executions/{execution_id}",
                auth=(HARBOR_USER, HARBOR_PASSWORD),
                timeout=30
            )
            status_response.raise_for_status()
            status = status_response.json().get("status")
            if status == "Succeed":
                print(f"Successfully promoted {staging_uri} to {prod_uri}")
                return True
            elif status == "Failed":
                print("::error::Replication failed for {staging_uri}")
                return False
            else:
                print(f"Replication status: {status}, waiting...")
                time.sleep(10)

    except requests.exceptions.RequestException as e:
        print(f"::error::Harbor API request failed: {e}")
        return False

def main():
    if not all([HARBOR_USER, HARBOR_PASSWORD]):
        print("::error::Missing HARBOR_USER or HARBOR_PASSWORD environment variables")
        sys.exit(1)

    # Get image tag from command line argument
    if len(sys.argv) != 2:
        print(f"Usage: {sys.argv[0]} ")
        sys.exit(1)
    image_tag = sys.argv[1]

    # Full staging image URI
    staging_image = f"{STAGING_REGISTRY}/{IMAGE_NAME}:{image_tag}"
    print(f"Scanning image: {staging_image}")

    # Run Trivy scan
    scan_results = run_trivy_scan(staging_image)
    vuln_count = get_vulnerability_count(scan_results)

    if vuln_count > 0:
        print(f"::error::Found {vuln_count} critical/high vulnerabilities. Not promoting to prod.")
        sys.exit(1)
    else:
        print(f"No critical/high vulnerabilities found. Promoting to prod...")
        if promote_image_to_prod(image_tag):
            print(f"Image {image_tag} successfully promoted to prod registry")
            sys.exit(0)
        else:
            print(f"::error::Failed to promote image {image_tag} to prod")
            sys.exit(1)

if __name__ == "__main__":
    main()

RUN curl -sfL https://github.com/aquasecurity/trivy/releases/download/v0.50.0/trivy_0.50.0_Linux-64bit.tar.gz | tar -xz -C /usr/local/bin trivy
trivy --version | grep 0.50.0 || (echo "Version mismatch"; exit 1)

curl -X POST "https://harbor.internal.example.com/api/v2.0/projects/example/repositories/payment-processor/artifacts/sha256:abc123/signatures" \
  -H "Content-Type: application/json" \
  -u "$HARBOR_USER:$HARBOR_PASSWORD" \
  -d '{"signature_type": "cosign", "payload": {"critical": {"identity": {"docker-reference": "harbor.internal.example.com/prod/payment-processor"}}}}'

#!/bin/bash
# Pre-push hook to scan Docker images with Trivy 0.50
while read local_ref local_sha remote_ref remote_sha; do
  if [[ $remote_ref == refs/heads/main ]]; then
    IMAGE_TAG=$local_sha
    trivy image --severity CRITICAL,HIGH --exit-code 1 harbor.internal.example.com/staging/payment-processor:$IMAGE_TAG
    if [ $? -ne 0 ]; then
      echo "Push rejected: Critical/high issues found in image"
      exit 1
    fi
  fi
done