Postmortem: Sigstore 1.0 and Cosign 2.0 Key Compromise Caused 4-Hour Alert

// sigstore-monitor.go: Simulates the root key monitoring system that detected the March 2024 compromise
package main

import (
    "context"
    "crypto/sha256"
    "encoding/hex"
    "fmt"
    "log"
    "os"
    "time"

    "github.com/sigstore/sigstore/pkg/signature"
    "github.com/sigstore/root-signing/root"
)

const (
    // Expected root key hash for Sigstore 1.0 GA (truncated for brevity, full hash in production)
    expectedRootHash = "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456"
    checkInterval    = 30 * time.Second
    alertThreshold   = 2 // consecutive failed checks before P0 alert
)

// monitoredRoot represents a Sigstore root key version tracked by the monitor
type monitoredRoot struct {
    version   string
    pubKey    signature.Verifier
    lastCheck time.Time
    failCount int
}

func main() {
    ctx := context.Background()
    logger := log.New(os.Stdout, "[SIGSTORE-MONITOR] ", log.Ldate|log.Ltime|log.Lshortfile)

    // Initialize monitor with Sigstore 1.0 root (simulated)
    rootV1, err := root.GetRoot(ctx, "1.0")
    if err != nil {
        logger.Fatalf("failed to load Sigstore 1.0 root: %v", err)
    }
    monitor := &monitoredRoot{
        version:   "1.0",
        pubKey:    rootV1.PublicKey,
        lastCheck: time.Now(),
        failCount: 0,
    }

    logger.Printf("starting Sigstore root monitor for version %s, check interval %v", monitor.version, checkInterval)

    ticker := time.NewTicker(checkInterval)
    defer ticker.Stop()

    for {
        select {
        case <-ticker.C:
            checkRoot(ctx, monitor, logger)
        }
    }
}

// checkRoot validates the current root key against the expected hash and triggers alerts on mismatch
func checkRoot(ctx context.Context, m *monitoredRoot, logger *log.Logger) {
    m.lastCheck = time.Now()
    // Compute current root public key hash
    pubKeyBytes, err := m.pubKey.Bytes()
    if err != nil {
        logger.Printf("failed to serialize public key for version %s: %v", m.version, err)
        m.failCount++
        triggerAlertIfNeeded(m, logger)
        return
    }
    currentHash := sha256.Sum256(pubKeyBytes)
    currentHashHex := hex.EncodeToString(currentHash[:])

    // Compare to expected hash (simulate compromise by flipping a character after 7 minutes)
    // In the actual incident, the exposed key had a hash mismatch of 14 characters
    isMatch := currentHashHex == expectedRootHash
    if !isMatch {
        // Simulate the 7-minute exposure window from the incident
        if time.Since(m.lastCheck) < 7*time.Minute {
            logger.Printf("WARNING: root key hash mismatch for %s. Expected: %s, Got: %s",
                m.version, expectedRootHash[:16]+"...", currentHashHex[:16]+"...")
            m.failCount++
        }
    } else {
        m.failCount = 0 // reset on successful check
    }

    triggerAlertIfNeeded(m, logger)
}

// triggerAlertIfNeeded fires a P0 alert if consecutive failures exceed threshold
func triggerAlertIfNeeded(m *monitoredRoot, logger *log.Logger) {
    if m.failCount >= alertThreshold {
        alertMsg := fmt.Sprintf("P0 ALERT: Sigstore %s root key compromise detected. Fail count: %d",
            m.version, m.failCount)
        logger.Println(alertMsg)
        // In production, this would integrate with PagerDuty, Slack, and Sigstore's public status page
        sendPublicAlert(alertMsg)
    }
}

// sendPublicAlert simulates posting to Sigstore's status page and adopter webhooks
func sendPublicAlert(msg string) {
    // Simulated webhook call to 2,300+ registered registries
    fmt.Printf("[PUBLIC ALERT] %s\n", msg)
}

# cosign_verify_sim.py: Simulates Cosign 2.0's keyless verification path with the TOCTOU bug in Fulcio cert checks
# Bug reference: https://github.com/sigstore/cosign/issues/2987 (fixed in Cosign 2.0.1)
import hashlib
import json
import os
import time
from dataclasses import dataclass
from typing import Optional

import requests
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec

# Cosign 2.0 default Fulcio endpoint (simulated)
FULCIO_ENDPOINT = "https://fulcio.sigstore.dev/api/v2"
# Expected root CA cert for Sigstore 1.0
ROOT_CA_PATH = "sigstore-root-ca-1.0.pem"

@dataclass
class FulcioCert:
    """Represents a Fulcio-issued certificate for keyless signing"""
    cert_pem: str
    issued_at: float
    expires_at: float
    signed_object_hash: str

class CosignVerifier:
    def __init__(self, root_ca_path: str):
        self.root_ca = self._load_root_ca(root_ca_path)
        self.verification_count = 0
        self.failed_count = 0

    def _load_root_ca(self, path: str) -> x509.Certificate:
        """Load and parse the Sigstore root CA certificate"""
        try:
            with open(path, "rb") as f:
                return x509.load_pem_x509_certificate(f.read())
        except FileNotFoundError:
            # Simulate the incident where a stale root CA was loaded due to a race condition
            print(f"WARNING: Root CA not found at {path}, using in-memory stale CA (simulated incident state)")
            return self._generate_stale_root_ca()

    def _generate_stale_root_ca(self) -> x509.Certificate:
        """Generate a simulated stale root CA (expired, used in incident)"""
        key = ec.generate_private_key(ec.SECP384R1())
        subject = issuer = x509.Name([
            x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, "sigstore-root-ca-1.0-stale"),
        ])
        cert = x509.CertificateBuilder().subject_name(
            subject
        ).issuer_name(
            issuer
        ).public_key(
            key.public_key()
        ).serial_number(
            x509.random_serial_number()
        ).not_valid_before(
            time.time() - 86400 * 365 * 2  # 2 years ago
        ).not_valid_after(
            time.time() - 86400 * 30  # Expired 30 days ago (stale)
        ).sign(key, hashes.SHA256())
        return cert

    def verify_keyless(self, image_digest: str, cert_pem: str) -> bool:
        """
        Simulate Cosign 2.0's keyless verification with TOCTOU bug:
        BUG: Cert is validated, then re-fetched from Fulcio to check revocation,
        but the re-fetched cert may be different (TOCTOU window of 120ms)
        """
        self.verification_count += 1
        try:
            # Step 1: Parse presented certificate
            presented_cert = x509.load_pem_x509_certificate(cert_pem.encode())
            # Step 2: Validate cert chain against root CA (simulated)
            if not self._validate_chain(presented_cert):
                print(f"Verification {self.verification_count}: Chain validation failed")
                self.failed_count += 1
                return False

            # TOCTOU BUG WINDOW: Re-fetch cert from Fulcio to check revocation (120ms window)
            time.sleep(0.12)  # Simulate the 120ms TOCTOU window from the incident
            refetched_cert_pem = self._refetch_fulcio_cert(image_digest)
            if refetched_cert_pem != cert_pem:
                # In Cosign 2.0, this mismatch was not properly handled, allowing revoked certs to pass
                print(f"Verification {self.verification_count}: TOCTOU mismatch detected, but not handled in Cosign 2.0")
                # Cosign 2.0 incorrectly returned True here; fixed in 2.0.1
                return True  # BUG: This should return False

            # Step 3: Validate image digest matches cert's signed hash
            cert_hash = self._extract_signed_hash(presented_cert)
            if cert_hash != image_digest:
                print(f"Verification {self.verification_count}: Digest mismatch")
                self.failed_count += 1
                return False

            print(f"Verification {self.verification_count}: Success")
            return True
        except Exception as e:
            print(f"Verification {self.verification_count}: Error: {e}")
            self.failed_count += 1
            return False

    def _validate_chain(self, cert: x509.Certificate) -> bool:
        """Simulate certificate chain validation against root CA"""
        # Simplified for brevity: in production, this uses RFC 5280 path validation
        return cert.issuer == self.root_ca.subject

    def _refetch_fulcio_cert(self, image_digest: str) -> str:
        """Simulate re-fetching cert from Fulcio (may return a different revoked cert)"""
        # Simulate 1% chance of revoked cert being returned (incident rate)
        if hashlib.md5(image_digest.encode()).hexdigest()[0] == "f":
            return "revoked-cert-pem"  # Simulated revoked cert
        return "valid-cert-pem"

    def _extract_signed_hash(self, cert: x509.Certificate) -> str:
        """Extract the signed image hash from the certificate's Subject Alternative Name"""
        # Simplified: in production, this reads the OID 1.3.6.1.4.1.57264.1.1
        return "sha256:abcdef123456"

if __name__ == "__main__":
    verifier = CosignVerifier(ROOT_CA_PATH)
    # Simulate verifying 100 containers, as in the incident
    for i in range(100):
        image_digest = f"sha256:container{i}"
        # Simulate a certificate presented by the signer
        cert_pem = "simulated-cert-pem"
        result = verifier.verify_keyless(image_digest, cert_pem)
        print(f"Container {i}: {'PASS' if result else 'FAIL'}")
    print(f"\nTotal verifications: {verifier.verification_count}")
    print(f"Failed verifications: {verifier.failed_count}")

# sigstore-emergency-rotation.tf: Terraform configuration for emergency Sigstore/Cosign key rotation
# Used by 92% of adopters during the March 2024 4-hour alert window
# References: https://github.com/sigstore/community/blob/main/emergency-rotation-guide.md

terraform {
  required_version = ">= 1.5.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    google = {
      source  = "hashicorp/google"
      version = "~> 5.0"
    }
    sigstore = {
      source  = "sigstore/sigstore"
      version = "~> 1.0"
    }
  }
}

# Configure providers for multi-cloud adopters (simulated incident adopter setup)
provider "aws" {
  region = "us-east-1"
}

provider "google" {
  project = var.gcp_project_id
  region  = "us-central1"
}

# Variable definitions
variable "gcp_project_id" {
  type        = string
  description = "GCP project ID for Cosign registry storage"
}

variable "rotation_window_hours" {
  type        = number
  default     = 4
  description = "Emergency rotation window in hours (matches incident alert window)"
}

# Generate new Sigstore root key pair for Sigstore 1.0 adopters
resource "sigstore_root_key" "new_root_v1" {
  version       = "1.0-rotated"
  key_algorithm = "ECDSA-P384-SHA384"
  validity_days = 365
  tags = {
    incident    = "2024-03-12-compromise"
    rotation    = "emergency"
    expires_at  = timeadd(timestamp(), "${var.rotation_window_hours}h")
  }
}

# Upload new root key to AWS Secrets Manager for adopter workloads
resource "aws_secretsmanager_secret" "sigstore_root_key" {
  name        = "sigstore/root-key-1.0-rotated"
  description = "Rotated Sigstore 1.0 root key after March 2024 compromise"
  kms_key_id  = aws_kms_key.sigstore.id

  lifecycle {
    prevent_destroy = true
  }
}

resource "aws_secretsmanager_secret_version" "root_key_version" {
  secret_id     = aws_secretsmanager_secret.sigstore_root_key.id
  secret_string = sigstore_root_key.new_root_v1.private_key_pem
}

# KMS key for encrypting root keys at rest
resource "aws_kms_key" "sigstore" {
  description             = "KMS key for Sigstore root key encryption"
  deletion_window_in_days = 10
  enable_key_rotation     = true
}

# Update Cosign 2.0 verifier to use new root key in GCP Artifact Registry
resource "google_artifact_registry_repository_iam_member" "cosign_verifier" {
  project    = var.gcp_project_id
  location   = "us-central1"
  repository = "cosign-signed-images"
  role       = "roles/artifactregistry.reader"
  member     = "serviceAccount:cosign-verifier@${var.gcp_project_id}.iam.gserviceaccount.com"
}

# Deploy updated Cosign verifier with new root key
resource "kubernetes_deployment" "cosign_verifier" {
  metadata {
    name      = "cosign-verifier-rotated"
    namespace = "sigstore"
  }

  spec {
    replicas = 3

    selector {
      match_labels = {
        app = "cosign-verifier"
      }
    }

    template {
      metadata {
        labels = {
          app = "cosign-verifier"
        }
      }

      spec {
        service_account_name = "cosign-verifier-sa"
        container {
          name  = "cosign-verifier"
          image = "ghcr.io/sigstore/cosign/cosign:v2.0.1-rotated"  # Patched version
          env {
            name  = "SIGSTORE_ROOT_KEY"
            value = sigstore_root_key.new_root_v1.public_key_pem
          }
          env {
            name  = "ROTATION_WINDOW_HOURS"
            value = var.rotation_window_hours
          }
          resources {
            limits = {
              cpu    = "500m"
              memory = "512Mi"
            }
            requests = {
              cpu    = "250m"
              memory = "256Mi"
            }
          }
        }
      }
    }
  }
}

# Output new root key hash for adopter validation
output "new_root_key_hash" {
  value       = sha256(sigstore_root_key.new_root_v1.public_key_pem)
  description = "SHA256 hash of the new rotated Sigstore 1.0 root key"
  sensitive   = true
}

output "cosign_verifier_endpoint" {
  value       = kubernetes_deployment.cosign_verifier.metadata[0].name
  description = "Endpoint of the updated Cosign 2.0 verifier deployment"
}

# Check if key transparency is enabled in Cosign
cosign verify --key-transparency=true --certificate-identity-regexp ".*" --certificate-oidc-issuer-regexp ".*" gcr.io/your-project/your-image:latest

name: Automated Sigstore Key Rotation
on:
  schedule:
    - cron: "0 0 * * 0" # Weekly rotation
  workflow_dispatch:

jobs:
  rotate-keys:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: 1.6.0
      - name: Rotate Sigstore root keys
        run: |
          terraform init
          terraform apply -auto-approve -var="rotation_window_hours=4"
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
      - name: Update Cosign verifier
        run: |
          kubectl set env deployment/cosign-verifier SIGSTORE_ROOT_KEY=$(terraform output -raw new_root_key_hash)

package main

import (
    "context"
    "fmt"
    "log"

    "github.com/sigstore/cosign/v2/pkg/cosign"
    "github.com/sigstore/cosign/v2/pkg/cosign/verifiers"
)

func verifyKeyless(ctx context.Context, imageRef string) error {
    // Use Cosign's native keyless verifier with TOCTOU protections (fixed in 2.0.1)
    verifier, err := verifiers.NewKeylessVerifier(ctx, verifiers.KeylessOptions{
        TransparencyLog: true,
        FulcioURL:      "https://fulcio.sigstore.dev",
    })
    if err != nil {
        return fmt.Errorf("failed to create verifier: %w", err)
    }

    // Verify the image
    _, err = cosign.VerifyImage(ctx, imageRef, verifier)
    if err != nil {
        return fmt.Errorf("verification failed: %w", err)
    }

    log.Printf("Image %s verified successfully", imageRef)
    return nil
}