War Story: Debugging Sigstore 1.10 Signature Verification Failure for Docker 25 Images

import os
import sys
import docker
import json
from sigstore.verify import Verifier
from sigstore.oidc import detect_credential
from sigstore.models import Bundle
from docker.errors import DockerException, APIError

def reproduce_verification_failure(image_name: str, tag: str = "latest") -> None:
    \"\"\"
    Reproduces Sigstore 1.10 verification failure for Docker 25 images.
    Requires: sigstore==1.10.0, docker==25.0.0, valid OIDC credential.
    \"\"\"
    # Initialize Docker 25 client
    try:
        client = docker.from_env(version="25.0.0")
        print(f"Connected to Docker daemon version: {client.version()['Version']}")
    except DockerException as e:
        print(f"FATAL: Failed to connect to Docker daemon: {e}", file=sys.stderr)
        sys.exit(1)

    # Pull the target image (Docker 25-pushed OCI v1.1.0 image)
    full_image = f"{image_name}:{tag}"
    try:
        print(f"Pulling image {full_image}...")
        image = client.images.pull(full_image)
        print(f"Pulled image ID: {image.id}")
    except APIError as e:
        print(f"FATAL: Failed to pull image {full_image}: {e}", file=sys.stderr)
        sys.exit(1)

    # Extract image digest and signature bundle (Sigstore 1.10 format)
    try:
        digest = image.attrs["RepoDigests"][0].split("@")[-1]
        # Assume bundle is stored in image label (Sigstore 1.10 convention)
        bundle_raw = image.labels.get("sigstore.bundle")
        if not bundle_raw:
            print("ERROR: No Sigstore bundle found in image labels", file=sys.stderr)
            sys.exit(1)
        bundle = Bundle.from_json(json.loads(bundle_raw))
    except (KeyError, json.JSONDecodeError) as e:
        print(f"FATAL: Failed to extract image metadata: {e}", file=sys.stderr)
        sys.exit(1)

    # Initialize Sigstore 1.10 verifier (default OCI v1.0.2 validation)
    try:
        credential = detect_credential()
        verifier = Verifier.production(oidc_credential=credential)
        print(f"Initialized Sigstore 1.10 verifier for identity: {credential.issuer}")
    except Exception as e:
        print(f"FATAL: Failed to initialize Sigstore verifier: {e}", file=sys.stderr)
        sys.exit(1)

    # Attempt verification (expected to fail for Docker 25 OCI v1.1.0 images)
    print(f"Verifying digest {digest} with Sigstore 1.10...")
    try:
        result = verifier.verify(digest, bundle)
        print(f"UNEXPECTED: Verification succeeded: {result}")
    except Exception as e:
        print(f"REPRODUCED FAILURE: Verification failed as expected: {e}", file=sys.stderr)
        # Log the OCI spec version of the image manifest
        manifest = client.images.get_registry_data(full_image).manifest()
        print(f"Image OCI spec version: {manifest.get('mediaType', 'unknown')}")
        sys.exit(0)

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print(f"Usage: {sys.argv[0]} ", file=sys.stderr)
        sys.exit(1)
    reproduce_verification_failure(sys.argv[1])

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "os"

    "github.com/sigstore/sigstore/pkg/verify"
    "github.com/opencontainers/image-spec/specs-go/v1"
    "github.com/docker/distribution/manifest/manifestlist"
    "github.com/docker/docker/client"
)

// patchedOCIValidator overrides Sigstore 1.10's default OCI v1.0.2 validation
// to support Docker 25's OCI Image Manifest v1.1.0 media types.
type patchedOCIValidator struct {
    originalValidator *verify.OCIValidator
}

// ValidateManifest checks if the manifest is a supported OCI v1.1.0 or v1.0.2 type
func (p *patchedOCIValidator) ValidateManifest(ctx context.Context, manifestBytes []byte) error {
    // First try original Sigstore 1.10 validation (v1.0.2 only)
    err := p.originalValidator.ValidateManifest(ctx, manifestBytes)
    if err == nil {
        return nil
    }

    // If original validation fails, check for OCI v1.1.0 media types (Docker 25 default)
    var manifest map[string]interface{}
    if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
        return fmt.Errorf("failed to unmarshal manifest: %w", err)
    }

    mediaType, ok := manifest["mediaType"].(string)
    if !ok {
        return fmt.Errorf("manifest missing mediaType: %w", err)
    }

    // Supported OCI v1.1.0 media types from Docker 25
    supportedV1_1Types := []string{
        v1.MediaTypeImageManifest,
        v1.MediaTypeImageIndex,
        manifestlist.MediaTypeManifestList,
    }

    for _, t := range supportedV1_1Types {
        if mediaType == t {
            log.Printf("Patched validator: accepted OCI v1.1.0 media type %s", t)
            return nil
        }
    }

    return fmt.Errorf("unsupported media type %s: %w", mediaType, err)
}

func main() {
    // Initialize Docker 25 client
    cli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion("25.0.0"))
    if err != nil {
        log.Fatalf("Failed to create Docker client: %v", err)
    }

    // Pull a Docker 25-pushed image
    imageName := "my-registry/my-app:latest"
    reader, err := cli.ImagePull(context.Background(), imageName, client.ImagePullOptions{})
    if err != nil {
        log.Fatalf("Failed to pull image %s: %v", imageName, err)
    }
    defer reader.Close()

    // Get image manifest bytes
    inspect, _, err := cli.ImageInspectWithRaw(context.Background(), imageName)
    if err != nil {
        log.Fatalf("Failed to inspect image: %v", err)
    }
    manifestBytes := inspect.RawManifest

    // Initialize original Sigstore 1.10 OCI validator
    originalVal := verify.NewOCIValidator()
    patchedVal := &patchedOCIValidator{originalValidator: originalVal}

    // Validate manifest with patched validator
    err = patchedVal.ValidateManifest(context.Background(), manifestBytes)
    if err != nil {
        log.Fatalf("Patched validation failed: %v", err)
    }

    fmt.Println("SUCCESS: Manifest validated with patched Sigstore 1.10 validator")
}

#!/bin/bash
set -euo pipefail

# Configuration
SIGSTORE_VERSION="1.10.0"
DOCKER_VERSION="25.0.0"
PATCH_URL="https://github.com/sigstore/sigstore/releases/download/v1.10.1/sigstore-patch-oci-v1.1.0.sh"
REGISTRY="my-registry.example.com"
KUBECONFIG="${KUBECONFIG:-~/.kube/config}"

# Logging functions
log_info() { echo "[INFO] $(date +%Y-%m-%dT%H:%M:%S%z) $1"; }
log_error() { echo "[ERROR] $(date +%Y-%m-%dT%H:%M:%S%z) $1" >&2; }

# Check prerequisites
check_prerequisites() {
    log_info "Checking prerequisites..."
    command -v kubectl >/dev/null 2>&1 || { log_error "kubectl not installed"; exit 1; }
    command -v docker >/dev/null 2>&1 || { log_error "docker not installed"; exit 1; }
    kubectl version --client >/dev/null 2>&1 || { log_error "kubectl not configured"; exit 1; }

    # Verify Docker version
    local docker_ver=$(docker version --format '{{.Server.Version}}' 2>/dev/null || echo "unknown")
    if [[ "$docker_ver" != "$DOCKER_VERSION"* ]]; then
        log_error "Docker version $docker_ver does not match target $DOCKER_VERSION"
        exit 1
    fi
    log_info "Prerequisites satisfied: Docker $docker_ver, kubectl configured"
}

# Patch Sigstore on a single node
patch_node() {
    local node_name="$1"
    log_info "Patching node $node_name..."

    # Drain node to avoid disruption
    kubectl drain "$node_name" --ignore-daemonsets --delete-emptydir-data --timeout=300s

    # Run patch script on node via kubectl debug
    kubectl debug -it "$node_name" --image=alpine:latest -- sh -c "
        wget -q $PATCH_URL -O /tmp/sigstore-patch.sh
        chmod +x /tmp/sigstore-patch.sh
        /tmp/sigstore-patch.sh --sigstore-version $SIGSTORE_VERSION --docker-version $DOCKER_VERSION
        rm /tmp/sigstore-patch.sh
    "

    # Restart Docker daemon to apply changes
    kubectl debug -it "$node_name" --image=alpine:latest -- sh -c "
        if [ -f /etc/init.d/docker ]; then
            /etc/init.d/docker restart
        elif command -v systemctl >/dev/null; then
            systemctl restart docker
        else
            kill -HUP \$(pidof dockerd)
        fi
    "

    # Uncordon node
    kubectl uncordon "$node_name"
    log_info "Node $node_name patched successfully"
}

# Main execution
main() {
    check_prerequisites

    # Get all Docker 25 nodes
    log_info "Fetching Docker 25 nodes..."
    nodes=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}')
    target_nodes=()
    for node in $nodes; do
        # Get Docker version on node
        node_docker_ver=$(kubectl debug -it "$node" --image=docker:25.0.0 -- docker version --format '{{.Server.Version}}' 2>/dev/null || echo "unknown")
        if [[ "$node_docker_ver" == "$DOCKER_VERSION"* ]]; then
            target_nodes+=("$node")
            log_info "Target node found: $node (Docker $node_docker_ver)"
        fi
    done

    if [ ${#target_nodes[@]} -eq 0 ]; then
        log_error "No Docker $DOCKER_VERSION nodes found"
        exit 1
    fi

    # Patch all target nodes
    log_info "Patching ${#target_nodes[@]} nodes..."
    for node in "${target_nodes[@]}"; do
        patch_node "$node"
    done

    # Verify patch across cluster
    log_info "Verifying patch deployment..."
    kubectl get pods -A -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '
' | grep "$REGISTRY" | while read -r image; do
        log_info "Verifying image $image..."
        docker buildx imagetools inspect "$image" --format '{{.Manifest.MediaType}}' | grep -q "v1.1.0" && log_info "Image $image uses OCI v1.1.0" || log_error "Image $image uses unsupported manifest"
    done

    log_info "All nodes patched successfully. Verification failure rate should drop to 0%."
}

main "$@"

- name: Install Sigstore 1.10.0
  run: pip install sigstore==1.10.0
- name: Install Docker 25.0.0
  run: curl -fsSL https://get.docker.com | sh -s -- --version 25.0.0

- name: Log OCI media type
  run: |
    skopeo inspect docker://$IMAGE_NAME --format '{{.MediaType}}' | tee oci-media-type.log
    echo "::set-output name=media-type::$(cat oci-media-type.log)"

- name: Canary verification check
  run: |
    docker pull $SIGNED_IMAGE
    sigstore verify identity --image $SIGNED_IMAGE --certificate-identity $IDENTITY --certificate-oidc-issuer $ISSUER || exit 1