Benchmark: Cert-Manager 1.14 vs AWS ACM 2026 for Certificate Issuance Time on K8s 1.32

// cert-manager-benchmark.go
// Benchmarks Cert-Manager 1.14 certificate issuance time
// Usage: go run cert-manager-benchmark.go --kubeconfig ~/.kube/config --namespace cert-tests --count 10000
package main

import (
    \"context\"
    \"flag\"
    \"fmt\"
    \"os\"
    \"sort\"
    \"time\"

    metav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"
    \"k8s.io/apimachinery/pkg/util/wait\"
    \"k8s.io/client-go/kubernetes\"
    \"k8s.io/client-go/tools/clientcmd\"
    certmanager \"github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1\"
    certmanagerclient \"github.com/cert-manager/cert-manager/pkg/client/clientset/versioned\"
)

var (
    kubeconfig  string
    namespace   string
    certCount   int
    issuerName  string
    dnsName     string
)

func init() {
    flag.StringVar(&kubeconfig, \"kubeconfig\", \"\", \"Path to kubeconfig file\")
    flag.StringVar(&namespace, \"namespace\", \"default\", \"Namespace to create certificates in\")
    flag.IntVar(&certCount, \"count\", 100, \"Number of certificates to issue\")
    flag.StringVar(&issuerName, \"issuer\", \"letsencrypt-staging\", \"Cert-Manager ClusterIssuer name\")
    flag.StringVar(&dnsName, \"dns\", \"bench.example.com\", \"DNS name for certificates\")
}

func main() {
    flag.Parse()

    // Validate flags
    if certCount <= 0 {
        fmt.Fprintf(os.Stderr, \"Error: cert count must be positive\\n\")
        os.Exit(1)
    }

    // Build k8s config
    config, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfig)
    if err != nil {
        fmt.Fprintf(os.Stderr, \"Error building kubeconfig: %v\\n\", err)
        os.Exit(1)
    }

    // Create cert-manager client
    cmClient, err := certmanagerclient.NewForConfig(config)
    if err != nil {
        fmt.Fprintf(os.Stderr, \"Error creating cert-manager client: %v\\n\", err)
        os.Exit(1)
    }

    // Create k8s client for general operations
    k8sClient, err := kubernetes.NewForConfig(config)
    if err != nil {
        fmt.Fprintf(os.Stderr, \"Error creating k8s client: %v\\n\", err)
        os.Exit(1)
    }

    // Clean up old certificates first
    err = cmClient.CertmanagerV1().Certificates(namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
        LabelSelector: \"bench=cert-manager\",
    })
    if err != nil {
        fmt.Fprintf(os.Stderr, \"Warning: failed to clean up old certs: %v\\n\", err)
    }

    issuanceTimes := make([]float64, 0, certCount)
    fmt.Printf(\"Issuing %d certificates...\\n\", certCount)

    for i := 0; i < certCount; i++ {
        certName := fmt.Sprintf(\"bench-cert-%d\", i)
        dns := fmt.Sprintf(\"%s-%d\", dnsName, i)

        // Create Certificate resource
        cert := &certmanager.Certificate{
            ObjectMeta: metav1.ObjectMeta{
                Name:      certName,
                Namespace: namespace,
                Labels: map[string]string{
                    \"bench\": \"cert-manager\",
                },
            },
            Spec: certmanager.CertificateSpec{
                SecretName: fmt.Sprintf(\"%s-secret\", certName),
                IssuerRef: certmanager.IssuerRef{
                    Name: issuerName,
                    Kind: \"ClusterIssuer\",
                },
                DNSNames: []string{dns},
                Usages: []certmanager.KeyUsage{
                    certmanager.UsageDigitalSignature,
                    certmanager.UsageKeyEncipherment,
                },
            },
        }

        start := time.Now()
        _, err := cmClient.CertmanagerV1().Certificates(namespace).Create(context.Background(), cert, metav1.CreateOptions{})
        if err != nil {
            fmt.Fprintf(os.Stderr, \"Error creating cert %s: %v\\n\", certName, err)
            continue
        }

        // Wait for certificate to be ready
        err = wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
            c, err := cmClient.CertmanagerV1().Certificates(namespace).Get(ctx, certName, metav1.GetOptions{})
            if err != nil {
                return false, err
            }
            for _, cond := range c.Status.Conditions {
                if cond.Type == certmanager.CertificateConditionReady && cond.Status == metav1.ConditionTrue {
                    return true, nil
                }
            }
            return false, nil
        })

        if err != nil {
            fmt.Fprintf(os.Stderr, \"Error waiting for cert %s to be ready: %v\\n\", certName, err)
            continue
        }

        elapsed := time.Since(start).Seconds()
        issuanceTimes = append(issuanceTimes, elapsed)

        // Clean up cert to avoid resource exhaustion
        err = cmClient.CertmanagerV1().Certificates(namespace).Delete(context.Background(), certName, metav1.DeleteOptions{})
        if err != nil {
            fmt.Fprintf(os.Stderr, \"Warning: failed to delete cert %s: %v\\n\", certName, err)
        }
    }

    // Calculate percentiles
    sort.Float64s(issuanceTimes)
    if len(issuanceTimes) == 0 {
        fmt.Fprintf(os.Stderr, \"Error: no successful certificate issuances\\n\")
        os.Exit(1)
    }

    p50 := percentile(issuanceTimes, 50)
    p90 := percentile(issuanceTimes, 90)
    p99 := percentile(issuanceTimes, 99)

    fmt.Printf(\"\\nBenchmark Results for Cert-Manager 1.14:\\n\")
    fmt.Printf(\"Total successful issuances: %d\\n\", len(issuanceTimes))
    fmt.Printf(\"p50 issuance time: %.2fs\\n\", p50)
    fmt.Printf(\"p90 issuance time: %.2fs\\n\", p90)
    fmt.Printf(\"p99 issuance time: %.2fs\\n\", p99)
}

func percentile(data []float64, p int) float64 {
    if len(data) == 0 {
        return 0
    }
    index := (float64(p) / 100) * float64(len(data)-1)
    lower := int(index)
    upper := lower + 1
    if upper >= len(data) {
        return data[len(data)-1]
    }
    weight := index - float64(lower)
    return data[lower]*(1-weight) + data[upper]*weight
}

# acm-benchmark.py
# Benchmarks AWS ACM 2026 certificate issuance via aws-acm-controller
# Usage: python acm-benchmark.py --region us-east-1 --namespace cert-tests --count 10000
import argparse
import boto3
import time
import numpy as np
import sys
from kubernetes import client, config
from kubernetes.client.rest import ApiException

def parse_args():
    parser = argparse.ArgumentParser(description='Benchmark AWS ACM 2026 issuance time')
    parser.add_argument('--region', required=True, help='AWS region')
    parser.add_argument('--namespace', default='default', help='K8s namespace')
    parser.add_argument('--count', type=int, default=100, help='Number of certificates to issue')
    parser.add_argument('--kubeconfig', default=None, help='Path to kubeconfig')
    parser.add_argument('--issuer', default='acm-issuer', help='AWS ACM ClusterIssuer name')
    parser.add_argument('--dns-suffix', default='acm-bench.example.com', help='DNS suffix for certs')
    return parser.parse_args()

def main():
    args = parse_args()

    # Validate args
    if args.count <= 0:
        print(\"Error: cert count must be positive\", file=sys.stderr)
        sys.exit(1)

    # Load kube config
    try:
        config.load_kube_config(config_file=args.kubeconfig)
    except Exception as e:
        print(f\"Error loading kubeconfig: {e}\", file=sys.stderr)
        sys.exit(1)

    # Create k8s API clients
    api = client.CustomObjectsApi()
    core_api = client.CoreV1Api()

    # Create AWS ACM client
    acm = boto3.client('acm', region_name=args.region)

    # Clean up old certificates
    try:
        certs = api.list_namespaced_custom_object(
            group='cert-manager.io',
            version='v1',
            namespace=args.namespace,
            plural='certificates',
            label_selector='bench=acm'
        )
        for item in certs.get('items', []):
            cert_name = item['metadata']['name']
            try:
                api.delete_namespaced_custom_object(
                    group='cert-manager.io',
                    version='v1',
                    namespace=args.namespace,
                    plural='certificates',
                    name=cert_name,
                    body=client.V1DeleteOptions()
                )
            except ApiException as e:
                print(f\"Warning: failed to delete cert {cert_name}: {e}\")
    except ApiException as e:
        print(f\"Warning: failed to list old certs: {e}\")

    issuance_times = []
    print(f\"Issuing {args.count} certificates...\")

    for i in range(args.count):
        cert_name = f\"acm-bench-cert-{i}\"
        dns_name = f\"{args.dns_suffix}-{i}\"

        # Certificate resource
        cert_body = {
            \"apiVersion\": \"cert-manager.io/v1\",
            \"kind\": \"Certificate\",
            \"metadata\": {
                \"name\": cert_name,
                \"namespace\": args.namespace,
                \"labels\": {\"bench\": \"acm\"}
            },
            \"spec\": {
                \"secretName\": f\"{cert_name}-secret\",
                \"issuerRef\": {
                    \"name\": args.issuer,
                    \"kind\": \"ClusterIssuer\"
                },
                \"dnsNames\": [dns_name],
                \"usages\": [\"digital signature\", \"key encipherment\"]
            }
        }

        start = time.time()
        try:
            api.create_namespaced_custom_object(
                group='cert-manager.io',
                version='v1',
                namespace=args.namespace,
                plural='certificates',
                body=cert_body
            )
        except ApiException as e:
            print(f\"Error creating cert {cert_name}: {e}\", file=sys.stderr)
            continue

        # Wait for cert to be ready
        timeout = time.time() + 30  # 30s timeout
        ready = False
        while time.time() < timeout:
            try:
                cert = api.get_namespaced_custom_object(
                    group='cert-manager.io',
                    version='v1',
                    namespace=args.namespace,
                    plural='certificates',
                    name=cert_name
                )
                conditions = cert.get('status', {}).get('conditions', [])
                for cond in conditions:
                    if cond['type'] == 'Ready' and cond['status'] == 'True':
                        ready = True
                        break
                if ready:
                    break
                time.sleep(1)
            except ApiException as e:
                print(f\"Error checking cert {cert_name}: {e}\", file=sys.stderr)
                time.sleep(1)

        if not ready:
            print(f\"Error: cert {cert_name} not ready within 30s\", file=sys.stderr)
            continue

        elapsed = time.time() - start
        issuance_times.append(elapsed)

        # Clean up cert
        try:
            api.delete_namespaced_custom_object(
                group='cert-manager.io',
                version='v1',
                namespace=args.namespace,
                plural='certificates',
                name=cert_name,
                body=client.V1DeleteOptions()
            )
        except ApiException as e:
            print(f\"Warning: failed to delete cert {cert_name}: {e}\")

    # Calculate percentiles
    if not issuance_times:
        print(\"Error: no successful issuances\", file=sys.stderr)
        sys.exit(1)

    issuance_times.sort()
    p50 = np.percentile(issuance_times, 50)
    p90 = np.percentile(issuance_times, 90)
    p99 = np.percentile(issuance_times, 99)

    print(\"\\nBenchmark Results for AWS ACM 2026:\")
    print(f\"Total successful issuances: {len(issuance_times)}\")
    print(f\"p50 issuance time: {p50:.2f}s\")
    print(f\"p90 issuance time: {p90:.2f}s\")
    print(f\"p99 issuance time: {p99:.2f}s\")

if __name__ == '__main__':
    main()

#!/bin/bash
# hybrid-cert-manager.sh
# Automates certificate management using Cert-Manager for internal and AWS ACM for external certs
# Usage: ./hybrid-cert-manager.sh --namespace prod --internal-suffix internal.example.com --external-suffix example.com

set -euo pipefail

# Default values
NAMESPACE=\"default\"
INTERNAL_SUFFIX=\"internal.example.com\"
EXTERNAL_SUFFIX=\"example.com\"
CERT_MANAGER_ISSUER=\"letsencrypt-prod\"
ACM_ISSUER=\"acm-issuer\"
RENEW_BEFORE=\"720h\"  # 30 days
LOG_FILE=\"cert-manager.log\"

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --namespace)
            NAMESPACE=\"$2\"
            shift 2
            ;;
        --internal-suffix)
            INTERNAL_SUFFIX=\"$2\"
            shift 2
            ;;
        --external-suffix)
            EXTERNAL_SUFFIX=\"$2\"
            shift 2
            ;;
        --log-file)
            LOG_FILE=\"$2\"
            shift 2
            ;;
        *)
            echo \"Unknown argument: $1\" >&2
            exit 1
            ;;
    esac
done

# Validate dependencies
for cmd in kubectl aws jq; do
    if ! command -v $cmd &> /dev/null; then
        echo \"Error: $cmd is not installed\" >&2
        exit 1
    fi
done

# Log function
log() {
    echo \"[$(date +'%Y-%m-%d %H:%M:%S')] $1\" | tee -a \"$LOG_FILE\"
}

# Check certificate expiry
check_expiry() {
    local secret_name=$1
    local namespace=$2

    # Get secret
    secret=$(kubectl get secret \"$secret_name\" -n \"$namespace\" -o json 2>/dev/null)
    if [ -z \"$secret\" ]; then
        log \"Secret $secret_name not found in $namespace\"
        return 1
    fi

    # Decode cert and check expiry
    cert=$(echo \"$secret\" | jq -r '.data.\"tls.crt\"' | base64 -d 2>/dev/null)
    if [ -z \"$cert\" ]; then
        log \"Failed to decode cert for $secret_name\"
        return 1
    fi

    expiry=$(echo \"$cert\" | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
    if [ -z \"$expiry\" ]; then
        log \"Failed to get expiry for $secret_name\"
        return 1
    fi

    expiry_epoch=$(date -d \"$expiry\" +%s 2>/dev/null)
    if [ -z \"$expiry_epoch\" ]; then
        log \"Failed to parse expiry date for $secret_name\"
        return 1
    fi

    current_epoch=$(date +%s)
    remaining=$(( (expiry_epoch - current_epoch) / 3600 ))  # hours remaining

    echo \"$remaining\"
    return 0
}

# Renew certificate
renew_cert() {
    local cert_name=$1
    local namespace=$2

    log \"Renewing certificate $cert_name in $namespace\"
    kubectl delete certificate \"$cert_name\" -n \"$namespace\" --ignore-not-found=true
    log \"Certificate $cert_name renewed\"
}

# Process internal certificates (Cert-Manager)
log \"Processing internal certificates for *.$INTERNAL_SUFFIX\"
internal_certs=$(kubectl get certificate -n \"$NAMESPACE\" -l \"cert-type=internal\" -o json 2>/dev/null)
if [ -n \"$internal_certs\" ]; then
    echo \"$internal_certs\" | jq -r '.items[] | .metadata.name' | while read -r cert_name; do
        secret_name=$(echo \"$internal_certs\" | jq -r \".items[] | select(.metadata.name == \\\"$cert_name\\\") | .spec.secretName\")
        remaining=$(check_expiry \"$secret_name\" \"$NAMESPACE\")
        if [ -n \"$remaining\" ] && [ \"$remaining\" -lt \"$RENEW_BEFORE\" ]; then
            log \"Internal cert $cert_name has $remaining hours remaining, renewing...\"
            renew_cert \"$cert_name\" \"$NAMESPACE\"
        else
            log \"Internal cert $cert_name is valid for $remaining hours, skipping\"
        fi
    done
else
    log \"No internal certificates found in $NAMESPACE\"
fi

# Process external certificates (AWS ACM)
log \"Processing external certificates for *.$EXTERNAL_SUFFIX\"
external_certs=$(kubectl get certificate -n \"$NAMESPACE\" -l \"cert-type=external\" -o json 2>/dev/null)
if [ -n \"$external_certs\" ]; then
    echo \"$external_certs\" | jq -r '.items[] | .metadata.name' | while read -r cert_name; do
        secret_name=$(echo \"$external_certs\" | jq -r \".items[] | select(.metadata.name == \\\"$cert_name\\\") | .spec.secretName\")
        remaining=$(check_expiry \"$secret_name\" \"$NAMESPACE\")
        if [ -n \"$remaining\" ] && [ \"$remaining\" -lt \"$RENEW_BEFORE\" ]; then
            log \"External cert $cert_name has $remaining hours remaining, renewing...\"
            renew_cert \"$cert_name\" \"$NAMESPACE\"
        else
            log \"External cert $cert_name is valid for $remaining hours, skipping\"
        fi
    done
else
    log \"No external certificates found in $NAMESPACE\"
fi

# Create new certificates for unmatched ingresses
log \"Checking for ingresses without certificates\"
ingresses=$(kubectl get ingress -n \"$NAMESPACE\" -o json 2>/dev/null)
if [ -n \"$ingresses\" ]; then
    echo \"$ingresses\" | jq -r '.items[] | .metadata.name' | while read -r ingress_name; do
        # Check if ingress has a matching certificate
        has_cert=$(echo \"$ingresses\" | jq -r \".items[] | select(.metadata.name == \\\"$ingress_name\\\") | .metadata.annotations[\\\"cert-manager.io/issuer\\\"]\")
        if [ -z \"$has_cert\" ]; then
            log \"Ingress $ingress_name has no certificate, creating...\"
            # Determine if internal or external
            host=$(echo \"$ingresses\" | jq -r \".items[] | select(.metadata.name == \\\"$ingress_name\\\") | .spec.rules[0].host\")
            if [[ \"$host\" == *\"$INTERNAL_SUFFIX\"* ]]; then
                issuer=\"$CERT_MANAGER_ISSUER\"
                cert_type=\"internal\"
            else
                issuer=\"$ACM_ISSUER\"
                cert_type=\"external\"
            fi
            # Create certificate (simplified, actual would use kubectl apply)
            log \"Creating certificate for $host with issuer $issuer\"
        fi
    done
fi

log \"Certificate check complete\"

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: pre-provisioned-canary-cert
  namespace: prod
spec:
  secretName: canary-cert-secret
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - canary-myapp.prod.internal.example.com
  renewBefore: 720h
  usages:
    - digital signature
    - key encipherment

resource \"aws_iam_role\" \"acm_controller\" {
  name = \"acm-controller-role\"
  assume_role_policy = jsonencode({
    Version = \"2012-10-17\"
    Statement = [
      {
        Effect = \"Allow\"
        Principal = {
          Federated = aws_iam_openid_connect_provider.eks.arn
        }
        Action = \"sts:AssumeRoleWithWebIdentity\"
        Condition = {
          StringEquals = {
            \"${aws_iam_openid_connect_provider.eks.url}:sub\" = \"system:serviceaccount:cert-manager:cert-manager\"
          }
        }
      }
    ]
  })
}

resource \"aws_iam_role_policy\" \"acm_controller\" {
  role = aws_iam_role.acm_controller.name
  policy = jsonencode({
    Version = \"2012-10-17\"
    Statement = [
      {
        Effect = \"Allow\"
        Action = [\"acm:RequestCertificate\", \"acm:DescribeCertificate\", \"acm:DeleteCertificate\"]
        Resource = \"*\"
      }
    ]
  })
}

groups:
- name: cert-manager-alerts
  rules:
  - alert: CertificateExpirySoon
    expr: certmanager_certificate_expiry_seconds < 604800  # 7 days
    for: 1h
    labels:
      severity: warning
    annotations:
      summary: \"Certificate {{ $labels.name }} in {{ $labels.namespace }} expires soon\"
      description: \"Certificate expires in {{ $value | humanizeDuration }}\"

  - alert: CertificateRequestFailed
    expr: rate(certmanager_certificate_request_failure_count[5m]) > 0
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: \"Certificate request failed for {{ $labels.name }} in {{ $labels.namespace }}\"
      description: \"{{ $labels.name }} has failed certificate requests for the last 10 minutes\"