Postmortem: A Misconfigured Istio 1.24 Sidecar Caused mTLS Errors – Here's How We Debugged It

// validate-istio-mtls.go
// Validates Istio sidecar mTLS configurations against a live Kubernetes cluster
// Usage: go run validate-istio-mtls.go --kubeconfig ~/.kube/config --namespace production
package main

import (
    "context"
    "flag"
    "fmt"
    "os"
    "strings"

    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    istio "istio.io/client-go/pkg/apis/networking/v1beta1"
    istioClient "istio.io/client-go/pkg/clientset/versioned"
)

var (
    kubeconfig   string
    namespace    string
    istioVersion string
)

func init() {
    flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to kubeconfig file")
    flag.StringVar(&namespace, "namespace", "default", "Kubernetes namespace to scan")
    flag.StringVar(&istioVersion, "istio-version", "1.24.1", "Expected Istio version")
    flag.Parse()
}

func main() {
    // Load kubeconfig
    config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Failed to load kubeconfig: %v\n", err)
        os.Exit(1)
    }

    // Create Kubernetes client
    k8sClient, err := kubernetes.NewForConfig(config)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Failed to create Kubernetes client: %v\n", err)
        os.Exit(1)
    }

    // Create Istio client
    istioClientSet, err := istioClient.NewForConfig(config)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Failed to create Istio client: %v\n", err)
        os.Exit(1)
    }

    ctx := context.Background()

    // Fetch all pods in the namespace
    pods, err := k8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
    if err != nil {
        fmt.Fprintf(os.Stderr, "Failed to list pods: %v\n", err)
        os.Exit(1)
    }

    // Track validation errors
    errorCount := 0

    // Iterate over pods to check for Istio sidecars
    for _, pod := range pods.Items {
        // Check if pod has Istio sidecar
        hasSidecar := false
        for _, container := range pod.Spec.Containers {
            if strings.Contains(container.Image, "proxyv2") {
                hasSidecar = true
                break
            }
        }
        if !hasSidecar {
            continue
        }

        // Check sidecar version
        sidecarVersion := pod.Labels["istio.io/version"]
        if sidecarVersion != istioVersion {
            fmt.Fprintf(os.Stderr, "Pod %s/%s has mismatched Istio version: expected %s, got %s\n",
                pod.Namespace, pod.Name, istioVersion, sidecarVersion)
            errorCount++
        }

        // Fetch PeerAuthentication policies for the pod's namespace
        peerAuths, err := istioClientSet.SecurityV1beta1().PeerAuthentications(pod.Namespace).List(ctx, metav1.ListOptions{})
        if err != nil {
            fmt.Fprintf(os.Stderr, "Failed to list PeerAuthentication for namespace %s: %v\n", pod.Namespace, err)
            errorCount++
            continue
        }

        // Check if STRICT mTLS is enforced and validate SANs
        for _, pa := range peerAuths.Items {
            if pa.Spec.Mtls == nil || pa.Spec.Mtls.Mode != istio.PeerAuthentication_MutualTLS_STRICT {
                continue
            }
            // Check for SAN mismatches (simplified check)
            if pa.Spec.Port == nil {
                // Namespace-wide STRICT mTLS: check pod's service account
                saName := pod.Spec.ServiceAccountName
                if saName == "" {
                    fmt.Fprintf(os.Stderr, "Pod %s/%s has no service account, STRICT mTLS will fail\n", pod.Namespace, pod.Name)
                    errorCount++
                }
            }
        }
    }

    // Output summary
    if errorCount > 0 {
        fmt.Fprintf(os.Stderr, "Found %d mTLS configuration errors\n", errorCount)
        os.Exit(1)
    }
    fmt.Println("All Istio sidecar mTLS configurations are valid")
}

global:
  hub: gcr.io/istio-release
  tag: 1.24.1
  proxy:
    image: proxyv2
    version: 1.24.1
istioctl:
  version: 1.24.1
  # Enforce digest pinning for air-gapped clusters
  imageDigest: sha256:5f4d7a9b8c3d2e1f0a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4e3f2a1b0c9d8e7f

apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: payment-service-mirror
  namespace: production
spec:
  hosts:
  - payment.production.svc.cluster.local
  http:
  - route:
    - destination:
        host: payment.production.svc.cluster.local
        subset: v1
      weight: 100
    mirror:
      host: payment.staging.svc.cluster.local
      subset: v1
    mirrorPercentage:
      value: 5.0
    # Mirror all mTLS traffic including headers for debugging
    mirrorHeaders:
      forward: true

#!/usr/bin/env bpftrace
#include 
#include 

BEGIN {
  printf("Tracing TLS handshake failures... Hit Ctrl-C to exit.\n");
}

// Trace TCP connection resets on port 443 (default mTLS port)
kprobe:tcp_send_active_reset {
  $sk = (struct sock *)arg0;
  $dport = $sk->__sk_common.skc_dport;
  $sport = $sk->__sk_common.skc_portpair >> 16;
  // Convert to host byte order
  $dport_host = ($dport >> 8) | (($dport & 0xFF) << 8);
  $sport_host = ($sport >> 8) | (($sport & 0xFF) << 8);
  if ($dport_host == 443 || $sport_host == 443) {
    printf("TLS handshake failure: src=%s:%d dst=%s:%d pid=%d\n",
           ntop($sk->__sk_common.skc_rcv_saddr), $sport_host,
           ntop($sk->__sk_common.skc_daddr), $dport_host,
           pid);
  }
}

// Trace OpenSSL TLS errors (if Envoy uses OpenSSL)
uprobe:/usr/lib/x86_64-linux-gnu/libssl.so.3:SSL_get_error {
  $ssl = (void *)arg0;
  $err = arg1;
  if ($err != SSL_ERROR_NONE) {
    printf("OpenSSL TLS error: %d, pid=%d\n", $err, pid);
  }
}

# parse-istio-mtls-logs.py
# Parses Istio Envoy proxy logs to extract mTLS error patterns, outputs Prometheus metrics
# Usage: kubectl logs -n production deployment/payment-service -c istio-proxy | python3 parse-istio-mtls-logs.py

import sys
import re
import time
from collections import defaultdict

# Regex patterns for mTLS errors in Envoy logs
MTLS_ERROR_PATTERNS = {
    "san_mismatch": re.compile(r"SSL error: error:1000007d:SSL routines:OPENSSL_internal:CERTIFICATE_VERIFY_FAILED"),
    "handshake_timeout": re.compile(r"TLS handshake timeout"),
    "cert_expired": re.compile(r"certificate has expired"),
    "wrong_peer": re.compile(r"peer not authorized"),
    "port_mismatch": re.compile(r"connection refused on port 443")
}

# Track error counts
error_counts = defaultdict(int)
total_lines = 0
mtls_connections = 0

def parse_log_line(line):
    """Parse a single Envoy log line for mTLS errors"""
    global total_lines, mtls_connections
    total_lines += 1

    # Check if line is related to mTLS (contains 443 or mTLS keywords)
    if "443" not in line and "mTLS" not in line and "TLS" not in line:
        return

    mtls_connections += 1

    # Check against all error patterns
    for error_type, pattern in MTLS_ERROR_PATTERNS.items():
        if pattern.search(line):
            error_counts[error_type] += 1
            # Extract timestamp and pod name if present
            timestamp_match = re.search(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+Z)", line)
            pod_match = re.search(r"pod:(\S+)", line)
            timestamp = timestamp_match.group(1) if timestamp_match else "unknown"
            pod = pod_match.group(1) if pod_match else "unknown"
            print(f"mTLS error: type={error_type} pod={pod} timestamp={timestamp} line={line.strip()}")

def output_metrics():
    """Output Prometheus-formatted metrics"""
    print("\n# HELP istio_mtls_total_connections Total mTLS connections observed")
    print(f"# TYPE istio_mtls_total_connections counter")
    print(f"istio_mtls_total_connections {mtls_connections}")

    print("\n# HELP istio_mtls_error_total Total mTLS errors by type")
    print("# TYPE istio_mtls_error_total counter")
    for error_type, count in error_counts.items():
        print(f"istio_mtls_error_total{{error_type=\"{error_type}\"}} {count}")

    print("\n# HELP istio_log_lines_total Total log lines processed")
    print("# TYPE istio_log_lines_total counter")
    print(f"istio_log_lines_total {total_lines}")

def main():
    """Main entry point: read logs from stdin"""
    print("Parsing Istio proxy logs for mTLS errors...\n")
    try:
        for line in sys.stdin:
            parse_log_line(line.strip())
    except KeyboardInterrupt:
        pass
    except Exception as e:
        print(f"Error reading logs: {e}", file=sys.stderr)
        sys.exit(1)

    # Output summary
    print("\n=== Summary ===")
    print(f"Total log lines processed: {total_lines}")
    print(f"Total mTLS connections: {mtls_connections}")
    print("mTLS error counts:")
    for error_type, count in error_counts.items():
        print(f"  {error_type}: {count}")
        if count > 0:
            print(f"  ALERT: {error_type} detected, investigate immediately")

    # Output Prometheus metrics
    output_metrics()

if __name__ == "__main__":
    main()

// reproduce-mtls-error_test.go
// Integration test to reproduce the Istio 1.24 mTLS misconfiguration error
// Run with: go test -v -tags integration reproduce-mtls-error_test.go
// Prerequisites: kind cluster with Istio 1.24.0 installed, kubeconfig set

package main

import (
    "context"
    "crypto/tls"
    "fmt"
    "net/http"
    "os"
    "strings"
    "testing"
    "time"

    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    istio "istio.io/client-go/pkg/apis/networking/v1beta1"
    istioClient "istio.io/client-go/pkg/clientset/versioned"
)

const (
    testNamespace = "mtls-test"
    serviceName   = "test-service"
    istioVersion  = "1.24.0" // Version with the SAN bug
)

func setupCluster(t *testing.T) (kubernetes.Interface, istioClient.Interface) {
    t.Helper()
    // Load kubeconfig
    config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
    if err != nil {
        t.Fatalf("Failed to load kubeconfig: %v", err)
    }

    // Create clients
    k8sClient, err := kubernetes.NewForConfig(config)
    if err != nil {
        t.Fatalf("Failed to create Kubernetes client: %v", err)
    }
    istioCS, err := istioClient.NewForConfig(config)
    if err != nil {
        t.Fatalf("Failed to create Istio client: %v", err)
    }

    // Create test namespace
    _, err = k8sClient.CoreV1().Namespaces().Create(context.Background(), &metav1.Namespace{
        ObjectMeta: metav1.ObjectMeta{
            Name: testNamespace,
        },
    }, metav1.CreateOptions{})
    if err != nil {
        t.Fatalf("Failed to create namespace: %v", err)
    }

    // Deploy PeerAuthentication with STRICT mTLS
    _, err = istioCS.SecurityV1beta1().PeerAuthentications(testNamespace).Create(context.Background(), &istio.PeerAuthentication{
        ObjectMeta: metav1.ObjectMeta{
            Name:      "strict-mtls",
            Namespace: testNamespace,
        },
        Spec: istio.PeerAuthenticationSpec{
            Mtls: &istio.PeerAuthentication_MutualTLS{
                Mode: istio.PeerAuthentication_MutualTLS_STRICT,
            },
        },
    }, metav1.CreateOptions{})
    if err != nil {
        t.Fatalf("Failed to create PeerAuthentication: %v", err)
    }

    // Deploy test service with misconfigured sidecar (1.24.0 with SAN bug)
    // In real test, deploy a pod with sidecar version 1.24.0
    // Simplified for brevity: assume pod is deployed
    time.Sleep(10 * time.Second) // Wait for pod to start

    return k8sClient, istioCS
}

func teardownCluster(t *testing.T, k8sClient kubernetes.Interface) {
    t.Helper()
    err := k8sClient.CoreV1().Namespaces().Delete(context.Background(), testNamespace, metav1.DeleteOptions{})
    if err != nil {
        t.Fatalf("Failed to delete namespace: %v", err)
    }
}

func TestMTLSMisconfiguration(t *testing.T) {
    // Skip if not integration test
    if os.Getenv("INTEGRATION_TEST") != "true" {
        t.Skip("Skipping integration test")
    }

    k8sClient, _ := setupCluster(t)
    defer teardownCluster(t, k8sClient)

    // Get pod IP
    pods, err := k8sClient.CoreV1().Pods(testNamespace).List(context.Background(), metav1.ListOptions{})
    if err != nil {
        t.Fatalf("Failed to list pods: %v", err)
    }
    if len(pods.Items) == 0 {
        t.Fatal("No pods found in test namespace")
    }
    podIP := pods.Items[0].Status.PodIP

    // Try to connect via mTLS (should fail with 1.24.0 sidecar)
    client := &http.Client{
        Transport: &http.Transport{
            TLSClientConfig: &tls.Config{
                InsecureSkipVerify: true, // We expect failure, skip cert verification
            },
        },
        Timeout: 5 * time.Second,
    }

    // Attempt connection to pod IP on 443 (mTLS port)
    resp, err := client.Get(fmt.Sprintf("https://%s:443", podIP))
    if err == nil {
        resp.Body.Close()
        t.Fatal("Expected mTLS connection to fail, but succeeded")
    }

    // Check if error is mTLS-related
    if !strings.Contains(err.Error(), "certificate verify failed") && !strings.Contains(err.Error(), "handshake failure") {
        t.Fatalf("Unexpected error: %v", err)
    }

    t.Logf("Successfully reproduced mTLS misconfiguration error: %v", err)

    // Now test with fixed version (1.24.1)
    // In real test, upgrade sidecar to 1.24.1 and retry
    // Expect connection to succeed
    t.Log("Test passed: mTLS misconfiguration error reproduced")
}