Postmortem: How a Cilium 1.15.0 Policy Bug Allowed Unauthorized Access to Our K8s 1.31 Pod

package cilium_policy_test

import (
    \"context\"
    \"fmt\"
    \"sync\"
    \"sync/atomic\"
    \"testing\"
    \"time\"

    \"github.com/cilium/cilium/api/v1/models\"
    \"github.com/cilium/cilium/pkg/client\"
    \"github.com/stretchr/testify/require\"
)

// TestCilium1150PolicyRace reproduces the Cilium 1.15.0 eBPF policy map
// race condition where concurrent policy updates and packet inspections
// return false positives for policy allow checks.
func TestCilium1150PolicyRace(t *testing.T) {
    // Initialize Cilium client with local agent connection
    // Note: This test assumes a local Cilium agent running 1.15.0 for reproduction
    // In CI, we use a containerized Cilium 1.15.0 agent for isolation
    cli, err := client.NewClient(\"unix:///var/run/cilium/cilium.sock\")
    require.NoError(t, err, \"failed to initialize Cilium client\")

    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Define a test policy that allows ingress from namespace \"frontend\" to \"payment\"
    testPolicy := &models.Policy{
        Name:        \"test-payment-ingress\",
        Namespace:   \"payment\",
        Labels:      []string{\"app=payment\", \"env=prod\"},
        Description: \"Allow frontend to payment ingress\",
        Rules: &models.PolicyRules{
            Ingress: []*models.PolicyIngressRule{
                {
                    FromEndpoints: []models.EndpointSelector{
                        {MatchLabels: map[string]string{\"namespace\": \"frontend\"}},
                    },
                    ToPorts: []models.PortRule{
                        {Port: \"8080\", Protocol: \"TCP\"},
                    },
                },
            },
        },
    }

    // Track race condition occurrences
    var raceDetections atomic.Int64
    // Track total policy checks
    var totalChecks atomic.Int64

    // Number of concurrent workers to simulate production load
    const numWorkers = 50
    // Number of iterations per worker
    const iterations = 1000

    var wg sync.WaitGroup
    wg.Add(numWorkers)

    // Start concurrent workers that update policies and check enforcement
    for i := 0; i < numWorkers; i++ {
        go func(workerID int) {
            defer wg.Done()
            for j := 0; j < iterations; j++ {
                // Update policy with a unique label to trigger map update
                testPolicy.Labels = []string{fmt.Sprintf(\"worker-%d-iter-%d\", workerID, j)}
                _, err := cli.Policy().Update(ctx, testPolicy)
                if err != nil {
                    t.Logf(\"worker %d: policy update failed: %v\", workerID, err)
                    continue
                }

                // Check policy enforcement for a non-frontend namespace (should be denied)
                // This is where the 1.15.0 race condition returns false positive \"allowed\"
                allowed, err := cli.Policy().Evaluate(
                    ctx,
                    \"payment\",
                    \"backend\",
                    \"8080\",
                    \"TCP\",
                )
                if err != nil {
                    t.Logf(\"worker %d: policy evaluate failed: %v\", workerID, err)
                    continue
                }

                totalChecks.Add(1)
                if allowed {
                    raceDetections.Add(1)
                    t.Logf(\"worker %d: race condition detected at iter %d\", workerID, j)
                }

                // Small sleep to simulate real-world request spacing
                time.Sleep(10 * time.Millisecond)
            }
        }(i)
    }

    wg.Wait()

    // Calculate failure rate
    failureRate := float64(raceDetections.Load()) / float64(totalChecks.Load()) * 100
    t.Logf(\"Total policy checks: %d\", totalChecks.Load())
    t.Logf(\"Race condition detections: %d\", raceDetections.Load())
    t.Logf(\"Failure rate: %.4f%%\", failureRate)

    // Assert failure rate matches Cilium 1.15.0's known 0.03% rate
    // We allow 0.01% margin of error for test variance
    require.InDelta(t, 0.03, failureRate, 0.01,
        \"failure rate does not match expected Cilium 1.15.0 race condition rate\")
}

#!/usr/bin/env python3
\"\"\"
Cilium 1.15.0 Policy Bug Audit Script
Checks all CiliumNetworkPolicy resources in a cluster for conditions that
trigger the 1.15.0 eBPF map race condition.
Requires: kubernetes>=28.1.0, pyyaml>=6.0.1
\"\"\"

import sys
import yaml
from kubernetes import client, config
from kubernetes.client.rest import ApiException

def load_kube_config():
    \"\"\"Load kubeconfig from default path or in-cluster config.\"\"\"
    try:
        config.load_kube_config()
        print(\"Loaded local kubeconfig\")
    except Exception as e:
        print(f\"Failed to load local kubeconfig: {e}\")
        try:
            config.load_in_cluster_config()
            print(\"Loaded in-cluster config\")
        except Exception as e:
            print(f\"Failed to load in-cluster config: {e}\")
            sys.exit(1)

def get_cilium_version():
    \"\"\"Retrieve Cilium agent version via Kubernetes API.\"\"\"
    apps_v1 = client.AppsV1Api()
    try:
        deployment = apps_v1.read_namespaced_deployment(
            name=\"cilium\",
            namespace=\"kube-system\"
        )
        for container in deployment.spec.template.spec.containers:
            if container.name == \"cilium-agent\":
                # Extract version from image tag (e.g., v1.15.0)
                image = container.image
                tag = image.split(\":\")[-1]
                return tag.lstrip(\"v\")
        return None
    except ApiException as e:
        print(f\"Failed to get Cilium deployment: {e}\")
        return None

def list_cilium_network_policies():
    \"\"\"List all CiliumNetworkPolicy resources across all namespaces.\"\"\"
    crd_api = client.CustomObjectsApi()
    policies = []
    try:
        # CiliumNetworkPolicy CRD group and version
        group = \"cilium.io\"
        version = \"v2\"
        plural = \"ciliumnetworkpolicies\"
        # List across all namespaces
        policy_list = crd_api.list_cluster_custom_object(
            group=group,
            version=version,
            plural=plural
        )
        for item in policy_list.get(\"items\", []):
            policies.append(item)
        return policies
    except ApiException as e:
        print(f\"Failed to list CiliumNetworkPolicies: {e}\")
        return []

def check_policy_for_bug(policy):
    \"\"\"Check if a policy triggers the Cilium 1.15.0 race condition.
    The bug is triggered when a policy has:
    1. Ingress rules with FromEndpoints selectors
    2. More than 5 port rules per ingress rule
    3. Frequent updates (tracked via annotation)
    \"\"\"
    policy_name = policy.get(\"metadata\", {}).get(\"name\", \"unknown\")
    policy_namespace = policy.get(\"metadata\", {}).get(\"namespace\", \"default\")
    spec = policy.get(\"spec\", {})
    ingress_rules = spec.get(\"ingress\", [])

    bug_triggered = False
    trigger_reasons = []

    for rule in ingress_rules:
        # Check for FromEndpoints selectors (required for bug trigger)
        from_endpoints = rule.get(\"fromEndpoints\", [])
        if not from_endpoints:
            continue

        # Check port rule count
        to_ports = rule.get(\"toPorts\", [])
        if len(to_ports) > 5:
            trigger_reasons.append(f\"Rule has {len(to_ports)} port rules (threshold: 5)\")
            bug_triggered = True

        # Check for frequent update annotation
        annotations = policy.get(\"metadata\", {}).get(\"annotations\", {})
        update_count = annotations.get(\"cilium.io/policy-update-count\", \"0\")
        try:
            if int(update_count) > 10:
                trigger_reasons.append(f\"Policy updated {update_count} times (threshold: 10)\")
                bug_triggered = True
        except ValueError:
            pass

    if bug_triggered:
        return {
            \"name\": policy_name,
            \"namespace\": policy_namespace,
            \"triggers\": trigger_reasons
        }
    return None

def main():
    load_kube_config()

    # Check Cilium version first
    cilium_version = get_cilium_version()
    if not cilium_version:
        print(\"ERROR: Could not determine Cilium version\")
        sys.exit(1)
    print(f\"Detected Cilium version: {cilium_version}\")

    # Only warn for 1.15.0 specifically
    if cilium_version == \"1.15.0\":
        print(\"WARNING: Cilium 1.15.0 is affected by the policy race condition bug\")
    elif cilium_version.startswith(\"1.15\"):
        print(f\"INFO: Cilium {cilium_version} is patched, but audit is still recommended\")
    else:
        print(f\"INFO: Cilium {cilium_version} is not affected by this bug\")

    # List and check all policies
    policies = list_cilium_network_policies()
    print(f\"Found {len(policies)} CiliumNetworkPolicy resources\")

    affected_policies = []
    for policy in policies:
        result = check_policy_for_bug(policy)
        if result:
            affected_policies.append(result)

    # Output results
    if affected_policies:
        print(\"\\n=== AFFECTED POLICIES ===\")
        for p in affected_policies:
            print(f\"Policy: {p['namespace']}/{p['name']}\")
            for reason in p[\"triggers\"]:
                print(f\"  - {reason}\")
        print(f\"\\nTotal affected policies: {len(affected_policies)}\")
        sys.exit(1)
    else:
        print(\"\\nNo policies triggering Cilium 1.15.0 bug found.\")
        sys.exit(0)

if __name__ == \"__main__\":
    main()

#!/bin/bash
#
# Cilium 1.15.0 to 1.15.1 Upgrade Script
# Includes pre-upgrade benchmarks, health checks, rollback, and post-upgrade validation
# Requires: kubectl, helm, cilium CLI, perf (for benchmarks)
set -euo pipefail

# Configuration
CILIUM_NAMESPACE=\"kube-system\"
HELM_RELEASE_NAME=\"cilium\"
HELM_CHART=\"cilium/cilium\"
OLD_VERSION=\"1.15.0\"
NEW_VERSION=\"1.15.1\"
BENCHMARK_DURATION=30  # seconds
HEALTH_CHECK_RETRIES=30
HEALTH_CHECK_DELAY=10  # seconds

# Logging function
log() {
    echo \"[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*\"
}

# Error handling
trap 'log \"ERROR: Upgrade failed at line $LINENO. Starting rollback...\"; rollback' ERR

rollback() {
    log \"Rolling back to Cilium ${OLD_VERSION}\"
    helm rollback \"${HELM_RELEASE_NAME}\" 0 \
        --namespace \"${CILIUM_NAMESPACE}\" \
        --wait
    log \"Rollback complete. Restoring old benchmarks...\"
    # Restore pre-upgrade benchmark results
    if [ -f \"pre-upgrade-benchmarks.json\" ]; then
        kubectl create configmap cilium-benchmarks \
            --from-file=pre-upgrade-benchmarks.json \
            --namespace \"${CILIUM_NAMESPACE}\" \
            --dry-run=client -o yaml | kubectl apply -f -
    fi
    exit 1
}

# Pre-upgrade checks
pre_upgrade_checks() {
    log \"Running pre-upgrade checks...\"
    # Check if Cilium is installed
    if ! helm list --namespace \"${CILIUM_NAMESPACE}\" | grep -q \"${HELM_RELEASE_NAME}\"; then
        log \"ERROR: Cilium Helm release not found\"
        exit 1
    fi

    # Check current Cilium version
    current_version=$(helm list --namespace \"${CILIUM_NAMESPACE}\" \
        --output json | jq -r '.[] | select(.name==\"cilium\") | .app_version')
    if [ \"${current_version}\" != \"v${OLD_VERSION}\" ]; then
        log \"ERROR: Current Cilium version is ${current_version}, expected v${OLD_VERSION}\"
        exit 1
    fi

    # Check cluster health
    log \"Checking cluster health...\"
    kubectl get nodes --no-headers | while read node status rest; do
        if [ \"${status}\" != \"Ready\" ]; then
            log \"ERROR: Node ${node} is not Ready\"
            exit 1
        fi
    done

    log \"Pre-upgrade checks passed\"
}

# Run pre-upgrade benchmarks
run_benchmarks() {
    log \"Running pre-upgrade benchmarks for ${BENCHMARK_DURATION} seconds...\"
    # Capture policy check throughput
    cilium-bench policy \
        --duration \"${BENCHMARK_DURATION}s\" \
        --output json > pre-upgrade-benchmarks.json
    # Capture eBPF map update latency
    kubectl exec -n \"${CILIUM_NAMESPACE}\" deploy/cilium \
        -- perf record -g -F 99 -o /tmp/perf.data -- sleep \"${BENCHMARK_DURATION}\" \
        || log \"WARNING: perf benchmark failed, continuing anyway\"
    log \"Pre-upgrade benchmarks saved to pre-upgrade-benchmarks.json\"
}

# Upgrade Cilium
upgrade_cilium() {
    log \"Upgrading Cilium from ${OLD_VERSION} to ${NEW_VERSION}...\"
    helm upgrade \"${HELM_RELEASE_NAME}\" \"${HELM_CHART}\" \
        --version \"${NEW_VERSION}\" \
        --namespace \"${CILIUM_NAMESPACE}\" \
        --set image.tag=\"v${NEW_VERSION}\" \
        --set operator.image.tag=\"v${NEW_VERSION}\" \
        --wait \
        --timeout 10m
    log \"Helm upgrade complete\"
}

# Post-upgrade health checks
post_upgrade_health_checks() {
    log \"Running post-upgrade health checks...\"
    for i in $(seq 1 \"${HEALTH_CHECK_RETRIES}\"); do
        log \"Health check attempt ${i}/${HEALTH_CHECK_RETRIES}\"
        # Check Cilium agent health
        if ! cilium status --wait; then
            log \"Cilium status check failed, retrying in ${HEALTH_CHECK_DELAY}s...\"
            sleep \"${HEALTH_CHECK_DELAY}\"
            continue
        fi
        # Check all Cilium pods are running
        if ! kubectl get pods -n \"${CILIUM_NAMESPACE}\" -l app=cilium \
            --no-headers | grep -q \"Running\"; then
            log \"Cilium pods not running, retrying in ${HEALTH_CHECK_DELAY}s...\"
            sleep \"${HEALTH_CHECK_DELAY}\"
            continue
        fi
        log \"Health checks passed\"
        return 0
    done
    log \"ERROR: Post-upgrade health checks failed after ${HEALTH_CHECK_RETRIES} attempts\"
    return 1
}

# Run post-upgrade benchmarks
post_upgrade_benchmarks() {
    log \"Running post-upgrade benchmarks...\"
    cilium-bench policy \
        --duration \"${BENCHMARK_DURATION}s\" \
        --output json > post-upgrade-benchmarks.json
    # Compare benchmarks
    log \"Comparing benchmarks...\"
    python3 -c \"
import json
pre = json.load(open('pre-upgrade-benchmarks.json'))
post = json.load(open('post-upgrade-benchmarks.json'))
print(f'Pre-upgrade throughput: {pre[\\\"throughput\\\"]} checks/s')
print(f'Post-upgrade throughput: {post[\\\"throughput\\\"]} checks/s')
print(f'Pre-upgrade latency p99: {pre[\\\"latency_p99\\\"]}ms')
print(f'Post-upgrade latency p99: {post[\\\"latency_p99\\\"]}ms')
if post['throughput'] < pre['throughput'] * 0.9:
    print('WARNING: Throughput dropped by more than 10%')
if post['latency_p99'] > pre['latency_p99'] * 1.1:
    print('WARNING: Latency increased by more than 10%')
\"
}

# Main execution
main() {
    log \"Starting Cilium upgrade from ${OLD_VERSION} to ${NEW_VERSION}\"
    pre_upgrade_checks
    run_benchmarks
    upgrade_cilium
    post_upgrade_health_checks
    post_upgrade_benchmarks
    log \"Upgrade completed successfully!\"
    log \"Post-upgrade benchmarks saved to post-upgrade-benchmarks.json\"
}

main