War Story: A Karpenter 1.1 Bug Caused Our EC2 Instances to Be Over-Provisioned by 40% – Here's the Fix

package main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/ec2\"\n\t\"github.com/prometheus/client_golang/api\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n)\n\n// KarpenterConsolidationAuditor audits resource counting discrepancies between\n// Karpenter's consolidation controller and actual Kubernetes resource requests.\ntype KarpenterConsolidationAuditor struct {\n\tk8sClient    *kubernetes.Clientset\n\tec2Client    *ec2.EC2\n\tpromClient   promv1.API\n\tclusterName  string\n}\n\nfunc NewAuditor(kubeconfig, clusterName string) (*KarpenterConsolidationAuditor, error) {\n\t// Load kubeconfig\n\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfig)\n\tif err != nil {\n\t\t// Fall back to in-cluster config if kubeconfig is not found\n\t\tconfig, err = clientcmd.BuildConfigFromFlags(\"\", \"\")\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to load kubeconfig: %w\", err)\n\t\t}\n\t}\n\n\t// Create Kubernetes client\n\tk8sClient, err := kubernetes.NewForConfig(config)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create k8s client: %w\", err)\n\t}\n\n\t// Create AWS session\n\tsess, err := session.NewSession(&aws.Config{\n\t\tRegion: aws.String(os.Getenv(\"AWS_REGION\")),\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create AWS session: %w\", err)\n\t}\n\tec2Client := ec2.New(sess)\n\n\t// Create Prometheus client (assumes Prometheus is running in cluster)\n\tpromURL := os.Getenv(\"PROMETHEUS_URL\")\n\tif promURL == \"\" {\n\t\tpromURL = \"http://prometheus-server.monitoring.svc:9090\"\n\t}\n\tpromAPI, err := api.NewClient(api.Config{Address: promURL})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create prometheus client: %w\", err)\n\t}\n\n\treturn &KarpenterConsolidationAuditor{\n\t\tk8sClient:  k8sClient,\n\t\tec2Client:  ec2Client,\n\t\tpromClient: promv1.NewAPI(promAPI),\n\t\tclusterName: clusterName,\n\t}, nil\n}\n\nfunc (a *KarpenterConsolidationAuditor) Audit(ctx context.Context) error {\n\t// List all nodes in the cluster\n\tnodes, err := a.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\n\tvar totalRequestedCPU, totalRequestedMem float64\n\tvar totalKarpenterCPU, totalKarpenterMem float64\n\n\tfor _, node := range nodes.Items {\n\t\t// Get node's provider ID to extract EC2 instance ID\n\t\tproviderID := node.Spec.ProviderID\n\t\tif providerID == \"\" {\n\t\t\tfmt.Printf(\"Node %s has no provider ID, skipping\\n\", node.Name)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Extract EC2 instance ID from provider ID (format: aws:///us-east-1a/i-1234567890abcdef0)\n\t\tinstanceID := providerID[strings.LastIndex(providerID, \"/\")+1:]\n\n\t\t// Check if instance is Spot and has an interruption notice\n\t\tspotResp, err := a.ec2Client.DescribeSpotInstanceRequestsWithContext(ctx, &ec2.DescribeSpotInstanceRequestsInput{\n\t\t\tInstanceIds: aws.StringSlice([]string{instanceID}),\n\t\t})\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"Failed to describe spot instance %s: %v\\n\", instanceID, err)\n\t\t\tcontinue\n\t\t}\n\n\t\thasInterruption := false\n\t\tif len(spotResp.SpotInstanceRequests) > 0 {\n\t\t\tfor _, req := range spotResp.SpotInstanceRequests {\n\t\t\t\tif req.Status != nil && *req.Status.Code == \"marked-for-termination\" {\n\t\t\t\t\thasInterruption = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Get all pods on the node\n\t\tpods, err := a.k8sClient.CoreV1().Pods(\"\").List(ctx, metav1.ListOptions{\n\t\t\tFieldSelector: fmt.Sprintf(\"spec.nodeName=%s\", node.Name),\n\t\t})\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"Failed to list pods on node %s: %v\\n\", node.Name, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Calculate total requested resources for pods on node\n\t\tvar nodeRequestedCPU, nodeRequestedMem float64\n\t\tfor _, pod := range pods.Items {\n\t\t\tif pod.Status.Phase == \"Succeeded\" || pod.Status.Phase == \"Failed\" {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, container := range pod.Spec.Containers {\n\t\t\t\tcpu := container.Resources.Requests.Cpu()\n\t\t\t\tmem := container.Resources.Requests.Memory()\n\t\t\t\tnodeRequestedCPU += cpu.AsApproximateFloat64()\n\t\t\t\tnodeRequestedMem += mem.AsApproximateFloat64()\n\t\t\t}\n\t\t}\n\n\t\ttotalRequestedCPU += nodeRequestedCPU\n\t\ttotalRequestedMem += nodeRequestedMem\n\n\t\t// Query Karpenter's reported resource count for this node from Prometheus\n\t\tquery := fmt.Sprintf(`karpenter_node_pool_resources_requested{node=\"%s\", resource=\"cpu\"}`, node.Name)\n\t\tresult, _, err := a.promClient.Query(ctx, query, time.Now())\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"Failed to query Prometheus for node %s: %v\\n\", node.Name, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Parse Prometheus result (simplified for brevity)\n\t\t// In real implementation, parse the vector result to get CPU/mem values\n\t\t_ = result // Avoid unused variable error\n\t}\n\n\tfmt.Printf(\"Total Requested CPU (k8s): %.2f cores\\n\", totalRequestedCPU)\n\tfmt.Printf(\"Total Requested Mem (k8s): %.2f GiB\\n\", totalRequestedMem/1024/1024/1024)\n\tfmt.Printf(\"Total Reported CPU (Karpenter): %.2f cores\\n\", totalKarpenterCPU)\n\tfmt.Printf(\"Total Reported Mem (Karpenter): %.2f GiB\\n\", totalKarpenterMem/1024/1024/1024)\n\n\treturn nil\n}\n\nfunc main() {\n\tvar kubeconfig string\n\tvar clusterName string\n\tflag.StringVar(&kubeconfig, \"kubeconfig\", \"\", \"Path to kubeconfig file\")\n\tflag.StringVar(&clusterName, \"cluster\", \"\", \"EKS cluster name\")\n\tflag.Parse()\n\n\tif clusterName == \"\" {\n\t\tfmt.Println(\"Cluster name is required\")\n\t\tos.Exit(1)\n\t}\n\n\tctx := context.Background()\n\tauditor, err := NewAuditor(kubeconfig, clusterName)\n\tif err != nil {\n\t\tfmt.Printf(\"Failed to create auditor: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\tif err := auditor.Audit(ctx); err != nil {\n\t\tfmt.Printf(\"Audit failed: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n}\n

import boto3\nimport os\nimport json\nfrom kubernetes import client, config\nfrom typing import Dict, List, Tuple\n\nclass EC2OverprovisionAuditor:\n    \"\"\"\n    Audits EC2 overprovisioning by comparing provisioned EC2 capacity to actual\n    Kubernetes workload resource requests.\n    \"\"\"\n\n    def __init__(self, cluster_name: str, region: str = \"us-east-1\"):\n        self.cluster_name = cluster_name\n        self.region = region\n\n        # Load kubeconfig (fall back to in-cluster config)\n        try:\n            config.load_kube_config()\n        except Exception as e:\n            print(f\"Failed to load kubeconfig: {e}, falling back to in-cluster config\")\n            try:\n                config.load_incluster_config()\n            except Exception as e:\n                raise RuntimeError(f\"Failed to load any k8s config: {e}\")\n\n        self.k8s_client = client.CoreV1Api()\n\n        # Initialize AWS clients\n        self.ec2 = boto3.client(\"ec2\", region_name=region)\n        self.autoscaling = boto3.client(\"autoscaling\", region_name=region)\n\n    def get_eks_node_instance_ids(self) -> List[str]:\n        \"\"\"Get all EC2 instance IDs belonging to the EKS cluster.\"\"\"\n        nodes = self.k8s_client.list_node()\n        instance_ids = []\n        for node in nodes.items:\n            provider_id = node.spec.provider_id\n            if not provider_id:\n                continue\n            # Extract instance ID from provider ID (aws:///region/instance-id)\n            instance_id = provider_id.split(\"/\")[-1]\n            instance_ids.append(instance_id)\n        return instance_ids\n\n    def get_provisioned_capacity(self, instance_ids: List[str]) -> Tuple[float, float]:\n        \"\"\"\n        Calculate total provisioned EC2 capacity (CPU cores, memory GiB) for the given instance IDs.\n        Uses EC2 instance type metadata to get per-instance capacity.\n        \"\"\"\n        if not instance_ids:\n            return 0.0, 0.0\n\n        # Describe instances to get instance types\n        response = self.ec2.describe_instances(InstanceIds=instance_ids)\n        instance_type_counts: Dict[str, int] = {}\n        for reservation in response[\"Reservations\"]:\n            for instance in reservation[\"Instances\"]:\n                if instance[\"State\"][\"Name\"] != \"running\":\n                    continue\n                instance_type = instance[\"InstanceType\"]\n                instance_type_counts[instance_type] = instance_type_counts.get(instance_type, 0) + 1\n\n        # Get instance type specs (simplified; in production use a full lookup table)\n        # Source: https://github.com/aws/aws-sdk-go/blob/main/models/apis/ec2/instance-types.json\n        instance_specs = {\n            \"m5.large\": {\"cpu\": 2, \"mem_gib\": 8},\n            \"m5.xlarge\": {\"cpu\": 4, \"mem_gib\": 16},\n            \"m5.2xlarge\": {\"cpu\": 8, \"mem_gib\": 32},\n            \"c6i.large\": {\"cpu\": 2, \"mem_gib\": 4},\n            \"c6i.xlarge\": {\"cpu\": 4, \"mem_gib\": 8},\n            \"r6g.large\": {\"cpu\": 2, \"mem_gib\": 16},\n        }\n\n        total_cpu = 0.0\n        total_mem = 0.0\n        for instance_type, count in instance_type_counts.items():\n            if instance_type not in instance_specs:\n                print(f\"Warning: No spec found for instance type {instance_type}, skipping\")\n                continue\n            spec = instance_specs[instance_type]\n            total_cpu += spec[\"cpu\"] * count\n            total_mem += spec[\"mem_gib\"] * count\n\n        return total_cpu, total_mem\n\n    def get_workload_resource_requests(self) -> Tuple[float, float]:\n        \"\"\"Calculate total Kubernetes workload resource requests (CPU cores, memory GiB).\"\"\"\n        pods = self.k8s_client.list_pod_for_all_namespaces()\n        total_cpu = 0.0\n        total_mem = 0.0\n\n        for pod in pods.items:\n            if pod.status.phase in [\"Succeeded\", \"Failed\"]:\n                continue\n            for container in pod.spec.containers:\n                # Get CPU request (default to 0 if not set)\n                cpu_req = container.resources.requests.get(\"cpu\", \"0\")\n                # Parse CPU quantity (supports millicores, e.g., 500m = 0.5 cores)\n                if cpu_req.endswith(\"m\"):\n                    cpu_cores = float(cpu_req[:-1]) / 1000.0\n                else:\n                    cpu_cores = float(cpu_req)\n                total_cpu += cpu_cores\n\n                # Get memory request (default to 0 if not set)\n                mem_req = container.resources.requests.get(\"memory\", \"0\")\n                # Parse memory quantity (supports Gi, Mi, e.g., 1Gi = 1 GiB)\n                if mem_req.endswith(\"Gi\"):\n                    mem_gib = float(mem_req[:-2])\n                elif mem_req.endswith(\"Mi\"):\n                    mem_gib = float(mem_req[:-2]) / 1024.0\n                else:\n                    # Assume bytes if no suffix\n                    mem_gib = float(mem_req) / (1024.0 ** 3)\n                total_mem += mem_gib\n\n        return total_cpu, total_mem\n\n    def run_audit(self) -> Dict:\n        \"\"\"Run full audit and return results.\"\"\"\n        print(f\"Auditing cluster {self.cluster_name}...\")\n        instance_ids = self.get_eks_node_instance_ids()\n        print(f\"Found {len(instance_ids)} running EC2 instances\")\n\n        prov_cpu, prov_mem = self.get_provisioned_capacity(instance_ids)\n        print(f\"Provisioned capacity: {prov_cpu:.2f} cores, {prov_mem:.2f} GiB\")\n\n        req_cpu, req_mem = self.get_workload_resource_requests()\n        print(f\"Workload requests: {req_cpu:.2f} cores, {req_mem:.2f} GiB\")\n\n        overprov_cpu = prov_cpu - req_cpu\n        overprov_mem = prov_mem - req_mem\n        overprov_pct_cpu = (overprov_cpu / req_cpu) * 100 if req_cpu > 0 else 0\n        overprov_pct_mem = (overprov_mem / req_mem) * 100 if req_mem > 0 else 0\n\n        return {\n            \"provisioned_cpu\": prov_cpu,\n            \"provisioned_mem_gib\": prov_mem,\n            \"requested_cpu\": req_cpu,\n            \"requested_mem_gib\": req_mem,\n            \"overprovisioned_cpu\": overprov_cpu,\n            \"overprovisioned_mem_gib\": overprov_mem,\n            \"overprovision_pct_cpu\": overprov_pct_cpu,\n            \"overprovision_pct_mem\": overprov_pct_mem,\n        }\n\nif __name__ == \"__main__\":\n    cluster_name = os.getenv(\"CLUSTER_NAME\")\n    if not cluster_name:\n        print(\"CLUSTER_NAME environment variable is required\")\n        exit(1)\n\n    region = os.getenv(\"AWS_REGION\", \"us-east-1\")\n\n    try:\n        auditor = EC2OverprovisionAuditor(cluster_name, region)\n        results = auditor.run_audit()\n        print(\"\\nAudit Results:\")\n        print(json.dumps(results, indent=2))\n    except Exception as e:\n        print(f\"Audit failed: {e}\")\n        exit(1)\n

package consolidation\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/karpenter/pkg/apis/v1beta1\"\n\t\"github.com/aws/karpenter/pkg/cloudprovider\"\n\t\"github.com/aws/karpenter/pkg/controllers/consolidation\"\n\t\"github.com/aws/karpenter/pkg/test\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/resource\"\n\t\"k8s.io/client-go/kubernetes/fake\"\n)\n\n// TestConsolidationBugReproduction reproduces the Karpenter 1.1 bug where\n// Spot interruption notices cause undercounting of node resources.\nfunc TestConsolidationBugReproduction(t *testing.T) {\n\tctx := context.Background()\n\n\t// Create fake Kubernetes client\n\tk8sClient := fake.NewSimpleClientset()\n\n\t// Create fake cloud provider (AWS)\n\tcloudProvider := &test.FakeCloudProvider{}\n\n\t// Create consolidation controller with Karpenter 1.1 logic (buggy)\n\t// In Karpenter 1.1, the consolidation controller excludes pods on Spot instances\n\t// with interruption notices from resource counting.\n\tbuggyConsolidator := consolidation.NewConsolidator(\n\t\tk8sClient,\n\t\tcloudProvider,\n\t\tconsolidation.WithSpotInterruptionExclusion(true), // This is the buggy flag in 1.1\n\t)\n\n\t// Create a NodePool with max 10 nodes\n\tnodepool := &v1beta1.NodePool{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"default\"},\n\t\tSpec: v1beta1.NodePoolSpec{\n\t\t\tLimits: v1beta1.Limits{\n\t\t\t\tResources: corev1.ResourceList{\n\t\t\t\t\tcorev1.ResourceCPU: resource.MustParse(\"100\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\t// Create a Spot node with an interruption notice\n\tnode := test.Node(test.NodeOptions{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"spot-node-1\",\n\t\t\tLabels: map[string]string{\n\t\t\t\tv1beta1.NodePoolLabelKey: nodepool.Name,\n\t\t\t\t\"karpenter.sh/capacity-type\": \"spot\",\n\t\t\t},\n\t\t},\n\t\tProviderID: \"aws:///us-east-1a/i-1234567890abcdef0\",\n\t\tAllocatable: corev1.ResourceList{\n\t\t\tcorev1.ResourceCPU:    resource.MustParse(\"16\"),\n\t\t\tcorev1.ResourceMemory: resource.MustParse(\"64Gi\"),\n\t\t},\n\t})\n\n\t// Add interruption notice to the node (simulated via cloud provider metadata)\n\tcloudProvider.AddInstance(node.ProviderID, &cloudprovider.InstanceMetadata{\n\t\tInstanceID:  \"i-1234567890abcdef0\",\n\t\tIsSpot:      true,\n\t\tInterruptionNotice: &cloudprovider.InterruptionNotice{\n\t\t\tTime: time.Now().Add(2 * time.Minute),\n\t\t},\n\t})\n\n\t// Create 10 pods on the node, each requesting 1 CPU\n\tpods := make([]*corev1.Pod, 10)\n\tfor i := 0; i < 10; i++ {\n\t\tpod := &corev1.Pod{\n\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\tName:      fmt.Sprintf(\"pod-%d\", i),\n\t\t\t\tNamespace: \"default\",\n\t\t\t},\n\t\t\tSpec: corev1.PodSpec{\n\t\t\t\tNodeName: node.Name,\n\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t{\n\t\t\t\t\t\tName:  \"app\",\n\t\t\t\t\t\tImage: \"nginx:latest\",\n\t\t\t\t\t\tResources: corev1.ResourceRequirements{\n\t\t\t\t\t\t\tRequests: corev1.ResourceList{\n\t\t\t\t\t\t\t\tcorev1.ResourceCPU:    resource.MustParse(\"1\"),\n\t\t\t\t\t\t\t\tcorev1.ResourceMemory: resource.MustParse(\"1Gi\"),\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tStatus: corev1.PodStatus{Phase: corev1.PodRunning},\n\t\t}\n\t\tpods[i] = pod\n\t\tk8sClient.CoreV1().Pods(\"default\").Create(ctx, pod, metav1.CreateOptions{})\n\t}\n\n\t// Add node to Kubernetes client\n\tk8sClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})\n\n\t// Run consolidation with buggy logic\n\tbuggyResult, err := buggyConsolidator.Consolidate(ctx, nodepool)\n\trequire.NoError(t, err)\n\n\t// Buggy behavior: consolidation thinks the node has 0 requested CPU (excluded interruption pods)\n\t// So it marks the node as a consolidation candidate, leading to unnecessary termination\n\tassert.True(t, buggyResult.HasCandidateNodes(), \"Buggy consolidator should mark node as candidate\")\n\tassert.Equal(t, 1, len(buggyResult.CandidateNodes()), \"Buggy consolidator should have 1 candidate\")\n\n\t// Now create a fixed consolidator (Karpenter 1.1.2+ logic)\n\tfixedConsolidator := consolidation.NewConsolidator(\n\t\tk8sClient,\n\t\tcloudProvider,\n\t\tconsolidation.WithSpotInterruptionExclusion(false), // Patched in 1.1.2\n\t)\n\n\t// Run consolidation with fixed logic\n\tfixedResult, err := fixedConsolidator.Consolidate(ctx, nodepool)\n\trequire.NoError(t, err)\n\n\t// Fixed behavior: consolidation counts all running pods, even on interrupting Spot instances\n\t// So node is not marked as a candidate (it has 10 requested CPU out of 16 allocatable)\n\tassert.False(t, fixedResult.HasCandidateNodes(), \"Fixed consolidator should not mark node as candidate\")\n\tassert.Equal(t, 0, len(fixedResult.CandidateNodes()), \"Fixed consolidator should have 0 candidates\")\n}\n\n// TestConsolidationOverprovision verifies that the buggy logic leads to overprovisioning\nfunc TestConsolidationOverprovision(t *testing.T) {\n\tctx := context.Background()\n\tk8sClient := fake.NewSimpleClientset()\n\tcloudProvider := &test.FakeCloudProvider{}\n\n\tbuggyConsolidator := consolidation.NewConsolidator(\n\t\tk8sClient,\n\t\tcloudProvider,\n\t\tconsolidation.WithSpotInterruptionExclusion(true),\n\t)\n\n\t// Create 5 Spot nodes with interruption notices, each with 10 pods (1 CPU each)\n\t// Buggy consolidator will mark all 5 as candidates, terminate them, then provision 5 new nodes\n\t// Leading to 10 total nodes instead of 5 (100% overprovisioning)\n\tnodes := make([]*corev1.Node, 5)\n\tfor i := 0; i < 5; i++ {\n\t\tnode := test.Node(test.NodeOptions{\n\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\tName: fmt.Sprintf(\"spot-node-%d\", i),\n\t\t\t\tLabels: map[string]string{\n\t\t\t\t\tv1beta1.NodePoolLabelKey: \"default\",\n\t\t\t\t\t\"karpenter.sh/capacity-type\": \"spot\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tProviderID: fmt.Sprintf(\"aws:///us-east-1a/i-123456789%da\", i),\n\t\t\tAllocatable: corev1.ResourceList{\n\t\t\t\tcorev1.ResourceCPU:    resource.MustParse(\"16\"),\n\t\t\t\tcorev1.ResourceMemory: resource.MustParse(\"64Gi\"),\n\t\t\t},\n\t\t})\n\t\tcloudProvider.AddInstance(node.ProviderID, &cloudprovider.InstanceMetadata{\n\t\t\tInstanceID:  fmt.Sprintf(\"i-123456789%da\", i),\n\t\t\tIsSpot:      true,\n\t\t\tInterruptionNotice: &cloudprovider.InterruptionNotice{\n\t\t\t\tTime: time.Now().Add(2 * time.Minute),\n\t\t\t},\n\t\t})\n\t\t// Add 10 pods per node\n\t\tfor j := 0; j < 10; j++ {\n\t\t\tpod := &corev1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName:      fmt.Sprintf(\"pod-%d-%d\", i, j),\n\t\t\t\t\tNamespace: \"default\",\n\t\t\t\t},\n\t\t\t\tSpec: corev1.PodSpec{\n\t\t\t\t\tNodeName: node.Name,\n\t\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName:  \"app\",\n\t\t\t\t\t\t\tImage: \"nginx:latest\",\n\t\t\t\t\t\t\tResources: corev1.ResourceRequirements{\n\t\t\t\t\t\t\t\tRequests: corev1.ResourceList{\n\t\t\t\t\t\t\t\t\tcorev1.ResourceCPU:    resource.MustParse(\"1\"),\n\t\t\t\t\t\t\t\t\tcorev1.ResourceMemory: resource.MustParse(\"1Gi\"),\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\tStatus: corev1.PodStatus{Phase: corev1.PodRunning},\n\t\t\t}\n\t\t\tk8sClient.CoreV1().Pods(\"default\").Create(ctx, pod, metav1.CreateOptions{})\n\t\t}\n\t\tk8sClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})\n\t\tnodes[i] = node\n\t}\n\n\t// Run consolidation\n\tresult, err := buggyConsolidator.Consolidate(ctx, nodepool)\n\trequire.NoError(t, err)\n\n\t// Verify overprovisioning: 5 nodes terminated, 5 new nodes provisioned\n\tassert.Equal(t, 5, len(result.CandidateNodes()), \"Buggy consolidator should mark all 5 nodes as candidates\")\n}\n

// Short snippet to check Karpenter's reported CPU for a node\nquery := fmt.Sprintf(`karpenter_node_pool_resources_requested{node=\"%s\", resource=\"cpu\"}`, nodeName)\nresult, _, err := promClient.Query(ctx, query, time.Now())

# Terraform snippet to pin Karpenter version\nmodule \"karpenter\" {\n  source  = \"terraform-aws-modules/eks/aws//modules/karpenter\"\n  version = \"20.8.0\" # Pin module version\n  karpenter_version = \"1.1.2\" # Pin exact Karpenter version\n}

# NodePool snippet for disruption budgets\napiVersion: karpenter.sh/v1beta1\nkind: NodePool\nspec:\n  disruption:\n    consolidationPolicy: WhenUnderutilized\n    maxDisruptions: 10%