Postmortem: How a Bad Hire Cost Our Team $500K in AWS Graviton4 and Kubernetes 1.31 Costs

# Copyright 2024 Platform Team. Licensed under MIT.
# terraform/main.tf: Provision EKS 1.31 node groups with Graviton4 instances
# Implements K8s 1.31 arm64 scheduler optimizations to avoid $500K-style waste
terraform {
  required_version = ">= 1.7.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.50"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.30"
    }
  }
  # Prevent accidental destruction of production node groups
  lifecycle {
    prevent_destroy = true
  }
}

# Configure AWS provider for us-east-1 (Graviton4 GA region)
provider "aws" {
  region = "us-east-1"
  default_tags {
    tags = {
      Team      = "Platform"
      CostCenter = "1001"
      ManagedBy  = "Terraform"
    }
  }
}

# Fetch latest EKS 1.31 optimized Graviton4 AMI
data "aws_ami" "eks_graviton4" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["amazon-eks-node-1.31-v*"]
  }

  filter {
    name   = "architecture"
    values = ["arm64"]
  }

  filter {
    name   = "root-device-type"
    values = ["ebs"]
  }
}

# EKS Cluster (1.31 control plane)
resource "aws_eks_cluster" "prod" {
  name     = "prod-eks-1-31"
  role_arn = aws_iam_role.eks_cluster.arn
  version  = "1.31"

  vpc_config {
    subnet_ids = aws_subnet.private[*].id
    endpoint_private_access = true
    endpoint_public_access  = false
  }

  # Enable K8s 1.31’s native arm64 scheduler
  enabled_cluster_log_types = ["api", "audit", "authenticator", "scheduler"]
}

# Graviton4 managed node group (the one our bad hire misconfigured)
resource "aws_eks_node_group" "graviton4_prod" {
  cluster_name    = aws_eks_cluster.prod.name
  node_group_name = "graviton4-prod-ng"
  node_role_arn   = aws_iam_role.eks_node.arn
  subnet_ids      = aws_subnet.private[*].id

  # Graviton4 instance type: c8g.4xlarge (16 vCPU, 32GB RAM, arm64)
  instance_types = ["c8g.4xlarge"]

  ami_type       = "AL2_ARM_64"
  capacity_type  = "ON_DEMAND" # Bad hire used SPOT without proper draining, added $120K waste
  disk_size      = 100

  # K8s 1.31 labels to enable scheduler awareness
  labels = {
    "node.kubernetes.io/instance-type" = "c8g.4xlarge"
    "topology.kubernetes.io/arch"      = "arm64"
    "k8s.io/cluster-autoscaler/enabled" = "true"
  }

  # Taints to prevent non-arm64 pods from scheduling (critical fix)
  taint {
    key    = "arch"
    value  = "arm64"
    effect = "NO_SCHEDULE"
  }

  scaling_config {
    desired_size = 12
    max_size     = 48
    min_size     = 6
  }

  # Force node group replacement on AMI change to avoid stale configs
  lifecycle {
    create_before_destroy = true
    ignore_changes = [
      scaling_config[0].desired_size,
    ]
  }

  depends_on = [aws_eks_cluster.prod]
}

# IAM roles (abbreviated for brevity, full code in linked repo)
resource "aws_iam_role" "eks_cluster" {
  name = "eks-cluster-prod-1-31"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action = "sts:AssumeRole"
      Effect = "Allow"
      Principal = { Service = "eks.amazonaws.com" }
    }]
  })
}

// Copyright 2024 Platform Team. Licensed under MIT.
// cmd/admission-webhook/main.go: K8s 1.31 admission webhook to prevent resource waste
// Blocks pods with unset or excessive CPU requests on Graviton4 nodes
package main

import (
    "context"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "os"
    "os/signal"
    "syscall"
    "time"

    admissionv1 "k8s.io/api/admission/v1"
    corev1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/rest"
    "k8s.io/klog/v2"
)

const (
    // Max allowed CPU request per pod on Graviton4 (prevents overprovisioning)
    maxGraviton4CPURequest = "4" // 4 vCPU, matches c8g.4xlarge allocation
    webhookPort            = 8443
    certDir                = "/etc/webhook/certs"
)

// AdmissionWebhook validates and mutates pod creation requests
type AdmissionWebhook struct {
    clientset *kubernetes.Clientset
}

// ValidatePod checks resource requests for Graviton4 compatibility
func (w *AdmissionWebhook) ValidatePod(ctx context.Context, pod *corev1.Pod) (bool, string, error) {
    // Skip system pods
    if pod.Namespace == "kube-system" || pod.Namespace == "kube-public" {
        return true, "system pod skipped", nil
    }

    // Check if pod is targeting Graviton4 nodes via node selector or affinity
    targetsGraviton := false
    if pod.Spec.NodeSelector["topology.kubernetes.io/arch"] == "arm64" {
        targetsGraviton = true
    }
    if pod.Spec.Affinity != nil && pod.Spec.Affinity.NodeAffinity != nil && pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
        for _, term := range pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
            for _, expr := range term.MatchExpressions {
                if expr.Key == "topology.kubernetes.io/arch" && expr.Operator == corev1.NodeSelectorOpIn {
                    for _, val := range expr.Values {
                        if val == "arm64" {
                            targetsGraviton = true
                        }
                    }
                }
            }
        }
    }

    if !targetsGraviton {
        return true, "non-arm64 pod skipped", nil
    }

    // Validate CPU requests for all containers
    for _, container := range pod.Spec.Containers {
        cpuRequest := container.Resources.Requests.Cpu()
        if cpuRequest.IsZero() {
            return false, fmt.Sprintf("container %s has no CPU request set; Graviton4 requires explicit requests", container.Name), nil
        }
        if cpuRequest.MilliValue() > 4000 { // 4 vCPU = 4000 millicores
            return false, fmt.Sprintf("container %s CPU request %dm exceeds max allowed %s for Graviton4", container.Name, cpuRequest.MilliValue(), maxGraviton4CPURequest), nil
        }
    }

    return true, "pod passed Graviton4 resource validation", nil
}

// HandleAdmission processes admission review requests
func (w *AdmissionWebhook) HandleAdmission(writer http.ResponseWriter, request *http.Request) {
    body, err := io.ReadAll(request.Body)
    if err != nil {
        klog.Errorf("Failed to read request body: %v", err)
        http.Error(writer, "failed to read request", http.StatusBadRequest)
        return
    }

    var admissionReview admissionv1.AdmissionReview
    if err := json.Unmarshal(body, &admissionReview); err != nil {
        klog.Errorf("Failed to unmarshal admission review: %v", err)
        http.Error(writer, "invalid admission review", http.StatusBadRequest)
        return
    }

    if admissionReview.Request == nil {
        klog.Error("Admission review request is nil")
        http.Error(writer, "nil admission request", http.StatusBadRequest)
        return
    }

    // Decode pod from admission request
    var pod corev1.Pod
    if err := json.Unmarshal(admissionReview.Request.Object.Raw, &pod); err != nil {
        klog.Errorf("Failed to unmarshal pod: %v", err)
        http.Error(writer, "invalid pod object", http.StatusBadRequest)
        return
    }

    // Validate pod
    allowed, message, err := w.ValidatePod(request.Context(), &pod)
    if err != nil {
        klog.Errorf("Validation error: %v", err)
        allowed = false
        message = "internal validation error"
    }

    // Build admission response
    response := admissionv1.AdmissionReview{
        TypeMeta: metav1.TypeMeta{
            APIVersion: "admission.k8s.io/v1",
            Kind:       "AdmissionReview",
        },
        Response: &admissionv1.AdmissionResponse{
            UID:     admissionReview.Request.UID,
            Allowed: allowed,
            Result: &metav1.Status{
                Message: message,
            },
        },
    }

    respBytes, err := json.Marshal(response)
    if err != nil {
        klog.Errorf("Failed to marshal response: %v", err)
        http.Error(writer, "failed to build response", http.StatusInternalServerError)
        return
    }

    writer.Header().Set("Content-Type", "application/json")
    writer.Write(respBytes)
}

func main() {
    // Initialize k8s client
    config, err := rest.InClusterConfig()
    if err != nil {
        klog.Fatalf("Failed to get in-cluster config: %v", err)
    }
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        klog.Fatalf("Failed to create k8s client: %v", err)
    }

    webhook := &AdmissionWebhook{clientset: clientset}

    // Start HTTP server with TLS
    mux := http.NewServeMux()
    mux.HandleFunc("/validate-pod", webhook.HandleAdmission)

    server := &http.Server{
        Addr:    fmt.Sprintf(":%d", webhookPort),
        Handler: mux,
    }

    // Graceful shutdown
    sigChan := make(chan os.Signal, 1)
    signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

    go func() {
        klog.Infof("Starting admission webhook on port %d", webhookPort)
        if err := server.ListenAndServeTLS(certDir+"/tls.crt", certDir+"/tls.key"); err != nil && err != http.ErrServerClosed {
            klog.Fatalf("Webhook server failed: %v", err)
        }
    }()

    <-sigChan
    klog.Info("Shutting down webhook server")
    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    defer cancel()
    if err := server.Shutdown(ctx); err != nil {
        klog.Errorf("Failed to shutdown server: %v", err)
    }
}

# Copyright 2024 Platform Team. Licensed under MIT.
# scripts/cost_analyzer.py: Detect Graviton4 and K8s 1.31 waste via AWS Cost Explorer
# Benchmarks our $500K loss against current spend
import boto3
import json
from datetime import datetime, timedelta
from typing import Dict, List
import logging

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# AWS configuration
REGION = "us-east-1"
COST_EXPLORER_GRANULARITY = "DAILY"
LOOKBACK_DAYS = 90 # Analyze last 3 months of spend

class Graviton4CostAnalyzer:
    def __init__(self):
        try:
            self.ce_client = boto3.client("ce", region_name=REGION)
            self.ec2_client = boto3.client("ec2", region_name=REGION)
            logger.info("Initialized AWS clients for Cost Explorer and EC2")
        except Exception as e:
            logger.error(f"Failed to initialize AWS clients: {e}")
            raise

    def get_graviton4_instance_ids(self) -> List[str]:
        """Fetch all running Graviton4 (c8g, m8g, r8g) instances"""
        instance_ids = []
        paginator = self.ec2_client.get_paginator("describe_instances")
        try:
            for page in paginator.paginate(
                Filters=[
                    {"Name": "instance-state-name", "Values": ["running"]},
                    {"Name": "architecture", "Values": ["arm64"]},
                    {"Name": "instance-type", "Values": ["c8g.*", "m8g.*", "r8g.*"]}
                ]
            ):
                for reservation in page["Reservations"]:
                    for instance in reservation["Instances"]:
                        instance_ids.append(instance["InstanceId"])
            logger.info(f"Found {len(instance_ids)} running Graviton4 instances")
            return instance_ids
        except Exception as e:
            logger.error(f"Failed to fetch Graviton4 instances: {e}")
            return []

    def get_cost_for_service(self, service: str, start_date: str, end_date: str) -> float:
        """Fetch total cost for a specific AWS service via Cost Explorer"""
        try:
            response = self.ce_client.get_cost_and_usage(
                TimePeriod={"Start": start_date, "End": end_date},
                Granularity=COST_EXPLORER_GRANULARITY,
                Metrics=["UnblendedCost"],
                Filter={"Dimensions": {"Key": "SERVICE", "Values": [service]}}
            )
            total_cost = 0.0
            for result in response["ResultsByTime"]:
                total_cost += float(result["Total"]["UnblendedCost"]["Amount"])
            logger.info(f"Total {service} cost from {start_date} to {end_date}: ${total_cost:.2f}")
            return total_cost
        except Exception as e:
            logger.error(f"Failed to fetch cost for {service}: {e}")
            return 0.0

    def get_eks_131_cost(self) -> float:
        """Fetch EKS 1.31 specific control plane and node costs"""
        end_date = datetime.now().strftime("%Y-%m-%d")
        start_date = (datetime.now() - timedelta(days=LOOKBACK_DAYS)).strftime("%Y-%m-%d")
        # EKS control plane cost is $0.10 per hour per cluster, 1.31 specific
        eks_cost = self.get_cost_for_service("Amazon Elastic Kubernetes Service", start_date, end_date)
        # Add node group costs (EC2)
        ec2_cost = self.get_cost_for_service("Amazon Elastic Compute Cloud - Compute", start_date, end_date)
        return eks_cost + ec2_cost

    def detect_waste(self) -> Dict:
        """Compare actual spend to optimized baseline to detect waste"""
        end_date = datetime.now().strftime("%Y-%m-%d")
        start_date = (datetime.now() - timedelta(days=LOOKBACK_DAYS)).strftime("%Y-%m-%d")
        total_spend = self.get_eks_131_cost()
        # Optimized baseline: 30% cheaper than x86, no overprovisioning
        optimized_baseline = total_spend * 0.7 # Graviton4 is 30% cheaper when configured correctly
        waste = total_spend - optimized_baseline
        return {
            "total_spend": round(total_spend, 2),
            "optimized_baseline": round(optimized_baseline, 2),
            "wasted_spend": round(waste, 2),
            "waste_percentage": round((waste / total_spend) * 100, 2) if total_spend > 0 else 0.0,
            "graviton4_instances": len(self.get_graviton4_instance_ids())
        }

    def generate_report(self, output_path: str = "cost_report.json"):
        """Generate JSON report of waste analysis"""
        try:
            report = self.detect_waste()
            with open(output_path, "w") as f:
                json.dump(report, f, indent=2)
            logger.info(f"Generated cost report at {output_path}")
            # Print summary
            print(f"\n=== Graviton4 & K8s 1.31 Cost Analysis ===")
            print(f"Total Spend (90d): ${report['total_spend']}")
            print(f"Optimized Baseline: ${report['optimized_baseline']}")
            print(f"Wasted Spend: ${report['wasted_spend']} ({report['waste_percentage']}%)")
            print(f"Graviton4 Instances: {report['graviton4_instances']}")
        except Exception as e:
            logger.error(f"Failed to generate report: {e}")
            raise

if __name__ == "__main__":
    try:
        analyzer = Graviton4CostAnalyzer()
        analyzer.generate_report()
    except Exception as e:
        logger.error(f"Cost analyzer failed: {e}")
        exit(1)

package kubernetes.admission

deny[msg] {
    input.request.kind.kind == "Pod"
    container := input.request.object.spec.containers[_]
    not container.resources.requests.cpu
    msg := sprintf("container %v has no CPU request set; arm64 nodes require explicit requests", [container.name])
}

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cpuManagerPolicy: static
cpuManagerReconcilePeriod: 5s
memoryManagerPolicy: Static
topologyManagerPolicy: single-numa-node
systemReserved:
  cpu: "500m"
  memory: "1Gi"
kubeReserved:
  cpu: "500m"
  memory: "1Gi"

package test

import (
    "testing"
    "github.com/gruntwork-io/terratest/modules/terraform"
    "github.com/stretchr/testify/assert"
)

func TestGraviton4NodeGroup(t *testing.T) {
    t.Parallel()

    terraformOptions := &terraform.Options{
        TerraformDir: "../terraform",
    }

    defer terraform.Destroy(t, terraformOptions)
    terraform.InitAndApply(t, terraformOptions)

    // Verify node group uses Graviton4 instance type
    instanceTypes := terraform.OutputList(t, terraformOptions, "node_group_instance_types")
    assert.Contains(t, instanceTypes, "c8g.4xlarge")

    // Verify capacity type is ON_DEMAND (no unapproved SPOT)
    capacityType := terraform.Output(t, terraformOptions, "node_group_capacity_type")
    assert.Equal(t, "ON_DEMAND", capacityType)

    // Verify arm64 taint is applied
    aints := terraform.OutputList(t, terraformOptions, "node_group_taints")
    assert.Contains(t, taints, "arch=arm64:NoSchedule")
}