War Story: Running 1M Containers on K8s 1.33 with AWS Graviton4 in 2026

package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "os"
    "strings"
    "time"

    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/rest"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/apimachinery/pkg/util/retry"
)

// ContainerAuditConfig holds configuration for the audit tool
type ContainerAuditConfig struct {
    kubeconfig string
    namespace  string
    labelSelector string
    outputFormat string
}

func main() {
    // Parse CLI flags
    config := ContainerAuditConfig{}
    flag.StringVar(&config.kubeconfig, "kubeconfig", "", "Path to kubeconfig file (uses in-cluster if empty)")
    flag.StringVar(&config.namespace, "namespace", "", "Namespace to audit (all if empty)")
    flag.StringVar(&config.labelSelector, "labels", "", "Label selector to filter pods (e.g. app=myapp)")
    flag.StringVar(&config.outputFormat, "output", "text", "Output format: text, json, csv")
    flag.Parse()

    // Initialize Kubernetes client
    var clientset *kubernetes.Clientset
    var err error
    if config.kubeconfig == "" {
        // Try in-cluster config first
        inClusterConfig, err := rest.InClusterConfig()
        if err == nil {
            clientset, err = kubernetes.NewForConfig(inClusterConfig)
            if err != nil {
                log.Fatalf("Failed to create in-cluster client: %v", err)
            }
        } else {
            // Fall back to default kubeconfig
            apiConfig, err := clientcmd.LoadFromFile(clientcmd.RecommendedHomeFile)
            if err != nil {
                log.Fatalf("Failed to load default kubeconfig: %v", err)
            }
            restConfig, err := clientcmd.NewDefaultClientConfig(apiConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
            if err != nil {
                log.Fatalf("Failed to create REST config: %v", err)
            }
            clientset, err = kubernetes.NewForConfig(restConfig)
            if err != nil {
                log.Fatalf("Failed to create Kubernetes client: %v", err)
            }
        }
    } else {
        // Load kubeconfig from file
        apiConfig, err := clientcmd.LoadFromFile(config.kubeconfig)
        if err != nil {
            log.Fatalf("Failed to load kubeconfig: %v", err)
        }
        restConfig, err := clientcmd.NewDefaultClientConfig(apiConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
        if err != nil {
            log.Fatalf("Failed to create REST config: %v", err)
        }
        clientset, err = kubernetes.NewForConfig(restConfig)
        if err != nil {
            log.Fatalf("Failed to create Kubernetes client: %v", err)
        }
    }

    // Retry logic for API calls to handle transient failures
    var totalContainers int
    err = retry.OnError(retry.NewBackoffDelay(5, time.Second, 0), func(err error) bool {
        // Retry on 5xx errors or rate limits
        return strings.Contains(err.Error(), "500") || strings.Contains(err.Error(), "429")
    }, func() error {
        // List all pods matching the filter
        pods, err := clientset.CoreV1().Pods(config.namespace).List(context.Background(), metav1.ListOptions{
            LabelSelector: config.labelSelector,
        })
        if err != nil {
            return fmt.Errorf("failed to list pods: %w", err)
        }

        // Count containers across all pods
        for _, pod := range pods.Items {
            // Count init containers
            totalContainers += len(pod.Spec.InitContainers)
            // Count regular containers
            totalContainers += len(pod.Spec.Containers)
            // Count ephemeral containers if present
            totalContainers += len(pod.Spec.EphemeralContainers)
        }
        return nil
    })

    if err != nil {
        log.Fatalf("Failed to audit containers after retries: %v", err)
    }

    // Output results
    switch config.outputFormat {
    case "json":
        fmt.Printf(`{"total_containers": %d, "namespace": "%s", "label_selector": "%s"}`, totalContainers, config.namespace, config.labelSelector)
    case "csv":
        fmt.Printf("namespace,label_selector,total_containers\n%s,%s,%d", config.namespace, config.labelSelector, totalContainers)
    default:
        fmt.Printf("Total active containers: %d\n", totalContainers)
        fmt.Printf("Namespace: %s\n", config.namespace)
        fmt.Printf("Label selector: %s\n", config.labelSelector)
    }
}

# Terraform configuration for AWS Graviton4 Kubernetes cluster infrastructure
# Provider version pinning to ensure reproducibility
terraform {
  required_version = ">= 1.10.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 6.0.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.30.0"
    }
  }
  # Store state in S3 with DynamoDB lock for team collaboration
  backend "s3" {
    bucket         = "our-prod-terraform-state-2026"
    key            = "k8s-graviton4/terraform.tfstate"
    region         = "us-east-1"
    dynamodb_table = "terraform-lock-k8s-graviton4"
    encrypt        = true
  }
}

# Configure AWS provider with Graviton4-supported region
provider "aws" {
  region = var.aws_region
  default_tags {
    tags = {
      Project     = "prod-k8s-graviton4"
      Environment = "production"
      ManagedBy   = "terraform"
      Year        = "2026"
    }
  }
}

# Variables for configuration
variable "aws_region" {
  type        = string
  default     = "us-east-1"
  description = "AWS region to deploy resources to"
}

variable "cluster_name" {
  type        = string
  default     = "prod-graviton4-k8s-1-33"
  description = "Name of the Kubernetes cluster"
}

variable "vpc_cidr" {
  type        = string
  default     = "10.0.0.0/16"
  description = "CIDR block for the VPC"
}

# Create VPC for the cluster
resource "aws_vpc" "k8s_vpc" {
  cidr_block           = var.vpc_cidr
  enable_dns_support   = true
  enable_dns_hostnames = true
  tags = {
    Name = "${var.cluster_name}-vpc"
  }
}

# Public subnets for load balancers
resource "aws_subnet" "public_subnets" {
  count                   = 3
  vpc_id                  = aws_vpc.k8s_vpc.id
  cidr_block              = cidrsubnet(var.vpc_cidr, 8, count.index + 1)
  availability_zone       = data.aws_availability_zones.available.names[count.index]
  map_public_ip_on_launch = true
  tags = {
    Name = "${var.cluster_name}-public-subnet-${count.index}"
    "kubernetes.io/role/elb" = "1"
  }
}

# Private subnets for Graviton4 nodes
resource "aws_subnet" "private_subnets" {
  count                   = 3
  vpc_id                  = aws_vpc.k8s_vpc.id
  cidr_block              = cidrsubnet(var.vpc_cidr, 8, count.index + 10)
  availability_zone       = data.aws_availability_zones.available.names[count.index]
  map_public_ip_on_launch = false
  tags = {
    Name = "${var.cluster_name}-private-subnet-${count.index}"
    "kubernetes.io/role/internal-elb" = "1"
  }
}

# Data source to get available AZs
data "aws_availability_zones" "available" {
  state = "available"
}

# IAM role for Graviton4 worker nodes
resource "aws_iam_role" "worker_node_role" {
  name = "${var.cluster_name}-worker-node-role"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })
  tags = {
    Name = "${var.cluster_name}-worker-node-role"
  }
}

# Attach required policies to worker node role
resource "aws_iam_role_policy_attachment" "worker_node_policies" {
  for_each = toset([
    "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
    "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
    "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
    "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
  ])
  policy_arn = each.value
  role       = aws_iam_role.worker_node_role.name
}

# Launch template for Graviton4 instances (c8g.4xlarge)
resource "aws_launch_template" "graviton4_lt" {
  name_prefix   = "${var.cluster_name}-graviton4-lt-"
  description   = "Launch template for Graviton4 worker nodes"
  image_id      = var.graviton4_ami_id # AMI pre-configured with K8s 1.33 and Graviton4 drivers
  instance_type = "c8g.4xlarge" # Graviton4 instance type, 16 vCPU, 32GB RAM
  iam_instance_profile {
    name = aws_iam_instance_profile.worker_node_profile.name
  }
  network_interfaces {
    security_groups = [aws_security_group.worker_node_sg.id]
  }
  tag_specifications {
    resource_type = "instance"
    tags = {
      Name = "${var.cluster_name}-graviton4-worker"
      "kubernetes.io/cluster/${var.cluster_name}" = "owned"
    }
  }
  user_data = base64encode(templatefile("${path.module}/worker-node-userdata.sh", {
    cluster_name = var.cluster_name
    k8s_version  = "1.33.0"
  }))
}

# IAM instance profile for worker nodes
resource "aws_iam_instance_profile" "worker_node_profile" {
  name = "${var.cluster_name}-worker-profile"
  role = aws_iam_role.worker_node_role.name
}

# Security group for worker nodes
resource "aws_security_group" "worker_node_sg" {
  name        = "${var.cluster_name}-worker-sg"
  vpc_id      = aws_vpc.k8s_vpc.id
  ingress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    self        = true
  }
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
  tags = {
    Name = "${var.cluster_name}-worker-sg"
  }
}

variable "graviton4_ami_id" {
  type        = string
  description = "AMI ID for Graviton4 nodes with K8s 1.33 pre-installed"
  default     = "ami-0a1b2c3d4e5f6g7h8" # Example AMI ID, replace with real one
}

#!/usr/bin/env python3
"""
Prometheus metrics scraper for Kubernetes container startup time analysis.
Calculates p50, p95, p99 startup times for Graviton4 vs x86 nodes.
"""

import argparse
import logging
import os
import sys
import time
from typing import Dict, List, Optional
import requests
import numpy as np
from dataclasses import dataclass

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

@dataclass
class StartupMetric:
    """Holds container startup metric data."""
    container_name: str
    pod_name: str
    node_arch: str # "arm64" for Graviton4, "amd64" for x86
    startup_time_ms: float
    namespace: str

class PrometheusScraper:
    """Scrapes Prometheus for container startup time metrics."""

    def __init__(self, prometheus_url: str, timeout: int = 10):
        """
        Initialize the Prometheus scraper.

        Args:
            prometheus_url: Base URL of the Prometheus instance
            timeout: Request timeout in seconds
        """
        self.prometheus_url = prometheus_url.rstrip("/")
        self.timeout = timeout
        self.session = requests.Session()
        # Retry failed requests up to 3 times
        self.session.mount("http://", requests.adapters.HTTPAdapter(max_retries=3))
        self.session.mount("https://", requests.adapters.HTTPAdapter(max_retries=3))

    def query_range(self, query: str, start: int, end: int, step: str = "1m") -> Optional[List[Dict]]:
        """
        Query Prometheus range API.

        Args:
            query: PromQL query string
            start: Start timestamp (Unix epoch seconds)
            end: End timestamp (Unix epoch seconds)
            step: Query resolution step

        Returns:
            List of time series data, or None if query fails
        """
        endpoint = f"{self.prometheus_url}/api/v1/query_range"
        params = {
            "query": query,
            "start": start,
            "end": end,
            "step": step
        }
        try:
            response = self.session.get(endpoint, params=params, timeout=self.timeout)
            response.raise_for_status()
            data = response.json()
            if data.get("status") != "success":
                logger.error(f"Prometheus query failed: {data.get('error', 'Unknown error')}")
                return None
            return data.get("data", {}).get("result", [])
        except requests.exceptions.RequestException as e:
            logger.error(f"Failed to query Prometheus: {e}")
            return None

    def get_container_startup_metrics(self, time_window_hours: int = 24) -> List[StartupMetric]:
        """
        Fetch container startup time metrics for the last N hours.

        Args:
            time_window_hours: Number of hours to look back

        Returns:
            List of StartupMetric objects
        """
        now = int(time.time())
        start = now - (time_window_hours * 3600)
        end = now
        # PromQL query for container startup time (kubelet_container_start_duration_seconds)
        query = """
        kubelet_container_start_duration_seconds{
            job="kubelet",
            metrics_path="/metrics"
        } * 1000 # Convert to milliseconds
        """
        results = self.query_range(query.strip(), start, end)
        if not results:
            logger.warning("No startup metrics found")
            return []

        metrics = []
        for series in results:
            # Extract labels
            labels = series.get("metric", {})
            container_name = labels.get("container", "unknown")
            pod_name = labels.get("pod", "unknown")
            namespace = labels.get("namespace", "unknown")
            node = labels.get("node", "")
            # Determine node architecture from node labels (we tag Graviton4 nodes with arch=arm64)
            node_arch = "arm64" if "arm64" in node else "amd64" # Simplified, in reality query node labels

            # Calculate average startup time for this series
            values = series.get("values", [])
            if not values:
                continue
            # Values are [timestamp, value] pairs, extract the metric values
            startup_times = [float(v[1]) for v in values if v[1] != "NaN"]
            if not startup_times:
                continue
            avg_startup = np.mean(startup_times)
            metrics.append(StartupMetric(
                container_name=container_name,
                pod_name=pod_name,
                node_arch=node_arch,
                startup_time_ms=avg_startup,
                namespace=namespace
            ))
        return metrics

    def calculate_percentiles(self, metrics: List[StartupMetric]) -> Dict[str, Dict[str, float]]:
        """
        Calculate p50, p95, p99 startup times per architecture.

        Args:
            metrics: List of StartupMetric objects

        Returns:
            Dict with arch as key, percentiles as value
        """
        arch_times: Dict[str, List[float]] = {}
        for metric in metrics:
            if metric.node_arch not in arch_times:
                arch_times[metric.node_arch] = []
            arch_times[metric.node_arch].append(metric.startup_time_ms)

        percentiles = {}
        for arch, times in arch_times.items():
            if not times:
                continue
            times_array = np.array(times)
            percentiles[arch] = {
                "p50": np.percentile(times_array, 50),
                "p95": np.percentile(times_array, 95),
                "p99": np.percentile(times_array, 99),
                "count": len(times)
            }
        return percentiles

def main():
    parser = argparse.ArgumentParser(description="Scrape Prometheus for container startup times")
    parser.add_argument("--prometheus-url", required=True, help="Prometheus base URL (e.g. http://prometheus:9090)")
    parser.add_argument("--time-window", type=int, default=24, help="Time window in hours to query")
    parser.add_argument("--output", default="text", choices=["text", "json"], help="Output format")
    args = parser.parse_args()

    scraper = PrometheusScraper(args.prometheus_url)
    logger.info(f"Fetching startup metrics for last {args.time_window} hours")
    metrics = scraper.get_container_startup_metrics(args.time_window)
    if not metrics:
        logger.error("No metrics found")
        sys.exit(1)

    percentiles = scraper.calculate_percentiles(metrics)
    if args.output == "json":
        import json
        print(json.dumps(percentiles, indent=2))
    else:
        print(f"Container Startup Time Percentiles (last {args.time_window} hours):")
        print("-" * 60)
        for arch, percs in percentiles.items():
            print(f"Architecture: {arch} (sample count: {percs['count']})")
            print(f"  p50: {percs['p50']:.2f} ms")
            print(f"  p95: {percs['p95']:.2f} ms")
            print(f"  p99: {percs['p99']:.2f} ms")
            print("-" * 60)

if __name__ == "__main__":
    main()

docker buildx create --name multiarch --driver docker-container --use
docker buildx inspect --bootstrap
docker buildx build \
  --platform linux/amd64,linux/arm64 \
  -t our-registry.com/myapp:$(git rev-parse --short HEAD) \
  --push \
  .

# /etc/kubernetes/kubelet.conf snippet
kubelet:
  memoryThreshold: 85%
  maxPods: 120
  containerRuntimeEndpoint: unix:///var/run/containerd/containerd.sock
  systemReserved:
    cpu: "1"
    memory: "2Gi"
  kubeReserved:
    cpu: "0.5"
    memory: "1Gi"

# cilium-config configmap snippet
datapath-mode: "lb-sock"
bpf-lb-sock-services: "true"
bpf-lb-algorithm: "random"
enable-ipv4: "true"
enable-ipv6: "false"
disable-iptables-fallback: "true"
arm64-jit: "true"
x86-jit: "false"