War Story: Running 10k Kubernetes 1.32 Pods on AWS Graviton5 – Lessons Learned

package main

import (
    "context"
    "flag"
    "fmt"
    "io/ioutil"
    "net/http"
    "os"
    "os/exec"
    "path/filepath"
    "strconv"
    "time"

    v1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
)

const (
    kubeletConfigPath = "/etc/kubernetes/kubelet.conf"
    maxRetries       = 3
    retryDelay       = 2 * time.Second
)

var (
    kubeconfig *string
    nodeName   *string
    podLimit   *int
)

func init() {
    kubeconfig = flag.String("kubeconfig", filepath.Join(os.Getenv("HOME"), ".kube", "config"), "path to kubeconfig file")
    nodeName = flag.String("node", "", "node name to tune (required)")
    podLimit = flag.Int("pod-limit", 140, "max pods per node for Graviton5 c8g.24xlarge")
    flag.Parse()

    if *nodeName == "" {
        fmt.Fprintf(os.Stderr, "error: --node flag is required\n")
        flag.Usage()
        os.Exit(1)
    }
}

func main() {
    // Load kubeconfig and create clientset
    config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
    if err != nil {
        fmt.Fprintf(os.Stderr, "failed to load kubeconfig: %v\n", err)
        os.Exit(1)
    }
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        fmt.Fprintf(os.Stderr, "failed to create clientset: %v\n", err)
        os.Exit(1)
    }

    // Validate node exists and is Graviton5
    node, err := clientset.CoreV1().Nodes().Get(context.Background(), *nodeName, metav1.GetOptions{})
    if err != nil {
        fmt.Fprintf(os.Stderr, "failed to get node %s: %v\n", *nodeName, err)
        os.Exit(1)
    }
    instanceType, ok := node.Labels["node.kubernetes.io/instance-type"]
    if !ok || instanceType != "c8g.24xlarge" {
        fmt.Fprintf(os.Stderr, "node %s is not a Graviton5 c8g.24xlarge (type: %s)\n", *nodeName, instanceType)
        os.Exit(1)
    }

    // Tune kubelet max pods and startup parameters
    if err := tuneKubelet(*nodeName, *podLimit); err != nil {
        fmt.Fprintf(os.Stderr, "failed to tune kubelet: %v\n", err)
        os.Exit(1)
    }

    // Verify pod limit is applied
    verifyPodLimit(clientset, *nodeName, *podLimit)
}

func tuneKubelet(nodeName string, podLimit int) error {
    // Retry kubelet config update up to maxRetries times
    var err error
    for i := 0; i < maxRetries; i++ {
        err = updateKubeletConfig(nodeName, podLimit)
        if err == nil {
            return nil
        }
        time.Sleep(retryDelay)
        fmt.Printf("retry %d/%d: %v\n", i+1, maxRetries, err)
    }
    return fmt.Errorf("failed to update kubelet config after %d retries: %w", maxRetries, err)
}

func updateKubeletConfig(nodeName string, podLimit int) error {
    // Use SSH to run commands on the target node (simplified for example)
    // In production, use Node Agent or kubelet API
    cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", fmt.Sprintf("core@%s", nodeName),
        fmt.Sprintf("sudo sed -i 's/--max-pods=.*/--max-pods=%d/' %s && sudo systemctl restart kubelet", podLimit, kubeletConfigPath))
    output, err := cmd.CombinedOutput()
    if err != nil {
        return fmt.Errorf("ssh command failed: %v, output: %s", err, string(output))
    }

    // Verify kubelet restarted successfully
    healthURL := fmt.Sprintf("https://%s:10250/healthz", nodeName)
    resp, err := http.Get(healthURL)
    if err != nil {
        return fmt.Errorf("kubelet health check failed: %v", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        body, _ := ioutil.ReadAll(resp.Body)
        return fmt.Errorf("kubelet unhealthy: status %d, body: %s", resp.StatusCode, string(body))
    }
    return nil
}

func verifyPodLimit(clientset *kubernetes.Clientset, nodeName string, expectedLimit int) {
    // Wait for kubelet to report updated pod limit
    timeout := time.After(30 * time.Second)
    ticker := time.NewTicker(2 * time.Second)
    defer ticker.Stop()

    for {
        select {
        case <-timeout:
            fmt.Fprintf(os.Stderr, "timeout waiting for pod limit update on node %s\n", nodeName)
            os.Exit(1)
        case <-ticker.C:
            node, err := clientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
            if err != nil {
                fmt.Fprintf(os.Stderr, "failed to get node: %v\n", err)
                continue
            }
            // Check node capacity for pods
            pods, ok := node.Status.Capacity.Pods()
            if !ok {
                fmt.Fprintf(os.Stderr, "pods not found in node capacity\n")
                continue
            }
            if int(pods.Value()) == expectedLimit {
                fmt.Printf("successfully updated node %s pod limit to %d\n", nodeName, expectedLimit)
                return
            }
            fmt.Printf("waiting for pod limit update: current %d, expected %d\n", pods.Value(), expectedLimit)
        }
    }
}

#!/bin/bash
# tune-graviton5-kernel.sh: Optimizes Linux kernel parameters for Kubernetes 1.32 on AWS Graviton5 Arm CPUs
# Requires root privileges, tested on Ubuntu 22.04 LTS with kernel 6.5.0-1019-aws

set -euo pipefail
IFS=$'\n\t'

# Configuration parameters (tuned for c8g.24xlarge with 80+ pods per node)
KERNEL_PARAMS=(
    "kernel.sched_min_granularity_ns=1000000"
    "kernel.sched_wakeup_granularity_ns=1500000"
    "vm.swappiness=0"
    "vm.overcommit_memory=1"
    "net.core.somaxconn=65535"
    "net.ipv4.tcp_tw_reuse=1"
    "net.ipv4.ip_local_port_range=1024 65535"
    "fs.file-max=2097152"
    "fs.inotify.max_user_watches=524288"
)

LOG_FILE="/var/log/graviton5-tune.log"
MAX_RETRIES=3
RETRY_DELAY=2

# Logging function
log() {
    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" | tee -a "$LOG_FILE"
}

# Error handling
trap 'log "ERROR: Script failed at line $LINENO"; exit 1' ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    log "ERROR: This script must be run as root"
    exit 1
fi

# Check if running on Graviton5 (Arm CPU)
CPU_MODEL=$(lscpu | grep "Model name" | cut -d: -f2 | xargs)
if [[ ! "$CPU_MODEL" =~ "Graviton5" ]]; then
    log "ERROR: Not running on AWS Graviton5 CPU (detected: $CPU_MODEL)"
    exit 1
fi

# Check K8s version is 1.32+
K8S_VERSION=$(kubelet --version | cut -d' ' -f2)
if [[ ! "$K8S_VERSION" =~ ^v1\.32\. ]]; then
    log "ERROR: Kubernetes version must be 1.32+ (detected: $K8S_VERSION)"
    exit 1
fi

# Apply kernel parameters with retry logic
for param in "${KERNEL_PARAMS[@]}"; do
    key=$(echo "$param" | cut -d= -f1)
    value=$(echo "$param" | cut -d= -f2)

    log "Applying $key=$value"

    for ((i=1; i<=MAX_RETRIES; i++)); do
        if sysctl -w "$param" >> "$LOG_FILE" 2>&1; then
            # Persist to sysctl.conf
            if grep -q "^$key=" /etc/sysctl.conf; then
                sed -i "s/^$key=.*/$param/" /etc/sysctl.conf
            else
                echo "$param" >> /etc/sysctl.conf
            fi
            log "Successfully applied $param"
            break
        else
            log "Retry $i/$MAX_RETRIES: Failed to apply $param"
            sleep "$RETRY_DELAY"
            if [[ $i -eq $MAX_RETRIES ]]; then
                log "ERROR: Failed to apply $param after $MAX_RETRIES retries"
                exit 1
            fi
        fi
    done
done

# Verify parameters are applied
log "Verifying applied kernel parameters..."
for param in "${KERNEL_PARAMS[@]}"; do
    key=$(echo "$param" | cut -d= -f1)
    expected=$(echo "$param" | cut -d= -f2)
    actual=$(sysctl -n "$key")

    if [[ "$actual" != "$expected" ]]; then
        log "ERROR: Verification failed for $key: expected $expected, got $actual"
        exit 1
    fi
done

# Restart kubelet to pick up kernel changes
log "Restarting kubelet..."
systemctl restart kubelet

# Wait for kubelet to become healthy
log "Waiting for kubelet health check..."
for ((i=1; i<=30; i++)); do
    if curl -sk https://localhost:10250/healthz | grep -q "ok"; then
        log "kubelet is healthy"
        log "All kernel parameters applied successfully"
        exit 0
    fi
    sleep 2
done

log "ERROR: kubelet failed to become healthy after 60 seconds"
exit 1

#!/usr/bin/env python3
"""
scale_test_graviton5.py: Automates scaling K8s pods to 10k and collects startup latency metrics
Requires: kubernetes>=28.1.0, pandas>=2.1.0, prometheus-api-client>=0.5.0
"""

import argparse
import logging
import os
import sys
import time
from datetime import datetime
from typing import List, Dict

from kubernetes import client, config
from kubernetes.client.rest import ApiException
import pandas as pd
from prometheus_api_client import PrometheusConnect

# Configuration
DEFAULT_NAMESPACE = "scale-test"
POD_TEMPLATE = {
    "apiVersion": "v1",
    "kind": "Pod",
    "metadata": {"name": "scale-test-pod", "namespace": DEFAULT_NAMESPACE},
    "spec": {
        "containers": [{
            "name": "nginx",
            "image": "nginx:1.25-alpine",
            "resources": {"requests": {"cpu": "100m", "memory": "128Mi"}},
        }],
        "restartPolicy": "Never",
    },
}

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

def load_kube_config(kubeconfig_path: str = None) -> None:
    """Load Kubernetes config from file or in-cluster"""
    try:
        if kubeconfig_path:
            config.load_kube_config(config_file=kubeconfig_path)
        else:
            config.load_incluster_config()
        logger.info("Kubernetes config loaded successfully")
    except Exception as e:
        logger.error(f"Failed to load Kubernetes config: {e}")
        sys.exit(1)

def create_namespace(api: client.CoreV1Api) -> None:
    """Create scale test namespace if it doesn't exist"""
    try:
        api.create_namespace(client.V1Namespace(metadata=client.V1ObjectMeta(name=DEFAULT_NAMESPACE)))
        logger.info(f"Created namespace {DEFAULT_NAMESPACE}")
    except ApiException as e:
        if e.status == 409:
            logger.info(f"Namespace {DEFAULT_NAMESPACE} already exists")
        else:
            logger.error(f"Failed to create namespace: {e}")
            sys.exit(1)

def deploy_pods(api: client.CoreV1Api, count: int, batch_size: int = 100) -> List[str]:
    """Deploy pods in batches and return list of pod names"""
    pod_names = []
    for i in range(0, count, batch_size):
        batch_end = min(i + batch_size, count)
        logger.info(f"Deploying pods {i} to {batch_end}...")
        for pod_num in range(i, batch_end):
            pod_name = f"scale-test-{pod_num}"
            pod_manifest = POD_TEMPLATE.copy()
            pod_manifest["metadata"]["name"] = pod_name
            try:
                api.create_namespaced_pod(namespace=DEFAULT_NAMESPACE, body=pod_manifest)
                pod_names.append(pod_name)
            except ApiException as e:
                logger.error(f"Failed to create pod {pod_name}: {e}")
                # Retry once
                try:
                    api.create_namespaced_pod(namespace=DEFAULT_NAMESPACE, body=pod_manifest)
                    pod_names.append(pod_name)
                except ApiException as e2:
                    logger.error(f"Retry failed for pod {pod_name}: {e2}")
        # Wait for batch to start
        time.sleep(2)
    return pod_names

def collect_metrics(prom: PrometheusConnect, start_time: datetime, end_time: datetime) -> pd.DataFrame:
    """Collect pod startup latency metrics from Prometheus"""
    query = """
        histogram_quantile(0.99, sum(rate(kubelet_pod_startup_latency_seconds_bucket[5m])) by (le))
    """
    try:
        metrics = prom.custom_query_range(
            query=query,
            start_time=start_time,
            end_time=end_time,
            step="1m"
        )
        if not metrics:
            logger.warning("No metrics returned from Prometheus")
            return pd.DataFrame()
        # Parse metrics into DataFrame
        data = []
        for metric in metrics:
            for value in metric["values"]:
                timestamp = datetime.fromtimestamp(value[0])
                latency = float(value[1])
                data.append({"timestamp": timestamp, "p99_latency": latency})
        return pd.DataFrame(data)
    except Exception as e:
        logger.error(f"Failed to collect metrics: {e}")
        return pd.DataFrame()

def main():
    parser = argparse.ArgumentParser(description="Scale test K8s pods on Graviton5")
    parser.add_argument("--count", type=int, default=10000, help="Number of pods to deploy")
    parser.add_argument("--kubeconfig", type=str, help="Path to kubeconfig file")
    parser.add_argument("--prometheus-url", type=str, default="http://prometheus:9090", help="Prometheus URL")
    args = parser.parse_args()

    # Load config
    load_kube_config(args.kubeconfig)
    api = client.CoreV1Api()
    prom = PrometheusConnect(url=args.prometheus_url, disable_ssl=True)

    # Create namespace
    create_namespace(api)

    # Deploy pods
    start_time = datetime.now()
    logger.info(f"Starting scale test to {args.count} pods...")
    pod_names = deploy_pods(api, args.count)
    end_time = datetime.now()
    logger.info(f"Deployed {len(pod_names)} pods in {end_time - start_time}")

    # Collect metrics
    logger.info("Collecting startup latency metrics...")
    metrics_df = collect_metrics(prom, start_time, datetime.now())
    if not metrics_df.empty:
        avg_latency = metrics_df["p99_latency"].mean()
        logger.info(f"Average p99 pod startup latency: {avg_latency:.2f}s")
        metrics_df.to_csv(f"scale_test_metrics_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False)

    # Cleanup (optional)
    logger.info("Scale test complete. Run with --cleanup to delete pods")

if __name__ == "__main__":
    main()

# GitHub Actions step to validate Arm image support
- name: Validate Arm Image Compatibility
  run: |
    IMAGE="ghcr.io/myorg/myapp:${{ github.sha }}"
    # Pull image manifest
    manifest=$(docker manifest inspect $IMAGE)
    # Check for linux/arm64 platform
    if ! echo "$manifest" | grep -q "linux/arm64"; then
      echo "ERROR: Image $IMAGE does not support linux/arm64"
      exit 1
    fi
    echo "Image $IMAGE supports Arm64"

# DaemonSet snippet to apply kernel tuning
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: graviton5-kernel-tuner
spec:
  selector:
    matchLabels:
      app: kernel-tuner
  template:
    spec:
      containers:
      - name: tuner
        image: ghcr.io/myorg/graviton5-tuner:v1.0.0
        securityContext:
          privileged: true
        volumeMounts:
        - name: sysctl-conf
          mountPath: /etc/sysctl.conf
      volumes:
      - name: sysctl-conf
        hostPath:
          path: /etc/sysctl.conf

# Kubelet config snippet to enable fast path
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
featureGates:
  PodStartupFastPath: true
evictionHard:
  memory.available: "512Mi"
  nodefs.available: "10%"