War Story: We Saved $150k/Year by Migrating K8s Clusters to Graviton4 with Kubecost 2.1

# Terraform configuration for provisioning Graviton4 (c7g) node groups in EKS
# with Kubecost 2.1 cost allocation labels and validation
# Requires Terraform >= 1.7.0, AWS provider ~> 5.0, Kubecost Helm chart 2.1+

terraform {
  required_version = ">= 1.7.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.12"
    }
  }
}

variable "cluster_name" {
  type        = string
  description = "Name of the target EKS cluster"
  validation {
    condition     = length(var.cluster_name) > 0 && length(var.cluster_name) <= 100
    error_message = "Cluster name must be 1-100 characters long."
  }
}

variable "graviton_instance_types" {
  type        = list(string)
  description = "Graviton4 instance types for node group"
  default     = ["c7g.4xlarge", "c7g.8xlarge"]
  validation {
    condition = alltrue([
      for t in var.graviton_instance_types : contains(["c7g.4xlarge", "c7g.8xlarge", "c7g.16xlarge"], t)
    ])
    error_message = "Only Graviton4 c7g instance types are supported for this node group."
  }
}

variable "kubecost_namespace" {
  type        = string
  default     = "kubecost"
  description = "Namespace where Kubecost 2.1 is deployed"
}

resource "aws_eks_node_group" "graviton4_production" {
  cluster_name    = var.cluster_name
  node_group_name = "graviton4-prod-ng-01"
  node_role_arn   = aws_iam_role.eks_node_role.arn
  subnet_ids      = aws_subnet.private[*].id
  instance_types  = var.graviton_instance_types

  scaling_config {
    desired_size = 20
    max_size     = 50
    min_size     = 10
  }

  disk_size = 100

  labels = {
    "kubernetes.io/arch"               = "arm64"
    "node.kubernetes.io/instance-type"  = "graviton4"
    "kubecost.io/cost-center"           = "production-ecommerce"
    "kubecost.io/environment"           = "prod"
    "kubecost.io/version"               = "2.1"
  }

  taint {
    key    = "arch"
    value  = "arm64"
    effect = "NO_SCHEDULE"
  }

  depends_on = [aws_iam_role_policy_attachment.eks_worker_node_policy]

  lifecycle {
    ignore_changes = [scaling_config[0].desired_size]
  }

  tags = {
    "Environment" = "prod"
    "CostCenter"  = "k8s-production"
    "ManagedBy"   = "terraform"
  }
}

resource "aws_iam_role" "eks_node_role" {
  name = "${var.cluster_name}-graviton4-node-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })

  tags = {
    Name = "${var.cluster_name}-graviton4-node-role"
  }
}

resource "aws_iam_role_policy_attachment" "eks_worker_node_policy" {
  role       = aws_iam_role.eks_node_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
}

resource "aws_iam_role_policy_attachment" "eks_cni_policy" {
  role       = aws_iam_role.eks_node_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
}

resource "aws_iam_role_policy_attachment" "ecr_read_policy" {
  role       = aws_iam_role.eks_node_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
}

output "graviton4_node_group_arn" {
  value       = aws_eks_node_group.graviton4_production.arn
  description = "ARN of the provisioned Graviton4 node group"
}

#!/usr/bin/env python3
"""
Kubecost 2.1 Savings Reporter: Compares x86 vs Graviton4 node costs
and calculates projected annual savings.
Requires: requests>=2.31.0, python-dateutil>=2.8.2
"""

import requests
import json
import os
from datetime import datetime, timedelta
from typing import Dict, List, Optional

# Configuration from environment variables
KUBECOST_URL = os.getenv("KUBECOST_URL", "http://kubecost.kubecost.svc.cluster.local:9090")
KUBECOST_API_TOKEN = os.getenv("KUBECOST_API_TOKEN")
SAVINGS_THRESHOLD = float(os.getenv("SAVINGS_THRESHOLD", 0.1))  # 10% minimum savings to report

if not KUBECOST_API_TOKEN:
    raise ValueError("KUBECOST_API_TOKEN environment variable is required")

class KubecostClient:
    """Client for Kubecost 2.1 Allocation API"""
    def __init__(self, base_url: str, api_token: str):
        self.base_url = base_url.rstrip("/")
        self.session = requests.Session()
        self.session.headers.update({
            "Authorization": f"Bearer {api_token}",
            "Content-Type": "application/json"
        })
        self.session.verify = False  # Disable SSL verify for in-cluster service, remove in prod

    def get_allocation_data(self, start: datetime, end: datetime, filters: Optional[Dict] = None) -> Dict:
        """
        Fetch allocation data from Kubecost Allocation API v2
        https://github.com/kubecost/docs/blob/main/apis/allocation.md
        """
        endpoint = f"{self.base_url}/api/v2/allocation"
        params = {
            "window": f"{start.isoformat()}/{end.isoformat()}",
            "aggregate": "node",
            "filterClusters": filters.get("clusters") if filters else None,
            "filterLabels": filters.get("labels") if filters else None
        }
        # Remove None params
        params = {k: v for k, v in params.items() if v is not None}

        try:
            response = self.session.get(endpoint, params=params, timeout=30)
            response.raise_for_status()  # Raise HTTPError for bad responses (4xx, 5xx)
            return response.json()
        except requests.exceptions.Timeout:
            print(f"Error: Request to {endpoint} timed out after 30 seconds")
            raise
        except requests.exceptions.HTTPError as e:
            print(f"Error: HTTP {response.status_code} from Kubecost API: {e}")
            raise
        except json.JSONDecodeError:
            print(f"Error: Invalid JSON response from Kubecost API")
            raise

    def calculate_savings(self, x86_cost: float, arm_cost: float) -> Dict:
        """Calculate savings percentage and annual projected savings"""
        if x86_cost == 0:
            return {"savings_pct": 0.0, "annual_savings": 0.0}
        savings_pct = ((x86_cost - arm_cost) / x86_cost) * 100
        # Project to annual: window is 7 days, so multiply by 52
        annual_savings = (x86_cost - arm_cost) * 52
        return {
            "savings_pct": round(savings_pct, 2),
            "annual_savings": round(annual_savings, 2),
            "x86_weekly_cost": round(x86_cost, 2),
            "arm_weekly_cost": round(arm_cost, 2)
        }

def main():
    # Calculate 7-day window ending now
    end_time = datetime.utcnow()
    start_time = end_time - timedelta(days=7)

    # Initialize Kubecost client
    client = KubecostClient(KUBECOST_URL, KUBECOST_API_TOKEN)

    # Fetch x86 node costs (amd64 arch)
    print(f"Fetching x86 node allocation data from {start_time} to {end_time}...")
    x86_filters = {
        "labels": [{"key": "kubernetes.io/arch", "value": "amd64"}]
    }
    x86_data = client.get_allocation_data(start_time, end_time, x86_filters)

    # Fetch Graviton4 (arm64) node costs
    print(f"Fetching Graviton4 node allocation data from {start_time} to {end_time}...")
    arm_filters = {
        "labels": [{"key": "kubernetes.io/arch", "value": "arm64"}, {"key": "node.kubernetes.io/instance-type", "value": "graviton4"}]
    }
    arm_data = client.get_allocation_data(start_time, end_time, arm_filters)

    # Sum total costs for each architecture
    x86_total = sum(node.get("cost", 0) for node in x86_data.get("data", []))
    arm_total = sum(node.get("cost", 0) for node in arm_data.get("data", []))

    # Calculate savings
    savings = client.calculate_savings(x86_total, arm_total)

    # Generate report
    report = {
        "report_period": f"{start_time.date()} to {end_time.date()}",
        "x86_weekly_cost_usd": savings["x86_weekly_cost"],
        "graviton4_weekly_cost_usd": savings["arm_weekly_cost"],
        "savings_percentage": savings["savings_pct"],
        "projected_annual_savings_usd": savings["annual_savings"],
        "meets_threshold": savings["savings_pct"] >= (SAVINGS_THRESHOLD * 100)
    }

    # Print report
    print("\n=== Kubecost 2.1 Graviton4 Savings Report ===")
    print(json.dumps(report, indent=2))

    # Alert if savings meet threshold
    if report["meets_threshold"]:
        print(f"\n✅ Savings of {savings['savings_pct']}% exceed threshold of {SAVINGS_THRESHOLD*100}%. Projected annual savings: ${savings['annual_savings']}")

if __name__ == "__main__":
    main()

#!/bin/bash
"""
ARM Compatibility Validator: Checks if container images are compatible with Graviton4 (arm64)
before deploying to EKS. Fails CI/CD pipeline if incompatible images are found.
Requires: docker >= 24.0, buildx plugin, crane >= 0.15 (optional for remote images)
"""

set -euo pipefail  # Exit on error, undefined vars, pipe fail
trap 'echo "Error: Script failed at line $LINENO"; exit 1' ERR

# Configuration
REGISTRY="${REGISTRY:-123456789012.dkr.ecr.us-east-1.amazonaws.com}"
NAMESPACE="${NAMESPACE:-production}"
MANIFEST_DIR="${MANIFEST_DIR:-./k8s/manifests}"
REQUIRED_ARCH="arm64"
LOG_FILE="arm-compat-check-$(date +%Y%m%d-%H%M%S).log"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

check_docker_available() {
    if ! command -v docker &> /dev/null; then
        log "${RED}Error: docker is not installed or not in PATH${NC}"
        exit 1
    fi
    if ! docker buildx version &> /dev/null; then
        log "${RED}Error: docker buildx plugin is not installed${NC}"
        exit 1
    fi
    log "${GREEN}docker and buildx are available${NC}"
}

check_image_arch() {
    local image="$1"
    log "Checking image: $image for $REQUIRED_ARCH compatibility"

    # Pull image manifest using docker buildx
    # We use buildx because it supports multi-arch manifests
    manifest_output=$(docker buildx imagetools inspect "$image" 2>&1)
    if [ $? -ne 0 ]; then
        log "${RED}Error: Failed to inspect image $image. Is it a valid image?${NC}"
        return 1
    fi

    # Check if arm64 is in the manifest
    if echo "$manifest_output" | grep -q "$REQUIRED_ARCH"; then
        log "${GREEN}✅ Image $image supports $REQUIRED_ARCH${NC}"
        return 0
    else
        log "${RED}❌ Image $image does NOT support $REQUIRED_ARCH${NC}"
        # List supported architectures
        supported_archs=$(echo "$manifest_output" | grep -oP '"architecture": "\K[^"]+' | sort -u)
        log "Supported architectures: $supported_archs"
        return 1
    fi
}

validate_manifests() {
    log "Scanning K8s manifests in $MANIFEST_DIR for container images..."
    local failed=0

    # Find all deployment, statefulset, daemonset manifests
    while IFS= read -r -d '' manifest; do
        log "Processing manifest: $manifest"
        # Extract image fields from manifest (handles multiple containers per pod)
        images=$(yq e '.spec.template.spec.containers[].image' "$manifest" 2>/dev/null)
        if [ -z "$images" ]; then
            log "${YELLOW}Warning: No container images found in $manifest${NC}"
            continue
        fi

        while IFS= read -r image; do
            # Skip empty lines
            [ -z "$image" ] && continue
            if ! check_image_arch "$image"; then
                failed=1
            fi
        done <<< "$images"
    done < <(find "$MANIFEST_DIR" -name "*.yaml" -o -name "*.yml" -print0)

    return $failed
}

generate_report() {
    log "Generating compatibility report..."
    total_images=$(grep -r "image:" "$MANIFEST_DIR" | wc -l)
    compatible_images=$(grep -c "${GREEN}✅" "$LOG_FILE" || echo 0)
    incompatible_images=$(grep -c "${RED}❌" "$LOG_FILE" || echo 0)

    echo -e "\n=== ARM Compatibility Report ===" | tee -a "$LOG_FILE"
    echo "Total images scanned: $total_images" | tee -a "$LOG_FILE"
    echo "Compatible images: $compatible_images" | tee -a "$LOG_FILE"
    echo "Incompatible images: $incompatible_images" | tee -a "$LOG_FILE"
    echo "Log file: $LOG_FILE" | tee -a "$LOG_FILE"
}

main() {
    log "Starting ARM compatibility check for Graviton4 migration"
    check_docker_available

    if [ ! -d "$MANIFEST_DIR" ]; then
        log "${RED}Error: Manifest directory $MANIFEST_DIR does not exist${NC}"
        exit 1
    fi

    validate_manifests
    local validation_result=$?

    generate_report

    if [ $validation_result -ne 0 ]; then
        log "${RED}❌ Validation failed: Incompatible images found. Fix before deploying to Graviton4 nodes.${NC}"
        exit 1
    else
        log "${GREEN}✅ All images are compatible with Graviton4 (arm64). Safe to deploy.${NC}"
        exit 0
    fi
}

main

- name: Check ARM Compatibility
  run: |
    chmod +x ./scripts/arm-compat-check.sh
    REGISTRY=${{ secrets.ECR_REGISTRY }} ./scripts/arm-compat-check.sh

curl -H "Authorization: Bearer $KUBECOST_TOKEN" \
  "$KUBECOST_URL/api/v2/migration/simulate?targetArch=arm64&window=7d" \
  > migration-simulator-results.json

apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
  name: require-kubecost-labels
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment"]
  parameters:
    labels:
      - key: kubecost.io/cost-center
        allowedValues: ["production-ecommerce", "unallocated"]