War Story: We Cut AWS Bills by 50% in 2026 Using Kubecost 2.0 and Graviton4 Instances

// kubecost-exporter.go
// Exports Kubecost 2.0 namespace-level cost allocation metrics to Prometheus
// Requires KUBECOST_URL and PROMETHEUS_PORT environment variables
package main

import (
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "os"
    "strconv"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

// KubecostAllocationResponse represents the response from Kubecost's /allocation API
type KubecostAllocationResponse struct {
    Code    int    `json:"code"`
    Message string `json:"message"`
    Data    []struct {
        Name       string  `json:"name"`
        Cost       float64 `json:"cost"`
        CPUCost    float64 `json:"cpuCost"`
        RAMCost    float64 `json:"ramCost"`
        NetworkCost float64 `json:"networkCost"`
        Namespace  string  `json:"namespace"`
    } `json:"data"`
}

var (
    // Prometheus metric for total namespace cost
    namespaceCost = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "kubecost_namespace_total_cost_usd",
            Help: "Total cost per namespace in USD over the last 24 hours",
        },
        []string{"namespace"},
    )
    // Prometheus metric for CPU cost per namespace
    cpuCost = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "kubecost_namespace_cpu_cost_usd",
            Help: "CPU cost per namespace in USD over the last 24 hours",
        },
        []string{"namespace"},
    )
)

func init() {
    prometheus.MustRegister(namespaceCost)
    prometheus.MustRegister(cpuCost)
}

func fetchKubecostData() (*KubecostAllocationResponse, error) {
    kubecostURL := os.Getenv("KUBECOST_URL")
    if kubecostURL == "" {
        return nil, fmt.Errorf("KUBECOST_URL environment variable not set")
    }

    // Query last 24 hours of allocation data
    queryURL := fmt.Sprintf("%s/api/v2/allocation?window=24h&aggregate=namespace", kubecostURL)
    resp, err := http.Get(queryURL)
    if err != nil {
        return nil, fmt.Errorf("failed to fetch Kubecost data: %\w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("kubecost API returned non-200 status: %d", resp.StatusCode)
    }

    var allocationResp KubecostAllocationResponse
    if err := json.NewDecoder(resp.Body).Decode(&allocationResp); err != nil {
        return nil, fmt.Errorf("failed to decode Kubecost response: %\w", err)
    }

    if allocationResp.Code != 200 {
        return nil, fmt.Errorf("kubecost API error: %s (code %d)", allocationResp.Message, allocationResp.Code)
    }

    return &allocationResp, nil
}

func updateMetrics() {
    data, err := fetchKubecostData()
    if err != nil {
        log.Printf("Error fetching Kubecost data: %v", err)
        return
    }

    // Reset all metrics before updating to avoid stale data
    namespaceCost.Reset()
    cpuCost.Reset()

    for _, item := range data.Data {
        namespaceCost.WithLabelValues(item.Namespace).Set(item.Cost)
        cpuCost.WithLabelValues(item.Namespace).Set(item.CPUCost)
    }
    log.Printf("Updated metrics for %d namespaces", len(data.Data))
}

func main() {
    promPort := os.Getenv("PROMETHEUS_PORT")
    if promPort == "" {
        promPort = "9091"
    }

    // Validate port is numeric
    if _, err := strconv.Atoi(promPort); err != nil {
        log.Fatalf("Invalid PROMETHEUS_PORT: %v", err)
    }

    // Update metrics every 5 minutes
    ticker := time.NewTicker(5 * time.Minute)
    defer ticker.Stop()

    // Initial update
    updateMetrics()

    // Start Prometheus metrics endpoint
    http.Handle("/metrics", promhttp.Handler())
    go func() {
        log.Printf("Starting Prometheus metrics endpoint on :%s", promPort)
        if err := http.ListenAndServe(fmt.Sprintf(":%s", promPort), nil); err != nil {
            log.Fatalf("Failed to start metrics server: %v", err)
        }
    }()

    // Periodic metric updates
    for range ticker.C {
        updateMetrics()
    }
}

# graviton4-node-group.tf
# Deploys EKS managed node groups with Graviton4 (m7g) instances, spot fallback, and ARM taints
# Requires AWS provider ~> 5.0, EKS cluster already provisioned

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
}

variable "cluster_name" {
  type        = string
  description = "Name of the target EKS cluster"
  default     = "prod-eks-cluster-2026"
}

variable "vpc_id" {
  type        = string
  description = "VPC ID where EKS nodes will be deployed"
}

variable "private_subnet_ids" {
  type        = list(string)
  description = "List of private subnet IDs for node group placement"
}

# IAM role for EKS nodes
resource "aws_iam_role" "graviton_node_role" {
  name = "${var.cluster_name}-graviton-node-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })

  tags = {
    Environment = "production"
    ManagedBy   = "terraform"
    Arch        = "arm64"
  }
}

# Attach required EKS node IAM policies
resource "aws_iam_role_policy_attachment" "graviton_worker_node_policy" {
  role       = aws_iam_role.graviton_node_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
}

resource "aws_iam_role_policy_attachment" "graviton_cni_policy" {
  role       = aws_iam_role.graviton_node_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
}

resource "aws_iam_role_policy_attachment" "graviton_ecr_policy" {
  role       = aws_iam_role.graviton_node_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
}

# EKS managed node group for Graviton4 instances
resource "aws_eks_node_group" "graviton4_group" {
  cluster_name    = var.cluster_name
  node_group_name = "graviton4-m7g-prod"
  node_role_arn   = aws_iam_role.graviton_node_role.arn
  subnet_ids      = var.private_subnet_ids

  # Mixed instance policy: Graviton4 m7g.2xlarge as on-demand, m7g.4xlarge as spot fallback
  instance_types = ["m7g.2xlarge", "m7g.4xlarge"]

  # Spot instance configuration (up to 30% of nodes)
  capacity_type  = "ON_DEMAND" # Base capacity is on-demand, spot via expander

  scaling_config {
    desired_size = 40
    max_size     = 80
    min_size     = 20
  }

  # Taint nodes as ARM64 to ensure only compatible workloads are scheduled
  taint {
    key    = "kubernetes.io/arch"
    value  = "arm64"
    effect = "NO_SCHEDULE"
  }

  # Labels for workload selection
  labels = {
    "node.kubernetes.io/instance-type" = "graviton4"
    "kubernetes.io/arch"               = "arm64"
    "cost-optimized"                   = "true"
  }

  # Node IAM role ARN (required for EKS)
  remote_access {
    ec2_ssh_key = "prod-eks-ssh-key"
  }

  # Update configuration: rolling update with max 10% unavailable
  update_config {
    max_unavailable = 4
  }

  tags = {
    Environment = "production"
    Arch        = "arm64"
    ManagedBy   = "terraform"
    CostCenter  = "infrastructure"
  }

  # Ensure IAM policies are attached before creating node group
  depends_on = [
    aws_iam_role_policy_attachment.graviton_worker_node_policy,
    aws_iam_role_policy_attachment.graviton_cni_policy,
    aws_iam_role_policy_attachment.graviton_ecr_policy,
  ]
}

# Output the node group ARN for reference
output "graviton_node_group_arn" {
  value       = aws_eks_node_group.graviton4_group.arn
  description = "ARN of the Graviton4 EKS node group"
}

# cost-reconciler.py
# Reconciles Kubecost 2.0 cost data with AWS Cost Explorer API, generates savings recommendations
# Requires KUBECOST_URL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, SLACK_WEBHOOK_URL env vars
# Install dependencies: pip install boto3 requests pandas slack_sdk

import os
import json
import logging
from datetime import datetime, timedelta
import requests
import boto3
import pandas as pd
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Initialize AWS Cost Explorer client
ce_client = boto3.client(
    "ce",
    region_name="us-east-1",
    aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
    aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
)

# Initialize Slack client
slack_client = WebClient(token=os.getenv("SLACK_WEBHOOK_URL")) if os.getenv("SLACK_WEBHOOK_URL") else None

def fetch_kubecost_costs() -> pd.DataFrame:
    """Fetch 30-day namespace cost data from Kubecost 2.0 API"""
    kubecost_url = os.getenv("KUBECOST_URL")
    if not kubecost_url:
        raise ValueError("KUBECOST_URL environment variable not set")

    query_url = f"{kubecost_url}/api/v2/allocation?window=30d&aggregate=namespace"
    try:
        response = requests.get(query_url, timeout=10)
        response.raise_for_status()
    except requests.exceptions.RequestException as e:
        logger.error(f"Failed to fetch Kubecost data: {e}")
        raise

    data = response.json()
    if data.get("code") != 200:
        raise ValueError(f"Kubecost API error: {data.get('message')}")

    # Convert to DataFrame
    df = pd.DataFrame([{
        "namespace": item["namespace"],
        "kubecost_total_cost": item["cost"],
        "kubecost_cpu_cost": item["cpuCost"],
        "kubecost_ram_cost": item["ramCost"]
    } for item in data["data"]])

    logger.info(f"Fetched {len(df)} namespaces from Kubecost")
    return df

def fetch_aws_costs() -> pd.DataFrame:
    """Fetch 30-day AWS cost data grouped by Kubernetes namespace tag"""
    end_date = datetime.now().strftime("%Y-%m-%d")
    start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")

    try:
        response = ce_client.get_cost_and_usage(
            TimePeriod={"Start": start_date, "End": end_date},
            Granularity="MONTHLY",
            Metrics=["UnblendedCost"],
            GroupBy=[{"Type": "TAG", "Key": "kubernetes_namespace"}]
        )
    except Exception as e:
        logger.error(f"Failed to fetch AWS Cost Explorer data: {e}")
        raise

    # Convert to DataFrame
    rows = []
    for group in response["ResultsByTime"][0]["Groups"]:
        namespace = group["Keys"][0].split(":")[-1] if group["Keys"] else "untagged"
        cost = float(group["Metrics"]["UnblendedCost"]["Amount"])
        rows.append({"namespace": namespace, "aws_total_cost": cost})

    df = pd.DataFrame(rows)
    logger.info(f"Fetched {len(df)} namespaces from AWS Cost Explorer")
    return df

def generate_recommendations(reconciled_df: pd.DataFrame) -> list:
    """Generate cost savings recommendations based on reconciled data"""
    recommendations = []

    # Find namespaces where Kubecost cost is higher than AWS reported (over-provisioned)
    over_provisioned = reconciled_df[reconciled_df["cost_delta"] > 10] # Delta > $10
    for _, row in over_provisioned.iterrows():
        recommendations.append(
            f"Namespace {row['namespace']} has $10+ cost delta: consider rightsizing pods, "
            f"current Kubecost cost: ${row['kubecost_total_cost']:.2f}, AWS reported: ${row['aws_total_cost']:.2f}"
        )

    # Find untagged AWS costs (not tracked by Kubecost)
    untagged = reconciled_df[reconciled_df["namespace"] == "untagged"]
    if not untagged.empty:
        total_untagged = untagged["aws_total_cost"].sum()
        recommendations.append(
            f"Found ${total_untagged:.2f} in untagged AWS costs: apply Kubernetes namespace tags to all resources "
            f"to enable full cost allocation via Kubecost"
        )

    # Find namespaces with high idle costs
    high_idle = reconciled_df[reconciled_df["kubecost_idle_cost"] > 50]
    for _, row in high_idle.iterrows():
        recommendations.append(
            f"Namespace {row['namespace']} has ${row['kubecost_idle_cost']:.2f} idle cost: "
            f"enable pod autoscaling or delete unused deployments"
        )

    return recommendations

def send_slack_alert(recommendations: list):
    """Send cost recommendations to Slack"""
    if not slack_client:
        logger.warning("Slack webhook not configured, skipping alert")
        return

    if not recommendations:
        logger.info("No recommendations to send")
        return

    message = "🚨 *Monthly Cost Optimization Recommendations* 🚨\n" + "\n".join([f"• {r}" for r in recommendations])
    try:
        slack_client.chat_postMessage(
            channel="#infrastructure-costs",
            text=message
        )
        logger.info("Sent Slack alert with %d recommendations", len(recommendations))
    except SlackApiError as e:
        logger.error(f"Failed to send Slack alert: {e}")

def main():
    try:
        # Fetch data from both sources
        kubecost_df = fetch_kubecost_costs()
        aws_df = fetch_aws_costs()

        # Reconcile data
        reconciled_df = pd.merge(
            kubecost_df,
            aws_df,
            on="namespace",
            how="outer"
        ).fillna(0)

        # Calculate cost delta (Kubecost - AWS)
        reconciled_df["cost_delta"] = reconciled_df["kubecost_total_cost"] - reconciled_df["aws_total_cost"]

        # Add idle cost (simplified: 20% of total cost for this example)
        reconciled_df["kubecost_idle_cost"] = reconciled_df["kubecost_total_cost"] * 0.2

        # Generate recommendations
        recommendations = generate_recommendations(reconciled_df)

        # Send to Slack
        send_slack_alert(recommendations)

        # Save reconciled data to CSV
        output_path = f"cost_reconciliation_{datetime.now().strftime('%Y%m%d')}.csv"
        reconciled_df.to_csv(output_path, index=False)
        logger.info(f"Saved reconciled data to {output_path}")

    except Exception as e:
        logger.error(f"Cost reconciler failed: {e}")
        raise

if __name__ == "__main__":
    main()

# kubecost-cur-integration.yaml
kubecost:
  aws:
    cur:
      enabled: true
      bucketName: "our-org-aws-cur-2026"
      region: "us-east-1"
      path: "cur/kubecost"
      iamRoleArn: "arn:aws:iam::123456789012:role/kubecost-cur-reader"

# Terraform mixed instance policy snippet
resource "aws_eks_node_group" "mixed_group" {
  instance_types = ["m7g.2xlarge", "m6i.2xlarge"] # Graviton4 + Intel x86
  labels = {
    "migration-phase" = "canary"
  }
}

# Kubecost alert rule example
alerts:
  - name: namespace-budget-exceeded
    condition: "namespace_cost > budget * 0.8"
    window: "24h"
    notify:
      - slack:
          channel: "#infrastructure-costs"
          message: "Namespace {{.Namespace}} is at 80% of its monthly budget"