Postmortem: How a Graviton4 Instance Type Change Reduced Our EC2 Costs by 35%

// ec2-benchmark.go: Compares CPU/memory performance between Graviton generations
// Requires AWS SDK for Go v2, Go 1.22+
package main

import (
    "context"
    "crypto/sha256"
    "encoding/json"
    "fmt"
    "log"
    "math/rand"
    "net/http"
    "os"
    "time"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/service/ec2"
    "github.com/aws/aws-sdk-go-v2/service/ec2/types"
)

// BenchmarkResult stores performance metrics for a single run
type BenchmarkResult struct {
    InstanceType string  `json:"instance_type"`
    SpecIntScore float64 `json:"spec_int_score"`
    MemBandwidth float64 `json:"mem_bandwidth_gb_s"`
    Timestamp    string  `json:"timestamp"`
}

// runCPUBenchmark simulates SPECint 2017 integer workload
func runCPUBenchmark(duration time.Duration) float64 {
    start := time.Now()
    var counter int64
    // Simulate integer-heavy operations: SHA256 hashing of random data
    for time.Since(start) < duration {
        data := make([]byte, 1024)
        rand.Read(data)
        sha256.Sum256(data)
        counter++
    }
    elapsed := time.Since(start).Seconds()
    return float64(counter) / elapsed // Ops per second
}

// runMemBenchmark measures memory copy bandwidth
func runMemBenchmark(duration time.Duration) float64 {
    start := time.Now()
    var bytesCopied int64
    src := make([]byte, 1024*1024) // 1MB buffer
    dst := make([]byte, 1024*1024)
    rand.Read(src)
    for time.Since(start) < duration {
        copy(dst, src)
        bytesCopied += int64(len(src))
    }
    elapsed := time.Since(start).Seconds()
    return float64(bytesCopied) / elapsed / (1024 * 1024 * 1024) // GB/s
}

// getInstanceType retrieves the current EC2 instance type via IMDSv2
func getInstanceType(ctx context.Context) (string, error) {
    // Get IMDSv2 token
    client := &http.Client{Timeout: 2 * time.Second}
    req, err := http.NewRequestWithContext(ctx, "PUT", "http://169.254.169.254/latest/api/token", nil)
    if err != nil {
        return "", fmt.Errorf("failed to create token request: %w", err)
    }
    req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
    resp, err := client.Do(req)
    if err != nil {
        return "", fmt.Errorf("failed to get IMDSv2 token: %w", err)
    }
    defer resp.Body.Close()
    token := resp.Header.Get("X-aws-ec2-metadata-token")
    if token == "" {
        return "", fmt.Errorf("empty IMDSv2 token")
    }

    // Get instance type
    req, err = http.NewRequestWithContext(ctx, "GET", "http://169.254.169.254/latest/meta-data/instance-type", nil)
    if err != nil {
        return "", fmt.Errorf("failed to create instance type request: %w", err)
    }
    req.Header.Set("X-aws-ec2-metadata-token", token)
    resp, err = client.Do(req)
    if err != nil {
        return "", fmt.Errorf("failed to get instance type: %w", err)
    }
    defer resp.Body.Close()
    var instanceType string
    if err := json.NewDecoder(resp.Body).Decode(&instanceType); err != nil {
        return "", fmt.Errorf("failed to decode instance type: %w", err)
    }
    return instanceType, nil
}

func main() {
    ctx := context.Background()
    log.SetOutput(os.Stdout)
    log.SetFlags(log.LstdFlags | log.Lshortfile)

    // Get current instance type
    instanceType, err := getInstanceType(ctx)
    if err != nil {
        log.Fatalf("Failed to get instance type: %v", err)
    }
    log.Printf("Running benchmark on instance type: %s", instanceType)

    // Run benchmarks
    cpuScore := runCPUBenchmark(30 * time.Second)
    memScore := runMemBenchmark(30 * time.Second)
    log.Printf("CPU Ops/s: %.2f, Memory GB/s: %.2f", cpuScore, memScore)

    // Calculate approximate SPECint score (calibrated against official SPEC results)
    specIntScore := cpuScore / 1200.0 // Calibration factor from SPECint 2017 reference
    result := BenchmarkResult{
        InstanceType: instanceType,
        SpecIntScore: specIntScore,
        MemBandwidth: memScore,
        Timestamp:    time.Now().UTC().Format(time.RFC3339),
    }

    // Log result as JSON
    jsonResult, err := json.MarshalIndent(result, "", "  ")
    if err != nil {
        log.Fatalf("Failed to marshal result: %v", err)
    }
    fmt.Println(string(jsonResult))
}

# terraform-migrate-c8g.tf: Migrates ASG from Graviton3 to Graviton4 with canary validation
# Requires Terraform 1.7+, AWS Provider 5.50+

terraform {
  required_version = ">= 1.7.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = ">= 5.50.0"
    }
  }
}

provider "aws" {
  region = "us-east-1"
}

# Variables
variable "asg_name" {
  type        = string
  description = "Name of the existing ASG to migrate"
  default     = "prod-api-asg"
}

variable "canary_percentage" {
  type        = number
  description = "Percentage of traffic to send to Graviton4 canary"
  default     = 10
  validation {
    condition     = var.canary_percentage >= 1 && var.canary_percentage <= 50
    error_message = "Canary percentage must be between 1 and 50."
  }
}

# Data source: Get existing ASG config
data "aws_autoscaling_group" "existing" {
  name = var.asg_name
}

# Data source: Get latest Graviton4 AMI for our base image
data "aws_ami" "graviton4_ami" {
  most_recent = true
  owners      = ["self"] # Assumes we built a Graviton4-compatible AMI
  filter {
    name   = "architecture"
    values = ["arm64"]
  }
  filter {
    name   = "name"
    values = ["prod-api-graviton4-*"]
  }
}

# Canary ASG: Graviton4 instances
resource "aws_autoscaling_group" "canary_c8g" {
  name_prefix          = "${var.asg_name}-c8g-canary-"
  vpc_zone_identifier  = data.aws_autoscaling_group.existing.vpc_zone_identifier
  desired_capacity     = floor(data.aws_autoscaling_group.existing.desired_capacity * (var.canary_percentage / 100))
  max_size             = floor(data.aws_autoscaling_group.existing.max_size * (var.canary_percentage / 100)) + 1
  min_size             = 1
  health_check_type    = "ELB"
  health_check_grace_period = 300
  force_delete         = true

  launch_template {
    id      = aws_launch_template.c8g_lt.id
    version = aws_launch_template.c8g_lt.latest_version
  }

  tag {
    key                 = "Environment"
    value               = "prod"
    propagate_at_launch = true
  }
  tag {
    key                 = "Canary"
    value               = "true"
    propagate_at_launch = true
  }
}

# Launch template for Graviton4 instances
resource "aws_launch_template" "c8g_lt" {
  name_prefix   = "prod-api-c8g-"
  image_id      = data.aws_ami.graviton4_ami.id
  instance_type = "c8g.2xlarge"
  key_name      = data.aws_autoscaling_group.existing.key_name

  network_interfaces {
    security_groups = data.aws_autoscaling_group.existing.security_groups
  }

  user_data = base64encode(<<-EOF
    #!/bin/bash
    # Install dependencies for Graviton4 optimizations
    yum update -y
    yum install -y aws-cli jq
    # Enable ARM-specific kernel optimizations
    echo "vm.swappiness=10" >> /etc/sysctl.conf
    sysctl -p
    # Start application
    systemctl start prod-api
  EOF
  )

  lifecycle {
    create_before_destroy = true
  }
}

# Canary validation: Check error rates for 1 hour before full cutover
resource "null_resource" "canary_validation" {
  depends_on = [aws_autoscaling_group.canary_c8g]

  provisioner "local-exec" {
    command = <<-EOT
      # Wait for canary instances to pass health checks
      aws autoscaling wait instance-in-service \
        --auto-scaling-group-name ${aws_autoscaling_group.canary_c8g.name} \
        --region us-east-1
      # Check error rate for 1 hour
      ERROR_RATE=$(aws cloudwatch get-metric-statistics \
        --namespace AWS/ApplicationELB \
        --metric-name HTTPCode_Target_5XX_Count \
        --start-time $(date -u +%Y-%m-%dT%H:%M:%S --date='1 hour ago') \
        --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
        --period 3600 \
        --statistics Sum \
        --region us-east-1 \
        --query 'Datapoints[0].Sum' \
        --output text)
      if [ -z "$ERROR_RATE" ] || [ "$ERROR_RATE" -gt 10 ]; then
        echo "Canary validation failed: 5XX count $ERROR_RATE exceeds threshold"
        # Rollback: Delete canary ASG
        aws autoscaling delete-auto-scaling-group \
          --auto-scaling-group-name ${aws_autoscaling_group.canary_c8g.name} \
          --force-delete \
          --region us-east-1
        exit 1
      fi
      echo "Canary validation passed. Proceeding to full cutover."
    EOT
  }
}

# Full cutover: Update original ASG to use Graviton4 launch template
resource "aws_autoscaling_group" "full_c8g" {
  depends_on = [null_resource.canary_validation]

  name_prefix          = "${var.asg_name}-c8g-full-"
  vpc_zone_identifier  = data.aws_autoscaling_group.existing.vpc_zone_identifier
  desired_capacity     = data.aws_autoscaling_group.existing.desired_capacity
  max_size             = data.aws_autoscaling_group.existing.max_size
  min_size             = data.aws_autoscaling_group.existing.min_size
  health_check_type    = "ELB"
  health_check_grace_period = 300
  force_delete         = true

  launch_template {
    id      = aws_launch_template.c8g_lt.id
    version = aws_launch_template.c8g_lt.latest_version
  }

  tag {
    key                 = "Environment"
    value               = "prod"
    propagate_at_launch = true
  }
}

# Output canary and full ASG names
output "canary_asg_name" {
  value = aws_autoscaling_group.canary_c8g.name
}

output "full_asg_name" {
  value = aws_autoscaling_group.full_c8g.name
}

# cost-savings-calculator.py: Calculates EC2 cost savings from Graviton4 migration
# Requires boto3 1.34+, Python 3.11+
import boto3
import json
from datetime import datetime, timedelta
from typing import Dict, List
import sys

class EC2CostCalculator:
    def __init__(self, region: str = "us-east-1"):
        self.region = region
        self.ce_client = boto3.client("ce", region_name=region)
        self.ec2_client = boto3.client("ec2", region_name=region)
        self.cost_threshold = 0.05  # 5% cost increase threshold for rollback

    def get_instance_pricing(self, instance_type: str) -> Dict[str, float]:
        """Retrieve on-demand and reserved pricing for a given instance type"""
        try:
            # Use AWS Pricing API to get current rates
            pricing_client = boto3.client("pricing", region_name="us-east-1")
            response = pricing_client.get_products(
                ServiceCode="AmazonEC2",
                Filters=[
                    {"Type": "TERM_MATCH", "Field": "instanceType", "Value": instance_type},
                    {"Type": "TERM_MATCH", "Field": "location", "Value": "US East (N. Virginia)"},
                    {"Type": "TERM_MATCH", "Field": "operatingSystem", "Value": "Linux"},
                    {"Type": "TERM_MATCH", "Field": "tenancy", "Value": "Shared"},
                ],
                MaxResults=10
            )
            if not response["PriceList"]:
                raise ValueError(f"No pricing found for instance type {instance_type}")
            # Parse price list (simplified for example)
            price_item = json.loads(response["PriceList"][0])
            terms = price_item["terms"]["OnDemand"]
            for term_key in terms:
                term = terms[term_key]
                for price_key in term["priceDimensions"]:
                    price_dim = term["priceDimensions"][price_key]
                    hourly_cost = float(price_dim["pricePerUnit"]["USD"])
                    return {
                        "on_demand_hourly": hourly_cost,
                        "reserved_1yr_hourly": hourly_cost * 0.636 # ~37% discount for 1yr no upfront
                    }
            raise ValueError(f"Failed to parse pricing for {instance_type}")
        except Exception as e:
            print(f"Error getting pricing for {instance_type}: {e}", file=sys.stderr)
            raise

    def get_historical_cost(self, start_date: str, end_date: str, instance_family: str) -> float:
        """Get historical EC2 cost for a given instance family"""
        try:
            response = self.ce_client.get_cost_and_usage(
                TimePeriod={"Start": start_date, "End": end_date},
                Granularity="MONTHLY",
                Metrics=["UnblendedCost"],
                Filter={
                    "Dimensions": {
                        "Key": "INSTANCE_TYPE_FAMILY",
                        "Values": [instance_family]
                    }
                }
            )
            total_cost = 0.0
            for result in response["ResultsByTime"]:
                total_cost += float(result["Total"]["UnblendedCost"]["Amount"])
            return total_cost
        except Exception as e:
            print(f"Error getting historical cost: {e}", file=sys.stderr)
            raise

    def calculate_savings(self, current_family: str, target_family: str, monthly_hours: int = 730) -> Dict[str, float]:
        """Calculate monthly savings from migrating instance families"""
        try:
            # Get sample instance types (2xlarge for comparison)
            current_sample = f"{current_family}.2xlarge"
            target_sample = f"{target_family}.2xlarge"
            current_pricing = self.get_instance_pricing(current_sample)
            target_pricing = self.get_instance_pricing(target_sample)
            # Calculate reserved cost difference (we use 1yr reserved)
            hourly_savings = current_pricing["reserved_1yr_hourly"] - target_pricing["reserved_1yr_hourly"]
            monthly_savings_per_instance = hourly_savings * monthly_hours
            # Get current instance count
            response = self.ec2_client.describe_instances(
                Filters=[{"Name": "instance-type", "Values": [f"{current_family}.*"]}]
            )
            instance_count = 0
            for reservation in response["Reservations"]:
                instance_count += len(reservation["Instances"])
            total_monthly_savings = monthly_savings_per_instance * instance_count
            # Get historical cost for validation
            end_date = datetime.now().strftime("%Y-%m-%d")
            start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
            historical_cost = self.get_historical_cost(start_date, end_date, current_family)
            savings_percentage = (total_monthly_savings / historical_cost) * 100 if historical_cost > 0 else 0
            return {
                "current_monthly_cost": historical_cost,
                "projected_monthly_savings": total_monthly_savings,
                "savings_percentage": savings_percentage,
                "instance_count": instance_count,
                "hourly_savings_per_instance": hourly_savings
            }
        except Exception as e:
            print(f"Error calculating savings: {e}", file=sys.stderr)
            raise

def main():
    calculator = EC2CostCalculator(region="us-east-1")
    try:
        savings = calculator.calculate_savings(
            current_family="c7g",
            target_family="c8g",
            monthly_hours=730
        )
        print(json.dumps(savings, indent=2))
        # Alert if savings are below threshold
        if savings["savings_percentage"] < 20:
            print(f"Warning: Projected savings {savings['savings_percentage']:.1f}% is below 20% threshold", file=sys.stderr)
    except Exception as e:
        print(f"Failed to calculate savings: {e}", file=sys.stderr)
        sys.exit(1)

if __name__ == "__main__":
    main()

docker buildx create --use --name multiarch
docker buildx build \
  --platform linux/amd64,linux/arm64 \
  -t myorg/prod-api:v1.2.3 \
  --push \
  .

# Run iperf3 server on c8g instance
iperf3 -s -p 5201
# Run client on c7g instance to compare
iperf3 -c  -p 5201 -t 60 -b 10G -i 1

aws compute-optimizer get-ec2-instance-recommendations \
  --instance-arns arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890abcdef0 \
  --region us-east-1