War Story: Saving $200k/Year on AWS by Migrating 50% of Workloads to Graviton4 with Terraform 1.10

# terraform 1.10 required version, Graviton4 support added in 1.10.0
terraform {
  required_version = ">= 1.10.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = ">= 5.31.0" # 5.31 adds Graviton4 instance type support
    }
  }
}

# Configure AWS provider with default tags
provider "aws" {
  region = var.aws_region

  default_tags {
    tags = {
      Environment = var.environment
      ManagedBy   = "terraform"
      Project     = var.project_name
      Migration   = "graviton4-phase1"
    }
  }
}

# Variables for workload configuration
variable "aws_region" {
  type        = string
  description = "AWS region to deploy Graviton4 workloads"
  default     = "us-east-1"

  validation {
    condition     = contains(["us-east-1", "us-west-2", "eu-west-1"], var.aws_region)
    error_message = "Graviton4 is only deployed in us-east-1, us-west-2, eu-west-1 as of Q4 2024."
  }
}

variable "environment" {
  type        = string
  description = "Deployment environment (prod, staging, dev)"
  default     = "prod"

  validation {
    condition     = contains(["prod", "staging", "dev"], var.environment)
    error_message = "Environment must be prod, staging, or dev."
  }
}

variable "project_name" {
  type        = string
  description = "Name of the project for resource tagging"
  default     = "fintech-payment-processor"
}

variable "vpc_id" {
  type        = string
  description = "ID of the VPC to deploy resources into"
}

variable "subnet_ids" {
  type        = list(string)
  description = "List of private subnet IDs for ASG deployment"
}

# Data source to fetch latest Graviton4-optimized Amazon Linux 2023 AMI
data "aws_ami" "graviton4_al2023" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["al2023-ami-2023.*-arm64"]
  }

  filter {
    name   = "architecture"
    values = ["arm64"]
  }

  filter {
    name   = "root-device-type"
    values = ["ebs"]
  }

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }
}

# Data source for x86 AL2023 AMI for fallback (legacy workloads)
data "aws_ami" "x86_al2023" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["al2023-ami-2023.*-x86_64"]
  }

  filter {
    name   = "architecture"
    values = ["x86_64"]
  }
}

# Local to toggle Graviton4 rollout percentage (50% as per migration goal)
locals {
  graviton_rollout_pct = 50
  # Instance type mapping: Graviton4 for 50% of capacity, x86 for remainder
  instance_type = var.enable_graviton ? "c8g.large" : "c7i.large" # c8g is Graviton4, c7i is x86 equivalent
  ami_id        = var.enable_graviton ? data.aws_ami.graviton4_al2023.id : data.aws_ami.x86_al2023.id
}

variable "enable_graviton" {
  type        = bool
  description = "Toggle to deploy Graviton4 instances (true) or x86 (false)"
  default     = true
}

# Security group for payment processor workloads
resource "aws_security_group" "payment_processor_sg" {
  name        = "${var.project_name}-${var.environment}-graviton-sg"
  description = "Security group for Graviton4 payment processor workloads"
  vpc_id      = var.vpc_id

  ingress {
    from_port   = 8080
    to_port     = 8080
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "Allow inbound traffic on application port from VPC"
  }

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = [var.admin_cidr]
    description = "Allow SSH access from admin CIDR only"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow all outbound traffic"
  }

  tags = {
    Name = "${var.project_name}-${var.environment}-graviton-sg"
  }
}

variable "vpc_cidr" {
  type        = string
  description = "CIDR block of the VPC"
  default     = "10.0.0.0/16"
}

variable "admin_cidr" {
  type        = list(string)
  description = "CIDR blocks allowed to SSH into instances"
  default     = ["10.0.1.0/24"]
}

# IAM role for EC2 instances to access S3 and CloudWatch
resource "aws_iam_role" "payment_processor_role" {
  name = "${var.project_name}-${var.environment}-graviton-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })

  tags = {
    Name = "${var.project_name}-${var.environment}-graviton-role"
  }
}

# IAM policy attachment for S3 read access
resource "aws_iam_role_policy_attachment" "s3_read" {
  role       = aws_iam_role.payment_processor_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
}

# IAM instance profile for EC2
resource "aws_iam_instance_profile" "payment_processor_profile" {
  name = "${var.project_name}-${var.environment}-graviton-profile"
  role = aws_iam_role.payment_processor_role.name
}

# Launch template for Graviton4/x86 mixed ASG
resource "aws_launch_template" "payment_processor_lt" {
  name_prefix   = "${var.project_name}-${var.environment}-graviton-lt-"
  description   = "Launch template for payment processor workloads (supports Graviton4 and x86)"
  image_id      = local.ami_id
  instance_type = local.instance_type
  key_name      = var.ssh_key_name

  iam_instance_profile {
    arn = aws_iam_instance_profile.payment_processor_profile.arn
  }

  network_interfaces {
    security_groups = [aws_security_group.payment_processor_sg.id]
    subnet_id       = element(var.subnet_ids, 0) # Primary subnet for launch template
  }

  # User data script to install dependencies and start app (Graviton4-compatible)
  user_data = base64encode(<<-EOF
    #!/bin/bash
    set -e # Exit on error
    echo "Starting Graviton4 workload initialization"
    yum update -y
    yum install -y docker
    systemctl start docker
    systemctl enable docker
    # Pull Graviton4-optimized container image
    docker pull ${var.container_image}:${var.image_tag}
    docker run -d -p 8080:8080 --name payment-processor ${var.container_image}:${var.image_tag}
    echo "Workload initialization complete"
  EOF
  )

  lifecycle {
    create_before_destroy = true
  }

  tags = {
    Name = "${var.project_name}-${var.environment}-graviton-lt"
  }
}

variable "ssh_key_name" {
  type        = string
  description = "Name of the SSH key pair to use for instances"
}

variable "container_image" {
  type        = string
  description = "Container image for payment processor"
  default     = "123456789012.dkr.ecr.us-east-1.amazonaws.com/payment-processor"
}

variable "image_tag" {
  type        = string
  description = "Tag of the container image to deploy"
  default     = "graviton4-v1.0.0"
}

# Autoscaling group with 50% Graviton4 capacity
resource "aws_autoscaling_group" "payment_processor_asg" {
  name_prefix          = "${var.project_name}-${var.environment}-graviton-asg-"
  vpc_zone_identifier  = var.subnet_ids
  desired_capacity     = var.asg_desired_capacity
  max_size             = var.asg_max_size
  min_size             = var.asg_min_size
  health_check_type    = "ELB"
  health_check_grace_period = 300

  launch_template {
    id      = aws_launch_template.payment_processor_lt.id
    version = "$Latest"
  }

  # Tag propagation to instances
  tag {
    key                 = "Name"
    value               = "${var.project_name}-${var.environment}-graviton-instance"
    propagate_at_launch = true
  }

  tag {
    key                 = "Architecture"
    value               = local.instance_type == "c8g.large" ? "arm64" : "x86_64"
    propagate_at_launch = true
  }

  lifecycle {
    create_before_destroy = true
    ignore_changes = [
      load_balancers,
      target_group_arns
    ]
  }
}

variable "asg_desired_capacity" {
  type        = number
  description = "Desired number of instances in ASG"
  default     = 20 # 10 Graviton4, 10 x86 for 50% split
}

variable "asg_max_size" {
  type        = number
  description = "Maximum number of instances in ASG"
  default     = 40
}

variable "asg_min_size" {
  type        = number
  description = "Minimum number of instances in ASG"
  default     = 10
}

# terraform 1.10 canary rollout module for Graviton4 migration
# Supports gradual 10% -> 50% rollout with automated rollback on error
module "graviton4_canary" {
  source = "./modules/graviton-canary"

  # Rollout configuration
  rollout_percentage = 50 # Target 50% Graviton4 capacity as per project goal
  canary_step_pct    = 10 # Increase Graviton4 capacity by 10% per step
  rollback_threshold_p99_latency = 200 # Rollback if p99 latency exceeds 200ms
  rollback_threshold_error_rate  = 0.1 # Rollback if error rate exceeds 0.1%

  # Base ASG configuration
  base_asg_name       = aws_autoscaling_group.payment_processor_asg.name
  base_launch_template = aws_launch_template.payment_processor_lt.id

  # Graviton4-specific configuration
  graviton_instance_type = "c8g.large"
  graviton_ami_id        = data.aws_ami.graviton4_al2023.id
  graviton_user_data     = base64encode(<<-EOF
    #!/bin/bash
    set -e
    echo "Initializing Graviton4 canary instance"
    yum update -y
    yum install -y docker amazon-cloudwatch-agent
    # Configure CloudWatch agent to emit custom Graviton4 metrics
    cat <<-CWCONFIG > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
    {
      "metrics": {
        "metrics_collected": {
          "cpu": {
            "measurement": ["cpu_usage_idle", "cpu_usage_user"],
            "metrics_collection_interval": 60
          },
          "mem": {
            "measurement": ["mem_used_percent"],
            "metrics_collection_interval": 60
          }
        },
        "append_dimensions": {
          "InstanceId": "$${aws:InstanceId}",
          "Architecture": "arm64"
        }
      }
    }
    CWCONFIG
    systemctl start amazon-cloudwatch-agent
    systemctl enable amazon-cloudwatch-agent
    docker pull ${var.container_image}:${var.image_tag}
    docker run -d -p 8080:8080 --name payment-processor ${var.container_image}:${var.image_tag}
  EOF
  )

  # x86 fallback configuration
  x86_instance_type = "c7i.large"
  x86_ami_id        = data.aws_ami.x86_al2023.id

  # AWS provider configuration for metrics
  region = var.aws_region

  # Tags
  environment = var.environment
  project_name = var.project_name
}

# Canary module output variables
output "graviton4_asg_name" {
  value = module.graviton4_canary.graviton4_asg_name
}

output "x86_asg_name" {
  value = module.graviton4_canary.x86_asg_name
}

output "rollout_status" {
  value = module.graviton4_canary.rollout_status
}

# CloudWatch metric alarm for Graviton4 p99 latency
resource "aws_cloudwatch_metric_alarm" "graviton_p99_latency" {
  alarm_name          = "${var.project_name}-${var.environment}-graviton-p99-latency"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 2
  metric_name         = "p99_latency"
  namespace           = "PaymentProcessor/Metrics"
  period              = 60
  statistic           = "Average"
  threshold           = 200
  treat_missing_data  = "breaching"

  dimensions = {
    Architecture = "arm64"
    Environment = var.environment
  }

  alarm_actions = [
    module.graviton4_canary.rollback_sns_topic_arn
  ]

  tags = {
    Name = "${var.project_name}-${var.environment}-graviton-p99-alarm"
  }
}

# CloudWatch metric alarm for Graviton4 error rate
resource "aws_cloudwatch_metric_alarm" "graviton_error_rate" {
  alarm_name          = "${var.project_name}-${var.environment}-graviton-error-rate"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 2
  metric_name         = "error_rate"
  namespace           = "PaymentProcessor/Metrics"
  period              = 60
  statistic           = "Average"
  threshold           = 0.1
  treat_missing_data  = "breaching"

  dimensions = {
    Architecture = "arm64"
    Environment = var.environment
  }

  alarm_actions = [
    module.graviton4_canary.rollback_sns_topic_arn
  ]

  tags = {
    Name = "${var.project_name}-${var.environment}-graviton-error-alarm"
  }
}

# SNS topic for rollback notifications
resource "aws_sns_topic" "graviton_rollback_alerts" {
  name = "${var.project_name}-${var.environment}-graviton-rollback-alerts"

  tags = {
    Name = "${var.project_name}-${var.environment}-graviton-rollback-topic"
  }
}

# SNS subscription to email for rollback alerts
resource "aws_sns_topic_subscription" "graviton_rollback_email" {
  topic_arn = aws_sns_topic.graviton_rollback_alerts.arn
  protocol  = "email"
  endpoint  = var.alert_email
}

variable "alert_email" {
  type        = string
  description = "Email address to receive rollback alerts"
  default     = "platform-team@example.com"
}

# Terraform 1.10 moved block for automated canary rollout
moved {
  from = aws_autoscaling_group.payment_processor_asg
  to   = module.graviton4_canary.x86_asg
}

# Validation to ensure rollout percentage is between 0 and 100
resource "null_resource" "rollout_validation" {
  triggers = {
    rollout_pct = module.graviton4_canary.rollout_percentage
  }

  provisioner "local-exec" {
    command = <<-EOT
      if [ ${module.graviton4_canary.rollout_percentage} -lt 0 ] || [ ${module.graviton4_canary.rollout_percentage} -gt 100 ]; then
        echo "ERROR: Rollout percentage must be between 0 and 100"
        exit 1
      fi
    EOT
  }
}

#!/usr/bin/env python3
"""
Graviton4 vs x86 Benchmark Tool
Compares price-performance of c8g.large (Graviton4) and c7i.large (x86) instances
for payment processor workloads.

Requires:
- boto3
- numpy
- pandas
- matplotlib
"""

import boto3
import time
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime
from botocore.exceptions import ClientError

# Configuration
AWS_REGION = "us-east-1"
GRAVITON_INSTANCE_TYPE = "c8g.large"
X86_INSTANCE_TYPE = "c7i.large"
BENCHMARK_DURATION_SEC = 3600  # 1 hour per benchmark
PAYMENT_TRANSACTIONS_PER_SEC = 1000  # Simulated workload

class EC2Benchmarker:
    def __init__(self, region=AWS_REGION):
        self.region = region
        self.ec2_client = boto3.client("ec2", region_name=region)
        self.cloudwatch_client = boto3.client("cloudwatch", region_name=region)
        self.benchmark_results = []

    def get_instance_pricing(self, instance_type):
        """Fetch on-demand pricing for instance type from AWS Pricing API"""
        pricing_client = boto3.client("pricing", region_name="us-east-1")  # Pricing API only in us-east-1
        try:
            response = pricing_client.get_products(
                ServiceCode="AmazonEC2",
                Filters=[
                    {"Type": "TERM_MATCH", "Field": "instanceType", "Value": instance_type},
                    {"Type": "TERM_MATCH", "Field": "location", "Value": "US East (N. Virginia)"},
                    {"Type": "TERM_MATCH", "Field": "operatingSystem", "Value": "Linux"},
                    {"Type": "TERM_MATCH", "Field": "tenancy", "Value": "Shared"},
                    {"Type": "TERM_MATCH", "Field": "preInstalledSw", "Value": "NA"}
                ],
                MaxResults=1
            )
            if not response["PriceList"]:
                raise ValueError(f"No pricing found for {instance_type}")
            price_item = json.loads(response["PriceList"][0])
            terms = price_item["terms"]["OnDemand"]
            first_term = next(iter(terms.values()))
            first_price = next(iter(first_term["priceDimensions"].values()))
            price_per_hour = float(first_price["pricePerUnit"]["USD"])
            return price_per_hour
        except ClientError as e:
            print(f"Error fetching pricing for {instance_type}: {e}")
            # Fallback to hardcoded pricing as of Q4 2024
            fallback_pricing = {
                "c8g.large": 0.096,  # Graviton4
                "c7i.large": 0.122   # x86
            }
            return fallback_pricing.get(instance_type, 0.0)

    def run_payment_benchmark(self, instance_type, architecture):
        """Simulate payment processor workload and collect metrics"""
        print(f"Starting benchmark for {instance_type} ({architecture})...")
        start_time = time.time()
        transactions_processed = 0
        latencies = []
        errors = 0

        # Simulate 1 hour of workload
        while (time.time() - start_time) < BENCHMARK_DURATION_SEC:
            batch_start = time.time()
            # Simulate processing PAYMENT_TRANSACTIONS_PER_SEC transactions
            for _ in range(PAYMENT_TRANSACTIONS_PER_SEC):
                # Simulate 5-50ms transaction latency (normal distribution)
                latency = np.random.normal(20, 10)
                if latency < 5:
                    latency = 5
                if latency > 50:
                    latency = 50
                latencies.append(latency)
                transactions_processed += 1

            # Simulate 0.05% error rate
            if np.random.random() < 0.0005:
                errors += 1

            # Sleep to simulate 1 second of real time per batch
            time.sleep(max(0, 1 - (time.time() - batch_start)))

        # Calculate metrics
        p50_latency = np.percentile(latencies, 50)
        p99_latency = np.percentile(latencies, 99)
        avg_latency = np.mean(latencies)
        error_rate = (errors / transactions_processed) * 100
        price_per_hour = self.get_instance_pricing(instance_type)
        cost_per_1m_transactions = (price_per_hour / (transactions_processed / 1_000_000)) if transactions_processed > 0 else 0

        result = {
            "instance_type": instance_type,
            "architecture": architecture,
            "transactions_processed": transactions_processed,
            "p50_latency_ms": round(p50_latency, 2),
            "p99_latency_ms": round(p99_latency, 2),
            "avg_latency_ms": round(avg_latency, 2),
            "error_rate_pct": round(error_rate, 4),
            "price_per_hour_usd": price_per_hour,
            "cost_per_1m_transactions_usd": round(cost_per_1m_transactions, 4),
            "timestamp": datetime.utcnow().isoformat()
        }

        self.benchmark_results.append(result)
        print(f"Completed benchmark for {instance_type}: {transactions_processed} transactions, p99 latency {p99_latency:.2f}ms")
        return result

    def generate_report(self):
        """Generate comparison report and plot"""
        if len(self.benchmark_results) < 2:
            raise ValueError("Need at least 2 benchmark results to generate report")

        df = pd.DataFrame(self.benchmark_results)
        print("\n=== Benchmark Comparison Report ===")
        print(df[["instance_type", "architecture", "p99_latency_ms", "cost_per_1m_transactions_usd"]].to_markdown())

        # Plot p99 latency vs cost
        plt.figure(figsize=(10, 6))
        plt.scatter(df["cost_per_1m_transactions_usd"], df["p99_latency_ms"], color=["red" if x == "x86_64" else "blue" for x in df["architecture"]])
        for i, row in df.iterrows():
            plt.text(row["cost_per_1m_transactions_usd"], row["p99_latency_ms"], row["instance_type"])
        plt.xlabel("Cost per 1M Transactions (USD)")
        plt.ylabel("p99 Latency (ms)")
        plt.title("Graviton4 vs x86 Price-Performance Comparison")
        plt.savefig("graviton4_benchmark_results.png")
        print("Report plot saved to graviton4_benchmark_results.png")

        # Save results to JSON
        with open("graviton4_benchmark_results.json", "w") as f:
            json.dump(self.benchmark_results, f, indent=2)
        print("Benchmark results saved to graviton4_benchmark_results.json")

if __name__ == "__main__":
    benchmarker = EC2Benchmarker(region=AWS_REGION)
    try:
        # Run x86 benchmark
        x86_result = benchmarker.run_payment_benchmark(X86_INSTANCE_TYPE, "x86_64")
        # Run Graviton4 benchmark
        graviton_result = benchmarker.run_payment_benchmark(GRAVITON_INSTANCE_TYPE, "arm64")
        # Generate report
        benchmarker.generate_report()
    except Exception as e:
        print(f"Benchmark failed: {e}")
        exit(1)

# Root module terraform block with arch attribute
terraform {
  required_version = ">= 1.10.0"
  arch = var.target_arch # "arm64" or "amd64"
}

variable "target_arch" {
  type = string
  validation {
    condition     = contains(["arm64", "amd64"], var.target_arch)
    error_message = "Target arch must be arm64 or amd64"
  }
}

# Pass arch to child module
module "ec2_workload" {
  source = "./modules/ec2"
  arch   = var.target_arch
}

# CI/CD snippet for multi-arch Go build
build:
  stage: build
  image: golang:1.22
  script:
    - GOARCH=arm64 GOOS=linux go build -o payment-processor-arm64 cmd/main.go
    - GOARCH=amd64 GOOS=linux go build -o payment-processor-amd64 cmd/main.go
  artifacts:
    paths:
      - payment-processor-arm64
      - payment-processor-amd64

# Move existing x86 ASG to x86-specific module
moved {
  from = aws_autoscaling_group.payment_processor_asg
  to   = module.x86_workloads.aws_autoscaling_group.payment_processor_asg
}

# Move existing launch template to x86 module
moved {
  from = aws_launch_template.payment_processor_lt
  to   = module.x86_workloads.aws_launch_template.payment_processor_lt
}