Hot Take: AWS Graviton5 Is a Waste of Money – Azure Cobalt 100 Beats It For Cost-Performance

// benchmark_runner.go
// Compiles with: go build -o benchmark_runner main.go
// Requires: Go 1.23+, access to Graviton5 and Cobalt 100 instances via SSH
package main

import (
    "context"
    "encoding/json"
    "errors"
    "fmt"
    "log"
    "os"
    "os/exec"
    "strconv"
    "strings"
    "time"
)

// InstanceConfig holds connection details for a benchmark target
type InstanceConfig struct {
    Host         string // IP or hostname of the instance
    User         string // SSH user (e.g., "ec2-user" for AWS, "azureuser" for Azure)
    KeyPath      string // Path to SSH private key
    InstanceType string // e.g., "m8g.medium" or "Dpsv6-medium"
    HourlyCost   float64 // USD per hour for the instance
}

// BenchmarkResult stores per-run metrics
type BenchmarkResult struct {
    InstanceType  string  `json:"instance_type"`
    ReqPerSec     float64 `json:"req_per_sec"`
    LatencyP99Ms  float64 `json:"latency_p99_ms"`
    CostPerReq    float64 `json:"cost_per_req"` // Hourly cost / (req/s * 3600)
}

// runSSHCommand executes a command on the remote instance via SSH
func runSSHCommand(ctx context.Context, cfg InstanceConfig, cmd string) (string, error) {
    sshArgs := []string{
        "-i", cfg.KeyPath,
        "-o", "StrictHostKeyChecking=no",
        "-o", "ConnectTimeout=10",
        fmt.Sprintf("%s@%s", cfg.User, cfg.Host),
        cmd,
    }
    sshCmd := exec.CommandContext(ctx, "ssh", sshArgs...)
    output, err := sshCmd.CombinedOutput()
    if err != nil {
        return "", fmt.Errorf("ssh command failed: %w, output: %s", err, string(output))
    }
    return strings.TrimSpace(string(output)), nil
}

// runHTTPBenchmark starts a Go HTTP load test against the remote instance
func runHTTPBenchmark(ctx context.Context, cfg InstanceConfig, duration time.Duration) (*BenchmarkResult, error) {
    // First, start the test HTTP server on the remote instance
    startServerCmd := "ulimit -n 65536 && ./http_server -port 8080 -payload-size 102400 &" // 100KB payload
    _, err := runSSHCommand(ctx, cfg, startServerCmd)
    if err != nil {
        return nil, fmt.Errorf("failed to start server: %w", err)
    }
    // Wait for server to start
    time.Sleep(2 * time.Second)

    // Run wrk load test from local machine (assumes wrk is installed)
    wrkCmd := exec.CommandContext(ctx, "wrk", "-t", "4", "-c", "1000", "-d", duration.String(), fmt.Sprintf("http://%s:8080", cfg.Host))
    wrkOutput, err := wrkCmd.CombinedOutput()
    if err != nil {
        return nil, fmt.Errorf("wrk failed: %w, output: %s", err, string(wrkOutput))
    }

    // Parse wrk output to extract req/s and p99 latency
    lines := strings.Split(string(wrkOutput), "\n")
    var reqPerSec, p99Latency float64
    for _, line := range lines {
        if strings.Contains(line, "Requests/sec:") {
            fields := strings.Fields(line)
            reqPerSec, err = strconv.ParseFloat(fields[2], 64)
            if err != nil {
                return nil, fmt.Errorf("failed to parse req/s: %w", err)
            }
        }
        if strings.Contains(line, "99%") {
            fields := strings.Fields(line)
            // Format: "99% 123ms" or "99% 1.23s"
            latStr := strings.TrimSuffix(fields[1], "ms")
            latStr = strings.TrimSuffix(latStr, "s")
            lat, err := strconv.ParseFloat(latStr, 64)
            if err != nil {
                return nil, fmt.Errorf("failed to parse p99 latency: %w", err)
            }
            if strings.Contains(fields[1], "s") {
                lat *= 1000 // convert to ms
            }
            p99Latency = lat
        }
    }

    if reqPerSec == 0 {
        return nil, errors.New("failed to extract req/s from wrk output")
    }

    // Calculate cost per request: hourly cost / (req/s * 3600 seconds)
    costPerReq := cfg.HourlyCost / (reqPerSec * 3600)

    return &BenchmarkResult{
        InstanceType: cfg.InstanceType,
        ReqPerSec:    reqPerSec,
        LatencyP99Ms: p99Latency,
        CostPerReq:   costPerReq,
    }, nil
}

func main() {
    // Load instance configs from environment variables
    gravitonHost := os.Getenv("GRAVITON_HOST")
    cobaltHost := os.Getenv("COBALT_HOST")
    sshKey := os.Getenv("SSH_KEY_PATH")
    if gravitonHost == "" || cobaltHost == "" || sshKey == "" {
        log.Fatal("Missing required env vars: GRAVITON_HOST, COBALT_HOST, SSH_KEY_PATH")
    }

    instances := []InstanceConfig{
        {
            Host:         gravitonHost,
            User:         "ec2-user",
            KeyPath:      sshKey,
            InstanceType: "m8g.medium",
            HourlyCost:   0.032,
        },
        {
            Host:         cobaltHost,
            User:         "azureuser",
            KeyPath:      sshKey,
            InstanceType: "Dpsv6-medium",
            HourlyCost:   0.028,
        },
    }

    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
    defer cancel()

    results := make([]BenchmarkResult, 0, len(instances))
    for _, inst := range instances {
        log.Printf("Running benchmark for %s...", inst.InstanceType)
        res, err := runHTTPBenchmark(ctx, inst, 60*time.Second)
        if err != nil {
            log.Printf("Benchmark failed for %s: %v", inst.InstanceType, err)
            continue
        }
        results = append(results, *res)
        // Stop the server
        _, _ = runSSHCommand(ctx, inst, "pkill -f http_server")
    }

    // Print results as JSON
    jsonOutput, err := json.MarshalIndent(results, "", "  ")
    if err != nil {
        log.Fatalf("Failed to marshal results: %v", err)
    }
    fmt.Println(string(jsonOutput))
}

// rust_axum_service.rs
// Compiles with: cargo build --release
// Requires: Rust 1.82+, tokio, axum, metrics crates
use axum::{
    extract::State,
    http::StatusCode,
    middleware::{self, Next},
    response::{IntoResponse, Response},
    routing::get,
    Router,
};
use metrics::{counter, gauge, histogram};
use metrics_exporter_prometheus::PrometheusBuilder;
use std::{
    env,
    net::SocketAddr,
    sync::Arc,
    time::{Duration, Instant},
};

// AppState holds shared application state, including instance cost metadata
#[derive(Clone)]
struct AppState {
    instance_type: String,
    hourly_cost_usd: f64,
    start_time: Instant,
}

// Cost tracking middleware: records request latency and calculates cost per request
async fn cost_tracking_middleware(
    State(state): State>,
    req: axum::extract::Request,
    next: Next,
) -> Response {
    let start = Instant::now();
    let response = next.run(req).await;
    let latency = start.elapsed();

    // Record metrics
    let status = response.status().as_u16().to_string();
    counter!("http_requests_total", "instance_type" => state.instance_type.clone(), "status" => status).increment(1);
    histogram!("http_request_duration_seconds", "instance_type" => state.instance_type.clone())
        .record(latency.as_secs_f64());

    // Calculate cost of this single request: hourly cost / (req/s * 3600) but simplified for single req
    let cost_per_req = state.hourly_cost_usd / (1.0 / latency.as_secs_f64() * 3600.0);
    gauge!("http_request_cost_usd", "instance_type" => state.instance_type.clone()).set(cost_per_req);

    response
}

// Health check endpoint: returns 200 with instance metadata
async fn health_check(State(state): State>) -> impl IntoResponse {
    (
        StatusCode::OK,
        format!(
            "Instance: {}, Hourly Cost: ${:.3}, Uptime: {:.2}s",
            state.instance_type,
            state.hourly_cost_usd,
            state.start_time.elapsed().as_secs_f64()
        ),
    )
}

// 100KB payload endpoint for benchmarking
async fn benchmark_payload() -> impl IntoResponse {
    let payload = vec![0u8; 102400]; // 100KB of zeros
    (StatusCode::OK, payload)
}

#[tokio::main]
async fn main() {
    // Initialize metrics exporter
    let builder = PrometheusBuilder::new();
    builder
        .with_http_listener(([0, 0, 0, 0], 9090))
        .install()
        .expect("Failed to install Prometheus metrics exporter");

    // Load environment variables for instance config
    let instance_type = env::var("INSTANCE_TYPE").unwrap_or_else(|_| "unknown".to_string());
    let hourly_cost_str = env::var("HOURLY_COST_USD").unwrap_or_else(|_| "0.0".to_string());
    let hourly_cost_usd: f64 = hourly_cost_str.parse().expect("HOURLY_COST_USD must be a valid float");

    // Validate inputs
    if hourly_cost_usd <= 0.0 {
        eprintln!("Error: HOURLY_COST_USD must be positive");
        std::process::exit(1);
    }

    let state = Arc::new(AppState {
        instance_type: instance_type.clone(),
        hourly_cost_usd,
        start_time: Instant::now(),
    });

    // Build router with middleware and routes
    let app = Router::new()
        .route("/health", get(health_check))
        .route("/benchmark", get(benchmark_payload))
        .route("/metrics", get(|| async { "Metrics exposed on :9090/metrics" }))
        .layer(middleware::from_fn_with_state(state.clone(), cost_tracking_middleware))
        .with_state(state);

    // Parse listen address from env or default to 0.0.0.0:8080
    let addr_str = env::var("LISTEN_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string());
    let addr: SocketAddr = addr_str.parse().expect("Invalid LISTEN_ADDR");

    println!("Starting Axum service on {} for instance type {}", addr, instance_type);
    println!("Metrics available at http://{}:9090/metrics", addr);

    // Start the server
    let listener = tokio::net::TcpListener::bind(addr).await.expect("Failed to bind to address");
    axum::serve(listener, app).await.expect("Failed to start server");
}

#!/bin/bash
# specjbb_setup.sh
# Sets up and runs SPECjbb2015 benchmark on Graviton5 or Cobalt 100 instances
# Requires: Java 17+, wget, unzip, SPECjbb2015 zip file (SPEC license required)
set -euo pipefail  # Exit on error, undefined variable, pipe failure

# Configuration - update these values before running
SSH_KEY="${SSH_KEY:-~/.ssh/id_rsa}"
INSTANCE_USER="${INSTANCE_USER:-ec2-user}"
INSTANCE_HOST="${INSTANCE_HOST:-}"
INSTANCE_TYPE="${INSTANCE_TYPE:-m8g.medium}"
HOURLY_COST="${HOURLY_COST:-0.032}"
SPECJBB_ZIP="${SPECJBB_ZIP:-/tmp/specjbb2015.zip}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/specjbb_results}"
JAVA_HOME="${JAVA_HOME:-/usr/lib/jvm/java-17-amazon-corretto}"  # Update for Azure: /usr/lib/jvm/java-17-azure

# Validate required parameters
if [ -z "$INSTANCE_HOST" ]; then
    echo "Error: INSTANCE_HOST must be set to the instance IP/hostname"
    exit 1
fi

if [ ! -f "$SPECJBB_ZIP" ]; then
    echo "Error: SPECJBB_ZIP file not found at $SPECJBB_ZIP. Obtain from SPEC: https://www.spec.org/jbb2015/"
    exit 1
fi

if [ ! -d "$JAVA_HOME" ]; then
    echo "Error: JAVA_HOME not found at $JAVA_HOME. Install Java 17+ first."
    exit 1
fi

# Create results directory
mkdir -p "$RESULTS_DIR"
chmod 777 "$RESULTS_DIR"

echo "=== Starting SPECjbb2015 benchmark setup for $INSTANCE_TYPE ==="
echo "Instance: $INSTANCE_HOST, Hourly Cost: \$$HOURLY_COST"

# Step 1: Copy SPECjbb2015 zip to remote instance
echo "Copying SPECjbb2015 to remote instance..."
scp -i "$SSH_KEY" -o StrictHostKeyChecking=no "$SPECJBB_ZIP" "$INSTANCE_USER@$INSTANCE_HOST:/tmp/"
if [ $? -ne 0 ]; then
    echo "Error: Failed to copy SPECjbb2015 zip to remote instance"
    exit 1
fi

# Step 2: Unzip and configure SPECjbb2015 on remote instance
echo "Unzipping and configuring SPECjbb2015 on remote instance..."
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "$INSTANCE_USER@$INSTANCE_HOST" << 'EOF'
    set -euo pipefail
    cd /tmp
    unzip -q specjbb2015.zip
    cd specjbb2015
    # Set Java home
    export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto  # Update for Azure if needed
    export PATH=$JAVA_HOME/bin:$PATH
    # Verify Java version
    java -version
    # Create benchmark config for composite mode (multi-jvm)
    cat > config/specjbb2015-composite.cfg << 'CONFIG'
        specjbb.controller.type = composite
        specjbb.controller.rtcurve = 0.1
        specjbb.controller.rtcurve.file = /tmp/specjbb2015/rtcurve.txt
        specjbb.group.count = 1
        specjbb.mapreducer.pool.size = 4
        specjbb.txi.pergroup.count = 1
        specjbb.controller.host = localhost
        specjbb.controller.port = 12345
CONFIG
    echo "SPECjbb2015 setup complete on remote instance"
EOF
if [ $? -ne 0 ]; then
    echo "Error: Failed to set up SPECjbb2015 on remote instance"
    exit 1
fi

# Step 3: Run SPECjbb2015 benchmark
echo "Running SPECjbb2015 benchmark (this will take ~30 minutes)..."
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "$INSTANCE_USER@$INSTANCE_HOST" << 'EOF'
    set -euo pipefail
    cd /tmp/specjbb2015
    export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto
    export PATH=$JAVA_HOME/bin:$PATH
    # Run benchmark in composite mode, log output
    ./run.sh -c config/specjbb2015-composite.cfg -l /tmp/specjbb2015/benchmark.log
    # Copy results to shared directory
    cp -r results/* /tmp/specjbb_results/
EOF
if [ $? -ne 0 ]; then
    echo "Error: SPECjbb2015 benchmark failed"
    exit 1
fi

# Step 4: Copy results back to local machine
echo "Copying results to local directory $RESULTS_DIR..."
scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -r "$INSTANCE_USER@$INSTANCE_HOST:/tmp/specjbb_results/*" "$RESULTS_DIR/"
if [ $? -ne 0 ]; then
    echo "Error: Failed to copy results from remote instance"
    exit 1
fi

# Step 5: Parse results to calculate bops/$ (business operations per second per dollar)
echo "Parsing results to calculate cost-performance..."
BENCH_LOG="$RESULTS_DIR/benchmark.log"
if [ ! -f "$BENCH_LOG" ]; then
    echo "Error: Benchmark log not found at $BENCH_LOG"
    exit 1
fi

# Extract max bops from log (look for "Composite result: X bops")
BOPS=$(grep "Composite result:" "$BENCH_LOG" | awk '{print $4}')
if [ -z "$BOPS" ]; then
    echo "Error: Failed to extract bops from benchmark log"
    exit 1
fi

# Calculate bops per dollar: bops / hourly cost
BOPS_PER_DOLLAR=$(echo "scale=2; $BOPS / $HOURLY_COST" | bc)
echo "=== Benchmark Results for $INSTANCE_TYPE ==="
echo "Max bops: $BOPS"
echo "Hourly Cost: \$$HOURLY_COST"
echo "Bops per Dollar: $BOPS_PER_DOLLAR"
echo "Results saved to $RESULTS_DIR"

# Save summary to JSON
SUMMARY_FILE="$RESULTS_DIR/summary.json"
cat > "$SUMMARY_FILE" << SUMMARY
{
    "instance_type": "$INSTANCE_TYPE",
    "hourly_cost_usd": $HOURLY_COST,
    "max_bops": $BOPS,
    "bops_per_dollar": $BOPS_PER_DOLLAR,
    "timestamp": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
}
SUMMARY

echo "Summary saved to $SUMMARY_FILE"
echo "=== Benchmark complete ==="

GOOS=linux GOARCH=arm64 GOARM=9 GOFLAGS="-tags=netgo -ldflags=-s -w" go build -o app-arm64 ./cmd/main.go

sum(rate(http_requests_total[5m])) by (instance_type) / (sum(kubecost_cluster_costs_per_hour) by (instance_type))

perf stat -e L2-dcache-loads,L2-dcache-load-misses,instructions,cycles -a -p $(pidof axum-service) sleep 60