War Story: We Migrated From AWS EC2 to GCP Axion Instances and Cut Compute Costs by 40%

# Terraform 1.6.0 configuration for GCP Axion T2A instance provisioning
# Includes auto-scaling, health checks, and cost allocation tags
terraform {
  required_version = ">= 1.6.0"
  required_providers {
    google = {
      source  = "hashicorp/google"
      version = "~> 5.0"
    }
  }
}

# Variables for environment-specific configuration
variable "env" {
  type        = string
  description = "Deployment environment (prod, staging, dev)"
  validation {
    condition     = contains(["prod", "staging", "dev"], var.env)
    error_message = "Env must be one of: prod, staging, dev."
  }
}

variable "project_id" {
  type        = string
  description = "GCP project ID"
  default     = "acme-analytics-prod"
}

variable "region" {
  type        = string
  description = "GCP region for deployment"
  default     = "us-central1"
}

# Enable required GCP APIs
resource "google_project_service" "compute" {
  project = var.project_id
  service = "compute.googleapis.com"
  disable_on_destroy = false
}

# Provision T2A instance template with Axion processor
resource "google_compute_instance_template" "t2a_go_worker" {
  name        = "t2a-go-worker-${var.env}-${timestamp()}"
  description = "Axion T2A instance template for Go analytics workers"
  project     = var.project_id
  region      = var.region

  machine_type = "t2a-standard-8" # 8 vCPU, 32GB RAM Axion instance
  tags         = ["go-worker", var.env, "axion"]

  # Boot disk with optimized image for Arm
  disk {
    source_image = "debian-cloud/debian-12-arm64"
    auto_delete  = true
    boot         = true
    disk_size_gb = 50
    disk_type    = "pd-ssd"
  }

  # Network interface with static IP for prod
  network_interface {
    network = "default"
    access_config {
      # Only assign public IP for non-prod environments
      count = var.env == "prod" ? 0 : 1
    }
  }

  # Startup script to install Go runtime and app
  metadata_startup_script = <<-SCRIPT
    #!/bin/bash
    set -euo pipefail
    apt-get update -y
    apt-get install -y golang-1.21 redis-tools
    export PATH=$PATH:/usr/lib/go-1.21/bin
    # Download and install app binary from GCS
    gsutil cp gs://acme-analytics-binaries/go-worker-v1.2.3-linux-arm64 /usr/local/bin/go-worker
    chmod +x /usr/local/bin/go-worker
    # Start worker as systemd service
    cat > /etc/systemd/system/go-worker.service <<-EOF
    [Unit]
    Description=Go Analytics Worker
    After=network.target
    [Service]
    User=root
    ExecStart=/usr/local/bin/go-worker --env ${var.env}
    Restart=always
    [Install]
    WantedBy=multi-user.target
    EOF
    systemctl enable go-worker
    systemctl start go-worker
  SCRIPT

  # Service account with least privilege
  service_account {
    email  = "go-worker-sa@${var.project_id}.iam.gserviceaccount.com"
    scopes = ["cloud-platform"]
  }

  # Lifecycle rule to prevent accidental deletion in prod
  lifecycle {
    create_before_destroy = true
    prevent_destroy       = var.env == "prod"
  }
}

# Auto-scaling group for T2A workers
resource "google_compute_region_autoscaler" "t2a_autoscaler" {
  name   = "t2a-go-worker-autoscaler-${var.env}"
  project = var.project_id
  region = var.region
  target = google_compute_region_instance_group_manager.t2a_ig_mgr.self_link

  autoscaling_policy {
    max_replicas    = var.env == "prod" ? 40 : 10
    min_replicas    = var.env == "prod" ? 20 : 2
    cooldown_period = 60

    cpu_utilization {
      target = 0.7
    }
  }
}

# Instance group manager to manage T2A instances
resource "google_compute_region_instance_group_manager" "t2a_ig_mgr" {
  name   = "t2a-go-worker-ig-${var.env}"
  project = var.project_id
  region = var.region
  version {
    instance_template = google_compute_instance_template.t2a_go_worker.self_link
  }
  base_instance_name = "t2a-go-worker"
  target_size        = var.env == "prod" ? 20 : 2
}

// bench_arm_x86.go: Benchmark compute-heavy workloads across architectures
// Compile: go build -o bench_arm_x86 -ldflags "-s -w" bench_arm_x86.go
// Run: ./bench_arm_x86 --iterations 100000 --workload json-parse
package main

import (
    "bytes"
    "compress/gzip"
    "context"
    "crypto/crc32"
    "encoding/json"
    "flag"
    "fmt"
    "os"
    "os/signal"
    "runtime"
    "syscall"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
)

// Workload types supported
type workloadType string

const (
    workloadJSONParse workloadType = "json-parse"
    workloadCRC32     workloadType = "crc32"
    workloadCompress  workloadType = "compress"
)

var (
    iterations  = flag.Int("iterations", 100000, "Number of benchmark iterations")
    workload    = flag.String("workload", "json-parse", "Workload type: json-parse, crc32, compress")
    arch        = flag.String("arch", runtime.GOARCH, "Architecture (amd64, arm64)")
    opsCounter  = promauto.NewCounterVec(prometheus.CounterOpts{
        Name: "bench_ops_total",
        Help: "Total benchmark operations completed",
    }, []string{"workload", "arch"})
    latencyHist = promauto.NewHistogramVec(prometheus.HistogramOpts{
        Name:    "bench_op_latency_ms",
        Help:    "Operation latency in milliseconds",
        Buckets: prometheus.DefBuckets,
    }, []string{"workload", "arch"})
)

// Sample JSON payload for parsing benchmark
var sampleJSON = []byte(`{"user_id": 12345, "event_type": "page_view", "timestamp": "2024-03-15T14:22:00Z", "metadata": {"page": "/dashboard", "referrer": "google.com", "duration_ms": 4500}}`)

func main() {
    flag.Parse()
    ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
    defer cancel()

    // Validate workload input
    switch workloadType(*workload) {
    case workloadJSONParse, workloadCRC32, workloadCompress:
        // valid
    default:
        fmt.Fprintf(os.Stderr, "invalid workload: %s. Must be one of: json-parse, crc32, compress\n", *workload)
        os.Exit(1)
    }

    fmt.Printf("Starting benchmark: arch=%s, workload=%s, iterations=%d\n", *arch, *workload, *iterations)
    start := time.Now()

    // Run selected workload
    switch workloadType(*workload) {
    case workloadJSONParse:
        runJSONParseBench(ctx, *iterations)
    case workloadCRC32:
        runCRC32Bench(ctx, *iterations)
    case workloadCompress:
        runCompressBench(ctx, *iterations)
    }

    elapsed := time.Since(start)
    opsPerSec := float64(*iterations) / elapsed.Seconds()
    fmt.Printf("Benchmark complete: %d ops in %v, %.2f ops/sec\n", *iterations, elapsed, opsPerSec)
    opsCounter.WithLabelValues(*workload, *arch).Add(float64(*iterations))
}

// runJSONParseBench parses sample JSON repeatedly
func runJSONParseBench(ctx context.Context, n int) {
    var data map[string]interface{}
    for i := 0; i < n; i++ {
        select {
        case <-ctx.Done():
            fmt.Println("benchmark cancelled")
            return
        default:
            start := time.Now()
            if err := json.Unmarshal(sampleJSON, &data); err != nil {
                fmt.Fprintf(os.Stderr, "json parse error: %v\n", err)
                os.Exit(1)
            }
            latencyHist.WithLabelValues("json-parse", *arch).Observe(float64(time.Since(start).Milliseconds()))
        }
    }
}

// runCRC32Bench computes CRC32 checksums repeatedly
func runCRC32Bench(ctx context.Context, n int) {
    payload := make([]byte, 1024) // 1KB payload
    for i := 0; i < n; i++ {
        select {
        case <-ctx.Done():
            fmt.Println("benchmark cancelled")
            return
        default:
            start := time.Now()
            crc32.ChecksumIEEE(payload)
            latencyHist.WithLabelValues("crc32", *arch).Observe(float64(time.Since(start).Milliseconds()))
        }
    }
}

// runCompressBench runs gzip compression repeatedly
func runCompressBench(ctx context.Context, n int) {
    payload := make([]byte, 4096) // 4KB payload
    for i := 0; i < n; i++ {
        select {
        case <-ctx.Done():
            fmt.Println("benchmark cancelled")
            return
        default:
            start := time.Now()
            var buf bytes.Buffer
            gz := gzip.NewWriter(&buf)
            if _, err := gz.Write(payload); err != nil {
                fmt.Fprintf(os.Stderr, "compress error: %v\n", err)
                os.Exit(1)
            }
            if err := gz.Close(); err != nil {
                fmt.Fprintf(os.Stderr, "gzip close error: %v\n", err)
                os.Exit(1)
            }
            latencyHist.WithLabelValues("compress", *arch).Observe(float64(time.Since(start).Milliseconds()))
        }
    }
}

// migrate_aws_gcp.go: Gradual traffic migration from AWS EC2 to GCP Axion
// Uses weighted round-robin load balancing with automatic rollback on error
// Compile: go build -o migrate_aws_gcp migrate_aws_gcp.go
package main

import (
    "context"
    "crypto/tls"
    "errors"
    "flag"
    "fmt"
    "log"
    "net/http"
    "os"
    "os/signal"
    "syscall"
    "time"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/elbv2"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
)

var (
    awsRegion    = flag.String("aws-region", "us-east-1", "AWS region")
    gcpRegion    = flag.String("gcp-region", "us-central1", "GCP region")
    targetWeight = flag.Int("target-weight", 100, "Target GCP traffic weight (0-100)")
    step         = flag.Int("step", 10, "Weight increment per step")
    interval     = flag.Duration("interval", 5*time.Minute, "Interval between weight steps")
    healthPath   = flag.String("health-path", "/healthz", "Health check path")
)

var (
    migrationStep = promauto.NewGauge(prometheus.GaugeOpts{
        Name: "migration_step_current",
        Help: "Current migration step (0 = all AWS, 100 = all GCP)",
    })
    healthCheckErrors = promauto.NewCounterVec(prometheus.CounterOpts{
        Name: "migration_health_check_errors_total",
        Help: "Total health check errors by target",
    }, []string{"target"})
)

type targetGroup struct {
    name     string
    endpoint string
    weight   int
}

func main() {
    flag.Parse()
    ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
    defer cancel()

    // Validate inputs
    if *targetWeight < 0 || *targetWeight > 100 {
        log.Fatalf("target-weight must be between 0 and 100")
    }
    if *step <= 0 || *step > 100 {
        log.Fatalf("step must be between 1 and 100")
    }

    // Initialize AWS session
    awsSess, err := session.NewSession(&aws.Config{
        Region: aws.String(*awsRegion),
    })
    if err != nil {
        log.Fatalf("failed to create AWS session: %v", err)
    }
    elbClient := elbv2.New(awsSess)

    // Define target groups (simplified for example)
    awsTG := targetGroup{name: "aws-ec2-tg", endpoint: "https://ec2.acme-analytics.com", weight: 100}
    gcpTG := targetGroup{name: "gcp-axion-tg", endpoint: "https://axion.acme-analytics.com", weight: 0}

    log.Printf("Starting migration: target GCP weight %d%%, step %d%%, interval %v", *targetWeight, *step, *interval)

    // Run migration steps
    currentWeight := 0
    for currentWeight < *targetWeight {
        select {
        case <-ctx.Done():
            log.Println("Migration cancelled, rolling back to AWS only")
            updateWeights(elbClient, 100, 0)
            return
        default:
        }

        // Calculate next weight
        nextWeight := currentWeight + *step
        if nextWeight > *targetWeight {
            nextWeight = *targetWeight
        }

        // Health check both targets
        log.Printf("Step %d: shifting %d%% traffic to GCP", nextWeight/(*step), nextWeight)
        if err := healthCheck(awsTG.endpoint); err != nil {
            log.Printf("AWS health check failed: %v, aborting", err)
            healthCheckErrors.WithLabelValues("aws").Inc()
            return
        }
        if err := healthCheck(gcpTG.endpoint); err != nil {
            log.Printf("GCP health check failed: %v, rolling back", err)
            healthCheckErrors.WithLabelValues("gcp").Inc()
            updateWeights(elbClient, 100, 0)
            return
        }

        // Update load balancer weights
        if err := updateWeights(elbClient, 100-nextWeight, nextWeight); err != nil {
            log.Printf("Failed to update weights: %v, rolling back", err)
            updateWeights(elbClient, 100, 0)
            return
        }

        migrationStep.Set(float64(nextWeight))
        currentWeight = nextWeight
        log.Printf("Current weights: AWS %d%%, GCP %d%%", 100-nextWeight, nextWeight)

        // Wait for next step
        time.Sleep(*interval)
    }

    log.Println("Migration complete: 100% traffic to GCP Axion")
}

// healthCheck verifies a target is healthy
func healthCheck(endpoint string) error {
    client := &http.Client{
        Timeout: 5 * time.Second,
        Transport: &http.Transport{
            TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // Only for migration, disable in prod
        },
    }
    resp, err := client.Get(fmt.Sprintf("%s%s", endpoint, *healthPath))
    if err != nil {
        return fmt.Errorf("health check request failed: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
    }
    return nil
}

// updateWeights updates AWS ALB target group weights
func updateWeights(elbClient *elbv2.ELBV2, awsWeight, gcpWeight int) error {
    // Note: In real implementation, this would update GCP target group weights too
    // Simplified for example: update AWS TG weight
    _, err := elbClient.ModifyTargetGroupAttributes(&elbv2.ModifyTargetGroupAttributesInput{
        TargetGroupArn: aws.String("arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/aws-ec2-tg/123456"),
        Attributes: []*elbv2.TargetGroupAttribute{
            {
                Key:   aws.String("weight"),
                Value: aws.String(fmt.Sprintf("%d", awsWeight)),
            },
        },
    })
    if err != nil {
        return fmt.Errorf("failed to update AWS TG weight: %w", err)
    }
    log.Printf("Updated weights: AWS=%d, GCP=%d", awsWeight, gcpWeight)
    return nil
}

# CI snippet to build multi-arch Go binaries
GOARCH=arm64 go build -ldflags "-s -w" -o bin/app-linux-arm64 ./cmd/app
GOARCH=amd64 go build -ldflags "-s -w" -o bin/app-linux-amd64 ./cmd/app

# Create a GCP Axion spot instance with CUD eligibility
gcloud compute instances create axion-spot-test \
  --machine-type=t2a-standard-8 \
  --image-project=debian-cloud \
  --image=debian-12-arm64 \
  --region=us-central1 \
  --spot \
  --maintenance-policy=terminate

# Replay production traffic to Axion test instance
./arm-bench --replay-kafka \
  --brokers=kafka.acme-analytics.com:9092 \
  --topic=prod-events \
  --duration=72h \
  --target=https://axion-test.acme-analytics.com