War Story: A Terraform 1.10 State Corruption Outage Deleted Our EKS Cluster

package main

import (
    "encoding/json"
    "fmt"
    "os"
    "sync"
    "testing"
    "time"

    "github.com/hashicorp/terraform/states"
    "github.com/hashicorp/terraform/states/statefile"
)

// TestTerraform110StateCorruption reproduces the race condition in Terraform 1.10
// where concurrent state writes corrupt the state file, leading to resource deletion.
// This matches the exact bug we encountered in production on Nov 12, 2024.
func TestTerraform110StateCorruption(t *testing.T) {
    // Initialize a mock state with an EKS cluster resource, matching our production setup
    eksResource := &states.Resource{
        Addr: states.ResourceAddr{
            Type: "aws_eks_cluster",
            Name: "production",
        },
        Instances: map[states.InstanceKey]*states.ResourceInstance{
            states.NoKey: {
                Current: &states.ResourceInstanceObject{
                    Value: map[string]interface{}{
                        "id":   "eks-prod-123456",
                        "name": "production-eks-cluster",
                        "node_groups": []interface{}{
                            map[string]interface{}{"name": "ng-1", "instance_types": []string{"m5.large"}},
                        },
                    },
                    Status: states.ObjectReady,
                },
            },
        },
    }

    // Create initial state file matching Terraform 1.10 serialization format
    sf := &statefile.File{
        TerraformVersion: "1.10.1",
        State:            states.NewState(),
    }
    sf.State.SetResourceInstanceCurrent(
        eksResource.Addr,
        eksResource.Instances[states.NoKey].Current,
        eksResource.Addr.Resource.Type,
    )

    // Write initial state to temp file
    tmpFile, err := os.CreateTemp("", "tf-state-*.json")
    if err != nil {
        t.Fatalf("failed to create temp file: %v", err)
    }
    defer os.Remove(tmpFile.Name())

    // Serialize state using Terraform 1.10's buggy serialization logic
    // The bug occurs when the JSON encoder writes partial data during concurrent access
    enc := json.NewEncoder(tmpFile)
    if err := enc.Encode(sf); err != nil {
        t.Fatalf("failed to write initial state: %v", err)
    }
    tmpFile.Close()

    // Simulate concurrent terraform apply runs (our CI pipeline had 2 concurrent runners due to misconfig)
    var wg sync.WaitGroup
    corrupted := false
    for i := 0; i < 2; i++ {
        wg.Add(1)
        go func(runnerID int) {
            defer wg.Done()
            // Each runner reads, modifies, and writes state concurrently
            f, err := os.OpenFile(tmpFile.Name(), os.O_RDWR|os.O_TRUNC, 0644)
            if err != nil {
                t.Errorf("runner %d failed to open file: %v", runnerID, err)
                return
            }
            defer f.Close()

            // Read state (simulates terraform init/refresh)
            dec := json.NewDecoder(f)
            var readSF statefile.File
            if err := dec.Decode(&readSF); err != nil {
                // This is where corruption happens: partial read returns invalid JSON
                t.Logf("runner %d encountered corrupt state read: %v", runnerID, err)
                corrupted = true
                return
            }

            // Simulate adding a new S3 bucket resource (the PR change that triggered the outage)
            newResource := &states.Resource{
                Addr: states.ResourceAddr{
                    Type: "aws_s3_bucket",
                    Name: "product-images",
                },
                Instances: map[states.InstanceKey]*states.ResourceInstance{
                    states.NoKey: {
                        Current: &states.ResourceInstanceObject{
                            Value: map[string]interface{}{
                                "id":   "s3-prod-images-789",
                                "name": "product-images-bucket",
                            },
                            Status: states.ObjectReady,
                        },
                    },
                },
            }
            readSF.State.SetResourceInstanceCurrent(
                newResource.Addr,
                newResource.Instances[states.NoKey].Current,
                newResource.Addr.Resource.Type,
            )

            // Truncate and write updated state (concurrent truncate causes data loss)
            if err := f.Truncate(0); err != nil {
                t.Errorf("runner %d failed to truncate: %v", runnerID, err)
                return
            }
            // Seek to start after truncate
            if _, err := f.Seek(0, 0); err != nil {
                t.Errorf("runner %d failed to seek: %v", runnerID, err)
                return
            }
            // Write updated state (if another runner truncates at the same time, data is lost)
            enc := json.NewEncoder(f)
            if err := enc.Encode(&readSF); err != nil {
                t.Errorf("runner %d failed to write state: %v", runnerID, err)
                return
            }
        }(i)
    }

    wg.Wait()

    // Verify if state was corrupted
    if corrupted {
        t.Log("State corruption reproduced: EKS cluster resource missing from state")
        // Read final state to confirm
        finalFile, err := os.Open(tmpFile.Name())
        if err != nil {
            t.Fatalf("failed to open final state: %v", err)
        }
        defer finalFile.Close()
        var finalSF statefile.File
        dec := json.NewDecoder(finalFile)
        if err := dec.Decode(&finalSF); err != nil {
            t.Fatalf("final state is invalid JSON: %v", err)
        }
        // Check if EKS cluster is still present
        eksAddr := states.ResourceAddr{Type: "aws_eks_cluster", Name: "production"}
        if finalSF.State.ResourceInstance(eksAddr) == nil {
            t.Errorf("EKS cluster resource missing from corrupted state: this triggers terraform destroy")
        }
    }
}

#!/usr/bin/env python3
"""
Terraform State Corruption Auditor
Scans S3-backed Terraform state files for signs of the 1.10 serialization bug,
validates state integrity, and outputs risk scores for infra teams.
"""

import json
import boto3
import hashlib
import argparse
from botocore.exceptions import ClientError
from datetime import datetime

class StateAuditor:
    def __init__(self, bucket: str, key_prefix: str, region: str = "us-east-1"):
        self.s3 = boto3.client("s3", region_name=region)
        self.bucket = bucket
        self.key_prefix = key_prefix
        self.risk_scores = {}

    def list_state_files(self) -> list:
        """List all .tfstate files in the S3 bucket under the given prefix."""
        state_files = []
        paginator = self.s3.get_paginator("list_objects_v2")
        try:
            for page in paginator.paginate(Bucket=self.bucket, Prefix=self.key_prefix):
                for obj in page.get("Contents", []):
                    if obj["Key"].endswith(".tfstate"):
                        state_files.append(obj["Key"])
        except ClientError as e:
            print(f"Error listing S3 objects: {e}")
            return []
        return state_files

    def validate_state_file(self, key: str) -> dict:
        """Validate a single state file for corruption and Terraform version risks."""
        result = {
            "key": key,
            "valid_json": False,
            "terraform_version": None,
            "has_eks": False,
            "corruption_risk": "LOW",
            "last_modified": None,
            "md5_hash": None,
        }

        try:
            # Fetch state file from S3
            response = self.s3.get_object(Bucket=self.bucket, Key=key)
            state_data = response["Body"].read()
            result["last_modified"] = response["LastModified"].isoformat()
            result["md5_hash"] = hashlib.md5(state_data).hexdigest()

            # Check if valid JSON
            state_json = json.loads(state_data)
            result["valid_json"] = True

            # Extract Terraform version
            result["terraform_version"] = state_json.get("version")
            tf_version = result["terraform_version"]

            # Check for EKS resources
            resources = state_json.get("resources", [])
            for resource in resources:
                if resource.get("type") == "aws_eks_cluster":
                    result["has_eks"] = True
                    break

            # Calculate corruption risk
            if tf_version and tf_version.startswith("1.10"):
                # 1.10.0-1.10.2 are high risk
                minor_patch = tf_version.split(".")[1:]
                if len(minor_patch) >= 2:
                    minor = int(minor_patch[0])
                    patch = int(minor_patch[1].split("-")[0])  # handle prerelease
                    if minor == 10 and patch <= 2:
                        result["corruption_risk"] = "HIGH"
                    elif minor == 10 and patch >= 3:
                        result["corruption_risk"] = "MEDIUM"
            elif not tf_version:
                result["corruption_risk"] = "MEDIUM"  # Unknown version

            # Check for concurrent write markers (partial JSON, truncated files)
            if not state_data.endswith(b"}"):
                result["corruption_risk"] = "HIGH"
                result["valid_json"] = False

        except json.JSONDecodeError:
            result["valid_json"] = False
            result["corruption_risk"] = "HIGH"
        except ClientError as e:
            result["error"] = str(e)
        except Exception as e:
            result["error"] = str(e)

        return result

    def audit_all(self) -> list:
        """Audit all state files in the bucket and return results."""
        state_files = self.list_state_files()
        print(f"Found {len(state_files)} state files to audit")
        results = []
        for key in state_files:
            print(f"Auditing {key}...")
            res = self.validate_state_file(key)
            results.append(res)
            # Add to risk scores
            risk = res["corruption_risk"]
            self.risk_scores[risk] = self.risk_scores.get(risk, 0) + 1
        return results

    def print_summary(self, results: list):
        """Print a human-readable summary of audit results."""
        print("\n=== Audit Summary ===")
        print(f"Total state files: {len(results)}")
        for risk in ["HIGH", "MEDIUM", "LOW"]:
            count = self.risk_scores.get(risk, 0)
            print(f"{risk} risk: {count} files")
        print("\n=== High Risk Files ===")
        for res in results:
            if res["corruption_risk"] == "HIGH":
                print(f"Key: {res['key']}")
                print(f"Version: {res['terraform_version']}")
                print(f"Has EKS: {res['has_eks']}")
                print(f"Last Modified: {res['last_modified']}")
                print("---")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Audit Terraform state files for corruption risks")
    parser.add_argument("--bucket", required=True, help="S3 bucket containing state files")
    parser.add_argument("--prefix", default="terraform/", help="S3 key prefix for state files")
    parser.add_argument("--region", default="us-east-1", help="AWS region")
    args = parser.parse_args()

    auditor = StateAuditor(args.bucket, args.prefix, args.region)
    results = auditor.audit_all()
    auditor.print_summary(results)

    # Exit with error if any high risk files found
    if auditor.risk_scores.get("HIGH", 0) > 0:
        print("\n[ERROR] High risk state files found. Downgrade Terraform or upgrade to 1.10.3+")
        exit(1)

# terraform/main.tf
# Production EKS cluster configuration with state corruption protections
# Version: 1.0.0
# Author: Senior Infra Team

terraform {
  # Enforce Terraform version to avoid 1.10.0-1.10.2 corruption bug
  required_version = ">= 1.9.8, < 1.10.0, >= 1.10.3"

  # S3 state backend with DynamoDB locking to prevent concurrent writes
  backend "s3" {
    bucket         = "our-company-terraform-state"
    key            = "prod/eks/terraform.tfstate"
    region         = "us-east-1"
    dynamodb_table = "terraform-state-lock"
    encrypt        = true
    kms_key_id     = "arn:aws:kms:us-east-1:123456789012:key/abcd-1234-efgh-5678"
  }

  # Require providers with version pins to prevent unexpected changes
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.20"
    }
    null = {
      source  = "hashicorp/null"
      version = "~> 3.2"
    }
  }
}

# Configure AWS provider for production account
provider "aws" {
  region = "us-east-1"
  assume_role {
    role_arn = "arn:aws:iam::123456789012:role/TerraformAdmin"
  }
}

# Pre-apply state validation null resource to catch corruption before changes
resource "null_resource" "state_validation" {
  # Run before any other resource creation
  triggers = {
    # Re-run validation on every apply
    timestamp = timestamp()
  }

  provisioner "local-exec" {
    command = </dev/null; then
        echo "ERROR: State file is invalid JSON (corruption detected)"
        exit 1
      fi
      # Check if EKS cluster is present (prevent accidental deletion)
      if ! jq '.resources[] | select(.type == "aws_eks_cluster")' "$STATE_FILE" >/dev/null 2>&1; then
        echo "ERROR: EKS cluster resource missing from state. Aborting apply."
        exit 1
      fi
      echo "State validation passed"
    EOT
  }

  # Ensure this runs before EKS cluster creation
  depends_on = [aws_eks_cluster.production]
}

# Production EKS cluster (the resource that was deleted in the outage)
resource "aws_eks_cluster" "production" {
  name     = "production-eks-cluster"
  role_arn = aws_iam_role.eks_cluster.arn
  version  = "1.29"

  vpc_config {
    subnet_ids = aws_subnet.production_private[*].id
    endpoint_private_access = true
    endpoint_public_access  = false
  }

  # Enable encryption at rest
  encryption_config {
    provider {
      key_arn = aws_kms_key.eks.arn
    }
    resources = ["secrets"]
  }

  tags = {
    Environment = "production"
    ManagedBy   = "terraform"
  }
}

# IAM role for EKS cluster
resource "aws_iam_role" "eks_cluster" {
  name = "eks-cluster-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "eks.amazonaws.com"
        }
      }
    ]
  })
}

# KMS key for EKS encryption
resource "aws_kms_key" "eks" {
  description             = "KMS key for EKS cluster encryption"
  deletion_window_in_days = 10
  enable_key_rotation     = true
}

# Output the EKS cluster ID to verify state persistence
output "eks_cluster_id" {
  value = aws_eks_cluster.production.id
  description = "ID of the production EKS cluster"
}

package terraform.version

deny[msg] {
  tf := input.resource.terraform[0]
  version := tf.required_version
  # Reject vulnerable 1.10.0-1.10.2 versions
  contains(version, "1.10.0") | contains(version, "1.10.1") | contains(version, "1.10.2")
  msg := sprintf("Terraform version %v is vulnerable to state corruption. Use 1.9.8 or 1.10.3+", [version])
}

resource "aws_dynamodb_table" "terraform_lock" {
  name           = "terraform-state-lock"
  billing_mode   = "PAY_PER_REQUEST"
  hash_key       = "LockID"

  attribute {
    name = "LockID"
    type = "S"
  }

  tags = {
    Environment = "production"
    Purpose     = "Terraform state locking"
  }
}

#!/bin/bash
# Pre-apply state validation
STATE_FILE="terraform.tfstate"
# Check valid JSON
if ! jq empty "$STATE_FILE" 2>/dev/null; then
  echo "ERROR: State file is corrupt (invalid JSON)"
  exit 1
fi
# Check EKS cluster exists
if ! jq '.resources[] | select(.type == "aws_eks_cluster")' "$STATE_FILE" >/dev/null 2>&1; then
  echo "ERROR: EKS cluster missing from state. Aborting apply."
  exit 1
fi
echo "State validation passed"