Postmortem: 2026 Cross-Region Outage Due to Route53 Misconfig – Fixed with Cloudflare DNS 1.0 and Terraform 1.9

# Terraform 1.9 configuration for Cloudflare DNS management
# Provider version pinned to ensure compatibility with DNS 1.0 features
terraform {
  required_version = ">= 1.9.0"
  required_providers {
    cloudflare = {
      source  = "cloudflare/cloudflare"
      version = "~> 4.0"
    }
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
  # Enable native drift detection for DNS resources (Terraform 1.9 feature)
  drift-detection {
    enabled = true
    resources = ["cloudflare_dns_record.*"]
  }
}

# Variables for environment configuration
variable "environment" {
  type        = string
  description = "Deployment environment (prod, staging, dev)"
  validation {
    condition     = contains(["prod", "staging", "dev"], var.environment)
    error_message = "Environment must be prod, staging, or dev."
  }
}

variable "domain_name" {
  type        = string
  description = "Primary domain name for DNS records"
  default     = "example-ecommerce.com"
}

variable "cloudflare_zone_id" {
  type        = string
  description = "Cloudflare Zone ID for the primary domain"
  sensitive   = true
}

# Configure Cloudflare provider with API token
provider "cloudflare" {
  api_token = var.cloudflare_api_token
}

# Configure AWS provider to compare with legacy Route53 records
provider "aws" {
  region = "us-east-1"
}

# Local variables for common DNS TTL and routing settings
locals {
  default_ttl = 60  # Low TTL for fast failover, supported by Cloudflare DNS 1.0
  regions = {
    us-east = "172.16.0.1"
    eu-west = "172.16.1.1"
    ap-south = "172.16.2.1"
  }
}

# Cloudflare DNS record for primary A record with geo-routing
resource "cloudflare_dns_record" "primary_a" {
  zone_id = var.cloudflare_zone_id
  name    = "@"
  type    = "A"
  content = local.regions["us-east"]
  ttl     = local.default_ttl
  proxied = false  # Disable Cloudflare proxy for direct origin routing

  # Geo-routing configuration (Cloudflare DNS 1.0 feature)
  data = {
    geo_routing = {
      us  = local.regions["us-east"]
      eu  = local.regions["eu-west"]
      asia = local.regions["ap-south"]
    }
  }

  # Error handling: prevent accidental deletion of production records
  lifecycle {
    prevent_destroy = var.environment == "prod" ? true : false
  }

  # Validation: ensure content is a valid IPv4 address
  validation {
    condition     = can(regex("^([0-9]{1,3}\\.){3}[0-9]{1,3}$", self.content))
    error_message = "Primary A record content must be a valid IPv4 address."
  }
}

# CNAME record for www subdomain, pointing to primary domain
resource "cloudflare_dns_record" "www_cname" {
  zone_id = var.cloudflare_zone_id
  name    = "www"
  type    = "CNAME"
  content = "${var.domain_name}."
  ttl     = local.default_ttl
  proxied = true

  lifecycle {
    prevent_destroy = var.environment == "prod" ? true : false
  }
}

# Output the Cloudflare nameservers for domain registration
output "cloudflare_nameservers" {
  value = cloudflare_zone.primary.name_servers
  description = "Cloudflare nameservers to configure at domain registrar"
}

# Compare with legacy Route53 records to detect drift
data "aws_route53_zone" "legacy" {
  name = "${var.domain_name}."
}

data "aws_route53_records" "legacy_a" {
  zone_id = data.aws_route53_zone.legacy.zone_id
  type    = "A"
  name    = "${var.domain_name}."
}

# Output drift between Cloudflare and legacy Route53
output "dns_drift" {
  value = length(data.aws_route53_records.legacy_a.records) > 0 ? "Legacy Route53 records still exist" : "No legacy DNS drift detected"
}

#!/usr/bin/env python3
"""
DNS Configuration Validator
Validates Cloudflare DNS records against Terraform state, checks for Route53 misconfigs,
and generates compliance reports.
"""

import os
import sys
import json
import logging
from typing import Dict, List, Optional
import boto3
from cloudflare import Cloudflare
from cloudflare.types.dns import RecordResponse
import subprocess

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

class DNSValidator:
    def __init__(self, cloudflare_api_token: str, aws_region: str = "us-east-1"):
        """Initialize validator with Cloudflare and AWS clients."""
        self.cf = Cloudflare(api_key=cloudflare_api_token)
        self.route53 = boto3.client("route53", region_name=aws_region)
        self.errors: List[str] = []
        self.warnings: List[str] = []

    def get_cloudflare_records(self, zone_id: str) -> List[RecordResponse]:
        """Fetch all DNS records from Cloudflare zone."""
        try:
            records = []
            page = 1
            while True:
                response = self.cf.dns.records.list(
                    zone_id=zone_id,
                    page=page,
                    per_page=100
                )
                records.extend(response.result)
                if response.result_info.page >= response.result_info.total_pages:
                    break
                page += 1
            logger.info(f"Fetched {len(records)} Cloudflare DNS records")
            return records
        except Exception as e:
            logger.error(f"Failed to fetch Cloudflare records: {e}")
            self.errors.append(f"Cloudflare API error: {str(e)}")
            return []

    def get_route53_records(self, zone_id: str) -> List[Dict]:
        """Fetch all DNS records from legacy Route53 zone."""
        try:
            paginator = self.route53.get_paginator("list_resource_record_sets")
            records = []
            for page in paginator.paginate(HostedZoneId=zone_id):
                records.extend(page["ResourceRecordSets"])
            logger.info(f"Fetched {len(records)} Route53 DNS records")
            return records
        except Exception as e:
            logger.error(f"Failed to fetch Route53 records: {e}")
            self.errors.append(f"Route53 API error: {str(e)}")
            return []

    def validate_terraform_state(self, tf_state_path: str = "terraform.tfstate") -> bool:
        """Validate Cloudflare records match Terraform state."""
        if not os.path.exists(tf_state_path):
            self.errors.append(f"Terraform state file not found at {tf_state_path}")
            return False
        try:
            with open(tf_state_path, "r") as f:
                tf_state = json.load(f)
            # Extract Cloudflare DNS records from Terraform state
            tf_records = []
            for resource in tf_state.get("resources", []):
                if resource.get("type") == "cloudflare_dns_record":
                    for instance in resource.get("instances", []):
                        tf_records.append(instance.get("attributes", {}))
            logger.info(f"Found {len(tf_records)} Cloudflare records in Terraform state")
            # Compare with live Cloudflare records (simplified for example)
            if len(tf_records) == 0:
                self.warnings.append("No Cloudflare DNS records found in Terraform state")
            return len(self.errors) == 0
        except Exception as e:
            logger.error(f"Failed to parse Terraform state: {e}")
            self.errors.append(f"Terraform state validation error: {str(e)}")
            return False

    def check_cname_misconfigs(self, records: List[RecordResponse]) -> None:
        """Check for CNAME records pointing to non-existent domains."""
        for record in records:
            if record.type == "CNAME":
                # Simplified check: ensure CNAME ends with a dot (fully qualified)
                if not record.content.endswith("."):
                    self.errors.append(
                        f"CNAME record {record.name} points to non-FQDN: {record.content}"
                    )
                    logger.error(f"Misconfigured CNAME: {record.name} -> {record.content}")

    def generate_report(self, output_path: str = "dns_report.json") -> None:
        """Generate JSON compliance report."""
        report = {
            "errors": self.errors,
            "warnings": self.warnings,
            "compliant": len(self.errors) == 0,
            "timestamp": logging.Formatter().formatTime(logging.LogRecord("", 0, "", 0, "", [], None))
        }
        with open(output_path, "w") as f:
            json.dump(report, f, indent=2)
        logger.info(f"Generated DNS compliance report at {output_path}")

def main():
    # Load environment variables
    cloudflare_token = os.getenv("CLOUDFLARE_API_TOKEN")
    if not cloudflare_token:
        logger.error("CLOUDFLARE_API_TOKEN environment variable not set")
        sys.exit(1)
    cloudflare_zone_id = os.getenv("CLOUDFLARE_ZONE_ID")
    if not cloudflare_zone_id:
        logger.error("CLOUDFLARE_ZONE_ID environment variable not set")
        sys.exit(1)
    route53_zone_id = os.getenv("ROUTE53_ZONE_ID")

    # Initialize validator
    validator = DNSValidator(cloudflare_token)

    # Fetch and validate records
    cf_records = validator.get_cloudflare_records(cloudflare_zone_id)
    validator.check_cname_misconfigs(cf_records)

    # Validate Terraform state
    validator.validate_terraform_state()

    # Check legacy Route53 if zone ID provided
    if route53_zone_id:
        route53_records = validator.get_route53_records(route53_zone_id)
        if len(route53_records) > 0:
            validator.warnings.append(f"Legacy Route53 zone still has {len(route53_records)} records")

    # Generate report
    validator.generate_report()

    # Exit with error code if issues found
    if len(validator.errors) > 0:
        logger.error(f"DNS validation failed with {len(validator.errors)} errors")
        sys.exit(1)
    logger.info("DNS validation passed successfully")

if __name__ == "__main__":
    main()

package main

import (
    "context"
    "fmt"
    "log"
    "net"
    "os"
    "os/exec"
    "time"

    "github.com/cloudflare/cloudflare-go/v4"
    "github.com/cloudflare/cloudflare-go/v4/dns"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/service/route53"
    "github.com/aws/aws-sdk-go-v2/service/route53/types"
)

// ChaosTestConfig holds configuration for DNS chaos testing
type ChaosTestConfig struct {
    CloudflareZoneID string
    Route53ZoneID    string
    DomainName       string
    FailoverDuration time.Duration
}

// DNSChaosTester runs DNS failover chaos experiments
type DNSChaosTester struct {
    cfClient  *cloudflare.Client
    route53Client *route53.Client
    config    ChaosTestConfig
    logger    *log.Logger
}

// NewDNSChaosTester initializes a new chaos tester
func NewDNSChaosTester(ctx context.Context, cfg ChaosTestConfig) (*DNSChaosTester, error) {
    // Initialize Cloudflare client
    cfClient := cloudflare.NewClient(
        cloudflare.WithAPIKey(os.Getenv("CLOUDFLARE_API_KEY")),
    )

    // Initialize AWS Route53 client
    awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-east-1"))
    if err != nil {
        return nil, fmt.Errorf("failed to load AWS config: %w", err)
    }
    route53Client := route53.NewFromConfig(awsCfg)

    return &DNSChaosTester{
        cfClient:  cfClient,
        route53Client: route53Client,
        config:    cfg,
        logger:    log.New(os.Stdout, "[DNS-CHAOS] ", log.LstdFlags),
    }, nil
}

// SimulateOutage simulates a Route53 outage by deleting all A records
func (t *DNSChaosTester) SimulateOutage(ctx context.Context) error {
    t.logger.Println("Simulating Route53 outage: deleting all A records")
    listParams := &route53.ListResourceRecordSetsInput{
        HostedZoneId: aws.String(t.config.Route53ZoneID),
    }
    paginator := route53.NewListResourceRecordSetsPaginator(t.route53Client, listParams)
    for paginator.HasMorePages() {
        page, err := paginator.NextPage(ctx)
        if err != nil {
            return fmt.Errorf("failed to list Route53 records: %w", err)
        }
        for _, record := range page.ResourceRecordSets {
            if *record.Type == types.RRTypeA {
                // Delete the A record
                _, err := t.route53Client.ChangeResourceRecordSets(ctx, &route53.ChangeResourceRecordSetsInput{
                    HostedZoneId: aws.String(t.config.Route53ZoneID),
                    ChangeBatch: &types.ChangeBatch{
                        Changes: []types.Change{
                            {
                                Action: types.ChangeActionDelete,
                                ResourceRecordSet: &record,
                            },
                        },
                    },
                })
                if err != nil {
                    t.logger.Printf("Failed to delete Route53 record %s: %v", *record.Name, err)
                } else {
                    t.logger.Printf("Deleted Route53 A record: %s", *record.Name)
                }
            }
        }
    }
    return nil
}

// TestFailover measures Cloudflare DNS failover time
func (t *DNSChaosTester) TestFailover(ctx context.Context) (time.Duration, error) {
    t.logger.Println("Starting failover test: measuring Cloudflare DNS failover time")
    start := time.Now()
    // Query DNS until Cloudflare record is served
    for {
        select {
        case <-ctx.Done():
            return 0, fmt.Errorf("failover test timed out")
        default:
            ips, err := net.LookupIP(t.config.DomainName)
            if err != nil {
                t.logger.Printf("DNS lookup failed: %v", err)
                time.Sleep(100 * time.Millisecond)
                continue
            }
            // Check if IP is Cloudflare anycast (simplified check)
            for _, ip := range ips {
                if ip.String() == "172.16.0.1" { // Cloudflare primary IP
                    elapsed := time.Since(start)
                    t.logger.Printf("Failover completed in %v", elapsed)
                    return elapsed, nil
                }
            }
            time.Sleep(100 * time.Millisecond)
        }
    }
}

// RestoreDNS restores original Route53 and Cloudflare records
func (t *DNSChaosTester) RestoreDNS(ctx context.Context) error {
    t.logger.Println("Restoring DNS records to original state")
    // Simplified restore: re-apply Terraform config
    cmd := exec.CommandContext(ctx, "terraform", "apply", "-auto-approve")
    cmd.Dir = "./terraform"
    cmd.Stdout = os.Stdout
    cmd.Stderr = os.Stderr
    if err := cmd.Run(); err != nil {
        return fmt.Errorf("failed to restore Terraform state: %w", err)
    }
    return nil
}

func main() {
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
    defer cancel()

    // Load config from environment
    cfg := ChaosTestConfig{
        CloudflareZoneID: os.Getenv("CLOUDFLARE_ZONE_ID"),
        Route53ZoneID:    os.Getenv("ROUTE53_ZONE_ID"),
        DomainName:       os.Getenv("DOMAIN_NAME"),
        FailoverDuration: 30 * time.Second,
    }
    if cfg.CloudflareZoneID == "" || cfg.Route53ZoneID == "" || cfg.DomainName == "" {
        log.Fatal("Missing required environment variables: CLOUDFLARE_ZONE_ID, ROUTE53_ZONE_ID, DOMAIN_NAME")
    }

    // Initialize tester
    tester, err := NewDNSChaosTester(ctx, cfg)
    if err != nil {
        log.Fatalf("Failed to initialize chaos tester: %v", err)
    }

    // Run test
    if err := tester.SimulateOutage(ctx); err != nil {
        log.Fatalf("Failed to simulate outage: %v", err)
    }
    failoverTime, err := tester.TestFailover(ctx)
    if err != nil {
        log.Fatalf("Failover test failed: %v", err)
    }
    fmt.Printf("Cloudflare DNS failover time: %v\n", failoverTime)

    // Restore
    if err := tester.RestoreDNS(ctx); err != nil {
        log.Fatalf("Failed to restore DNS: %v", err)
    }
}

validation {
  condition     = can(regex("^([0-9]{1,3}\\.){3}[0-9]{1,3}$", self.content))
  error_message = "A record content must be a valid IPv4 address."
}

ips, err := net.LookupIP(domainName)
if err != nil {
  log.Printf("DNS lookup failed: %v", err)
}

dig +short example-ecommerce.com
# Anycast output: 104.16.132.229, 104.16.133.229 (Cloudflare IPs)
# Unicast output: 205.251.192.123 (single Route53 IP)