Architecture Teardown: How Cloudflare Scales Load Balancing to 1M Requests per Second with HAProxy 3.0

# Cloudflare Edge Load Balancer Config - HAProxy 3.0
# Global settings optimized for 1M RPS per PoP
global
    log /dev/log local0
    log /dev/log local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    # Enable eBPF-based connection tracking (Cloudflare custom patch, upstreamed to 3.0)
    ebpf-mode on
    ebpf-path /usr/lib/haproxy/ebpf/cloudflare-conn-track.o
    # HTTP/3 over QUIC settings
    quic-enable on
    quic-port 443
    quic-max-connections 1000000
    quic-idle-timeout 300s
    # Performance tuning for 16-core AMD EPYC nodes
    nbproc 16
    nbthread 1
    cpu-map auto:1/1-16 0-15
    maxconn 2000000
    tune.bufsize 65536
    tune.maxrewrite 2048
    tune.http.cookielen 63

defaults
    log     global
    mode    http
    option  httplog
    option  dontlognull
    timeout connect 5s
    timeout client  30s
    timeout server  30s
    timeout http-keep-alive 10s
    timeout check 3s
    # Error handling: return 503 with custom page on backend failure
    errorfile 503 /etc/haproxy/errors/503-cloudflare.http
    errorfile 429 /etc/haproxy/errors/429-rate-limit.http
    # Health check defaults
    default-server inter 2s downinter 5s rise 3 fall 2 slowstart 10s

# Frontend for HTTP/3 and HTTP/2 traffic
frontend edge-lb-frontend
    bind :443 ssl crt /etc/haproxy/certs/cloudflare-edge.pem alpn h2,http/1.1
    bind quic:443 ssl crt /etc/haproxy/certs/cloudflare-edge.pem
    # Rate limiting: 1000 RPS per IP, burst 2000
    stick-table type ip size 1m expire 30s store http_req_rate(1s)
    http-request track-sc0 src
    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 1000 }
    # Route to backend based on host header
    use_backend static-backend if { hdr(host) -i static.cloudflare.com }
    use_backend api-backend if { hdr(host) -i api.cloudflare.com }
    default_backend dynamic-backend

# Static content backend (cached at edge)
backend static-backend
    balance roundrobin
    # Health check for static origin servers
    option httpchk GET /health HTTP/1.1\\r\\nHost:\\ static.cloudflare.com
    http-check expect status 200
    server origin-1 10.0.1.10:443 ssl verify none check maxconn 50000
    server origin-2 10.0.1.11:443 ssl verify none check maxconn 50000
    server origin-3 10.0.1.12:443 ssl verify none check maxconn 50000
    # Cache static responses for 1 hour
    http-response set-header Cache-Control \"public, max-age=3600\" if { path_end .css .js .png .jpg }

# API backend with leastconn balancing
backend api-backend
    balance leastconn
    option httpchk GET /api/health HTTP/1.1\\r\\nHost:\\ api.cloudflare.com
    http-check expect status 200
    # Sticky sessions for API clients
    stick-table type string len 64 size 1m expire 1h store server_id
    stick on req.hdr(X-Client-ID) if { hdr(X-Client-ID) -m found }
    server api-1 10.0.2.10:443 ssl verify full ca-file /etc/haproxy/certs/origin-ca.pem check maxconn 20000
    server api-2 10.0.2.11:443 ssl verify full ca-file /etc/haproxy/certs/origin-ca.pem check maxconn 20000
    server api-3 10.0.2.12:443 ssl verify full ca-file /etc/haproxy/certs/origin-ca.pem check maxconn 20000

# Dynamic content backend
backend dynamic-backend
    balance random
    option httpchk GET /health HTTP/1.1\\r\\nHost:\\ dynamic.cloudflare.com
    http-check expect status 200
    server dyn-1 10.0.3.10:443 ssl verify none check maxconn 30000
    server dyn-2 10.0.3.11:443 ssl verify none check maxconn 30000
    server dyn-3 10.0.3.12:443 ssl verify none check maxconn 30000

# Stats page for monitoring
listen stats
    bind :8404
    stats enable
    stats uri /stats
    stats refresh 10s
    stats admin if TRUE
    stats auth admin:CloudflareHAProxy3!

// haproxy-dynamic-weight-adjuster.go
// Dynamically adjusts HAProxy 3.0 backend server weights based on real-time p99 latency
// Compile: go build -o haproxy-adjuster haproxy-dynamic-weight-adjuster.go
// Run: ./haproxy-adjuster --api-socket /run/haproxy/admin.sock --check-interval 5s
package main

import (
    \"bufio\"
    \"context\"
    \"flag\"
    \"fmt\"
    \"log\"
    \"net\"
    \"os\"
    \"os/signal\"
    \"strconv\"
    \"strings\"
    \"sync\"
    \"time\"
)

// Config holds CLI flags
type Config struct {
    apiSocket    string
    checkInterval time.Duration
    latencyThresholdMs int
}

// ServerMetrics stores per-server latency stats
type ServerMetrics struct {
    ServerName string
    P99LatencyMs float64
    ActiveConnections int
    Weight int
}

func main() {
    // Parse CLI flags
    cfg := Config{}
    flag.StringVar(&cfg.apiSocket, \"api-socket\", \"/run/haproxy/admin.sock\", \"Path to HAProxy admin API socket\")
    flag.DurationVar(&cfg.checkInterval, \"check-interval\", 5*time.Second, \"Interval between latency checks\")
    flag.IntVar(&cfg.latencyThresholdMs, \"latency-threshold\", 100, \"P99 latency threshold in ms to trigger weight adjustment\")
    flag.Parse()

    log.Printf(\"Starting HAProxy 3.0 dynamic weight adjuster. API socket: %s, check interval: %v\", cfg.apiSocket, cfg.checkInterval)

    // Validate socket exists
    if _, err := os.Stat(cfg.apiSocket); os.IsNotExist(err) {
        log.Fatalf(\"HAProxy API socket %s does not exist: %v\", cfg.apiSocket, err)
    }

    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    // Handle SIGINT/SIGTERM for graceful shutdown
    sigChan := make(chan os.Signal, 1)
    signal.Notify(sigChan, os.Interrupt)
    go func() {
        <-sigChan
        log.Println(\"Shutdown signal received, stopping adjuster...\")
        cancel()
    }()

    // Run adjustment loop
    ticker := time.NewTicker(cfg.checkInterval)
    defer ticker.Stop()

    for {
        select {
        case <-ctx.Done():
            log.Println(\"Adjuster stopped\")
            return
        case <-ticker.C:
            metrics, err := fetchServerMetrics(cfg.apiSocket)
            if err != nil {
                log.Printf(\"Failed to fetch server metrics: %v\", err)
                continue
            }
            adjustWeights(ctx, cfg, metrics)
        }
    }
}

// fetchServerMetrics queries HAProxy API for per-server latency and connection stats
func fetchServerMetrics(socketPath string) ([]ServerMetrics, error) {
    conn, err := net.Dial(\"unix\", socketPath)
    if err != nil {
        return nil, fmt.Errorf(\"failed to connect to HAProxy socket: %w\", err)
    }
    defer conn.Close()

    // Send \"show stat\" command to get CSV stats
    fmt.Fprintf(conn, \"show stat\\n\")
    scanner := bufio.NewScanner(conn)
    scanner.Split(bufio.ScanLines)

    var metrics []ServerMetrics
    // Skip header line
    scanner.Scan()
    for scanner.Scan() {
        line := scanner.Text()
        // Split CSV line (HAProxy stat output is semicolon-separated)
        fields := strings.Split(line, \";\")
        if len(fields) < 60 {
            continue // Skip malformed lines
        }

        // Extract relevant fields: pxname (0), svname (1), latime (59 = p99 latency ms), scur (4 = active connections), weight (18)
        backendName := fields[0]
        serverName := fields[1]
        if serverName == \"BACKEND\" || serverName == \"FRONTEND\" {
            continue // Skip aggregate entries
        }

        p99LatencyMs, err := strconv.ParseFloat(fields[59], 64)
        if err != nil {
            log.Printf(\"Failed to parse p99 latency for %s/%s: %v\", backendName, serverName, err)
            continue
        }

        activeConns, err := strconv.Atoi(fields[4])
        if err != nil {
            log.Printf(\"Failed to parse active connections for %s/%s: %v\", backendName, serverName, err)
            continue
        }

        weight, err := strconv.Atoi(fields[18])
        if err != nil {
            log.Printf(\"Failed to parse weight for %s/%s: %v\", backendName, serverName, err)
            continue
        }

        metrics = append(metrics, ServerMetrics{
            ServerName: fmt.Sprintf(\"%s/%s\", backendName, serverName),
            P99LatencyMs: p99LatencyMs,
            ActiveConnections: activeConns,
            Weight: weight,
        })
    }

    if err := scanner.Err(); err != nil {
        return nil, fmt.Errorf(\"error reading HAProxy stats: %w\", err)
    }

    return metrics, nil
}

// adjustWeights modifies HAProxy server weights based on latency thresholds
func adjustWeights(ctx context.Context, cfg Config, metrics []ServerMetrics) {
    for _, m := range metrics {
        var newWeight int
        switch {
        case m.P99LatencyMs > float64(cfg.latencyThresholdMs*2):
            newWeight = 10 // Severely reduce weight for high latency
        case m.P99LatencyMs > float64(cfg.latencyThresholdMs):
            newWeight = 50 // Reduce weight for elevated latency
        default:
            newWeight = 100 // Full weight for healthy servers
        }

        if newWeight == m.Weight {
            continue // No change needed
        }

        // Send weight adjustment command to HAProxy API
        err := setServerWeight(cfg.apiSocket, m.ServerName, newWeight)
        if err != nil {
            log.Printf(\"Failed to set weight for %s: %v\", m.ServerName, err)
            continue
        }
        log.Printf(\"Adjusted weight for %s: %d -> %d (p99 latency: %.2fms)\", m.ServerName, m.Weight, newWeight, m.P99LatencyMs)
    }
}

// setServerWeight sends a command to HAProxy to update a server's weight
func setServerWeight(socketPath, serverName string, weight int) error {
    conn, err := net.Dial(\"unix\", socketPath)
    if err != nil {
        return fmt.Errorf(\"failed to connect to HAProxy socket: %w\", err)
    }
    defer conn.Close()

    // HAProxy command: set weight / 
    cmd := fmt.Sprintf(\"set weight %s %d\\n\", serverName, weight)
    _, err = fmt.Fprintf(conn, cmd)
    if err != nil {
        return fmt.Errorf(\"failed to send weight command: %w\", err)
    }

    // Read response to confirm
    scanner := bufio.NewScanner(conn)
    scanner.Scan()
    resp := scanner.Text()
    if !strings.Contains(resp, \"Done\") {
        return fmt.Errorf(\"unexpected response from HAProxy: %s\", resp)
    }

    return nil
}

#!/bin/bash
# haproxy-1m-rps-benchmark.sh
# Benchmark HAProxy 3.0 to validate 1M RPS throughput per PoP
# Requires: wrk2 (https://github.com/giltene/wrk2), jq, bc
# Usage: ./haproxy-1m-rps-benchmark.sh --target https://lb-test.cloudflare.com --threads 16 --connections 100000

set -euo pipefail

# Default configuration
TARGET=\"\"
THREADS=16
CONNECTIONS=100000
DURATION=\"60s\"
RATE=1000000 # Target 1M RPS
WARMUP_DURATION=\"10s\"
OUTPUT_DIR=\"./benchmark-results-$(date +%Y%m%d-%H%M%S)\"

# Parse CLI arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --target)
            TARGET=\"$2\"
            shift 2
            ;;
        --threads)
            THREADS=\"$2\"
            shift 2
            ;;
        --connections)
            CONNECTIONS=\"$2\"
            shift 2
            ;;
        --duration)
            DURATION=\"$2\"
            shift 2
            ;;
        --rate)
            RATE=\"$2\"
            shift 2
            ;;
        *)
            echo \"Unknown argument: $1\"
            exit 1
            ;;
    esac
done

# Validate required arguments
if [[ -z \"$TARGET\" ]]; then
    echo \"Error: --target is required\"
    exit 1
fi

# Check dependencies
for cmd in wrk2 jq bc; do
    if ! command -v $cmd &> /dev/null; then
        echo \"Error: $cmd is not installed. Please install it first.\"
        exit 1
    fi
done

# Create output directory
mkdir -p \"$OUTPUT_DIR\"
echo \"Benchmark results will be saved to $OUTPUT_DIR\"

# Warmup run to prime caches
echo \"Starting warmup run for $WARMUP_DURATION...\"
wrk2 -t \"$THREADS\" -c \"$CONNECTIONS\" -d \"$WARMUP_DURATION\" -R 500000 --latency \"$TARGET\" > /dev/null 2>&1
echo \"Warmup complete.\"

# Main benchmark run
echo \"Starting main benchmark: $DURATION at $RATE RPS...\"
BENCH_OUTPUT=\"$OUTPUT_DIR/benchmark-raw.txt\"
wrk2 -t \"$THREADS\" -c \"$CONNECTIONS\" -d \"$DURATION\" -R \"$RATE\" --latency \"$TARGET\" > \"$BENCH_OUTPUT\" 2>&1

# Parse results
echo \"Parsing benchmark results...\"
LATENCY_JSON=\"$OUTPUT_DIR/latency.json\"
jq -n --argfile bench \"$BENCH_OUTPUT\" '{
    target: \"'$TARGET'\",
    duration: \"'$DURATION'\",
    threads: \"'$THREADS'\",
    connections: \"'$CONNECTIONS'\",
    target_rps: \"'$RATE'\",
    actual_rps: ($bench | capture(\"Requests/sec:\\s+(?[0-9.]+)\") | .rps | tonumber),
    p50_latency: ($bench | capture(\"50%\\s+(?[0-9.]+ms)\") | .lat),
    p90_latency: ($bench | capture(\"90%\\s+(?[0-9.]+ms)\") | .lat),
    p99_latency: ($bench | capture(\"99%\\s+(?[0-9.]+ms)\") | .lat),
    max_latency: ($bench | capture(\"Max\\s+(?[0-9.]+ms)\") | .lat),
    errors: ($bench | capture(\"Errors:\\s+(?[0-9]+)\") | .err | tonumber)
}' > \"$LATENCY_JSON\"

# Validate throughput
ACTUAL_RPS=$(jq -r '.actual_rps' \"$LATENCY_JSON\")
ERRORS=$(jq -r '.errors' \"$LATENCY_JSON\")

echo \"Benchmark complete. Actual RPS: $ACTUAL_RPS, Errors: $ERRORS\"

if (( $(echo \"$ACTUAL_RPS < $RATE * 0.95\" | bc -l) )); then
    echo \"WARNING: Actual RPS is below 95% of target. Throughput not met.\"
    jq '.throughput_met = false' \"$LATENCY_JSON\" > \"$OUTPUT_DIR/final-results.json\"
else
    echo \"SUCCESS: Throughput target met.\"
    jq '.throughput_met = true' \"$LATENCY_JSON\" > \"$OUTPUT_DIR/final-results.json\"
fi

# Generate summary report
SUMMARY=\"$OUTPUT_DIR/summary.txt\"
cat > \"$SUMMARY\" << EOF
HAProxy 3.0 1M RPS Benchmark Summary
=====================================
Target: $TARGET
Duration: $DURATION
Threads: $THREADS
Connections: $CONNECTIONS
Target RPS: $RATE
Actual RPS: $ACTUAL_RPS
P50 Latency: $(jq -r '.p50_latency' \"$LATENCY_JSON\")
P90 Latency: $(jq -r '.p90_latency' \"$LATENCY_JSON\")
P99 Latency: $(jq -r '.p99_latency' \"$LATENCY_JSON\")
Max Latency: $(jq -r '.max_latency' \"$LATENCY_JSON\")
Errors: $ERRORS
Throughput Met: $(jq -r '.throughput_met' \"$OUTPUT_DIR/final-results.json\")
EOF

echo \"Summary report saved to $SUMMARY\"
cat \"$SUMMARY\"

global
    ebpf-mode on
    ebpf-path /usr/lib/haproxy/ebpf/haproxy-conn-track.o

frontend edge-lb-frontend
    bind quic:443 ssl crt /etc/haproxy/certs/edge.pem
    quic-enable on
    quic-max-connections 1000000

curl -X POST http://localhost:8404/api/set/weight/static-backend/origin-1/50

stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners