Retrospective: Migrating 100 Ingress Rules from HAProxy 3.0 to NGINX 1.25: 25% Lower Cost

# HAProxy 3.0 Enterprise Configuration (Pre-Migration)
# Global settings for 100 ingress rules across 3 backend clusters
global
    log /dev/log local0
    log /dev/log local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

    # Error handling: global timeout and retry settings
    timeout connect 5s
    timeout client 30s
    timeout server 30s
    retries 3
    option redispatch

    # SSL settings for 100+ ingress rules with SNI
    ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256
    ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets

defaults
    log global
    mode http
    option httplog
    option dontlognull
    timeout connect 5s
    timeout client 30s
    timeout server 30s

# Frontend for HTTP (port 80) - redirect to HTTPS
frontend http_front
    bind *:80
    # Error handling: return 301 for all HTTP requests to HTTPS
    redirect scheme https code 301 if !{ ssl_fc }

# Frontend for HTTPS (port 443) - SNI-based routing for 100 ingress rules
frontend https_front
    bind *:443 ssl crt /etc/haproxy/certs/ alpn h2,http/1.1
    # Enable HTTP/2 for all ingress rules
    option http-use-htx

    # Ingress Rule 1: api.example.com (backend: api-cluster)
    acl host_api hdr(host) -i api.example.com
    use_backend api_cluster if host_api

    # Ingress Rule 2: app.example.com (backend: app-cluster)
    acl host_app hdr(host) -i app.example.com
    use_backend app_cluster if host_app

    # Ingress Rule 3: admin.example.com (backend: admin-cluster, IP whitelist)
    acl host_admin hdr(host) -i admin.example.com
    acl whitelist_admin src 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16
    use_backend admin_cluster if host_admin whitelist_admin
    # Error handling: return 403 for admin access from non-whitelisted IPs
    http-request return 403 if host_admin !whitelist_admin

    # ... [Repeat for 97 additional ingress rules, truncated for brevity] ...
    # Ingress Rule 100: static.example.com (backend: static-cluster, cache enabled)
    acl host_static hdr(host) -i static.example.com
    use_backend static_cluster if host_static

# Backend: api-cluster (3 nodes, health check every 2s)
backend api_cluster
    balance roundrobin
    option httpchk GET /health HTTP/1.1\\r\\nHost:\\ api.example.com
    http-check expect status 200
    server api-1 10.0.1.10:8080 check inter 2s fall 3 rise 2
    server api-2 10.0.1.11:8080 check inter 2s fall 3 rise 2
    server api-3 10.0.1.12:8080 check inter 2s fall 3 rise 2
    # Error handling: return 503 if all backend nodes are down
    errorfile 503 /etc/haproxy/errors/503.http

# Backend: app-cluster (5 nodes, sticky sessions)
backend app_cluster
    balance roundrobin
    cookie app_session insert indirect nocache
    option httpchk GET /health HTTP/1.1\\r\\nHost:\\ app.example.com
    http-check expect status 200
    server app-1 10.0.2.10:8080 check inter 2s fall 3 rise 2 cookie app-1
    server app-2 10.0.2.11:8080 check inter 2s fall 3 rise 2 cookie app-2
    # ... [Additional app nodes] ...
    errorfile 503 /etc/haproxy/errors/503.http

# Backend: admin-cluster (2 nodes, mutual TLS)
backend admin_cluster
    balance leastconn
    option ssl-hello-chk
    server admin-1 10.0.3.10:8443 check ssl verify required ca-file /etc/haproxy/certs/ca.pem
    server admin-2 10.0.3.11:8443 check ssl verify required ca-file /etc/haproxy/certs/ca.pem
    errorfile 503 /etc/haproxy/errors/503.http

# Backend: static-cluster (4 nodes, cache static assets for 7 days)
backend static_cluster
    balance roundrobin
    http-response set-header Cache-Control \"public, max-age=604800\" if { path_end .css .js .png .jpg .gif }
    server static-1 10.0.4.10:8080 check inter 5s fall 3 rise 2
    # ... [Additional static nodes] ...
    errorfile 503 /etc/haproxy/errors/503.http

# NGINX 1.25 Open Source Configuration (Post-Migration)
# Global settings for 100 ingress rules, zero-downtime reloads
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;

# Error handling: global worker connections and timeout settings
events {
    worker_connections 4096;
    use epoll;
    multi_accept on;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Logging settings for 100 ingress rules
    log_format main '$remote_addr - $remote_user [$time_local] \"$request\" '
                    '$status $body_bytes_sent \"$http_referer\" '
                    '\"$http_user_agent\" \"$http_x_forwarded_for\" \"$host\"';
    access_log /var/log/nginx/access.log main;

    # Performance and error handling timeouts
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    client_max_body_size 10m;
    client_body_timeout 12s;
    client_header_timeout 12s;
    send_timeout 10s;

    # SSL settings for 100+ ingress rules with SNI
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256;
    ssl_prefer_server_ciphers on;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;

    # Frontend: HTTP (port 80) - redirect to HTTPS
    server {
        listen 80;
        server_name _;
        # Error handling: 301 redirect for all HTTP requests
        return 301 https://$host$request_uri;
    }

    # Frontend: HTTPS (port 443) - SNI-based routing for 100 ingress rules
    server {
        listen 443 ssl http2;
        server_name _;

        # Wildcard certificate for all 100 ingress rules (cost savings over per-rule certs)
        ssl_certificate /etc/nginx/certs/wildcard.example.com.pem;
        ssl_certificate_key /etc/nginx/certs/wildcard.example.com.key;

        # Ingress Rule 1: api.example.com (backend: api-cluster)
        location / {
            if ($host ~* ^api\\.example\\.com$) {
                proxy_pass http://api_cluster;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                # Error handling: 503 if backend is down
                proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
                proxy_connect_timeout 5s;
                proxy_send_timeout 30s;
                proxy_read_timeout 30s;
                break;
            }
        }

        # Ingress Rule 2: app.example.com (backend: app-cluster, sticky sessions)
        location / {
            if ($host ~* ^app\\.example\\.com$) {
                proxy_pass http://app_cluster;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                # Sticky session support via cookie
                sticky cookie app_session expires=1h domain=.example.com path=/;
                proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
                break;
            }
        }

        # Ingress Rule 3: admin.example.com (backend: admin-cluster, IP whitelist)
        location / {
            if ($host ~* ^admin\\.example\\.com$) {
                # Error handling: 403 for non-whitelisted IPs
                allow 10.0.0.0/8;
                allow 172.16.0.0/12;
                allow 192.168.0.0/16;
                deny all;
                proxy_pass https://admin_cluster;
                proxy_ssl_verify on;
                proxy_ssl_trusted_certificate /etc/nginx/certs/ca.pem;
                proxy_set_header Host $host;
                break;
            }
        }

        # ... [Repeat for 97 additional ingress rules, truncated for brevity] ...
        # Ingress Rule 100: static.example.com (backend: static-cluster, cache enabled)
        location / {
            if ($host ~* ^static\\.example\\.com$) {
                proxy_pass http://static_cluster;
                # Cache static assets for 7 days
                expires 7d;
                add_header Cache-Control \"public, max-age=604800\";
                proxy_set_header Host $host;
                break;
            }
        }

        # Error handling: custom 403 and 503 pages
        error_page 403 /403.html;
        location = /403.html {
            root /etc/nginx/errors;
            internal;
        }
        error_page 503 /503.html;
        location = /503.html {
            root /etc/nginx/errors;
            internal;
        }
    }

    # Upstream: api-cluster (3 nodes, health check every 2s)
    upstream api_cluster {
        server 10.0.1.10:8080 max_fails=3 fail_timeout=2s;
        server 10.0.1.11:8080 max_fails=3 fail_timeout=2s;
        server 10.0.1.12:8080 max_fails=3 fail_timeout=2s;
    }

    # Upstream: app-cluster (5 nodes, sticky sessions)
    upstream app_cluster {
        sticky cookie app_session expires=1h domain=.example.com path=/;
        server 10.0.2.10:8080 max_fails=3 fail_timeout=2s;
        server 10.0.2.11:8080 max_fails=3 fail_timeout=2s;
        # ... [Additional app nodes] ...
    }

    # Upstream: admin-cluster (2 nodes, mutual TLS)
    upstream admin_cluster {
        server 10.0.3.10:8443 max_fails=3 fail_timeout=2s;
        server 10.0.3.11:8443 max_fails=3 fail_timeout=2s;
    }

    # Upstream: static-cluster (4 nodes)
    upstream static_cluster {
        server 10.0.4.10:8080 max_fails=3 fail_timeout=5s;
        # ... [Additional static nodes] ...
    }
}

#!/usr/bin/env python3
\"\"\"
Migration Validation Script: Compare HAProxy 3.0 vs NGINX 1.25 Ingress Performance
Sends 12,000 synthetic requests across 100 ingress rules, outputs latency and error metrics.
Requires: requests, pandas, matplotlib (pip install requests pandas matplotlib)
\"\"\"

import requests
import time
import concurrent.futures
import pandas as pd
import matplotlib.pyplot as plt
from typing import List, Dict, Tuple
import logging
from requests.exceptions import RequestException, Timeout

# Configure logging for error handling
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s - %(levelname)s - %(message)s\"
)
logger = logging.getLogger(__name__)

# Configuration: 100 ingress rules to test (truncated to 5 for example, full list in prod)
INGRESS_RULES: List[Dict[str, str]] = [
    {\"host\": \"api.example.com\", \"path\": \"/health\", \"method\": \"GET\"},
    {\"host\": \"app.example.com\", \"path\": \"/\", \"method\": \"GET\"},
    {\"host\": \"admin.example.com\", \"path\": \"/dashboard\", \"method\": \"GET\"},
    {\"host\": \"static.example.com\", \"path\": \"/css/main.css\", \"method\": \"GET\"},
    # ... [96 additional ingress rules] ...
    {\"host\": \"webhook.example.com\", \"path\": \"/stripe\", \"method\": \"POST\"},
]

# Test parameters
TOTAL_REQUESTS = 12000
CONCURRENT_WORKERS = 50
HA_PROXY_URL = \"http://haproxy-lb.example.com\"
NGINX_URL = \"http://nginx-lb.example.com\"
REQUEST_TIMEOUT = 10  # seconds

def send_request(rule: Dict[str, str], base_url: str) -> Tuple[str, float, int]:
    \"\"\"
    Send a single request to a target URL, return host, latency (ms), status code.
    Includes error handling for timeouts and connection errors.
    \"\"\"
    host = rule[\"host\"]
    path = rule[\"path\"]
    method = rule[\"method\"]
    url = f\"{base_url}{path}\"
    headers = {\"Host\": host}

    start_time = time.perf_counter()
    try:
        if method == \"GET\":
            response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT)
        elif method == \"POST\":
            response = requests.post(url, headers=headers, timeout=REQUEST_TIMEOUT, json={})
        else:
            logger.warning(f\"Unsupported method {method} for {host}\")
            return host, 0.0, 405
        latency = (time.perf_counter() - start_time) * 1000  # ms
        return host, latency, response.status_code
    except Timeout:
        logger.error(f\"Timeout for {host}{path} to {base_url}\")
        return host, REQUEST_TIMEOUT * 1000, 0
    except RequestException as e:
        logger.error(f\"Request failed for {host}{path} to {base_url}: {str(e)}\")
        return host, 0.0, 0
    except Exception as e:
        logger.error(f\"Unexpected error for {host}{path}: {str(e)}\")
        return host, 0.0, 0

def run_load_test(base_url: str) -> pd.DataFrame:
    \"\"\"
    Run load test with TOTAL_REQUESTS across all ingress rules, return DataFrame of results.
    \"\"\"
    results = []
    # Distribute requests evenly across 100 ingress rules
    requests_per_rule = TOTAL_REQUESTS // len(INGRESS_RULES)
    test_rules = INGRESS_RULES * requests_per_rule

    logger.info(f\"Starting load test for {base_url}: {len(test_rules)} requests, {CONCURRENT_WORKERS} workers\")

    with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENT_WORKERS) as executor:
        futures = [executor.submit(send_request, rule, base_url) for rule in test_rules]
        for future in concurrent.futures.as_completed(futures):
            results.append(future.result())

    df = pd.DataFrame(results, columns=[\"host\", \"latency_ms\", \"status_code\"])
    logger.info(f\"Load test complete for {base_url}: {len(df)} results\")
    return df

def generate_report(haproxy_df: pd.DataFrame, nginx_df: pd.DataFrame) -> None:
    \"\"\"
    Generate latency comparison report and plot.
    \"\"\"
    # Calculate p50, p99, error rate for both targets
    haproxy_stats = {
        \"p50_latency\": haproxy_df[\"latency_ms\"].quantile(0.5),
        \"p99_latency\": haproxy_df[\"latency_ms\"].quantile(0.99),
        \"error_rate\": (haproxy_df[\"status_code\"] >= 400).mean() * 100,
        \"total_requests\": len(haproxy_df)
    }
    nginx_stats = {
        \"p50_latency\": nginx_df[\"latency_ms\"].quantile(0.5),
        \"p99_latency\": nginx_df[\"latency_ms\"].quantile(0.99),
        \"error_rate\": (nginx_df[\"status_code\"] >= 400).mean() * 100,
        \"total_requests\": len(nginx_df)
    }

    # Print stats
    print(\"\\n=== Migration Validation Report ===\")
    print(f\"HAProxy 3.0 Results: {haproxy_stats['total_requests']} requests\")
    print(f\"  p50 Latency: {haproxy_stats['p50_latency']:.2f}ms\")
    print(f\"  p99 Latency: {haproxy_stats['p99_latency']:.2f}ms\")
    print(f\"  Error Rate: {haproxy_stats['error_rate']:.2f}%\")
    print(f\"\\nNGINX 1.25 Results: {nginx_stats['total_requests']} requests\")
    print(f\"  p50 Latency: {nginx_stats['p50_latency']:.2f}ms\")
    print(f\"  p99 Latency: {nginx_stats['p99_latency']:.2f}ms\")
    print(f\"  Error Rate: {nginx_stats['error_rate']:.2f}%\")

    # Plot latency distribution
    plt.figure(figsize=(10, 6))
    plt.hist(haproxy_df[\"latency_ms\"], bins=50, alpha=0.5, label=\"HAProxy 3.0\")
    plt.hist(nginx_df[\"latency_ms\"], bins=50, alpha=0.5, label=\"NGINX 1.25\")
    plt.xlabel(\"Latency (ms)\")
    plt.ylabel(\"Number of Requests\")
    plt.title(\"Ingress Latency Distribution: HAProxy 3.0 vs NGINX 1.25\")
    plt.legend()
    plt.savefig(\"/tmp/ingress_latency_comparison.png\")
    logger.info(\"Saved latency plot to /tmp/ingress_latency_comparison.png\")

if __name__ == \"__main__\":
    # Run tests for both targets
    haproxy_results = run_load_test(HA_PROXY_URL)
    nginx_results = run_load_test(NGINX_URL)
    # Generate comparison report
    generate_report(haproxy_results, nginx_results)