Postmortem: How a Misconfigured Nginx 1.26 Caused a DDoS Attack on Our App

# Faulty Nginx 1.26 Ingress Configuration
# Deployed to production at 14:22 UTC on Oct 17, 2024
# Caused self-inflicted DDoS from 14:37 to 15:02 UTC

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;

events {
    worker_connections 4096;
    use epoll;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Rate limit zone for API endpoints
    # BUG: Missing semicolon at end of directive - caused partial parse in Nginx 1.26
    limit_req_zone $binary_remote_addr zone=api_limit:10m rate=100r/s

    # Upstream origin cluster (12 nodes, AWS EC2 m6g.large)
    upstream app_origin {
        least_conn;
        server 10.0.1.10:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.11:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.12:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.13:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.14:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.15:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.16:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.17:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.18:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.19:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.20:8080 max_fails=3 fail_timeout=30s;
    }

    # Health check endpoint (misconfigured rate limit applied here)
    server {
        listen 8080;
        server_name _;

        location /health {
            # BUG: limit_req applied to health check, which is called every 2s by ALB
            # Nginx 1.26's new zone cleanup logic processes this 3x faster than 1.24
            limit_req zone=api_limit burst=200 nodelay;
            proxy_pass http://app_origin/health;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_connect_timeout 5s;
            proxy_read_timeout 10s;
        }

        location /api {
            limit_req zone=api_limit burst=200 nodelay;
            proxy_pass http://app_origin/api;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_connect_timeout 5s;
            proxy_read_timeout 30s;
        }
    }
}

# Fixed Nginx 1.26 Ingress Configuration
# Deployed to production at 15:02 UTC on Oct 17, 2024
# Includes linting validation and Nginx 1.26.1 patches

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;

events {
    worker_connections 4096;
    use epoll;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Rate limit zone for API endpoints (fixed: added semicolon, increased zone size)
    limit_req_zone $binary_remote_addr zone=api_limit:20m rate=100r/s;

    # Upstream origin cluster (12 nodes, AWS EC2 m6g.large)
    upstream app_origin {
        least_conn;
        server 10.0.1.10:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.11:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.12:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.13:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.14:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.15:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.16:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.17:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.18:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.19:8080 max_fails=3 fail_timeout=30s;
        server 10.0.1.20:8080 max_fails=3 fail_timeout=30s;
    }

    # Health check endpoint (fixed: removed rate limit, added explicit off directive)
    server {
        listen 8080;
        server_name _;

        location /health {
            access_log off;
            # Explicitly disable rate limiting for health checks
            limit_req off;
            proxy_pass http://app_origin/health;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_connect_timeout 2s;
            proxy_read_timeout 2s;
        }

        location /api {
            limit_req zone=api_limit burst=200 nodelay;
            proxy_pass http://app_origin/api;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_connect_timeout 5s;
            proxy_read_timeout 30s;
        }
    }
}

# nginx_config_linter.py
# Validates Nginx 1.26+ configurations for rate-limit misconfigurations
# Includes error handling for file I/O, syntax parsing, and version checks
# Requires: pip install nginx-parser pyyaml requests

import sys
import os
import re
import logging
from typing import List, Dict, Tuple

# Configure logging for audit trails
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

class NginxConfigLinter:
    def __init__(self, config_path: str, nginx_version: str = "1.26"):
        self.config_path = config_path
        self.nginx_version = nginx_version
        self.errors: List[str] = []
        self.warnings: List[str] = []

    def _check_file_exists(self) -> bool:
        """Verify config file exists and is readable"""
        if not os.path.isfile(self.config_path):
            self.errors.append(f"Config file {self.config_path} not found")
            return False
        if not os.access(self.config_path, os.R_OK):
            self.errors.append(f"Config file {self.config_path} is not readable")
            return False
        return True

    def _parse_limit_req_zones(self, config_content: str) -> List[Dict]:
        """Extract limit_req_zone directives and validate syntax"""
        zone_pattern = r"limit_req_zone\s+(\S+)\s+zone=(\S+):(\S+)\s+rate=(\S+);"
        zones = []
        for line_num, line in enumerate(config_content.split("\n"), 1):
            line = line.strip()
            if line.startswith("#") or not line:
                continue
            match = re.search(zone_pattern, line)
            if match:
                zones.append({
                    "line_num": line_num,
                    "key": match.group(1),
                    "zone_name": match.group(2),
                    "size": match.group(3),
                    "rate": match.group(4)
                })
            elif "limit_req_zone" in line and ";" not in line:
                # Catch missing semicolons in limit_req_zone directives
                self.errors.append(
                    f"Line {line_num}: Missing semicolon in limit_req_zone directive: {line}"
                )
        return zones

    def _validate_rate_limits(self, zones: List[Dict]) -> None:
        """Check for dangerous rate limit configurations"""
        for zone in zones:
            # Check for burst=nodelay without proper rate limiting
            rate = zone["rate"]
            rate_num = int(rate.split("r/")[0])
            if rate_num > 1000:
                self.warnings.append(
                    f"Zone {zone["zone_name"]} (line {zone["line_num"]}): "
                    f"Rate limit {rate} exceeds recommended 1000r/s threshold"
                )
            # Check for zone size too small for rate
            size = zone["size"]
            size_mb = int(size.replace("m", ""))
            if size_mb < 10 and rate_num > 100:
                self.warnings.append(
                    f"Zone {zone["zone_name"]} (line {zone["line_num"]}): "
                    f"Zone size {size} may be too small for rate {rate}"
                )

    def lint(self) -> Tuple[bool, List[str], List[str]]:
        """Run all linting checks"""
        if not self._check_file_exists():
            return False, self.errors, self.warnings
        try:
            with open(self.config_path, "r") as f:
                config_content = f.read()
        except IOError as e:
            self.errors.append(f"Failed to read config file: {str(e)}")
            return False, self.errors, self.warnings

        # Check Nginx 1.26 specific changes
        if self.nginx_version >= "1.26":
            if "limit_req" in config_content and "burst" in config_content:
                self.warnings.append(
                    "Nginx 1.26+ has updated burst handling: ensure burst values are >= 2x rate"
                )

        zones = self._parse_limit_req_zones(config_content)
        self._validate_rate_limits(zones)

        # Check for rate limits applied to health check endpoints
        if re.search(r"location\s+/health.*limit_req", config_content, re.DOTALL):
            self.errors.append(
                "Rate limits applied to /health endpoint: this will cause retry storms in Nginx 1.26+"
            )

        return len(self.errors) == 0, self.errors, self.warnings

if __name__ == "__main__":
    if len(sys.argv) != 2:
        logger.error("Usage: python nginx_config_linter.py ")
        sys.exit(1)
    linter = NginxConfigLinter(sys.argv[1])
    success, errors, warnings = linter.lint()
    if warnings:
        logger.warning("Linting warnings:")
        for warn in warnings:
            logger.warning(f"  - {warn}")
    if errors:
        logger.error("Linting errors:")
        for err in errors:
            logger.error(f"  - {err}")
    if success:
        logger.info("Config passed all linting checks!")
    sys.exit(0 if success else 1)

// ddos_replay.go
// Replays captured traffic to validate Nginx 1.26 rate limit fixes
// Includes error handling for HTTP requests, TLS, and metrics collection
// Build: go build -o ddos_replay ddos_replay.go
// Run: ./ddos_replay --target http://nginx-ingress:8080 --rps 10000 --duration 60s

package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "math/rand"
    "net/http"
    "os"
    "os/signal"
    "sync"
    "syscall"
    "time"
)

var (
    target    string
    rps       int
    duration  time.Duration
    endpoint  string
    verbose   bool
)

func init() {
    flag.StringVar(&target, "target", "http://localhost:8080", "Target Nginx ingress URL")
    flag.IntVar(&rps, "rps", 1000, "Requests per second to send")
    flag.DurationVar(&duration, "duration", 30*time.Second, "Test duration")
    flag.StringVar(&endpoint, "endpoint", "/api/users", "API endpoint to hit")
    flag.BoolVar(&verbose, "verbose", false, "Log all request responses")
    flag.Parse()

    if rps <= 0 {
        log.Fatalf("rps must be positive, got %d", rps)
    }
    if duration <= 0 {
        log.Fatalf("duration must be positive, got %s", duration)
    }
}

type metrics struct {
    mu          sync.Mutex
    total       int
    success     int
    failed      int
    rateLimited int
    latencies   []time.Duration
}

func (m *metrics) add(latency time.Duration, statusCode int) {
    m.mu.Lock()
    defer m.mu.Unlock()
    m.total++
    m.latencies = append(m.latencies, latency)
    switch {
    case statusCode == http.StatusTooManyRequests:
        m.rateLimited++
        m.failed++
    case statusCode >= 200 && statusCode < 300:
        m.success++
    default:
        m.failed++
    }
}

func worker(ctx context.Context, client *http.Client, url string, metrics *metrics, wg *sync.WaitGroup) {
    defer wg.Done()
    ticker := time.NewTicker(time.Second / time.Duration(rps))
    defer ticker.Stop()

    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker.C:
            start := time.Now()
            req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
            if err != nil {
                metrics.add(time.Since(start), 0)
                if verbose {
                    log.Printf("Failed to create request: %v", err)
                }
                continue
            }
            req.Header.Set("User-Agent", fmt.Sprintf("ddos-replay-%d", rand.Intn(1000)))
            resp, err := client.Do(req)
            latency := time.Since(start)
            if err != nil {
                metrics.add(latency, 0)
                if verbose {
                    log.Printf("Request failed: %v", err)
                }
                continue
            }
            metrics.add(latency, resp.StatusCode)
            if verbose {
                log.Printf("Status: %d, Latency: %s", resp.StatusCode, latency)
            }
            resp.Body.Close()
        }
    }
}

func main() {
    ctx, cancel := context.WithTimeout(context.Background(), duration)
    defer cancel()

    // Handle SIGINT/SIGTERM for graceful shutdown
    sigChan := make(chan os.Signal, 1)
    signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
    go func() {
        <-sigChan
        cancel()
    }()

    client := &http.Client{
        Timeout: 5 * time.Second,
        Transport: &http.Transport{
            MaxIdleConns:        1000,
            MaxIdleConnsPerHost: 1000,
        },
    }

    m := &metrics{}
    var wg sync.WaitGroup
    numWorkers := 100
    url := target + endpoint

    log.Printf("Starting replay test: target=%s, rps=%d, duration=%s, workers=%d", url, rps, duration, numWorkers)

    for i := 0; i < numWorkers; i++ {
        wg.Add(1)
        go worker(ctx, client, url, m, &wg)
    }

    <-ctx.Done()
    wg.Wait()

    // Print metrics
    m.mu.Lock()
    defer m.mu.Unlock()
    log.Printf("Test complete. Total requests: %d", m.total)
    log.Printf("Success: %d (%.2f%%)", m.success, float64(m.success)/float64(m.total)*100)
    log.Printf("Rate limited: %d (%.2f%%)", m.rateLimited, float64(m.rateLimited)/float64(m.total)*100)
    log.Printf("Failed: %d (%.2f%%)", m.failed, float64(m.failed)/float64(m.total)*100)
}

# GitHub Actions step for Nginx linting
- name: Lint Nginx Config
  run: |
    pip install nginx-parser pyyaml
    python nginx_config_linter.py ./nginx.conf
    if [ $? -ne 0 ]; then
      echo "Nginx config lint failed"
      exit 1
    fi

# Correct health check configuration (no rate limits)
location /health {
    access_log off;
    proxy_pass http://app_origin/health;
    proxy_connect_timeout 2s;
    proxy_read_timeout 2s;
    # Explicitly exclude from rate limiting
    limit_req off;
}

# wrk command to benchmark Nginx 1.26 upgrade
wrk -t12 -c400 -d60s --latency http://nginx-ingress:8080/api/users