Postmortem: When Our vLLM 0.6 Instance Crashed Under 10k Concurrent Claude 4 Requests

import asyncio
import aiohttp
import time
import logging
from dataclasses import dataclass
from typing import List, Dict
import psutil  # For system memory monitoring
import GPUtil  # For GPU memory tracking

# Configure logging for crash reproduction
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

@dataclass
class RequestConfig:
    """Configuration for concurrent inference requests"""
    prompt: str = "Explain quantum entanglement in 2000 words"
    max_tokens: int = 2048
    temperature: float = 0.7
    top_p: float = 0.95
    n: int = 1  # Number of completions per request

class vLLMCrashReproducer:
    def __init__(self, vllm_endpoint: str = "http://localhost:8000", target_concurrency: int = 10000):
        self.endpoint = vllm_endpoint
        self.target_concurrency = target_concurrency
        self.success_count = 0
        self.fail_count = 0
        self.oom_count = 0
        self.latencies: List[float] = []
        # Track GPU memory before starting requests
        self.initial_gpu_mem = self._get_gpu_memory()

    def _get_gpu_memory(self) -> Dict[int, float]:
        """Fetch current GPU memory usage in MB per GPU"""
        gpus = GPUtil.getGPUs()
        return {gpu.id: gpu.memoryUsed for gpu in gpus}

    async def _send_single_request(self, session: aiohttp.ClientSession, req_id: int) -> None:
        """Send a single inference request and track results"""
        start_time = time.perf_counter()
        try:
            async with session.post(
                f"{self.endpoint}/v1/completions",
                json={
                    "model": "claude-4-sonnet",
                    "prompt": RequestConfig.prompt,
                    "max_tokens": RequestConfig.max_tokens,
                    "temperature": RequestConfig.temperature,
                    "top_p": RequestConfig.top_p,
                    "n": RequestConfig.n
                },
                timeout=aiohttp.ClientTimeout(total=30)  # 30s timeout per request
            ) as resp:
                if resp.status == 200:
                    self.success_count += 1
                    latency = time.perf_counter() - start_time
                    self.latencies.append(latency)
                elif resp.status == 500 and "OOM" in await resp.text():
                    self.oom_count += 1
                    logger.warning(f"Request {req_id} failed with OOM: {await resp.text()}")
                else:
                    self.fail_count += 1
                    logger.error(f"Request {req_id} failed with status {resp.status}: {await resp.text()}")
        except asyncio.TimeoutError:
            self.fail_count += 1
            logger.error(f"Request {req_id} timed out after 30s")
        except Exception as e:
            self.fail_count += 1
            logger.error(f"Request {req_id} raised exception: {str(e)}")

    async def run_load_test(self) -> None:
        """Run concurrent load test to reproduce vLLM 0.6 OOM crash"""
        logger.info(f"Starting load test with {self.target_concurrency} concurrent requests")
        logger.info(f"Initial GPU memory usage: {self.initial_gpu_mem} MB")

        # Create aiohttp session with connection limit matching target concurrency
        connector = aiohttp.TCPConnector(limit=self.target_concurrency)
        async with aiohttp.ClientSession(connector=connector) as session:
            # Send all requests concurrently
            tasks = [self._send_single_request(session, i) for i in range(self.target_concurrency)]
            await asyncio.gather(*tasks)

        # Calculate final metrics
        final_gpu_mem = self._get_gpu_memory()
        avg_latency = sum(self.latencies) / len(self.latencies) if self.latencies else 0
        p99_latency = sorted(self.latencies)[int(0.99 * len(self.latencies))] if self.latencies else 0

        logger.info(f"Load test complete. Success: {self.success_count}, Fail: {self.fail_count}, OOM: {self.oom_count}")
        logger.info(f"Average latency: {avg_latency:.2f}s, P99 latency: {p99_latency:.2f}s")
        logger.info(f"Final GPU memory usage: {final_gpu_mem} MB")
        logger.info(f"GPU memory delta: {final_gpu_mem.get(0, 0) - self.initial_gpu_mem.get(0, 0)} MB")

if __name__ == "__main__":
    # Note: Requires vLLM 0.6.0 running locally with Claude 4 Sonnet weights
    # Start vLLM with: python -m vllm.entrypoints.openai.api_server --model claude-4-sonnet --dtype bfloat16 --gpu-memory-utilization 0.95
    reproducer = vLLMCrashReproducer(target_concurrency=10000)
    try:
        asyncio.run(reproducer.run_load_test())
    except KeyboardInterrupt:
        logger.info("Load test interrupted by user")
    except Exception as e:
        logger.error(f"Load test failed with fatal error: {str(e)}")

#!/bin/bash
# vllm_safe_deploy.sh - Startup script for vLLM 0.6.2+ to avoid OOM under high concurrency
# Requires: vLLM 0.6.2+, NVIDIA GPU with 80GB+ VRAM, Claude 4 Sonnet weights

set -euo pipefail  # Exit on error, undefined vars, pipe failures

# --------------------------
# Configuration Variables
# --------------------------
MODEL_NAME="claude-4-sonnet"
VLLM_VERSION="0.6.2"
GPU_MEM_UTIL=0.85  # Lower than default 0.95 to leave headroom for spikes
MAX_NUM_SEQS=2048  # Max concurrent sequences per GPU, down from default 4096
MAX_NUM_BATCHED_TOKENS=8192  # Cap batched tokens to prevent memory spikes
DISABLE_PREFIX_CACHE=true  # Disable leaky prefix cache in vLLM 0.6.0-0.6.1
ENABLE_CHUNKED_PREFILL=true  # Reduce memory pressure for long prompts
CHUNKED_PREFILL_MAX_LEN=2048  # Max chunk size for prefill
API_PORT=8000
LOG_FILE="/var/log/vllm/vllm_$(date +%Y%m%d_%H%M%S).log"
HEALTH_CHECK_RETRIES=5
HEALTH_CHECK_INTERVAL=10  # Seconds between health checks

# --------------------------
# Pre-flight Checks
# --------------------------
echo "Starting vLLM ${VLLM_VERSION} safe deployment for ${MODEL_NAME}"

# Check if vLLM is installed
if ! python -c "import vllm; print(vllm.__version__)" | grep -q "${VLLM_VERSION}"; then
    echo "ERROR: vLLM version ${VLLM_VERSION} not found. Install with: pip install vllm==${VLLM_VERSION}"
    exit 1
fi

# Check GPU availability and memory
if ! nvidia-smi &> /dev/null; then
    echo "ERROR: No NVIDIA GPUs detected"
    exit 1
fi

TOTAL_GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1)
if [ "${TOTAL_GPU_MEM}" -lt 80000 ]; then
    echo "ERROR: GPU has less than 80GB VRAM (detected ${TOTAL_GPU_MEM}MB). Claude 4 requires 80GB+ VRAM"
    exit 1
fi

# Create log directory if not exists
mkdir -p "$(dirname "${LOG_FILE}")"

# --------------------------
# Start vLLM Server
# --------------------------
echo "Starting vLLM API server on port ${API_PORT}..."
nohup python -m vllm.entrypoints.openai.api_server \
    --model "${MODEL_NAME}" \
    --dtype bfloat16 \
    --gpu-memory-utilization "${GPU_MEM_UTIL}" \
    --max-num-seqs "${MAX_NUM_SEQS}" \
    --max-num-batched-tokens "${MAX_NUM_BATCHED_TOKENS}" \
    --disable-prefix-cache "${DISABLE_PREFIX_CACHE}" \
    --enable-chunked-prefill "${ENABLE_CHUNKED_PREFILL}" \
    --chunked-prefill-max-len "${CHUNKED_PREFILL_MAX_LEN}" \
    --port "${API_PORT}" \
    > "${LOG_FILE}" 2>&1 &

VLLM_PID=$!
echo "vLLM started with PID ${VLLM_PID}"

# --------------------------
# Post-start Health Check
# --------------------------
echo "Running health checks..."
for i in $(seq 1 "${HEALTH_CHECK_RETRIES}"); do
    echo "Health check attempt ${i}/${HEALTH_CHECK_RETRIES}"
    # Check if PID is still running
    if ! kill -0 "${VLLM_PID}" &> /dev/null; then
        echo "ERROR: vLLM process died. Check logs: ${LOG_FILE}"
        exit 1
    fi
    # Check API health endpoint
    if curl -s -f "http://localhost:${API_PORT}/health" &> /dev/null; then
        echo "vLLM API server is healthy"
        # Test a single inference request to verify model loading
        echo "Testing inference request..."
        RESPONSE=$(curl -s -X POST "http://localhost:${API_PORT}/v1/completions" \
            -H "Content-Type: application/json" \
            -d '{"model":"'${MODEL_NAME}'", "prompt":"test", "max_tokens":10}')
        if echo "${RESPONSE}" | grep -q "choices"; then
            echo "Inference test passed. Deployment successful!"
            exit 0
        else
            echo "ERROR: Inference test failed. Response: ${RESPONSE}"
            kill "${VLLM_PID}"
            exit 1
        fi
    else
        echo "API not healthy yet, retrying in ${HEALTH_CHECK_INTERVAL}s..."
        sleep "${HEALTH_CHECK_INTERVAL}"
    fi
done

echo "ERROR: vLLM failed to start after ${HEALTH_CHECK_RETRIES} retries"
kill "${VLLM_PID}" &> /dev/null || true
exit 1

import time
import threading
from prometheus_client import start_http_server, Gauge, Counter, Histogram
from typing import Dict
import GPUtil
import psutil
import aiohttp
import asyncio
import logging

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# --------------------------
# Prometheus Metrics Definitions
# --------------------------
# GPU metrics
GPU_MEM_USED = Gauge("vllm_gpu_mem_used_mb", "GPU memory used in MB", ["gpu_id"])
GPU_MEM_TOTAL = Gauge("vllm_gpu_mem_total_mb", "Total GPU memory in MB", ["gpu_id"])
GPU_UTIL = Gauge("vllm_gpu_util_percent", "GPU utilization percentage", ["gpu_id"])

# Request metrics
CONCURRENT_REQUESTS = Gauge("vllm_concurrent_requests", "Number of concurrent inference requests")
REQUEST_COUNT = Counter("vllm_requests_total", "Total inference requests", ["status"])  # success, fail, oom
REQUEST_LATENCY = Histogram("vllm_request_latency_seconds", "Inference request latency in seconds",
                            buckets=[0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0])

# vLLM internal metrics (scraped from /metrics endpoint if available)
VLLM_PREFIX_CACHE_SIZE = Gauge("vllm_prefix_cache_size", "Size of vLLM prefix cache in entries")
VLLM_BATCHED_TOKENS = Gauge("vllm_batched_tokens", "Number of tokens in current batch")

class vLLMMetricsExporter:
    def __init__(self, vllm_api_endpoint: str = "http://localhost:8000", scrape_interval: int = 5):
        self.vllm_endpoint = vllm_api_endpoint
        self.scrape_interval = scrape_interval
        self._stop_event = threading.Event()
        # Start Prometheus HTTP server on port 9090
        start_http_server(9090)
        logger.info("Prometheus metrics server started on port 9090")

    def _scrape_gpu_metrics(self) -> None:
        """Scrape GPU metrics via GPUtil and update Prometheus gauges"""
        try:
            gpus = GPUtil.getGPUs()
            for gpu in gpus:
                gpu_id = str(gpu.id)
                GPU_MEM_USED.labels(gpu_id=gpu_id).set(gpu.memoryUsed)
                GPU_MEM_TOTAL.labels(gpu_id=gpu_id).set(gpu.memoryTotal)
                GPU_UTIL.labels(gpu_id=gpu_id).set(gpu.load * 100)
        except Exception as e:
            logger.error(f"Failed to scrape GPU metrics: {str(e)}")

    async def _scrape_vllm_metrics(self) -> None:
        """Scrape vLLM internal metrics from /metrics endpoint"""
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(f"{self.vllm_endpoint}/metrics") as resp:
                    if resp.status == 200:
                        metrics_text = await resp.text()
                        # Parse prefix cache size (simplified, real implementation would use regex)
                        if "vllm_prefix_cache_entries" in metrics_text:
                            for line in metrics_text.split("\n"):
                                if line.startswith("vllm_prefix_cache_entries"):
                                    value = float(line.split(" ")[-1])
                                    VLLM_PREFIX_CACHE_SIZE.set(value)
                        # Parse batched tokens
                        if "vllm_batched_tokens" in metrics_text:
                            for line in metrics_text.split("\n"):
                                if line.startswith("vllm_batched_tokens"):
                                    value = float(line.split(" ")[-1])
                                    VLLM_BATCHED_TOKENS.set(value)
        except Exception as e:
            logger.error(f"Failed to scrape vLLM metrics: {str(e)}")

    def _scrape_system_metrics(self) -> None:
        """Scrape system metrics (CPU, RAM) for context"""
        # We don't export these to Prometheus in this example, but log for debugging
        cpu_util = psutil.cpu_percent()
        ram_used = psutil.virtual_memory().used / (1024 * 1024)  # MB
        logger.debug(f"System CPU: {cpu_util}%, RAM used: {ram_used}MB")

    def run_scraper(self) -> None:
        """Main loop to scrape metrics at regular intervals"""
        logger.info(f"Starting metrics scraper with interval {self.scrape_interval}s")
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        while not self._stop_event.is_set():
            self._scrape_gpu_metrics()
            loop.run_until_complete(self._scrape_vllm_metrics())
            self._scrape_system_metrics()
            time.sleep(self.scrape_interval)
        loop.close()
        logger.info("Metrics scraper stopped")

    def stop(self) -> None:
        """Signal the scraper to stop"""
        self._stop_event.set()

if __name__ == "__main__":
    exporter = vLLMMetricsExporter(scrape_interval=5)
    scraper_thread = threading.Thread(target=exporter.run_scraper, daemon=True)
    scraper_thread.start()
    try:
        # Keep main thread alive
        while scraper_thread.is_alive():
            scraper_thread.join(timeout=1)
    except KeyboardInterrupt:
        logger.info("Stopping metrics exporter...")
        exporter.stop()
        scraper_thread.join()
    except Exception as e:
        logger.error(f"Fatal error in metrics exporter: {str(e)}")
        exporter.stop()

from vllm import LLM, SamplingParams

# Disable prefix cache to avoid memory leaks
llm = LLM(
    model="claude-4-sonnet",
    disable_prefix_cache=True,  # Critical for high concurrency
    gpu_memory_utilization=0.85
)

sampling_params = SamplingParams(temperature=0.7, max_tokens=2048)
result = llm.generate(["Explain quantum entanglement"], sampling_params)

use Symfony\Component\RateLimiter\RateLimiterFactory;
use Symfony\Component\RateLimiter\Storage\RedisStorage;

// Configure token bucket to allow 8000 concurrent requests per vLLM pod
$storage = new RedisStorage(new \Redis(), 'vllm_concurrency_limiter');
$factory = new RateLimiterFactory([
    'id' => 'vllm_concurrent_requests',
    'strategy' => 'token_bucket',
    'limit' => 8000,  # Max burst/concurrent requests
    'rate' => ['interval' => '1 minute', 'limit' => 10000],  # Refill rate
], $storage);

// Check if request is allowed
$limiter = $factory->create("vllm-pod-1");  # Unique ID per pod
if (!$limiter->consume()->isAccepted()) {
    return new Response('Too many concurrent requests', 429);
}

# Alert if GPU memory increases by >5% over 10 minutes with stable request count
rate(vllm_gpu_mem_used_mb[10m]) > 0
and
changes(vllm_concurrent_requests[10m]) < 10  # Request count is stable