War Story: How a vLLM 0.6 OOM Error Crashed Our LLM Inference Service for 30 Minutes in 2026

import os
import sys
import signal
import logging
from typing import Dict, Any, Optional
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt, ExplicitEncoderDecoderPrompt
import torch

# Configure logging for production tracing
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s [%(levelname)s] %(name)s: %(message)s\",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(\"vllm-inference-server\")

# Global flag for graceful shutdown
SHUTDOWN_REQUESTED = False

def handle_shutdown_signal(signum: int, frame: Optional[Any]) -> None:
    \"\"\"Handle SIGTERM/SIGINT for graceful vLLM engine teardown\"\"\"
    global SHUTDOWN_REQUESTED
    logger.warning(f\"Received shutdown signal {signum}, draining requests...\")
    SHUTDOWN_REQUESTED = True

def init_vllm_engine() -> LLM:
    \"\"\"
    Initialize vLLM 0.6.0 engine with the configuration that triggered the OOM bug.
    Bug context: vLLM 0.6 PagedAttention scheduler miscalculates memory for multi-modal
    inputs with context windows >4096 tokens, leading to unaccounted GPU memory allocation.
    \"\"\"
    try:
        # vLLM 0.6.0 specific config - DO NOT USE IN PRODUCTION
        engine_args = {
            \"model\": \"meta-llama/Llama-3-70B-Instruct\",
            \"tensor_parallel_size\": 4,  # Spread across 4 A100s per instance
            \"gpu_memory_utilization\": 0.95,  # Aggressive utilization (part of the problem)
            \"max_model_len\": 8192,  # 8k context window, triggers the bug
            \"enable_multimodal\": True,  # Multi-modal support enabled (required for image inputs)
            \"block_size\": 16,  # PagedAttention block size, scheduler uses this for allocation math
            \"swap_space\": 4,  # GB of CPU swap for offloaded blocks
            \"disable_log_requests\": False,  # Enable request logging for debugging
            \"disable_log_stats\": False,  # Enable stats for memory monitoring
        }
        logger.info(f\"Initializing vLLM 0.6.0 engine with args: {engine_args}\")
        llm = LLM(**engine_args)
        logger.info(\"vLLM engine initialized successfully\")
        return llm
    except Exception as e:
        logger.error(f\"Failed to initialize vLLM engine: {str(e)}\", exc_info=True)
        sys.exit(1)

def main() -> None:
    # Register shutdown handlers
    signal.signal(signal.SIGTERM, handle_shutdown_signal)
    signal.signal(signal.SIGINT, handle_shutdown_signal)

    # Initialize engine
    llm = init_vllm_engine()

    # Sample sampling params for our chat workload
    sampling_params = SamplingParams(
        temperature=0.7,
        top_p=0.9,
        max_tokens=2048,
        stop=[\"\", \"<|end_of_text|>\"]
    )

    # Dummy warmup request to trigger memory allocation
    warmup_prompt = ExplicitEncoderDecoderPrompt(
        encoder_prompt=\"<|image|>Describe this image in detail.\",
        decoder_prompt=\"Describe the image:\"
    )
    try:
        logger.info(\"Running warmup request to allocate GPU memory\")
        llm.generate([warmup_prompt], sampling_params)
        logger.info(\"Warmup complete, server ready to accept requests\")
    except Exception as e:
        logger.error(f\"Warmup request failed: {str(e)}\", exc_info=True)
        sys.exit(1)

    # Keep server running until shutdown is requested
    while not SHUTDOWN_REQUESTED:
        signal.pause()

    # Graceful teardown
    logger.info(\"Shutting down vLLM engine...\")
    del llm
    torch.cuda.empty_cache()
    logger.info(\"Server shutdown complete\")

if __name__ == \"__main__\":
    main()

import time
import subprocess
import json
import logging
import sys
from typing import List, Dict, Any, Optional
from datetime import datetime
import requests

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s [%(levelname)s] memory-monitor: %(message)s\",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(\"memory-monitor\")

# vLLM metrics endpoint (default for vLLM 0.6+)
VLLM_METRICS_URL = \"http://localhost:8000/metrics\"
# Sampling interval in seconds
SAMPLE_INTERVAL = 10
# Threshold for unaccounted memory (reported vs actual) in %
MEMORY_DISCREPANCY_THRESHOLD = 5.0

def get_nvidia_smi_stats() -> Dict[str, Any]:
    \"\"\"Query nvidia-smi for actual GPU memory usage across all devices\"\"\"
    try:
        cmd = [
            \"nvidia-smi\",
            \"--query-gpu=index,name,memory.used,memory.total,utilization.gpu\",
            \"--format=csv,noheader, nounits\"
        ]
        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
        gpus = []
        for line in result.stdout.strip().split(\"\\n\"):
            parts = [p.strip() for p in line.split(\",\")]
            if len(parts) != 5:
                continue
            gpus.append({
                \"index\": int(parts[0]),
                \"name\": parts[1],
                \"memory_used_mb\": float(parts[2]),
                \"memory_total_mb\": float(parts[3]),
                \"gpu_utilization_pct\": float(parts[4])
            })
        return {\"gpus\": gpus, \"timestamp\": datetime.utcnow().isoformat()}
    except subprocess.CalledProcessError as e:
        logger.error(f\"nvidia-smi query failed: {e.stderr}\")
        return {}
    except Exception as e:
        logger.error(f\"Failed to parse nvidia-smi output: {str(e)}\")
        return {}

def get_vllm_memory_stats() -> Dict[str, Any]:
    \"\"\"Scrape vLLM's Prometheus metrics for reported memory usage\"\"\"
    try:
        response = requests.get(VLLM_METRICS_URL, timeout=5)
        response.raise_for_status()
        metrics = {}
        for line in response.text.split(\"\\n\"):
            if line.startswith(\"vllm_gpu_memory_usage_bytes\"):
                # Parse metric: vllm_gpu_memory_usage_bytes{device=\"0\"} 123456789
                parts = line.split(\" \")
                if len(parts) == 2:
                    metrics[\"reported_memory_bytes\"] = float(parts[1])
            elif line.startswith(\"vllm_gpu_cache_usage_pct\"):
                parts = line.split(\" \")
                if len(parts) == 2:
                    metrics[\"reported_cache_usage_pct\"] = float(parts[1])
        return metrics
    except requests.exceptions.RequestException as e:
        logger.error(f\"Failed to fetch vLLM metrics: {str(e)}\")
        return {}

def calculate_discrepancy(nvidia_stats: Dict[str, Any], vllm_stats: Dict[str, Any]) -> float:
    \"\"\"Calculate the % difference between reported and actual GPU memory usage\"\"\"
    if not nvidia_stats.get(\"gpus\") or not vllm_stats.get(\"reported_memory_bytes\"):
        return 0.0
    total_actual_mb = sum(gpu[\"memory_used_mb\"] for gpu in nvidia_stats[\"gpus\"])
    total_actual_bytes = total_actual_mb * 1024 * 1024
    reported_bytes = vllm_stats[\"reported_memory_bytes\"]
    if reported_bytes == 0:
        return 0.0
    discrepancy_pct = abs((total_actual_bytes - reported_bytes) / reported_bytes) * 100
    return discrepancy_pct

def log_discrepancy(discrepancy: float, nvidia_stats: Dict[str, Any], vllm_stats: Dict[str, Any]) -> None:
    \"\"\"Log discrepancy details if it exceeds the threshold\"\"\"
    if discrepancy > MEMORY_DISCREPANCY_THRESHOLD:
        logger.warning(f\"MEMORY DISCREPANCY DETECTED: {discrepancy:.2f}%\")
        logger.warning(f\"Actual GPU Memory (nvidia-smi): {sum(g['memory_used_mb'] for g in nvidia_stats['gpus']):.2f} MB\")
        logger.warning(f\"Reported GPU Memory (vLLM): {vllm_stats.get('reported_memory_bytes', 0) / (1024*1024):.2f} MB\")
        logger.warning(f\"vLLM Cache Usage: {vllm_stats.get('reported_cache_usage_pct', 0):.2f}%\")

def main() -> None:
    logger.info(f\"Starting GPU memory monitor. Sampling every {SAMPLE_INTERVAL}s.\")
    logger.info(f\"Discrepancy threshold: {MEMORY_DISCREPANCY_THRESHOLD}%\")
    while True:
        try:
            # Collect stats
            nvidia_stats = get_nvidia_smi_stats()
            vllm_stats = get_vllm_memory_stats()
            # Calculate discrepancy
            discrepancy = calculate_discrepancy(nvidia_stats, vllm_stats)
            # Log summary
            timestamp = datetime.utcnow().isoformat()
            total_actual_mb = sum(g[\"memory_used_mb\"] for g in nvidia_stats.get(\"gpus\", []))
            logger.info(
                f\"[{timestamp}] Actual GPU Mem: {total_actual_mb:.2f} MB | \"
                f\"vLLM Reported: {vllm_stats.get('reported_memory_usage_bytes', 0)/(1024*1024):.2f} MB | \"
                f\"Discrepancy: {discrepancy:.2f}%\"
            )
            # Check for threshold breach
            log_discrepancy(discrepancy, nvidia_stats, vllm_stats)
            # Sleep until next sample
            time.sleep(SAMPLE_INTERVAL)
        except KeyboardInterrupt:
            logger.info(\"Monitor stopped by user\")
            break
        except Exception as e:
            logger.error(f\"Unexpected error in monitor loop: {str(e)}\", exc_info=True)
            time.sleep(SAMPLE_INTERVAL)

if __name__ == \"__main__\":
    main()

import os
import sys
import signal
import logging
import time
from typing import Dict, Any, Optional, List
from vllm import LLM, SamplingParams
from vllm.engine.arg_utils import EngineArgs
import torch
import psutil  # For CPU memory checks

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s [%(levelname)s] hardened-vllm: %(message)s\",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(\"hardened-vllm\")

# Global state
SHUTDOWN_REQUESTED = False
MAX_MEMORY_DISCREPANCY_PCT = 3.0  # Fail fast if vLLM reports <3% of actual memory
MIN_FREE_GPU_MEM_MB = 2048  # Minimum free GPU memory before rejecting requests
ENGINE: Optional[LLM] = None

def handle_shutdown(signum: int, frame: Any) -> None:
    \"\"\"Graceful shutdown handler\"\"\"
    global SHUTDOWN_REQUESTED
    logger.warning(f\"Received signal {signum}, draining requests...\")
    SHUTDOWN_REQUESTED = True

def check_gpu_memory_safety() -> bool:
    \"\"\"Pre-flight check to ensure GPU memory is not over-allocated\"\"\"
    try:
        result = subprocess.run(
            [\"nvidia-smi\", \"--query-gpu=memory.free\", \"--format=csv,noheader,nounits\"],
            capture_output=True, text=True, check=True
        )
        free_mb = float(result.stdout.strip())
        if free_mb < MIN_FREE_GPU_MEM_MB:
            logger.error(f\"Insufficient free GPU memory: {free_mb} MB < {MIN_FREE_GPU_MEM_MB} MB minimum\")
            return False
        logger.info(f\"GPU memory check passed: {free_mb} MB free\")
        return True
    except Exception as e:
        logger.error(f\"GPU memory check failed: {str(e)}\")
        return False

def init_hardened_engine() -> LLM:
    \"\"\"Initialize vLLM 0.7.1 engine with memory guardrails\"\"\"
    # Engine args for vLLM 0.7.1 (fixed version)
    engine_args = EngineArgs(
        model=\"meta-llama/Llama-3-70B-Instruct\",
        tensor_parallel_size=4,
        gpu_memory_utilization=0.90,  # Reduced from 0.95 to add headroom
        max_model_len=16384,  # Now supported with fixed scheduler
        enable_multimodal=True,
        block_size=32,  # Larger block size reduces scheduler overhead
        swap_space=2,  # Reduced swap since we have more headroom
        disable_log_requests=False,
        disable_log_stats=False,
        # vLLM 0.7+ specific: enable memory budgeting
        enable_memory_budget=True,
        memory_budget_pct=85.0  # Hard cap on memory allocation
    )
    try:
        logger.info(f\"Initializing hardened vLLM engine with args: {engine_args}\")
        llm = LLM(engine_args)
        # Post-init memory check
        if not check_gpu_memory_safety():
            raise RuntimeError(\"Post-initialization GPU memory check failed\")
        logger.info(\"Hardened vLLM engine initialized successfully\")
        return llm
    except Exception as e:
        logger.error(f\"Engine initialization failed: {str(e)}\", exc_info=True)
        sys.exit(1)

def generate_with_guardrails(prompts: List[Any], params: SamplingParams) -> List[str]:
    \"\"\"Generate responses with pre-request memory checks\"\"\"
    if SHUTDOWN_REQUESTED:
        raise RuntimeError(\"Server is shutting down, rejecting request\")
    # Check GPU memory before processing
    if not check_gpu_memory_safety():
        raise RuntimeError(\"Insufficient GPU memory to process request\")
    # Check CPU memory (swap safety)
    cpu_mem = psutil.virtual_memory()
    if cpu_mem.percent > 90:
        logger.warning(f\"High CPU memory usage: {cpu_mem.percent}%, may impact swap performance\")
    # Generate
    try:
        start = time.time()
        outputs = ENGINE.generate(prompts, params)
        latency = time.time() - start
        logger.info(f\"Generated {len(prompts)} responses in {latency:.2f}s\")
        return [output.outputs[0].text for output in outputs]
    except Exception as e:
        logger.error(f\"Generation failed: {str(e)}\", exc_info=True)
        raise

def main() -> None:
    # Register signals
    signal.signal(signal.SIGTERM, handle_shutdown)
    signal.signal(signal.SIGINT, handle_shutdown)

    # Pre-flight checks
    if not check_gpu_memory_safety():
        sys.exit(1)

    # Initialize engine
    global ENGINE
    ENGINE = init_hardened_engine()

    # Warmup
    sampling_params = SamplingParams(max_tokens=128, temperature=0.1)
    warmup_prompt = \"What is the capital of France?\"
    try:
        logger.info(\"Running warmup request\")
        generate_with_guardrails([warmup_prompt], sampling_params)
        logger.info(\"Warmup complete, server ready\")
    except Exception as e:
        logger.error(f\"Warmup failed: {str(e)}\")
        sys.exit(1)

    # Keep alive
    while not SHUTDOWN_REQUESTED:
        time.sleep(1)

    # Teardown
    logger.info(\"Shutting down engine...\")
    del ENGINE
    torch.cuda.empty_cache()
    logger.info(\"Shutdown complete\")

if __name__ == \"__main__\":
    import subprocess  # Import here to avoid circular issues
    main()

# One-liner to compare vLLM reported vs actual memory
curl -s localhost:8000/metrics | grep vllm_gpu_memory_usage_bytes | awk '{print \"Reported: \" $2 / 1024 / 1024 \" MB\"}' && nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '{print \"Actual: \" $1 \" MB\"}'

- name: Run vLLM Memory Regression Test
  run: |
    kubectl apply -f staging-vllm-deployment.yaml
    kubectl wait --for=condition=ready pod -l app=vllm-staging --timeout=300s
    python run_synthetic_workload.py --requests 10000 --context-len 8192 --multimodal
    python check_memory_discrepancy.py --threshold 3.0

from vllm.engine.arg_utils import EngineArgs
engine_args = EngineArgs(
    enable_memory_budget=True,
    memory_budget_pct=85.0,  # Hard cap on vLLM memory allocation
    gpu_memory_utilization=0.90  # Fallback for older vLLM versions
)