Performance Test: Ollama 0.5.0 vs. vLLM 0.4.0 Local LLM Inference Latency on NVIDIA RTX 5090 and AMD Radeon RX 8900 in 2026

import ollama
import time
import statistics
import argparse
import logging
from typing import List, Dict
from dataclasses import dataclass

# Configure logging for error tracking
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s - %(levelname)s - %(message)s\"
)
logger = logging.getLogger(__name__)

@dataclass
class BenchmarkResult:
    \"\"\"Container for single inference request results\"\"\"
    latency_ms: float
    prompt_tokens: int
    completion_tokens: int
    success: bool
    error: str = \"\"

class OllamaBenchmarker:
    \"\"\"Benchmark Ollama 0.5.0 inference performance\"\"\"

    def __init__(self, model: str, base_url: str = \"http://localhost:11434\"):
        self.model = model
        self.client = ollama.Client(host=base_url)
        self._validate_model()

    def _validate_model(self) -> None:
        \"\"\"Check if target model is available in Ollama\"\"\"
        try:
            models = self.client.list()
            model_names = [m[\"name\"] for m in models[\"models\"]]
            if self.model not in model_names:
                raise ValueError(f\"Model {self.model} not found. Available: {model_names}\")
            logger.info(f\"Validated model {self.model} is available\")
        except Exception as e:
            logger.error(f\"Model validation failed: {e}\")
            raise

    def run_single_request(self, prompt: str, max_tokens: int = 128) -> BenchmarkResult:
        \"\"\"Execute a single inference request and measure latency\"\"\"
        start_time = time.perf_counter()
        try:
            response = self.client.generate(
                model=self.model,
                prompt=prompt,
                max_tokens=max_tokens,
                stream=False
            )
            end_time = time.perf_counter()
            latency_ms = (end_time - start_time) * 1000
            return BenchmarkResult(
                latency_ms=latency_ms,
                prompt_tokens=response[\"prompt_eval_count\"],
                completion_tokens=response[\"eval_count\"],
                success=True
            )
        except Exception as e:
            end_time = time.perf_counter()
            latency_ms = (end_time - start_time) * 1000
            logger.error(f\"Request failed: {e}\")
            return BenchmarkResult(
                latency_ms=latency_ms,
                prompt_tokens=0,
                completion_tokens=0,
                success=False,
                error=str(e)
            )

    def run_benchmark(self, num_requests: int, prompt: str, max_tokens: int = 128) -> Dict:
        \"\"\"Run full benchmark suite and aggregate results\"\"\"
        results: List[BenchmarkResult] = []
        for i in range(num_requests):
            logger.info(f\"Running request {i+1}/{num_requests}\")
            result = self.run_single_request(prompt, max_tokens)
            results.append(result)
            # Small delay to avoid rate limiting
            time.sleep(0.1)

        successful = [r for r in results if r.success]
        failed = [r for r in results if not r.success]

        if not successful:
            raise RuntimeError(\"All benchmark requests failed\")

        latencies = [r.latency_ms for r in successful]
        return {
            \"total_requests\": num_requests,
            \"successful_requests\": len(successful),
            \"failed_requests\": len(failed),
            \"mean_latency_ms\": statistics.mean(latencies),
            \"median_latency_ms\": statistics.median(latencies),
            \"p99_latency_ms\": sorted(latencies)[int(0.99 * len(latencies))],
            \"min_latency_ms\": min(latencies),
            \"max_latency_ms\": max(latencies),
            \"avg_prompt_tokens\": statistics.mean([r.prompt_tokens for r in successful]),
            \"avg_completion_tokens\": statistics.mean([r.completion_tokens for r in successful])
        }

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"Benchmark Ollama 0.5.0 inference\")
    parser.add_argument(\"--model\", type=str, default=\"llama3.1:8b\", help=\"Ollama model to benchmark\")
    parser.add_argument(\"--num-requests\", type=int, default=100, help=\"Number of requests to send\")
    parser.add_argument(\"--prompt\", type=str, default=\"Explain quantum computing in 3 sentences\", help=\"Prompt to use\")
    parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Max tokens per response\")
    args = parser.parse_args()

    try:
        benchmarker = OllamaBenchmarker(model=args.model)
        results = benchmarker.run_benchmark(
            num_requests=args.num_requests,
            prompt=args.prompt,
            max_tokens=args.max_tokens
        )
        print(\"\n=== Ollama 0.5.0 Benchmark Results ===\")
        for key, value in results.items():
            print(f\"{key}: {value:.2f}\" if isinstance(value, float) else f\"{key}: {value}\")
    except Exception as e:
        logger.error(f\"Benchmark failed: {e}\")
        exit(1)

import requests
import time
import statistics
import argparse
import logging
from typing import List, Dict
from dataclasses import dataclass

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s - %(levelname)s - %(message)s\"
)
logger = logging.getLogger(__name__)

@dataclass
class BenchmarkResult:
    \"\"\"Container for vLLM inference results\"\"\"
    latency_ms: float
    prompt_tokens: int
    completion_tokens: int
    success: bool
    error: str = \"\"

class VLLMBenchmarker:
    \"\"\"Benchmark vLLM 0.4.0 inference performance via OpenAI-compatible API\"\"\"

    def __init__(self, model: str, base_url: str = \"http://localhost:8000/v1\"):
        self.model = model
        self.base_url = base_url.rstrip(\"/\")
        self._validate_model()

    def _validate_model(self) -> None:
        \"\"\"Check if target model is loaded in vLLM\"\"\"
        try:
            response = requests.get(f\"{self.base_url}/models\", timeout=10)
            response.raise_for_status()
            models = response.json()[\"data\"]
            model_ids = [m[\"id\"] for m in models]
            if self.model not in model_ids:
                raise ValueError(f\"Model {self.model} not found. Available: {model_ids}\")
            logger.info(f\"Validated model {self.model} is available in vLLM\")
        except requests.exceptions.ConnectionError:
            raise RuntimeError(f\"Could not connect to vLLM at {self.base_url}. Is the server running?\")
        except Exception as e:
            logger.error(f\"Model validation failed: {e}\")
            raise

    def run_single_request(self, prompt: str, max_tokens: int = 128) -> BenchmarkResult:
        \"\"\"Execute a single inference request to vLLM\"\"\"
        start_time = time.perf_counter()
        try:
            payload = {
                \"model\": self.model,
                \"messages\": [{\"role\": \"user\", \"content\": prompt}],
                \"max_tokens\": max_tokens,
                \"stream\": False
            }
            response = requests.post(
                f\"{self.base_url}/chat/completions\",
                json=payload,
                timeout=30
            )
            response.raise_for_status()
            end_time = time.perf_counter()
            latency_ms = (end_time - start_time) * 1000

            result = response.json()
            prompt_tokens = result[\"usage\"][\"prompt_tokens\"]
            completion_tokens = result[\"usage\"][\"completion_tokens\"]

            return BenchmarkResult(
                latency_ms=latency_ms,
                prompt_tokens=prompt_tokens,
                completion_tokens=completion_tokens,
                success=True
            )
        except Exception as e:
            end_time = time.perf_counter()
            latency_ms = (end_time - start_time) * 1000
            logger.error(f\"Request failed: {e}\")
            return BenchmarkResult(
                latency_ms=latency_ms,
                prompt_tokens=0,
                completion_tokens=0,
                success=False,
                error=str(e)
            )

    def run_benchmark(self, num_requests: int, prompt: str, max_tokens: int = 128) -> Dict:
        \"\"\"Run full vLLM benchmark suite\"\"\"
        results: List[BenchmarkResult] = []
        for i in range(num_requests):
            logger.info(f\"Running request {i+1}/{num_requests}\")
            result = self.run_single_request(prompt, max_tokens)
            results.append(result)
            time.sleep(0.1)

        successful = [r for r in results if r.success]
        failed = [r for r in results if not r.success]

        if not successful:
            raise RuntimeError(\"All benchmark requests failed\")

        latencies = [r.latency_ms for r in successful]
        return {
            \"total_requests\": num_requests,
            \"successful_requests\": len(successful),
            \"failed_requests\": len(failed),
            \"mean_latency_ms\": statistics.mean(latencies),
            \"median_latency_ms\": statistics.median(latencies),
            \"p99_latency_ms\": sorted(latencies)[int(0.99 * len(latencies))],
            \"min_latency_ms\": min(latencies),
            \"max_latency_ms\": max(latencies),
            \"avg_prompt_tokens\": statistics.mean([r.prompt_tokens for r in successful]),
            \"avg_completion_tokens\": statistics.mean([r.completion_tokens for r in successful])
        }

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"Benchmark vLLM 0.4.0 inference\")
    parser.add_argument(\"--model\", type=str, default=\"meta-llama/Llama-3.1-8B-Instruct\", help=\"vLLM model ID\")
    parser.add_argument(\"--num-requests\", type=int, default=100, help=\"Number of requests to send\")
    parser.add_argument(\"--prompt\", type=str, default=\"Explain quantum computing in 3 sentences\", help=\"Prompt to use\")
    parser.add_argument(\"--max-tokens\", type=int, default=128, help=\"Max tokens per response\")
    parser.add_argument(\"--base-url\", type=str, default=\"http://localhost:8000/v1\", help=\"vLLM API base URL\")
    args = parser.parse_args()

    try:
        benchmarker = VLLMBenchmarker(model=args.model, base_url=args.base_url)
        results = benchmarker.run_benchmark(
            num_requests=args.num_requests,
            prompt=args.prompt,
            max_tokens=args.max_tokens
        )
        print(\"\n=== vLLM 0.4.0 Benchmark Results ===\")
        for key, value in results.items():
            print(f\"{key}: {value:.2f}\" if isinstance(value, float) else f\"{key}: {value}\")
    except Exception as e:
        logger.error(f\"Benchmark failed: {e}\")
        exit(1)

import subprocess
import json
import sys
import logging
from typing import Literal, Optional

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

GPUType = Literal[\"nvidia\", \"amd\", \"unknown\"]

def detect_gpu_type() -> GPUType:
    \"\"\"Detect if system has NVIDIA or AMD GPU using lspci\"\"\"
    try:
        result = subprocess.run(
            [\"lspci\"],
            capture_output=True,
            text=True,
            timeout=10
        )
        output = result.stdout.lower()
        if \"nvidia\" in output:
            return \"nvidia\"
        elif \"amd\" in output or \"radeon\" in output:
            return \"amd\"
        else:
            return \"unknown\"
    except Exception as e:
        logger.warning(f\"GPU detection failed: {e}\")
        return \"unknown\"

def get_gpu_vram() -> Optional[int]:
    \"\"\"Get total VRAM in GB for primary GPU\"\"\"
    gpu_type = detect_gpu_type()
    try:
        if gpu_type == \"nvidia\":
            result = subprocess.run(
                [\"nvidia-smi\", \"--query-gpu=memory.total\", \"--format=csv,noheader,nounits\"],
                capture_output=True,
                text=True,
                timeout=10
            )
            vram_mb = int(result.stdout.strip())
            return vram_mb // 1024  # Convert to GB
        elif gpu_type == \"amd\":
            # Use rocm-smi for AMD GPUs
            result = subprocess.run(
                [\"rocm-smi\", \"--showmeminfo\", \"vram\"],
                capture_output=True,
                text=True,
                timeout=10
            )
            # Parse output for total VRAM
            for line in result.stdout.split(\"\\n\"):
                if \"Total\" in line and \"MB\" in line:
                    vram_mb = int(line.split()[2])
                    return vram_mb // 1024
            return None
        else:
            return None
    except Exception as e:
        logger.warning(f\"VRAM detection failed: {e}\")
        return None

def select_runtime(
    model_size_b: int,
    gpu_type: GPUType,
    gpu_vram_gb: Optional[int],
    require_multi_gpu: bool = False
) -> Literal[\"ollama\", \"vllm\", \"unsupported\"]:
    \"\"\"
    Select optimal runtime based on hardware and workload.

    Args:
        model_size_b: Model size in billions of parameters
        gpu_type: Detected GPU type
        gpu_vram_gb: Total GPU VRAM in GB
        require_multi_gpu: If multi-GPU support is needed
    \"\"\"
    # vLLM requires more VRAM overhead: ~2GB per 7B params
    vllm_required_vram = model_size_b * 0.28  # ~280MB per B param for vLLM
    ollama_required_vram = model_size_b * 0.2  # ~200MB per B param for Ollama

    if require_multi_gpu:
        # Ollama only supports up to 2 GPUs experimentally, vLLM up to 8
        if gpu_type == \"nvidia\":
            return \"vllm\"
        else:
            logger.warning(\"Multi-GPU on AMD not fully supported in vLLM 0.4.0\")
            return \"unsupported\"

    if gpu_vram_gb is None:
        logger.warning(\"Could not detect VRAM, defaulting to Ollama\")
        return \"ollama\"

    # Check if Ollama can run the model
    ollama_fits = gpu_vram_gb >= ollama_required_vram
    # Check if vLLM can run the model
    vllm_fits = gpu_vram_gb >= vllm_required_vram

    if not ollama_fits and not vllm_fits:
        logger.error(f\"Model {model_size_b}B requires {ollama_required_vram:.1f}GB VRAM, only {gpu_vram_gb}GB available\")
        return \"unsupported\"

    # For small models on AMD, Ollama has better latency
    if model_size_b <= 13 and gpu_type == \"amd\":
        return \"ollama\"

    # For large models on NVIDIA, vLLM has better throughput
    if model_size_b >= 70 and gpu_type == \"nvidia\":
        return \"vllm\"

    # Default to Ollama for ease of use
    return \"ollama\"

if __name__ == \"__main__\":
    # Example usage
    gpu_type = detect_gpu_type()
    gpu_vram = get_gpu_vram()
    logger.info(f\"Detected GPU: {gpu_type}, VRAM: {gpu_vram}GB\")

    # Test with 7B model
    runtime_7b = select_runtime(model_size_b=7, gpu_type=gpu_type, gpu_vram_gb=gpu_vram)
    print(f\"7B model runtime: {runtime_7b}\")

    # Test with 70B model
    runtime_70b = select_runtime(model_size_b=70, gpu_type=gpu_type, gpu_vram_gb=gpu_vram)
    print(f\"70B model runtime: {runtime_70b}\")

    # Test multi-GPU requirement
    runtime_multi = select_runtime(model_size_b=13, gpu_type=gpu_type, gpu_vram_gb=gpu_vram, require_multi_gpu=True)
    print(f\"Multi-GPU 13B model runtime: {runtime_multi}\")

# Start vLLM 0.4.0 server with optimized settings for RTX 5090
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-70B-Instruct-GPTQ \
  --gpu-memory-utilization 0.85 \
  --tensor-parallel-size 1 \
  --max-model-len 4096 \
  --port 8000

# Start Ollama 0.5.0 server with model caching enabled for RX 8900
OLLAMA_MODEL_CACHE=\"/mnt/nvme/ollama-cache\" \
OLLAMA_CACHE_TTL=24h \
ollama serve

# Run Ollama benchmark for 70B model
python ollama_benchmark.py \
  --model llama3.1:70b-q4_0 \
  --num-requests 1000 \
  --prompt \"Summarize the 2026 State of the Union address in 5 bullet points\" \
  --max-tokens 256

# Run vLLM benchmark for same model
python vllm_benchmark.py \
  --model meta-llama/Llama-3.1-70B-Instruct-GPTQ \
  --num-requests 1000 \
  --prompt \"Summarize the 2026 State of the Union address in 5 bullet points\" \
  --max-tokens 256