The Unexpected migration in OpenVINO vs vLLM: A Head-to-Head

import os
import time
import argparse
from typing import List, Dict
import numpy as np
from openvino_genai import LLM, GenerationConfig, Tokenizer
from datasets import load_dataset

def run_openvino_benchmark(
    model_path: str,
    prompt_dataset: str,
    num_prompts: int,
    max_new_tokens: int = 256
) -> Dict[str, float]:
    \"\"\"
    Benchmark OpenVINO LLM inference throughput and latency.

    Args:
        model_path: Path to OpenVINO quantized model directory
        prompt_dataset: Name of HuggingFace dataset to load prompts from
        num_prompts: Number of prompts to run (truncated from dataset)
        max_new_tokens: Maximum number of new tokens to generate per prompt

    Returns:
        Dictionary with throughput (tokens/sec), p50_latency (ms), p99_latency (ms)
    \"\"\"
    # Load tokenizer and model with error handling
    try:
        tokenizer = Tokenizer(model_path)
        llm = LLM(model_path)
    except Exception as e:
        raise RuntimeError(f\"Failed to load OpenVINO model from {model_path}: {str(e)}\")

    # Load and preprocess prompts
    try:
        dataset = load_dataset(prompt_dataset, split=\"train\")
        prompts = [sample[\"chosen\"] for sample in dataset.select(range(num_prompts))]
    except Exception as e:
        raise RuntimeError(f\"Failed to load dataset {prompt_dataset}: {str(e)}\")

    # Warmup run to avoid cold start bias
    print(\"Running OpenVINO warmup...\")
    warmup_config = GenerationConfig(max_new_tokens=32, temperature=0.7)
    llm.generate(prompts[0], warmup_config)

    # Run benchmark
    latencies = []
    total_tokens = 0
    print(f\"Running OpenVINO benchmark with {num_prompts} prompts...\")

    for idx, prompt in enumerate(prompts):
        try:
            start_time = time.perf_counter()
            config = GenerationConfig(
                max_new_tokens=max_new_tokens,
                temperature=0.7,
                top_p=0.9,
                do_sample=True
            )
            result = llm.generate(prompt, config)
            end_time = time.perf_counter()

            # Calculate latency and token count
            latency_ms = (end_time - start_time) * 1000
            latencies.append(latency_ms)
            total_tokens += len(tokenizer.encode(result.text)) - len(tokenizer.encode(prompt))

            if (idx + 1) % 10 == 0:
                print(f\"Processed {idx + 1}/{num_prompts} prompts\")
        except Exception as e:
            print(f\"Warning: Failed to process prompt {idx}: {str(e)}\")
            continue

    # Calculate metrics
    if not latencies:
        raise RuntimeError(\"No successful inference runs completed\")

    throughput = total_tokens / (sum(latencies) / 1000)  # tokens per second
    p50_latency = np.percentile(latencies, 50)
    p99_latency = np.percentile(latencies, 99)

    return {
        \"throughput\": round(throughput, 2),
        \"p50_latency\": round(p50_latency, 2),
        \"p99_latency\": round(p99_latency, 2),
        \"total_tokens\": total_tokens,
        \"successful_runs\": len(latencies)
    }

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"OpenVINO LLM Benchmark\")
    parser.add_argument(\"--model-path\", type=str, required=True, help=\"Path to OpenVINO model\")
    parser.add_argument(\"--dataset\", type=str, default=\"Anthropic/hh-rlhf\", help=\"HuggingFace dataset name\")
    parser.add_argument(\"--num-prompts\", type=int, default=100, help=\"Number of prompts to process\")
    parser.add_argument(\"--max-new-tokens\", type=int, default=256, help=\"Max new tokens per prompt\")
    args = parser.parse_args()

    # Validate inputs
    if not os.path.exists(args.model_path):
        raise ValueError(f\"Model path {args.model_path} does not exist\")
    if args.num_prompts <= 0:
        raise ValueError(\"num_prompts must be positive\")

    results = run_openvino_benchmark(
        model_path=args.model_path,
        prompt_dataset=args.dataset,
        num_prompts=args.num_prompts,
        max_new_tokens=args.max_new_tokens
    )

    print(\"\\n=== OpenVINO Benchmark Results ===\")
    for key, value in results.items():
        print(f\"{key}: {value}\")

import os
import time
import argparse
from typing import List, Dict
import numpy as np
from vllm import LLM, SamplingParams
from datasets import load_dataset

def run_vllm_benchmark(
    model_name: str,
    prompt_dataset: str,
    num_prompts: int,
    max_new_tokens: int = 256,
    tensor_parallel_size: int = 1
) -> Dict[str, float]:
    \"\"\"
    Benchmark vLLM inference throughput and latency.

    Args:
        model_name: HuggingFace model name or local path
        prompt_dataset: Name of HuggingFace dataset to load prompts from
        num_prompts: Number of prompts to run (truncated from dataset)
        max_new_tokens: Maximum number of new tokens to generate per prompt
        tensor_parallel_size: Number of GPUs to use for tensor parallelism

    Returns:
        Dictionary with throughput (tokens/sec), p50_latency (ms), p99_latency (ms)
    \"\"\"
    # Initialize vLLM with error handling
    try:
        llm = LLM(
            model=model_name,
            tensor_parallel_size=tensor_parallel_size,
            trust_remote_code=True,
            dtype=\"auto\"
        )
    except Exception as e:
        raise RuntimeError(f\"Failed to initialize vLLM with model {model_name}: {str(e)}\")

    # Load and preprocess prompts
    try:
        dataset = load_dataset(prompt_dataset, split=\"train\")
        prompts = [sample[\"chosen\"] for sample in dataset.select(range(num_prompts))]
    except Exception as e:
        raise RuntimeError(f\"Failed to load dataset {prompt_dataset}: {str(e)}\")

    # Warmup run to avoid cold start bias
    print(\"Running vLLM warmup...\")
    warmup_params = SamplingParams(max_tokens=32, temperature=0.7, top_p=0.9)
    llm.generate([prompts[0]], warmup_params)

    # Run benchmark
    latencies = []
    total_tokens = 0
    print(f\"Running vLLM benchmark with {num_prompts} prompts...\")

    # Batch size for vLLM (adjust based on GPU memory)
    batch_size = 8 if tensor_parallel_size == 1 else 16

    for batch_start in range(0, len(prompts), batch_size):
        batch_end = min(batch_start + batch_size, len(prompts))
        batch_prompts = prompts[batch_start:batch_end]

        try:
            start_time = time.perf_counter()
            sampling_params = SamplingParams(
                max_tokens=max_new_tokens,
                temperature=0.7,
                top_p=0.9,
                do_sample=True
            )
            results = llm.generate(batch_prompts, sampling_params)
            end_time = time.perf_counter()

            # Calculate latency and token count
            batch_latency_ms = (end_time - start_time) * 1000
            per_prompt_latency = batch_latency_ms / len(batch_prompts)
            latencies.extend([per_prompt_latency] * len(batch_prompts))

            # Count tokens (vLLM returns token count in result)
            for res in results:
                total_tokens += len(res.outputs[0].token_ids)

            if (batch_end) % 10 == 0:
                print(f\"Processed {batch_end}/{num_prompts} prompts\")
        except Exception as e:
            print(f\"Warning: Failed to process batch {batch_start}-{batch_end}: {str(e)}\")
            continue

    # Calculate metrics
    if not latencies:
        raise RuntimeError(\"No successful inference runs completed\")

    throughput = total_tokens / (sum(latencies) / 1000)  # tokens per second
    p50_latency = np.percentile(latencies, 50)
    p99_latency = np.percentile(latencies, 99)

    return {
        \"throughput\": round(throughput, 2),
        \"p50_latency\": round(p50_latency, 2),
        \"p99_latency\": round(p99_latency, 2),
        \"total_tokens\": total_tokens,
        \"successful_runs\": len(latencies)
    }

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"vLLM Benchmark\")
    parser.add_argument(\"--model-name\", type=str, required=True, help=\"HuggingFace model name or path\")
    parser.add_argument(\"--dataset\", type=str, default=\"Anthropic/hh-rlhf\", help=\"HuggingFace dataset name\")
    parser.add_argument(\"--num-prompts\", type=int, default=100, help=\"Number of prompts to process\")
    parser.add_argument(\"--max-new-tokens\", type=int, default=256, help=\"Max new tokens per prompt\")
    parser.add_argument(\"--tensor-parallel-size\", type=int, default=1, help=\"Number of GPUs for tensor parallelism\")
    args = parser.parse_args()

    # Validate inputs
    if args.num_prompts <= 0:
        raise ValueError(\"num_prompts must be positive\")
    if args.tensor_parallel_size <= 0:
        raise ValueError(\"tensor_parallel_size must be positive\")

    results = run_vllm_benchmark(
        model_name=args.model_name,
        prompt_dataset=args.dataset,
        num_prompts=args.num_prompts,
        max_new_tokens=args.max_new_tokens,
        tensor_parallel_size=args.tensor_parallel_size
    )

    print(\"\\n=== vLLM Benchmark Results ===\")
    for key, value in results.items():
        print(f\"{key}: {value}\")

import json
import argparse
from typing import Dict, List
from openvino_benchmark import run_openvino_benchmark  # Assumes first script is saved as openvino_benchmark.py
from vllm_benchmark import run_vllm_benchmark        # Assumes second script is saved as vllm_benchmark.py

def generate_comparison_report(
    openvino_model: str,
    vllm_model: str,
    dataset: str,
    num_prompts: int,
    max_new_tokens: int,
    output_path: str
) -> None:
    \"\"\"
    Generate a head-to-head comparison report between OpenVINO and vLLM.

    Args:
        openvino_model: Path to OpenVINO quantized model
        vllm_model: HuggingFace model name for vLLM
        dataset: HuggingFace dataset name
        num_prompts: Number of prompts to process
        max_new_tokens: Max new tokens per prompt
        output_path: Path to save JSON report
    \"\"\"
    report = {
        \"metadata\": {
            \"openvino_model\": openvino_model,
            \"vllm_model\": vllm_model,
            \"dataset\": dataset,
            \"num_prompts\": num_prompts,
            \"max_new_tokens\": max_new_tokens
        },
        \"results\": {}
    }

    # Run OpenVINO benchmark
    print(\"\\n=== Running OpenVINO Benchmark ===\")
    try:
        openvino_results = run_openvino_benchmark(
            model_path=openvino_model,
            prompt_dataset=dataset,
            num_prompts=num_prompts,
            max_new_tokens=max_new_tokens
        )
        report[\"results\"][\"openvino\"] = openvino_results
    except Exception as e:
        print(f\"OpenVINO benchmark failed: {str(e)}\")
        report[\"results\"][\"openvino\"] = {\"error\": str(e)}

    # Run vLLM benchmark
    print(\"\\n=== Running vLLM Benchmark ===\")
    try:
        vllm_results = run_vllm_benchmark(
            model_name=vllm_model,
            prompt_dataset=dataset,
            num_prompts=num_prompts,
            max_new_tokens=max_new_tokens,
            tensor_parallel_size=1
        )
        report[\"results\"][\"vllm\"] = vllm_results
    except Exception as e:
        print(f\"vLLM benchmark failed: {str(e)}\")
        report[\"results\"][\"vllm\"] = {\"error\": str(e)}

    # Calculate relative performance
    if \"openvino\" in report[\"results\"] and \"vllm\" in report[\"results\"]:
        if \"throughput\" in report[\"results\"][\"openvino\"] and \"throughput\" in report[\"results\"][\"vllm\"]:
            openvino_throughput = report[\"results\"][\"openvino\"][\"throughput\"]
            vllm_throughput = report[\"results\"][\"vllm\"][\"throughput\"]
            report[\"relative_performance\"] = {
                \"openvino_vs_vllm_throughput\": round(openvino_throughput / vllm_throughput, 2) if vllm_throughput != 0 else \"N/A\",
                \"vllm_vs_openvino_throughput\": round(vllm_throughput / openvino_throughput, 2) if openvino_throughput != 0 else \"N/A\"
            }

    # Save report
    try:
        with open(output_path, \"w\") as f:
            json.dump(report, f, indent=2)
        print(f\"\\nReport saved to {output_path}\")
    except Exception as e:
        raise RuntimeError(f\"Failed to save report to {output_path}: {str(e)}\")

    # Print summary table
    print(\"\\n=== Comparison Summary ===\")
    print(f\"{'Metric':<20} {'OpenVINO':<15} {'vLLM':<15} {'Winner':<10}\")
    print(\"-\" * 60)

    metrics = [\"throughput\", \"p50_latency\", \"p99_latency\", \"total_tokens\"]
    for metric in metrics:
        openvino_val = report[\"results\"].get(\"openvino\", {}).get(metric, \"N/A\")
        vllm_val = report[\"results\"].get(\"vllm\", {}).get(metric, \"N/A\")

        if openvino_val != \"N/A\" and vllm_val != \"N/A\":
            if metric == \"throughput\":
                winner = \"OpenVINO\" if openvino_val > vllm_val else \"vLLM\"
            else:  # latency is lower is better
                winner = \"OpenVINO\" if openvino_val < vllm_val else \"vLLM\"
        else:
            winner = \"N/A\"

        print(f\"{metric:<20} {str(openvino_val):<15} {str(vllm_val):<15} {winner:<10}\")

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"OpenVINO vs vLLM Comparison Report\")
    parser.add_argument(\"--openvino-model\", type=str, required=True, help=\"Path to OpenVINO model\")
    parser.add_argument(\"--vllm-model\", type=str, required=True, help=\"HuggingFace model name for vLLM\")
    parser.add_argument(\"--dataset\", type=str, default=\"Anthropic/hh-rlhf\", help=\"Dataset name\")
    parser.add_argument(\"--num-prompts\", type=int, default=100, help=\"Number of prompts\")
    parser.add_argument(\"--max-new-tokens\", type=int, default=256, help=\"Max new tokens\")
    parser.add_argument(\"--output\", type=str, default=\"comparison_report.json\", help=\"Output JSON path\")
    args = parser.parse_args()

    generate_comparison_report(
        openvino_model=args.openvino_model,
        vllm_model=args.vllm_model,
        dataset=args.dataset,
        num_prompts=args.num_prompts,
        max_new_tokens=args.max_new_tokens,
        output_path=args.output
    )

# Quantize Llama 3 8B to INT8 with Neural Compressor for OpenVINO
from neural_compressor import PostTrainingQuantConfig, quantization
from transformers import AutoModelForCausalLM, AutoTokenizer

model_name = \"meta-llama/Meta-Llama-3-8B-Instruct\"
calibration_dataset = \"Anthropic/hh-rlhf\"

# Load model and tokenizer
model = AutoModelForCausalLM.from_pretrained(model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Configure INT8 PTQ with calibration
conf = PostTrainingQuantConfig(
    approach=\"static\",
    calibration_sampling_size=500,
    op_type_dict={\"embeddings\": {\"weight\": {\"algorithm\": \"minmax\"}}}
)

# Run quantization and save OpenVINO IR
q_model = quantization.fit(model, conf, calib_dataloader=calibration_dataset)
q_model.save(\"llama3-8b-int8-openvino\")

# Launch vLLM API server with tuned PagedAttention settings for 70B model
from vllm import LLM, SamplingParams

llm = LLM(
    model=\"meta-llama/Meta-Llama-3-70B-Instruct\",
    tensor_parallel_size=2,
    gpu_memory_utilization=0.9,
    max_num_seqs=256,
    dtype=\"float16\",
    trust_remote_code=True
)

# Run inference with tuned sampling params
params = SamplingParams(
    max_tokens=256,
    temperature=0.7,
    top_p=0.9,
    repetition_penalty=1.1
)
results = llm.generate([\"What is the return policy?\"], params)

# FastAPI router for hybrid OpenVINO (edge) + vLLM (cloud) deployment
from fastapi import FastAPI, HTTPException
from openvino_client import OpenVINOClient  # Custom client for edge
from vllm_client import vLLMClient        # Custom client for cloud

app = FastAPI()
openvino_client = OpenVINOClient(\"localhost:8001\")
vllm_client = vLLMClient(\"cloud-vllm.example.com:8000\")

@app.post(\"/generate\")
async def generate(prompt: str, latency_requirement_ms: int = 200):
    if latency_requirement_ms <= 200:
        # Route to edge OpenVINO for low latency
        try:
            return await openvino_client.generate(prompt)
        except Exception as e:
            # Fallback to cloud vLLM
            return await vllm_client.generate(prompt)
    else:
        # Route to cloud vLLM for high throughput
        return await vllm_client.generate(prompt)