The Hidden Cost of migration in fine-tuning vs OpenVINO: A Head-to-Head

import torch
from torch.utils.data import DataLoader
from datasets import load_dataset, load_metric
from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer, EarlyStoppingCallback
)
import numpy as np
import os
from typing import Dict

# Configuration
MODEL_NAME = "bert-base-uncased"
DATASET_NAME = "glue"
DATASET_CONFIG = "mrpc"
OUTPUT_DIR = "./fine-tuned-bert-mrpc"
BATCH_SIZE = 32
EPOCHS = 3
LEARNING_RATE = 2e-5
MAX_SEQ_LENGTH = 128

def tokenize_function(examples: Dict) -> Dict:
    \"\"\"Tokenize input sequences with padding and truncation.\"\"\"
    return tokenizer(
        examples["sentence1"],
        examples["sentence2"],
        padding="max_length",
        truncation=True,
        max_length=MAX_SEQ_LENGTH
    )

def compute_metrics(eval_pred: np.ndarray) -> Dict:
    \"\"\"Calculate accuracy and F1 score for validation.\"\"\"
    metric = load_metric("glue", "mrpc")
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

if __name__ == "__main__":
    # Initialize tokenizer and model
    try:
        tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
        model = AutoModelForSequenceClassification.from_pretrained(
            MODEL_NAME, num_labels=2
        )
    except Exception as e:
        raise RuntimeError(f"Failed to load model/tokenizer: {str(e)}")

    # Load and preprocess dataset
    try:
        dataset = load_dataset(DATASET_NAME, DATASET_CONFIG)
        tokenized_dataset = dataset.map(tokenize_function, batched=True)
    except Exception as e:
        raise RuntimeError(f"Failed to load/preprocess dataset: {str(e)}")

    # Training arguments
    training_args = TrainingArguments(
        output_dir=OUTPUT_DIR,
        evaluation_strategy="epoch",
        save_strategy="epoch",
        learning_rate=LEARNING_RATE,
        per_device_train_batch_size=BATCH_SIZE,
        per_device_eval_batch_size=BATCH_SIZE,
        num_train_epochs=EPOCHS,
        weight_decay=0.01,
        load_best_model_at_end=True,
        metric_for_best_model="f1",
        save_total_limit=2,
        fp16=torch.cuda.is_available(),
        report_to="none"  # Disable wandb/tensorboard for reproducibility
    )

    # Initialize trainer with early stopping
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=tokenized_dataset["train"],
        eval_dataset=tokenized_dataset["validation"],
        compute_metrics=compute_metrics,
        callbacks=[EarlyStoppingCallback(early_stopping_patience=2)]
    )

    # Fine-tune model
    print("Starting fine-tuning...")
    try:
        trainer.train()
    except Exception as e:
        raise RuntimeError(f"Fine-tuning failed: {str(e)}")

    # Evaluate and save
    eval_results = trainer.evaluate()
    print(f"Validation results: {eval_results}")
    trainer.save_model(OUTPUT_DIR)
    tokenizer.save_pretrained(OUTPUT_DIR)
    print(f"Fine-tuned model saved to {OUTPUT_DIR}")

import torch
import openvino as ov
from transformers import AutoTokenizer
import numpy as np
from typing import List, Dict
import os

# Configuration
FINE_TUNED_MODEL_DIR = "./fine-tuned-bert-mrpc"
OPENVINO_MODEL_DIR = "./openvino-bert-mrpc"
OPENVINO_MODEL_NAME = "bert-mrpc-fp32"
INPUT_NAMES = ["input_ids", "attention_mask", "token_type_ids"]
OUTPUT_NAMES = ["logits"]
MAX_SEQ_LENGTH = 128
DEVICE = "CPU"  # Change to "GPU" or "NPU" for other Intel hardware

def convert_to_openvino() -> ov.Model:
    \"\"\"Convert fine-tuned PyTorch model to OpenVINO IR format.\"\"\"
    # Load fine-tuned model and tokenizer
    try:
        tokenizer = AutoTokenizer.from_pretrained(FINE_TUNED_MODEL_DIR)
        from transformers import AutoConfig, AutoModelForSequenceClassification
        config = AutoConfig.from_pretrained(FINE_TUNED_MODEL_DIR)
        model = AutoModelForSequenceClassification.from_pretrained(
            FINE_TUNED_MODEL_DIR, config=config
        )
        model.eval()
    except Exception as e:
        raise RuntimeError(f"Failed to load fine-tuned model: {str(e)}")

    # Create dummy input for tracing
    dummy_text = ["Hello world", "This is a test"]
    inputs = tokenizer(
        dummy_text[0], dummy_text[1],
        padding="max_length",
        truncation=True,
        max_length=MAX_SEQ_LENGTH,
        return_tensors="pt"
    )
    # Convert inputs to tuple for tracing
    dummy_input = (
        inputs["input_ids"],
        inputs["attention_mask"],
        inputs["token_type_ids"]
    )

    # Convert to OpenVINO
    print("Converting model to OpenVINO IR...")
    try:
        ov_model = ov.convert_model(
            model,
            example_input=dummy_input,
            input_names=INPUT_NAMES,
            output_names=OUTPUT_NAMES
        )
    except Exception as e:
        raise RuntimeError(f"OpenVINO conversion failed: {str(e)}")

    # Save OpenVINO model
    os.makedirs(OPENVINO_MODEL_DIR, exist_ok=True)
    ov.save_model(ov_model, f"{OPENVINO_MODEL_DIR}/{OPENVINO_MODEL_NAME}.xml")
    print(f"OpenVINO model saved to {OPENVINO_MODEL_DIR}/{OPENVINO_MODEL_NAME}.xml")
    return ov_model

def deploy_openvino_model() -> ov.CompiledModel:
    \"\"\"Compile and deploy OpenVINO model to target device.\"\"\"
    try:
        core = ov.Core()
        model_path = f"{OPENVINO_MODEL_DIR}/{OPENVINO_MODEL_NAME}.xml"
        compiled_model = core.compile_model(model_path, DEVICE)
        print(f"Model compiled for device: {DEVICE}")
        return compiled_model
    except Exception as e:
        raise RuntimeError(f"Failed to compile OpenVINO model: {str(e)}")

def run_inference(compiled_model: ov.CompiledModel, sentence1: str, sentence2: str) -> List[int]:
    \"\"\"Run inference on input sentences.\"\"\"
    try:
        tokenizer = AutoTokenizer.from_pretrained(FINE_TUNED_MODEL_DIR)
        inputs = tokenizer(
            sentence1, sentence2,
            padding="max_length",
            truncation=True,
            max_length=MAX_SEQ_LENGTH,
            return_tensors="np"
        )
        # Prepare input dict for OpenVINO
        input_dict = {
            "input_ids": inputs["input_ids"],
            "attention_mask": inputs["attention_mask"],
            "token_type_ids": inputs["token_type_ids"]
        }
        # Run inference
        result = compiled_model(input_dict)
        logits = result["logits"]
        predictions = np.argmax(logits, axis=-1).tolist()
        return predictions
    except Exception as e:
        raise RuntimeError(f"Inference failed: {str(e)}")

if __name__ == "__main__":
    # Convert model
    ov_model = convert_to_openvino()
    # Deploy model
    compiled_model = deploy_openvino_model()
    # Test inference
    test_cases = [
        ("The cat sat on the mat.", "The feline rested on the rug."),
        ("I love machine learning.", "I hate artificial intelligence.")
    ]
    for s1, s2 in test_cases:
        pred = run_inference(compiled_model, s1, s2)
        print(f"Sentence 1: {s1}")
        print(f"Sentence 2: {s2}")
        print(f"Prediction (0=not equivalent, 1=equivalent): {pred[0]}\n")

import torch
import openvino as ov
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import time
import numpy as np
from typing import Dict, List
import psutil
import os

# Configuration
FINE_TUNED_MODEL_DIR = "./fine-tuned-bert-mrpc"
OPENVINO_MODEL_DIR = "./openvino-bert-mrpc"
OPENVINO_MODEL_NAME = "bert-mrpc-fp32"
BATCH_SIZES = [1, 8, 16, 32]
NUM_ITERATIONS = 100
WARMUP_ITERATIONS = 10
MAX_SEQ_LENGTH = 128
DEVICE = "cpu"  # Match hardware for fair comparison

class BenchmarkResult:
    \"\"\"Container for benchmark metrics.\"\"\"
    def __init__(self, framework: str, batch_size: int):
        self.framework = framework
        self.batch_size = batch_size
        self.latencies: List[float] = []
        self.throughputs: List[float] = []
        self.peak_memory_mb: float = 0.0

    def add_latency(self, latency: float):
        self.latencies.append(latency)

    def calculate_metrics(self):
        \"\"\"Calculate p50, p95, p99 latency and average throughput.\"\"\"
        self.p50_latency = np.percentile(self.latencies, 50)
        self.p95_latency = np.percentile(self.latencies, 95)
        self.p99_latency = np.percentile(self.latencies, 99)
        # Throughput: batch_size / latency (latency in seconds)
        self.avg_throughput = self.batch_size / np.mean(self.latencies)

def load_pytorch_model() -> tuple:
    \"\"\"Load fine-tuned PyTorch model and tokenizer.\"\"\"
    try:
        tokenizer = AutoTokenizer.from_pretrained(FINE_TUNED_MODEL_DIR)
        model = AutoModelForSequenceClassification.from_pretrained(FINE_TUNED_MODEL_DIR)
        model.eval()
        model.to(DEVICE)
        return model, tokenizer
    except Exception as e:
        raise RuntimeError(f"Failed to load PyTorch model: {str(e)}")

def load_openvino_model() -> tuple:
    \"\"\"Load compiled OpenVINO model and tokenizer.\"\"\"
    try:
        core = ov.Core()
        model_path = f"{OPENVINO_MODEL_DIR}/{OPENVINO_MODEL_NAME}.xml"
        compiled_model = core.compile_model(model_path, "CPU")
        tokenizer = AutoTokenizer.from_pretrained(FINE_TUNED_MODEL_DIR)
        return compiled_model, tokenizer
    except Exception as e:
        raise RuntimeError(f"Failed to load OpenVINO model: {str(e)}")

def generate_dummy_batch(tokenizer: AutoTokenizer, batch_size: int) -> Dict:
    \"\"\"Generate dummy input batch for benchmarking.\"\"\"
    sentences1 = ["Hello world"] * batch_size
    sentences2 = ["Hi there"] * batch_size
    inputs = tokenizer(
        sentences1, sentences2,
        padding="max_length",
        truncation=True,
        max_length=MAX_SEQ_LENGTH,
        return_tensors="pt" if DEVICE == "cpu" else "np"
    )
    return inputs

def benchmark_pytorch(model: AutoModelForSequenceClassification, tokenizer: AutoTokenizer, batch_size: int) -> BenchmarkResult:
    \"\"\"Benchmark PyTorch model inference.\"\"\"
    result = BenchmarkResult("PyTorch 2.3", batch_size)
    inputs = generate_dummy_batch(tokenizer, batch_size)
    # Warmup
    for _ in range(WARMUP_ITERATIONS):
        with torch.no_grad():
            _ = model(**inputs)
    # Benchmark
    process = psutil.Process(os.getpid())
    initial_memory = process.memory_info().rss / (1024 * 1024)  # MB
    for _ in range(NUM_ITERATIONS):
        start = time.perf_counter()
        with torch.no_grad():
            outputs = model(**inputs)
        end = time.perf_counter()
        latency = end - start
        result.add_latency(latency)
    final_memory = process.memory_info().rss / (1024 * 1024)
    result.peak_memory_mb = final_memory - initial_memory
    result.calculate_metrics()
    return result

def benchmark_openvino(compiled_model: ov.CompiledModel, tokenizer: AutoTokenizer, batch_size: int) -> BenchmarkResult:
    \"\"\"Benchmark OpenVINO model inference.\"\"\"
    result = BenchmarkResult("OpenVINO 2024.2", batch_size)
    inputs = generate_dummy_batch(tokenizer, batch_size)
    # Convert inputs to numpy for OpenVINO
    input_dict = {
        "input_ids": inputs["input_ids"].numpy(),
        "attention_mask": inputs["attention_mask"].numpy(),
        "token_type_ids": inputs["token_type_ids"].numpy()
    }
    # Warmup
    for _ in range(WARMUP_ITERATIONS):
        _ = compiled_model(input_dict)
    # Benchmark
    process = psutil.Process(os.getpid())
    initial_memory = process.memory_info().rss / (1024 * 1024)
    for _ in range(NUM_ITERATIONS):
        start = time.perf_counter()
        outputs = compiled_model(input_dict)
        end = time.perf_counter()
        latency = end - start
        result.add_latency(latency)
    final_memory = process.memory_info().rss / (1024 * 1024)
    result.peak_memory_mb = final_memory - initial_memory
    result.calculate_metrics()
    return result

if __name__ == "__main__":
    print("Loading models...")
    pytorch_model, pytorch_tokenizer = load_pytorch_model()
    openvino_model, openvino_tokenizer = load_openvino_model()

    print("\nStarting benchmarks...")
    for batch_size in BATCH_SIZES:
        print(f"\nBatch size: {batch_size}")
        # PyTorch benchmark
        pytorch_result = benchmark_pytorch(pytorch_model, pytorch_tokenizer, batch_size)
        print(f"PyTorch Results:")
        print(f"  P50 Latency: {pytorch_result.p50_latency*1000:.2f}ms")
        print(f"  P99 Latency: {pytorch_result.p99_latency*1000:.2f}ms")
        print(f"  Avg Throughput: {pytorch_result.avg_throughput:.2f} samples/sec")
        print(f"  Peak Memory Delta: {pytorch_result.peak_memory_mb:.2f} MB")
        # OpenVINO benchmark
        openvino_result = benchmark_openvino(openvino_model, openvino_tokenizer, batch_size)
        print(f"OpenVINO Results:")
        print(f"  P50 Latency: {openvino_result.p50_latency*1000:.2f}ms")
        print(f"  P99 Latency: {openvino_result.p99_latency*1000:.2f}ms")
        print(f"  Avg Throughput: {openvino_result.avg_throughput:.2f} samples/sec")
        print(f"  Peak Memory Delta: {openvino_result.peak_memory_mb:.2f} MB")
        # Print comparison
        print(f"Latency Improvement (P99): {((pytorch_result.p99_latency - openvino_result.p99_latency)/pytorch_result.p99_latency)*100:.1f}%")

# Validate input shapes before conversion
def validate_input_shapes(tokenizer, max_length):
    dummy_input = tokenizer(
        "Test sentence 1", "Test sentence 2",
        max_length=max_length, padding="max_length",
        truncation=True, return_tensors="pt"
    )
    print(f"Input shapes: { {k: v.shape for k, v in dummy_input.items()} }")
    return dummy_input

# Enable dynamic shape support for sequence length
dynamic_shape_config = [
    (1, 128),   # Min sequence length
    (512, 128), # Max sequence length
    (1, 1)      # Variable batch size (optional)
]
ov_model = ov.convert_model(
    model,
    example_input=dummy_input,
    input=ov.PartialShape([1, ov.Dimension(1, 512), 768])  # For BERT embeddings
)

# Automate regression testing
def check_regressions(pytorch_model, ov_model, tokenizer, test_set):
    mismatches = 0
    for sample in test_set:
        # PyTorch inference
        pt_inputs = tokenizer(sample, return_tensors="pt")
        with torch.no_grad():
            pt_pred = torch.argmax(pytorch_model(**pt_inputs).logits)
        # OpenVINO inference
        ov_inputs = {k: v.numpy() for k, v in pt_inputs.items()}
        ov_pred = np.argmax(ov_model(ov_inputs)["logits"])
        if pt_pred != ov_pred:
            mismatches +=1
    regression_rate = mismatches / len(test_set)
    print(f"Regression rate: {regression_rate*100:.2f}%")
    return regression_rate < 0.01  # Pass if <1% regressions