Why We Ditched Proprietary LLMs for Open-Source Llama 3.2 on Graviton4 Instances: 2026 Cost and Latency Data

import os
import logging
import argparse
from vllm import LLM, SamplingParams
from vllm.neuron import NeuronConfig
from typing import List, Dict, Any

# Configure logging for audit and debugging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(), logging.FileHandler("llama_inference.log")]
)
logger = logging.getLogger(__name__)

def init_neuron_llm(model_id: str, tensor_parallel_size: int = 4) -> LLM:
    """
    Initialize vLLM engine with AWS Neuron backend for Graviton4/Inferentia3.

    Args:
        model_id: HuggingFace model ID for Llama 3.2 70B
        tensor_parallel_size: Number of Neuron cores to use for tensor parallelism

    Returns:
        Initialized vLLM LLM engine
    """
    try:
        # Validate model ID is a supported Llama 3.2 variant
        if "llama-3.2" not in model_id.lower():
            raise ValueError(f"Unsupported model: {model_id}. Only Llama 3.2 variants are supported.")

        # Neuron-specific configuration for Graviton4 r8g instances
        neuron_config = NeuronConfig(
            tensor_parallel_size=tensor_parallel_size,
            max_num_seqs=32,  # Max concurrent requests for r8g.16xlarge (64 vCPUs, 512GB RAM)
            max_model_len=8192,  # Llama 3.2 70B supports 8k context by default
            dtype="bfloat16",  # Optimized for Neuron's BF16 support
            trust_remote_code=True
        )

        # Initialize vLLM engine with Neuron backend
        llm = LLM(
            model=model_id,
            neuron_config=neuron_config,
            gpu_memory_utilization=0.9,  # Neuron uses device memory, not GPU
            enforce_eager=False  # Use graph compilation for faster inference
        )
        logger.info(f"Successfully initialized LLM engine for model: {model_id}")
        return llm
    except ImportError as e:
        logger.error(f"Missing dependency: {e}. Install vllm-neuron with `pip install vllm-neuron-sdk`")
        raise
    except Exception as e:
        logger.error(f"Failed to initialize LLM engine: {str(e)}", exc_info=True)
        raise

def run_inference(llm: LLM, prompts: List[str], max_tokens: int = 256) -> List[str]:
    """
    Run batch inference on Llama 3.2 with error handling per prompt.

    Args:
        llm: Initialized vLLM engine
        prompts: List of input prompts
        max_tokens: Maximum number of tokens to generate per prompt

    Returns:
        List of generated responses, with error messages for failed prompts
    """
    sampling_params = SamplingParams(
        temperature=0.7,
        top_p=0.9,
        max_tokens=max_tokens,
        stop=["", "<|eot_id|>"]  # Llama 3.2 stop tokens
    )

    results = []
    try:
        # Batch generate responses
        outputs = llm.generate(prompts, sampling_params)
        for output in outputs:
            try:
                generated_text = output.outputs[0].text
                results.append(generated_text)
                logger.debug(f"Generated response for prompt: {output.prompt[:50]}...")
            except IndexError:
                error_msg = f"Empty output for prompt: {output.prompt[:50]}..."
                logger.warning(error_msg)
                results.append(f"ERROR: {error_msg}")
    except Exception as e:
        logger.error(f"Batch inference failed: {str(e)}", exc_info=True)
        results = [f"ERROR: Batch inference failed: {str(e)}"] * len(prompts)
    return results

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Run Llama 3.2 inference on Graviton4 with Neuron")
    parser.add_argument("--model-id", type=str, default="meta-llama/Llama-3.2-70B-Instruct", help="HuggingFace model ID")
    parser.add_argument("--tensor-parallel", type=int, default=4, help="Number of Neuron cores for tensor parallelism")
    parser.add_argument("--prompt-file", type=str, help="Path to file with prompts (one per line)")
    args = parser.parse_args()

    # Load prompts from file or use default
    if args.prompt_file:
        with open(args.prompt_file, "r") as f:
            prompts = [line.strip() for line in f if line.strip()]
    else:
        prompts = ["Explain the benefits of Arm-based cloud instances for LLM inference."]

    logger.info(f"Running inference for {len(prompts)} prompts on model {args.model_id}")
    llm = init_neuron_llm(args.model_id, args.tensor_parallel)
    responses = run_inference(llm, prompts)

    for i, (prompt, response) in enumerate(zip(prompts, responses)):
        print(f"Prompt {i+1}: {prompt[:100]}...")
        print(f"Response {i+1}: {response[:200]}...\n")

import time
import json
import os
import logging
from typing import List, Dict, Tuple
from openai import OpenAI, OpenAIError
from anthropic import Anthropic, AnthropicError
import requests
from requests.exceptions import RequestException

# Configure logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)

class LLMBenchmarker:
    """Benchmark LLM inference latency, cost, and output quality."""

    def __init__(self, vllm_endpoint: str = "http://localhost:8000"):
        self.vllm_endpoint = vllm_endpoint
        self.openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
        self.anthropic_client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
        self.results = []

    def benchmark_vllm(self, prompts: List[str], model_id: str = "meta-llama/Llama-3.2-70B-Instruct") -> Dict:
        """Benchmark self-hosted Llama 3.2 on Graviton4 via vLLM."""
        latencies = []
        errors = 0
        total_tokens = 0

        for prompt in prompts:
            payload = {
                "model": model_id,
                "prompt": prompt,
                "max_tokens": 256,
                "temperature": 0.7,
                "top_p": 0.9
            }
            start = time.perf_counter()
            try:
                response = requests.post(
                    f"{self.vllm_endpoint}/v1/completions",
                    json=payload,
                    headers={"Content-Type": "application/json"},
                    timeout=30
                )
                response.raise_for_status()
                end = time.perf_counter()
                latency = (end - start) * 1000  # ms
                latencies.append(latency)
                total_tokens += response.json()["usage"]["completion_tokens"]
                logger.debug(f"vLLM latency: {latency:.2f}ms for prompt: {prompt[:50]}...")
            except RequestException as e:
                logger.error(f"vLLM request failed: {str(e)}")
                errors += 1
                latencies.append(30000)  # 30s timeout as penalty

        # Calculate cost: $3.20 per 1M tokens for Llama 3.2 on Graviton4
        cost = (total_tokens / 1_000_000) * 3.20
        return {
            "model": f"Llama 3.2 70B (Graviton4)",
            "p50_latency_ms": sorted(latencies)[len(latencies)//2],
            "p99_latency_ms": sorted(latencies)[int(len(latencies)*0.99)],
            "avg_latency_ms": sum(latencies)/len(latencies),
            "error_rate": errors/len(prompts),
            "total_cost_usd": round(cost, 2),
            "total_tokens": total_tokens
        }

    def benchmark_openai(self, prompts: List[str], model: str = "gpt-4-turbo-2026-03") -> Dict:
        """Benchmark OpenAI GPT-4 Turbo."""
        latencies = []
        errors = 0
        total_tokens = 0

        for prompt in prompts:
            start = time.perf_counter()
            try:
                response = self.openai_client.completions.create(
                    model=model,
                    prompt=prompt,
                    max_tokens=256,
                    temperature=0.7,
                    top_p=0.9
                )
                end = time.perf_counter()
                latency = (end - start) * 1000
                latencies.append(latency)
                total_tokens += response.usage.completion_tokens
                logger.debug(f"OpenAI latency: {latency:.2f}ms")
            except OpenAIError as e:
                logger.error(f"OpenAI request failed: {str(e)}")
                errors +=1
                latencies.append(30000)

        # Cost: $10 per 1M input, $30 per 1M output
        input_cost = (len(prompts) * 50 / 1_000_000) * 10
        output_cost = (total_tokens / 1_000_000) * 30
        total_cost = input_cost + output_cost
        return {
            "model": f"GPT-4 Turbo (2026-03)",
            "p50_latency_ms": sorted(latencies)[len(latencies)//2],
            "p99_latency_ms": sorted(latencies)[int(len(latencies)*0.99)],
            "avg_latency_ms": sum(latencies)/len(latencies),
            "error_rate": errors/len(prompts),
            "total_cost_usd": round(total_cost, 2),
            "total_tokens": total_tokens
        }

    def benchmark_anthropic(self, prompts: List[str], model: str = "claude-3-opus-20240229") -> Dict:
        """Benchmark Anthropic Claude 3 Opus."""
        latencies = []
        errors = 0
        total_tokens = 0

        for prompt in prompts:
            start = time.perf_counter()
            try:
                response = self.anthropic_client.completions.create(
                    model=model,
                    prompt=f"\n\nHuman: {prompt}\n\nAssistant:",
                    max_tokens_to_sample=256,
                    temperature=0.7,
                    top_p=0.9
                )
                end = time.perf_counter()
                latency = (end - start) * 1000
                latencies.append(latency)
                total_tokens += response.usage.output_tokens
                logger.debug(f"Anthropic latency: {latency:.2f}ms")
            except AnthropicError as e:
                logger.error(f"Anthropic request failed: {str(e)}")
                errors +=1
                latencies.append(30000)

        # Cost: $15 per 1M input, $75 per 1M output
        input_cost = (len(prompts) * 50 / 1_000_000) * 15
        output_cost = (total_tokens / 1_000_000) * 75
        total_cost = input_cost + output_cost
        return {
            "model": f"Claude 3 Opus (2026-02)",
            "p50_latency_ms": sorted(latencies)[len(latencies)//2],
            "p99_latency_ms": sorted(latencies)[int(len(latencies)*0.99)],
            "avg_latency_ms": sum(latencies)/len(latencies),
            "error_rate": errors/len(prompts),
            "total_cost_usd": round(total_cost, 2),
            "total_tokens": total_tokens
        }

    def run_full_benchmark(self, prompts: List[str], output_file: str = "benchmark_results.json"):
        """Run benchmarks across all models and save results."""
        logger.info(f"Starting benchmark with {len(prompts)} prompts...")
        self.results.append(self.benchmark_vllm(prompts))
        self.results.append(self.benchmark_openai(prompts))
        self.results.append(self.benchmark_anthropic(prompts))

        with open(output_file, "w") as f:
            json.dump(self.results, f, indent=2)
        logger.info(f"Benchmark results saved to {output_file}")
        return self.results

if __name__ == "__main__":
    # Load 100 test prompts from file
    with open("test_prompts.txt", "r") as f:
        test_prompts = [line.strip() for line in f if line.strip()][:100]

    benchmarker = LLMBenchmarker(vllm_endpoint="http://localhost:8000")
    results = benchmarker.run_full_benchmark(test_prompts)

    print("\n=== Benchmark Results ===")
    for res in results:
        print(f"\nModel: {res['model']}")
        print(f"p50 Latency: {res['p50_latency_ms']:.2f}ms")
        print(f"p99 Latency: {res['p99_latency_ms']:.2f}ms")
        print(f"Avg Latency: {res['avg_latency_ms']:.2f}ms")
        print(f"Error Rate: {res['error_rate']*100:.1f}%")
        print(f"Total Cost: ${res['total_cost_usd']:.2f}")

terraform {
  required_version = ">= 1.5.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
  # Store state in S3 for team collaboration
  backend "s3" {
    bucket         = "llama-graviton4-terraform-state"
    key            = "llama-inference/terraform.tfstate"
    region         = "us-east-1"
    encrypt        = true
    dynamodb_table = "llama-terraform-locks"
  }
}

provider "aws" {
  region = var.aws_region
  default_tags {
    tags = {
      Project     = "llama-3.2-inference"
      Environment = var.environment
      ManagedBy   = "terraform"
    }
  }
}

# Data source: Get latest Amazon Linux 2023 AMI for Graviton4 (Arm)
data "aws_ami" "al2023_arm" {
  most_recent = true
  owners      = ["amazon"]
  filter {
    name   = "name"
    values = ["al2023-ami-2023.*-arm64"]
  }
  filter {
    name   = "architecture"
    values = ["arm64"]
  }
  filter {
    name   = "root-device-type"
    values = ["ebs"]
  }
}

# IAM role for EC2 instances to access S3, CloudWatch, and ECR
resource "aws_iam_role" "llama_ec2_role" {
  name = "llama-graviton4-ec2-role-${var.environment}"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action = "sts:AssumeRole"
      Effect = "Allow"
      Principal = { Service = "ec2.amazonaws.com" }
    }]
  })
  tags = { Name = "llama-graviton4-ec2-role" }
}

# Attach policy for S3 access (model weights)
resource "aws_iam_role_policy_attachment" "s3_access" {
  role       = aws_iam_role.llama_ec2_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
}

# Attach policy for CloudWatch Logs
resource "aws_iam_role_policy_attachment" "cloudwatch_logs" {
  role       = aws_iam_role.llama_ec2_role.name
  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
}

# Instance profile for EC2
resource "aws_iam_instance_profile" "llama_instance_profile" {
  name = "llama-graviton4-instance-profile-${var.environment}"
  role = aws_iam_role.llama_ec2_role.name
}

# Security group: Allow inbound HTTP (8000) and SSH (22) from internal CIDR
resource "aws_security_group" "llama_sg" {
  name        = "llama-graviton4-sg-${var.environment}"
  description = "Allow HTTP for vLLM and SSH for debugging"
  vpc_id      = var.vpc_id

  ingress {
    description = "vLLM API"
    from_port   = 8000
    to_port     = 8000
    protocol    = "tcp"
    cidr_blocks = var.internal_cidr_blocks
  }

  ingress {
    description = "SSH"
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = var.admin_cidr_blocks
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = { Name = "llama-graviton4-sg" }
}

# Launch template for Graviton4 r8g.16xlarge instances
resource "aws_launch_template" "llama_launch_template" {
  name_prefix   = "llama-graviton4-${var.environment}-"
  image_id      = data.aws_ami.al2023_arm.id
  instance_type = "r8g.16xlarge"  # 64 vCPUs, 512GB RAM, 4x Inferentia3 accelerators
  key_name      = var.ssh_key_name

  iam_instance_profile {
    arn = aws_iam_instance_profile.llama_instance_profile.arn
  }

  network_interfaces {
    security_groups = [aws_security_group.llama_sg.id]
    associate_public_ip_address = false
  }

  # User data script to install dependencies and start vLLM
  user_data = base64encode(<<-EOF
    #!/bin/bash
    set -e
    echo "Starting Llama 3.2 setup on Graviton4..."

    # Update packages
    dnf update -y

    # Install Docker, Neuron SDK, and vLLM
    dnf install -y docker git
    systemctl start docker
    systemctl enable docker

    # Install AWS Neuron drivers
    dnf install -y https://aws-neuron-sdk.s3.amazonaws.com/releases/neuron-2.20.1/aws-neuron-dkms-2.20.1.el2023.noarch.rpm
    dnf install -y aws-neuron-tools

    # Install vLLM with Neuron support
    pip3 install vllm-neuron-sdk==0.4.3

    # Pull Llama 3.2 70B model weights from S3 (encrypted bucket)
    aws s3 sync s3://llama-model-weights/llama-3.2-70b-instruct /models/llama-3.2-70b --region us-east-1

    # Start vLLM server with Neuron backend
    docker run -d \
      --name vllm-llama \
      --device /dev/neuron0 \
      --device /dev/neuron1 \
      --device /dev/neuron2 \
      --device /dev/neuron3 \
      -v /models:/models \
      -p 8000:8000 \
      vllm/vllm-neuron:0.4.3 \
      --model /models/llama-3.2-70b-instruct \
      --tensor-parallel-size 4 \
      --max-num-seqs 32 \
      --port 8000

    echo "vLLM server started successfully"
  EOF
  )

  lifecycle {
    create_before_destroy = true
  }

  tags = { Name = "llama-graviton4-launch-template" }
}

# Auto Scaling Group for Llama inference cluster
resource "aws_autoscaling_group" "llama_asg" {
  name_prefix          = "llama-graviton4-asg-${var.environment}-"
  vpc_zone_identifier  = var.private_subnet_ids
  desired_capacity     = var.asg_desired_capacity
  min_size             = var.asg_min_size
  max_size             = var.asg_max_size
  launch_template {
    id      = aws_launch_template.llama_launch_template.id
    version = "$Latest"
  }
  tag {
    key                 = "Name"
    value               = "llama-graviton4-instance"
    propagate_at_launch = true
  }
}

# Variables
variable "aws_region" { type = string; default = "us-east-1" }
variable "environment" { type = string; default = "prod" }
variable "vpc_id" { type = string }
variable "private_subnet_ids" { type = list(string) }
variable "internal_cidr_blocks" { type = list(string) }
variable "admin_cidr_blocks" { type = list(string) }
variable "ssh_key_name" { type = string }
variable "asg_desired_capacity" { type = number; default = 2 }
variable "asg_min_size" { type = number; default = 2 }
variable "asg_max_size" { type = number; default = 4 }

# Run Neuron profiler on Llama 3.2 70B
from vllm.neuron import NeuronProfiler
from vllm import LLM

llm = LLM(model="meta-llama/Llama-3.2-70B-Instruct", neuron_config=NeuronConfig(tensor_parallel_size=4))
profiler = NeuronProfiler(llm)
# Profile 100 sample prompts
with open("sample_prompts.txt") as f:
    prompts = [line.strip() for line in f][:100]
profiler.profile(prompts, output_file="neuron_profile.html")

# BERT-based prompt complexity classifier
from transformers import pipeline
import numpy as np

classifier = pipeline("text-classification", model="bert-base-uncased", device="cpu")  # Optimized for Arm

def get_prompt_complexity(prompt: str) -> int:
    """Return complexity score 1-5 for prompt routing."""
    result = classifier(prompt[:512])[0]  # Truncate to BERT max length
    # Map classifier output to 1-5 scale
    label_map = {"LABEL_0": 1, "LABEL_1": 2, "LABEL_2": 3, "LABEL_3": 4, "LABEL_4": 5}
    return label_map[result["label"]]

# LoRA fine-tuning for Llama 3.2 70B on Graviton4
from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
from trl import SFTTrainer

model_id = "meta-llama/Llama-3.2-70B-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype="bfloat16")

# LoRA configuration for Llama 3.2
lora_config = LoraConfig(
    r=64,  # Rank of LoRA matrices
    l_alpha=128,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)

# Training arguments
training_args = TrainingArguments(
    output_dir="./llama-3.2-70b-financial-lora",
    per_device_train_batch_size=1,
    gradient_accumulation_steps=16,
    learning_rate=2e-4,
    num_train_epochs=3,
    bf16=True,  # Optimized for Graviton4/Neuron
    save_steps=500,
    logging_steps=10
)

# Trainer
trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=load_dataset("csv", data_files="financial_transcripts.csv")["train"],
    tokenizer=tokenizer
)
trainer.train()
model.save_pretrained("./llama-3.2-70b-financial-lora")