Istio 1.23 vs Linkerd 2.15: Sidecar Proxy Overhead for 1k RPS per Microservice

apiVersion: v1
kind: Namespace
metadata:
  name: benchmark-workloads
  labels:
    istio-injection: enabled # Enable Istio sidecar injection for Istio runs
    linkerd.io/inject: enabled # Enable Linkerd sidecar injection for Linkerd runs
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: sample-service
  namespace: benchmark-workloads
  labels:
    app: sample-service
spec:
  replicas: 10 # 10 microservices to hit 10k total RPS (1k per service)
  selector:
    matchLabels:
      app: sample-service
  template:
    metadata:
      labels:
        app: sample-service
    spec:
      containers:
      - name: service
        image: nginx:1.25.3 # Lightweight HTTP server to return 1kB response
        ports:
        - containerPort: 80
        volumeMounts:
        - name: nginx-config
          mountPath: /etc/nginx/conf.d
      - name: response-generator # Sidecar to simulate 10ms processing + 1kB response
        image: alpine:3.19.1
        command: [\"/bin/sh\"]
        args:
        - -c
        - |
          while true; do
            # Simulate 10ms backend processing time
            usleep 10000
            # Return 1kB response (1024 bytes)
            dd if=/dev/urandom of=/tmp/response bs=1024 count=1 2>/dev/null
            # Simple HTTP server on port 8080
            (echo -ne \"HTTP/1.1 200 OK\\r\\nContent-Length: 1024\\r\\n\\r\\n\"; cat /tmp/response) | nc -l -p 8080 -q 1
          done
        ports:
        - containerPort: 8080
      volumes:
      - name: nginx-config
        configMap:
          name: nginx-config
      # Resource limits to prevent node overload
      - name: service
        resources:
          requests:
            cpu: 0.1
            memory: 128Mi
          limits:
            cpu: 0.5
            memory: 256Mi
      - name: response-generator
        resources:
          requests:
            cpu: 0.1
            memory: 64Mi
          limits:
            cpu: 0.3
            memory: 128Mi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-config
  namespace: benchmark-workloads
data:
  default.conf: |
    server {
      listen 80;
      location / {
        proxy_pass http://localhost:8080; # Forward to response generator sidecar
        proxy_set_header Host $host;
        # Error handling: return 503 if backend is unavailable
        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
      }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: fortio-loadgen
  namespace: benchmark-workloads
  labels:
    app: fortio
spec:
  replicas: 1
  selector:
    matchLabels:
      app: fortio
  template:
    metadata:
      labels:
        app: fortio
    spec:
      containers:
      - name: fortio
        image: fortio/fortio:1.52.0
        ports:
        - containerPort: 8080
        args:
        - server
        resources:
          requests:
            cpu: 1
            memory: 1Gi
          limits:
            cpu: 2
            memory: 2Gi
      # Deploy on separate node to avoid resource contention
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: workload-type
                operator: In
                values:
                - load-generator
---
apiVersion: batch/v1
kind: Job
metadata:
  name: fortio-benchmark
  namespace: benchmark-workloads
spec:
  template:
    spec:
      containers:
      - name: fortio-runner
        image: fortio/fortio:1.52.0
        command: [\"/bin/sh\"]
        args:
        - -c
        - |
          # Run 30-minute benchmark at 1k RPS per service, 10 services total
          fortio load -c 100 -qps 1000 -t 30m -labels \"istio-vs-linkerd\" http://sample-service.benchmark-workloads.svc.cluster.local:80
          # Exit after benchmark completes
          echo \"Benchmark completed successfully\"
        resources:
          requests:
            cpu: 0.5
            memory: 512Mi
      restartPolicy: Never
  backoffLimit: 1

import requests
import time
import csv
import os
from typing import Dict, List, Optional

# Configuration: Update these values to match your Prometheus deployment
PROMETHEUS_URL = os.getenv(\"PROMETHEUS_URL\", \"http://prometheus.istio-system.svc.cluster.local:9090\")
BENCHMARK_DURATION = 1800  # 30 minutes in seconds
SCRAPE_INTERVAL = 1  # 1 second scrape interval
OUTPUT_CSV = \"benchmark_results.csv\"

class PrometheusClient:
    def __init__(self, base_url: str):
        self.base_url = base_url.rstrip(\"/\")
        self.session = requests.Session()
        self.session.headers.update({\"Content-Type\": \"application/json\"})

    def query_range(self, query: str, start: float, end: float, step: str) -> Optional[List[dict]]:
        \"\"\"Query Prometheus range API with retry logic for transient failures.\"\"\"
        url = f\"{self.base_url}/api/v1/query_range\"
        params = {
            \"query\": query,
            \"start\": start,
            \"end\": end,
            \"step\": step
        }
        max_retries = 3
        for attempt in range(max_retries):
            try:
                response = self.session.get(url, params=params, timeout=10)
                response.raise_for_status()
                data = response.json()
                if data.get(\"status\") == \"success\":
                    return data.get(\"data\", {}).get(\"result\", [])
                else:
                    print(f\"Prometheus query failed: {data.get('error', 'Unknown error')}\")
                    return None
            except requests.exceptions.RequestException as e:
                print(f\"Attempt {attempt + 1} failed: {e}\")
                if attempt < max_retries - 1:
                    time.sleep(2 ** attempt)  # Exponential backoff
                else:
                    print(f\"Failed to query Prometheus after {max_retries} attempts\")
                    return None
        return None

def get_baseline_latency(prom_client: PrometheusClient, start: float, end: float) -> float:
    \"\"\"Calculate baseline p99 latency without service mesh sidecars.\"\"\"
    query = 'histogram_quantile(0.99, sum(rate(request_duration_seconds_bucket{job=\"sample-service\", mesh=\"none\"}[1m])) by (le))'
    results = prom_client.query_range(query, start, end, \"1m\")
    if not results:
        raise ValueError(\"No baseline latency metrics found. Ensure baseline benchmark was run without service mesh.\")
    # Extract the p99 value from the last data point
    for result in results:
        values = result.get(\"values\", [])
        if values:
            return float(values[-1][1])
    raise ValueError(\"Baseline latency metric has no data points.\")

def get_sidecar_metrics(prom_client: PrometheusClient, mesh_type: str, start: float, end: float) -> Dict[str, float]:
    \"\"\"Extract sidecar resource usage and latency overhead for a given service mesh.\"\"\"
    metrics = {}
    # p99 Latency
    latency_query = f'histogram_quantile(0.99, sum(rate(request_duration_seconds_bucket{{job=\"sample-service\", mesh=\"{mesh_type}\"}}[1m])) by (le))'
    latency_results = prom_client.query_range(latency_query, start, end, \"1m\")
    if latency_results:
        for result in latency_results:
            values = result.get(\"values\", [])
            if values:
                metrics[\"p99_latency\"] = float(values[-1][1])
                break
    # Sidecar Memory (idle)
    memory_query = f'sum(container_memory_working_set_bytes{{container=\"{mesh_type}-proxy\", namespace=\"benchmark-workloads\"}}) by (container)'
    memory_results = prom_client.query_range(memory_query, start, end, \"1m\")
    if memory_results:
        for result in memory_results:
            values = result.get(\"values\", [])
            if values:
                # Convert bytes to MB
                metrics[\"sidecar_memory_mb\"] = float(values[-1][1]) / (1024 * 1024)
                break
    # Sidecar CPU (idle)
    cpu_query = f'sum(rate(container_cpu_usage_seconds_total{{container=\"{mesh_type}-proxy\", namespace=\"benchmark-workloads\"}}[1m])) by (container)'
    cpu_results = prom_client.query_range(cpu_query, start, end, \"1m\")
    if cpu_results:
        for result in cpu_results:
            values = result.get(\"values\", [])
            if values:
                # Convert to vCPU (1 core = 1 vCPU)
                metrics[\"sidecar_cpu_vcpu\"] = float(values[-1][1])
                break
    # Validate all required metrics are present
    required = [\"p99_latency\", \"sidecar_memory_mb\", \"sidecar_cpu_vcpu\"]
    missing = [m for m in required if m not in metrics]
    if missing:
        raise ValueError(f\"Missing required metrics for {mesh_type}: {missing}\")
    return metrics

def main():
    # Calculate time range for the benchmark
    end_time = time.time()
    start_time = end_time - BENCHMARK_DURATION
    prom_client = PrometheusClient(PROMETHEUS_URL)

    print(\"Collecting baseline metrics...\")
    baseline_p99 = get_baseline_latency(prom_client, start_time, end_time)
    print(f\"Baseline p99 latency: {baseline_p99:.3f}ms\")

    results = [[\"Mesh\", \"p99_latency_ms\", \"p99_overhead_ms\", \"sidecar_memory_mb\", \"sidecar_cpu_vcpu\"]]
    for mesh in [\"istio\", \"linkerd\"]:
        print(f\"Collecting metrics for {mesh}...\")
        try:
            mesh_metrics = get_sidecar_metrics(prom_client, mesh, start_time, end_time)
            overhead = (mesh_metrics[\"p99_latency\"] * 1000) - (baseline_p99 * 1000)  # Convert to ms
            results.append([
                mesh,
                f\"{mesh_metrics['p99_latency'] * 1000:.1f}\",
                f\"{overhead:.1f}\",
                f\"{mesh_metrics['sidecar_memory_mb']:.1f}\",
                f\"{mesh_metrics['sidecar_cpu_vcpu']:.3f}\"
            ])
        except ValueError as e:
            print(f\"Failed to collect metrics for {mesh}: {e}\")
            continue

    # Write results to CSV
    with open(OUTPUT_CSV, \"w\", newline=\"\") as f:
        writer = csv.writer(f)
        writer.writerows(results)
    print(f\"Results written to {OUTPUT_CSV}\")

if __name__ == \"__main__\":
    main()

#!/bin/bash
set -euo pipefail  # Exit on error, undefined variable, pipe failure

# Configuration
ISTIO_VERSION=\"1.23.0\"
LINKERD_VERSION=\"2.15.0\"
KUBECONFIG=\"${KUBECONFIG:-$HOME/.kube/config}\"
BENCHMARK_NAMESPACE=\"benchmark-workloads\"
RESULTS_DIR=\"./benchmark-results\"

# Prerequisite checks
check_prerequisites() {
  echo \"Checking prerequisites...\"
  for cmd in kubectl istioctl linkerd curl; do
    if ! command -v $cmd &> /dev/null; then
      echo \"Error: $cmd is not installed. Please install it before running this script.\"
      exit 1
    fi
  done
  if ! kubectl cluster-info &> /dev/null; then
    echo \"Error: Cannot connect to Kubernetes cluster. Check KUBECONFIG.\"
    exit 1
  fi
  mkdir -p \"$RESULTS_DIR\"
  echo \"Prerequisites satisfied.\"
}

# Install Istio 1.23
install_istio() {
  echo \"Installing Istio $ISTIO_VERSION...\"
  # Download istioctl if not present
  if ! command -v istioctl &> /dev/null || ! istioctl version | grep -q \"$ISTIO_VERSION\"; then
    echo \"Downloading istioctl $ISTIO_VERSION...\"
    curl -L https://istio.io/downloadIstio | ISTIO_VERSION=$ISTIO_VERSION sh -
    export PATH=\"$PWD/istio-$ISTIO_VERSION/bin:$PATH\"
  fi
  # Install Istio with default profile (sidecar mode, no ambient)
  istioctl install --set profile=default -y
  # Enable sidecar injection for benchmark namespace
  kubectl label namespace $BENCHMARK_NAMESPACE istio-injection=enabled --overwrite
  echo \"Istio $ISTIO_VERSION installed successfully.\"
}

# Install Linkerd 2.15
install_linkerd() {
  echo \"Installing Linkerd $LINKERD_VERSION...\"
  # Download linkerd if not present
  if ! command -v linkerd &> /dev/null || ! linkerd version | grep -q \"Client version: $LINKERD_VERSION\"; then
    echo \"Downloading linkerd $LINKERD_VERSION...\"
    curl -sL https://run.linkerd.io/install | LINKERD_VERSION=$LINKERD_VERSION sh -
    export PATH=\"$HOME/.linkerd2/bin:$PATH\"
  fi
  # Install Linkerd with default configuration
  linkerd install | kubectl apply -f -
  # Validate installation
  linkerd check
  # Enable sidecar injection for benchmark namespace
  kubectl label namespace $BENCHMARK_NAMESPACE linkerd.io/inject=enabled --overwrite
  echo \"Linkerd $LINKERD_VERSION installed successfully.\"
}

# Run benchmark for a given mesh
run_benchmark() {
  local mesh=$1
  echo \"Running benchmark for $mesh...\"
  # Delete existing workload if present
  kubectl delete namespace $BENCHMARK_NAMESPACE --ignore-not-found=true
  kubectl create namespace $BENCHMARK_NAMESPACE
  # Apply mesh injection label
  if [ \"$mesh\" == \"istio\" ]; then
    kubectl label namespace $BENCHMARK_NAMESPACE istio-injection=enabled --overwrite
  elif [ \"$mesh\" == \"linkerd\" ]; then
    kubectl label namespace $BENCHMARK_NAMESPACE linkerd.io/inject=enabled --overwrite
  fi
  # Deploy workload
  kubectl apply -f benchmark-workload.yaml -n $BENCHMARK_NAMESPACE
  # Wait for workload to be ready
  kubectl wait --for=condition=ready pod -l app=sample-service -n $BENCHMARK_NAMESPACE --timeout=300s
  kubectl wait --for=condition=ready pod -l app=fortio -n $BENCHMARK_NAMESPACE --timeout=300s
  # Run Fortio benchmark
  kubectl apply -f fortio-benchmark-job.yaml -n $BENCHMARK_NAMESPACE
  # Wait for benchmark job to complete
  kubectl wait --for=condition=complete job/fortio-benchmark -n $BENCHMARK_NAMESPACE --timeout=3600s
  # Collect logs
  kubectl logs job/fortio-benchmark -n $BENCHMARK_NAMESPACE > \"$RESULTS_DIR/$mesh-fortio-logs.txt\"
  # Collect Prometheus metrics
  kubectl port-forward -n istio-system svc/prometheus 9090:9090 &
  PF_PID=$!
  sleep 5
  curl -s \"http://localhost:9090/api/v1/query?query=up\" > \"$RESULTS_DIR/$mesh-prometheus-up.json\"
  kill $PF_PID || true
  echo \"Benchmark for $mesh completed. Results saved to $RESULTS_DIR\"
}

# Tear down mesh
teardown_mesh() {
  local mesh=$1
  echo \"Tearing down $mesh...\"
  if [ \"$mesh\" == \"istio\" ]; then
    istioctl uninstall --purge -y
    kubectl delete namespace istio-system --ignore-not-found=true
  elif [ \"$mesh\" == \"linkerd\" ]; then
    linkerd uninstall | kubectl delete -f -
    kubectl delete namespace linkerd --ignore-not-found=true
  fi
}

# Main execution
main() {
  check_prerequisites
  # Run baseline (no mesh)
  echo \"Running baseline benchmark (no service mesh)...\"
  kubectl create namespace $BENCHMARK_NAMESPACE
  kubectl apply -f benchmark-workload.yaml -n $BENCHMARK_NAMESPACE
  kubectl wait --for=condition=ready pod -l app=sample-service -n $BENCHMARK_NAMESPACE --timeout=300s
  run_benchmark \"none\"
  # Run Istio benchmark
  install_istio
  run_benchmark \"istio\"
  teardown_mesh \"istio\"
  # Run Linkerd benchmark
  install_linkerd
  run_benchmark \"linkerd\"
  teardown_mesh \"linkerd\"
  echo \"All benchmarks completed. Results in $RESULTS_DIR\"
}

# Cleanup on exit
trap 'echo \"Script interrupted. Cleaning up...\"; teardown_mesh \"istio\"; teardown_mesh \"linkerd\"; exit 1' INT TERM

main()

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: linkerd-proxy-vpa
  namespace: benchmark-workloads
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: sample-service
  updatePolicy:
    updateMode: \"Off\" # Only recommend, don’t auto-update
  resourcePolicy:
    containerPolicies:
    - containerName: linkerd-proxy
      maxAllowed:
        cpu: 200m
        memory: 128Mi
      minAllowed:
        cpu: 10m
        memory: 8Mi

istioctl install --set profile=ambient -y
kubectl label namespace benchmark-workloads istio.io/dataplane-mode=ambient --overwrite

apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
metadata:
  namespace: istio-system
  name: example-istiocontrolplane
spec:
  meshConfig:
    accessLogFile: \"\" # Disable access logs
    enableEnvoyStats: false # Disable unused Envoy stats