The Hidden Cost of the scaling of Helm 4 and Docker 25: What Fails

package main

import (
    "context"
    "fmt"
    "helm.sh/helm/v4/pkg/action"
    "helm.sh/helm/v4/pkg/chart/loader"
    "helm.sh/helm/v4/pkg/cli"
    "log"
    "os"
    "time"
)

const (
    // ChartPath is the local path to the test Helm chart
    ChartPath = "./test-chart"
    // RenderIterations is the number of chart render cycles to run
    RenderIterations = 1000
    // TargetClusterContext is the k8s context to use for config loading
    TargetClusterContext = "benchmark-cluster"
)

func main() {
    // Initialize Helm settings with default config
    settings := cli.New()

    // Load the test chart from local path
    chart, err := loader.LoadDir(ChartPath)
    if err != nil {
        log.Fatalf("failed to load chart from %s: %v", ChartPath, err)
    }

    // Create a new Helm install action to use for rendering
    cfg, err := action.NewConfigFlags(false)
    if err != nil {
        log.Fatalf("failed to create Helm config: %v", err)
    }
    // Set the kubernetes context for the action config
    cfg.KubeContext = TargetClusterContext

    // Initialize the install action for rendering (dry-run)
    client := action.NewInstall(cfg)
    client.DryRun = true
    client.ReleaseName = "benchmark-render"
    client.Namespace = "default"

    // Track latency metrics
    var totalLatency time.Duration
    latencyBuckets := map[string]int{
        "0-100ms":  0,
        "100-300ms": 0,
        "300-500ms": 0,
        "500ms+":   0,
    }

    // Run render iterations
    for i := 0; i < RenderIterations; i++ {
        start := time.Now()

        // Render the chart with empty values (baseline test)
        _, err := client.Run(chart, nil)
        if err != nil {
            log.Printf("render iteration %d failed: %v", i, err)
            continue
        }

        // Calculate latency for this iteration
        latency := time.Since(start)
        totalLatency += latency

        // Bucket the latency
        switch {
        case latency < 100*time.Millisecond:
            latencyBuckets["0-100ms"]++
        case latency < 300*time.Millisecond:
            latencyBuckets["100-300ms"]++
        case latency < 500*time.Millisecond:
            latencyBuckets["300-500ms"]++
        default:
            latencyBuckets["500ms+"]++
        }
    }

    // Calculate and print results
    avgLatency := totalLatency / RenderIterations
    fmt.Printf("Helm 4 Chart Render Benchmark Results (%d iterations)\n", RenderIterations)
    fmt.Printf("Average Latency: %v\n", avgLatency)
    fmt.Printf("Total Latency: %v\n", totalLatency)
    fmt.Println("Latency Distribution:")
    for bucket, count := range latencyBuckets {
        fmt.Printf("  %s: %d iterations (%.2f%%)\n", bucket, count, float64(count)/float64(RenderIterations)*100)
    }

    // Exit with non-zero code if average latency exceeds 300ms
    if avgLatency > 300*time.Millisecond {
        fmt.Println("ERROR: Average render latency exceeds 300ms threshold")
        os.Exit(1)
    }
}

import docker
import time
import sys
import argparse
from typing import Dict, List
from dataclasses import dataclass

@dataclass
class ContainerMemoryStats:
    container_id: str
    name: str
    runtime: str
    mem_usage_bytes: int
    mem_limit_bytes: int
    timestamp: float

def get_docker_client() -> docker.DockerClient:
    """Initialize Docker client with error handling for connection failures."""
    try:
        client = docker.from_env()
        # Verify connection by pinging the daemon
        client.ping()
        return client
    except docker.errors.DockerException as e:
        print(f"FATAL: Failed to connect to Docker daemon: {e}", file=sys.stderr)
        sys.exit(1)

def collect_container_stats(client: docker.DockerClient, runtime_filter: str = None) -> List[ContainerMemoryStats]:
    """Collect memory stats for all running containers, optionally filtered by runtime."""
    stats_list = []
    containers = client.containers.list()

    for container in containers:
        try:
            # Get container details to check runtime
            container.reload()
            runtime = container.attrs.get("HostConfig", {}).get("Runtime", "runc")
            if runtime_filter and runtime != runtime_filter:
                continue

            # Stream stats for 1 second to get stable memory reading
            stats_stream = container.stats(stream=True, decode=True)
            stats = next(stats_stream)
            # Stop the stream after first reading
            container.client.api.close()

            # Calculate memory usage (usage - cache)
            mem_usage = stats["memory_stats"]["usage"]
            mem_cache = stats["memory_stats"].get("cache", 0)
            actual_usage = mem_usage - mem_cache
            mem_limit = stats["memory_stats"]["limit"]

            stats_list.append(ContainerMemoryStats(
                container_id=container.id[:12],
                name=container.name,
                runtime=runtime,
                mem_usage_bytes=actual_usage,
                mem_limit_bytes=mem_limit,
                timestamp=time.time()
            ))
        except StopIteration:
            print(f"WARNING: Failed to collect stats for container {container.name}: no stats stream", file=sys.stderr)
        except KeyError as e:
            print(f"WARNING: Missing stat key {e} for container {container.name}", file=sys.stderr)
        except Exception as e:
            print(f"WARNING: Unexpected error collecting stats for {container.name}: {e}", file=sys.stderr)

    return stats_list

def print_stats_report(stats: List[ContainerMemoryStats], runtime: str):
    """Print a formatted report of memory stats for a given runtime."""
    if not stats:
        print(f"No running containers found for runtime: {runtime}")
        return

    total_mem = sum(s.mem_usage_bytes for s in stats)
    avg_mem = total_mem / len(stats)
    container_count = len(stats)

    print(f"\n=== Docker {runtime} Memory Stats ===")
    print(f"Container Count: {container_count}")
    print(f"Total Memory Usage: {total_mem / 1024 / 1024:.2f} MiB")
    print(f"Average Memory Per Container: {avg_mem / 1024 / 1024:.2f} MiB")
    print(f"Sample Containers:")
    for stat in stats[:5]:  # Print first 5 samples
        print(f"  {stat.name} ({stat.container_id}): {stat.mem_usage_bytes / 1024 / 1024:.2f} MiB")

def main():
    parser = argparse.ArgumentParser(description="Docker 25 vs 24 Memory Usage Benchmark")
    parser.add_argument("--runtime", help="Filter containers by runtime (e.g., runc, runsc)")
    args = parser.parse_args()

    client = get_docker_client()
    print(f"Connected to Docker Daemon Version: {client.version()['Version']}")

    # Collect stats for all running containers
    all_stats = collect_container_stats(client, args.runtime)

    # Separate stats by Docker version (simulated for benchmark, in practice use version check)
    # For this benchmark, we assume Docker 25 uses runtime v2 and Docker 24 uses v1
    docker25_stats = [s for s in all_stats if s.runtime.startswith("runc-v2")]
    docker24_stats = [s for s in all_stats if s.runtime.startswith("runc-v1")]

    print_stats_report(docker25_stats, "25 (runc-v2)")
    print_stats_report(docker24_stats, "24 (runc-v1)")

    # Compare average memory usage
    if docker25_stats and docker24_stats:
        avg25 = sum(s.mem_usage_bytes for s in docker25_stats) / len(docker25_stats)
        avg24 = sum(s.mem_usage_bytes for s in docker24_stats) / len(docker24_stats)
        delta = ((avg25 - avg24) / avg24) * 100
        print(f"\nMemory Usage Delta: Docker 25 uses {delta:.2f}% more memory per container than Docker 24")

if __name__ == "__main__":
    main()

import subprocess
import time
import json
import argparse
import sys
from pathlib import Path
from typing import Dict, List

# Benchmark configuration
HELM_BIN = "helm4"  # Path to Helm 4 binary
KUSTOMIZE_BIN = "kustomize"  # Path to Kustomize 5.x binary
CHART_PATH = "./large-test-chart"  # Path to 1000+ resource Helm chart
KUSTOMIZE_OVERLAY = "./kustomize-overlay"  # Path to equivalent Kustomize overlay
ITERATIONS = 100  # Number of render iterations per tool

def run_command(cmd: List[str], timeout: int = 300) -> Dict:
    """Run a shell command and return parsed output with timing."""
    result = {
        "success": False,
        "stdout": "",
        "stderr": "",
        "duration_ms": 0
    }

    start = time.time()
    try:
        proc = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True
        )
        stdout, stderr = proc.communicate(timeout=timeout)
        result["duration_ms"] = (time.time() - start) * 1000
        result["success"] = proc.returncode == 0
        result["stdout"] = stdout
        result["stderr"] = stderr
    except subprocess.TimeoutExpired:
        proc.kill()
        result["stderr"] = "Command timed out"
    except Exception as e:
        result["stderr"] = str(e)
    return result

def benchmark_helm_render() -> List[float]:
    """Benchmark Helm 4 chart render times for large charts."""
    latencies = []
    print(f"Running Helm 4 render benchmark ({ITERATIONS} iterations)...")

    for i in range(ITERATIONS):
        # Helm 4 template command with OCI registry pull
        cmd = [
            HELM_BIN, "template", "benchmark-release",
            CHART_PATH,
            "--namespace", "default",
            "--set", "replicaCount=1000"  # Scale to 1000 replicas
        ]

        result = run_command(cmd)
        if not result["success"]:
            print(f"WARNING: Helm render iteration {i} failed: {result['stderr']}")
            continue

        latencies.append(result["duration_ms"])
        # Print progress every 10 iterations
        if (i + 1) % 10 == 0:
            print(f"  Completed {i+1}/{ITERATIONS} Helm iterations")

    return latencies

def benchmark_kustomize_render() -> List[float]:
    """Benchmark Kustomize render times for equivalent overlay."""
    latencies = []
    print(f"Running Kustomize render benchmark ({ITERATIONS} iterations)...")

    for i in range(ITERATIONS):
        # Kustomize build command for the overlay
        cmd = [KUSTOMIZE_BIN, "build", KUSTOMIZE_OVERLAY]

        result = run_command(cmd)
        if not result["success"]:
            print(f"WARNING: Kustomize render iteration {i} failed: {result['stderr']}")
            continue

        latencies.append(result["duration_ms"])
        if (i + 1) % 10 == 0:
            print(f"  Completed {i+1}/{ITERATIONS} Kustomize iterations")

    return latencies

def generate_report(helm_latencies: List[float], kustomize_latencies: List[float]):
    """Generate a comparison report for Helm vs Kustomize render times."""
    if not helm_latencies or not kustomize_latencies:
        print("ERROR: No valid benchmark data collected")
        sys.exit(1)

    # Calculate Helm stats
    helm_avg = sum(helm_latencies) / len(helm_latencies)
    helm_max = max(helm_latencies)
    helm_min = min(helm_latencies)

    # Calculate Kustomize stats
    kustomize_avg = sum(kustomize_latencies) / len(kustomize_latencies)
    kustomize_max = max(kustomize_latencies)
    kustomize_min = min(kustomize_latencies)

    # Print report
    print("\n=== Render Time Benchmark Report ===")
    print(f"Iterations per tool: {ITERATIONS}")
    print("\nHelm 4 Results:")
    print(f"  Average: {helm_avg:.2f} ms")
    print(f"  Min: {helm_min:.2f} ms")
    print(f"  Max: {helm_max:.2f} ms")
    print(f"  Valid Samples: {len(helm_latencies)}")

    print("\nKustomize 5.x Results:")
    print(f"  Average: {kustomize_avg:.2f} ms")
    print(f"  Min: {kustomize_min:.2f} ms")
    print(f"  Max: {kustomize_max:.2f} ms")
    print(f"  Valid Samples: {len(kustomize_latencies)}")

    # Calculate delta
    delta = ((helm_avg - kustomize_avg) / kustomize_avg) * 100
    print(f"\nHelm 4 is {delta:.2f}% slower than Kustomize for large renders")

    # Save report to JSON
    report = {
        "helm": {"avg_ms": helm_avg, "min_ms": helm_min, "max_ms": helm_max, "samples": len(helm_latencies)},
        "kustomize": {"avg_ms": kustomize_avg, "min_ms": kustomize_min, "max_ms": kustomize_max, "samples": len(kustomize_latencies)},
        "delta_percent": delta
    }
    with open("render-benchmark.json", "w") as f:
        json.dump(report, f, indent=2)
    print("Report saved to render-benchmark.json")

def main():
    parser = argparse.ArgumentParser(description="Helm 4 vs Kustomize Render Benchmark")
    parser.add_argument("--helm-bin", default=HELM_BIN, help="Path to Helm 4 binary")
    parser.add_argument("--kustomize-bin", default=KUSTOMIZE_BIN, help="Path to Kustomize binary")
    parser.add_argument("--iterations", type=int, default=ITERATIONS, help="Number of benchmark iterations")
    args = parser.parse_args()

    # Verify binaries exist
    if not Path(args.helm_bin).exists():
        print(f"ERROR: Helm binary not found at {args.helm_bin}")
        sys.exit(1)
    if not Path(args.kustomize_bin).exists():
        print(f"ERROR: Kustomize binary not found at {args.kustomize_bin}")
        sys.exit(1)

    # Run benchmarks
    helm_lats = benchmark_helm_render()
    kustomize_lats = benchmark_kustomize_render()

    # Generate report
    generate_report(helm_lats, kustomize_lats)

if __name__ == "__main__":
    main()

# Chart.yaml with dependencies pinned to local ECR mirror
apiVersion: v2
name: fintech-api
version: 1.2.4
dependencies:
  - name: redis
    version: 7.2.0
    repository: oci://123456789012.dkr.ecr.us-east-1.amazonaws.com/charts
  - name: postgres
    version: 16.1.0
    repository: oci://123456789012.dkr.ecr.us-east-1.amazonaws.com/charts
  - name: nginx-ingress
    version: 1.10.0
    repository: oci://123456789012.dkr.ecr.us-east-1.amazonaws.com/charts

# Ansible playbook to downgrade Docker to 24.0.7 on worker nodes
- hosts: k8s_workers
  become: yes
  tasks:
    - name: Stop Docker service
      systemd:
        name: docker
        state: stopped

    - name: Remove Docker 25 packages
      apt:
        name:
          - docker-ce
          - docker-ce-cli
          - containerd.io
        state: absent
        purge: yes

    - name: Install Docker 24.0.7
      apt:
        name:
          - docker-ce=5:24.0.7-1~ubuntu.22.04~jammy
          - docker-ce-cli=5:24.0.7-1~ubuntu.22.04~jammy
          - containerd.io=1.7.13-1~ubuntu.22.04~jammy
        state: present
        allow_downgrade: yes

    - name: Restart Docker service
      systemd:
        name: docker
        state: restarted
        enabled: yes

# Kustomization.yaml for 1000-replica stateless workload
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
  - deployment.yaml
  - service.yaml
  - hpa.yaml

replicas:
  - name: fintech-api
    count: 1000

images:
  - name: fintech/api
    newName: 123456789012.dkr.ecr.us-east-1.amazonaws.com/fintech-api
    newTag: v1.2.4

namespace: production