Under the Hood: Docker 25.0 BuildKit vs. Podman 5.0 Buildah Internals in 2026

#!/bin/bash
# build-benchmark.sh: Benchmark Docker 25.0 BuildKit vs Podman 5.0 Buildah build performance
# Methodology: 100 consecutive builds of a 3-stage Node.js 22 Dockerfile, 16 vCPU, 32GB RAM, Ubuntu 24.04 LTS
# Hardware: AWS c7g.4xlarge (ARM64) and c6i.4xlarge (x86_64), 1Gbps network
# Dependencies: docker 25.0.0, podman 5.0.0, buildah 1.33.0, jq 1.7

set -euo pipefail

# Configuration
DOCKER_VERSION="25.0.0"
PODMAN_VERSION="5.0.0"
BUILD_COUNT=100
DOCKERFILE_PATH="./node-multi-stage.Dockerfile"
IMAGE_NAME="node-bench"
RESULTS_DIR="./benchmark-results-$(date +%Y%m%d)"
mkdir -p "$RESULTS_DIR"

# Validate dependencies
validate_deps() {
  command -v docker >/dev/null 2>&1 || { echo "Docker $DOCKER_VERSION required"; exit 1; }
  command -v podman >/dev/null 2>&1 || { echo "Podman $PODMAN_VERSION required"; exit 1; }
  command -v buildah >/dev/null 2>&1 || { echo "Buildah 1.33.0 required"; exit 1; }
  command -v jq >/dev/null 2>&1 || { echo "jq 1.7 required"; exit 1; }
  # Check versions
  CURRENT_DOCKER=$(docker --version | awk '{print $3}' | tr -d ',')
  if [[ "$CURRENT_DOCKER" != "$DOCKER_VERSION" ]]; then
    echo "Docker version mismatch: expected $DOCKER_VERSION, got $CURRENT_DOCKER"
    exit 1
  fi
  CURRENT_PODMAN=$(podman --version | awk '{print $3}')
  if [[ "$CURRENT_PODMAN" != "$PODMAN_VERSION" ]]; then
    echo "Podman version mismatch: expected $PODMAN_VERSION, got $CURRENT_PODMAN"
    exit 1
  fi
}

# Run Docker BuildKit benchmark
run_docker_bench() {
  echo "Starting Docker BuildKit benchmark: $BUILD_COUNT builds"
  local results_file="$RESULTS_DIR/docker-build-times.json"
  echo "[]" > "$results_file"
  for i in $(seq 1 $BUILD_COUNT); do
    echo "Docker build $i/$BUILD_COUNT"
    # Clear build cache before each run to simulate cold start (optional: remove for warm cache)
    docker builder prune -af >/dev/null 2>&1 || true
    local start_time=$(date +%s%3N)
    docker buildx build --platform linux/amd64,linux/arm64 -t "$IMAGE_NAME:docker-$i" -f "$DOCKERFILE_PATH" . >/dev/null 2>&1
    local end_time=$(date +%s%3N)
    local duration=$((end_time - start_time))
    jq ". += [$duration]" "$results_file" > "$results_file.tmp" && mv "$results_file.tmp" "$results_file"
  done
  echo "Docker benchmark complete. Results: $results_file"
}

# Run Podman Buildah benchmark
run_podman_bench() {
  echo "Starting Podman Buildah benchmark: $BUILD_COUNT builds"
  local results_file="$RESULTS_DIR/podman-build-times.json"
  echo "[]" > "$results_file"
  for i in $(seq 1 $BUILD_COUNT); do
    echo "Podman build $i/$BUILD_COUNT"
    # Clear Buildah cache
    buildah rm --all >/dev/null 2>&1 || true
    buildah rmi --all >/dev/null 2>&1 || true
    local start_time=$(date +%s%3N)
    buildah bud --platform linux/amd64,linux/arm64 -t "$IMAGE_NAME:podman-$i" -f "$DOCKERFILE_PATH" . >/dev/null 2>&1
    local end_time=$(date +%s%3N)
    local duration=$((end_time - start_time))
    jq ". += [$duration]" "$results_file" > "$results_file.tmp" && mv "$results_file.tmp" "$results_file"
  done
  echo "Podman benchmark complete. Results: $results_file"
}

# Generate summary report
generate_report() {
  echo "Generating benchmark report..."
  local report_file="$RESULTS_DIR/summary.md"
  # Calculate Docker stats
  local docker_avg=$(jq 'add/length' "$RESULTS_DIR/docker-build-times.json")
  local docker_min=$(jq 'min' "$RESULTS_DIR/docker-build-times.json")
  local docker_max=$(jq 'max' "$RESULTS_DIR/docker-build-times.json")
  # Calculate Podman stats
  local podman_avg=$(jq 'add/length' "$RESULTS_DIR/podman-build-times.json")
  local podman_min=$(jq 'min' "$RESULTS_DIR/podman-build-times.json")
  local podman_max=$(jq 'max' "$RESULTS_DIR/podman-build-times.json")
  # Write report
  cat > "$report_file" << EOF
# Build Benchmark Summary
## Docker 25.0 BuildKit
- Average build time: ${docker_avg}ms
- Min build time: ${docker_min}ms
- Max build time: ${docker_max}ms

## Podman 5.0 Buildah
- Average build time: ${podman_avg}ms
- Min build time: ${podman_min}ms
- Max build time: ${podman_max}ms

## Delta
Docker is $(echo "scale=2; (($podman_avg - $docker_avg)/$podman_avg)*100" | bc)% faster on average.
EOF
  echo "Report generated: $report_file"
}

# Main execution
validate_deps
run_docker_bench
run_podman_bench
generate_report
echo "All benchmarks complete. Results in $RESULTS_DIR"

#!/usr/bin/env python3
# cache-analyzer.py: Compare Docker BuildKit LLB cache vs Podman Buildah layer cache efficiency
# Dependencies: docker 7.0.0, podman-client 0.3.0, python3-json, python3-sqlite3
# Methodology: Analyze cache hits/misses for 50 monorepo builds with 12 shared base images

import json
import subprocess
import sqlite3
import sys
from pathlib import Path
from typing import Dict, List, Tuple

class BuildCacheAnalyzer:
    def __init__(self, docker_version: str = "25.0.0", podman_version: str = "5.0.0"):
        self.docker_version = docker_version
        self.podman_version = podman_version
        self.results: Dict[str, Dict] = {"docker": {}, "podman": {}}

    def _validate_docker(self) -> None:
        """Validate Docker version and buildx availability"""
        try:
            docker_ver = subprocess.check_output(["docker", "--version"], text=True).split()[2].strip(",")
            if docker_ver != self.docker_version:
                raise ValueError(f"Docker version mismatch: expected {self.docker_version}, got {docker_ver}")
            # Check buildx is installed
            subprocess.check_call(["docker", "buildx", "version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        except subprocess.CalledProcessError as e:
            raise RuntimeError(f"Docker validation failed: {e.stderr}") from e

    def _validate_podman(self) -> None:
        """Validate Podman and Buildah versions"""
        try:
            podman_ver = subprocess.check_output(["podman", "--version"], text=True).split()[2]
            if podman_ver != self.podman_version:
                raise ValueError(f"Podman version mismatch: expected {self.podman_version}, got {podman_ver}")
            # Check buildah is available
            subprocess.check_call(["buildah", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        except subprocess.CalledProcessError as e:
            raise RuntimeError(f"Podman validation failed: {e.stderr}") from e

    def analyze_docker_cache(self, build_logs_dir: Path) -> Tuple[int, int]:
        """Parse Docker BuildKit build logs to count cache hits/misses"""
        hits = 0
        misses = 0
        for log_file in build_logs_dir.glob("docker-build-*.log"):
            with open(log_file, "r") as f:
                for line in f:
                    if "CACHED" in line:
                        hits += 1
                    elif "RUN" in line or "COPY" in line or "ADD" in line:
                        # Only count build steps that would use cache
                        if "CACHED" not in line:
                            misses += 1
        return hits, misses

    def analyze_podman_cache(self, build_logs_dir: Path) -> Tuple[int, int]:
        """Parse Buildah build logs to count cache hits/misses"""
        hits = 0
        misses = 0
        for log_file in build_logs_dir.glob("podman-build-*.log"):
            with open(log_file, "r") as f:
                for line in f:
                    if "Using cache" in line:
                        hits += 1
                    elif "RUN" in line or "COPY" in line or "ADD" in line:
                        if "Using cache" not in line:
                            misses += 1
        return hits, misses

    def analyze_docker_llb_cache(self, cache_db_path: Path) -> Dict:
        """Inspect Docker BuildKit's LLB content-addressable cache SQLite DB"""
        if not cache_db_path.exists():
            raise FileNotFoundError(f"Docker cache DB not found at {cache_db_path}")
        conn = sqlite3.connect(cache_db_path)
        cursor = conn.cursor()
        # Get total cache entries
        cursor.execute("SELECT COUNT(*) FROM cache_entries")
        total_entries = cursor.fetchone()[0]
        # Get cache size in bytes
        cursor.execute("SELECT SUM(size) FROM cache_entries")
        total_size = cursor.fetchone()[0] or 0
        conn.close()
        return {"total_entries": total_entries, "total_size_bytes": total_size}

    def analyze_buildah_cache(self, cache_dir: Path) -> Dict:
        """Inspect Buildah's layer cache directory"""
        if not cache_dir.exists():
            raise FileNotFoundError(f"Buildah cache dir not found at {cache_dir}")
        total_layers = sum(1 for _ in cache_dir.glob("*.tar"))
        total_size = sum(f.stat().st_size for f in cache_dir.glob("*.tar"))
        return {"total_layers": total_layers, "total_size_bytes": total_size}

    def run_analysis(self, logs_dir: Path, docker_cache_db: Path, buildah_cache_dir: Path) -> None:
        """Run full cache analysis"""
        self._validate_docker()
        self._validate_podman()

        print("Analyzing Docker BuildKit cache...")
        docker_hits, docker_misses = self.analyze_docker_cache(logs_dir)
        docker_llb = self.analyze_docker_llb_cache(docker_cache_db)
        self.results["docker"] = {
            "cache_hits": docker_hits,
            "cache_misses": docker_misses,
            "hit_rate": docker_hits / (docker_hits + docker_misses) if (docker_hits + docker_misses) > 0 else 0,
            "llb_entries": docker_llb["total_entries"],
            "llb_size_bytes": docker_llb["total_size_bytes"]
        }

        print("Analyzing Podman Buildah cache...")
        podman_hits, podman_misses = self.analyze_podman_cache(logs_dir)
        buildah_cache = self.analyze_buildah_cache(buildah_cache_dir)
        self.results["podman"] = {
            "cache_hits": podman_hits,
            "cache_misses": podman_misses,
            "hit_rate": podman_hits / (podman_hits + podman_misses) if (podman_hits + podman_misses) > 0 else 0,
            "layer_count": buildah_cache["total_layers"],
            "layer_size_bytes": buildah_cache["total_size_bytes"]
        }

    def generate_report(self, output_path: Path) -> None:
        """Generate JSON report of cache analysis"""
        with open(output_path, "w") as f:
            json.dump(self.results, f, indent=2)
        print(f"Cache analysis report written to {output_path}")

if __name__ == "__main__":
    try:
        analyzer = BuildCacheAnalyzer(docker_version="25.0.0", podman_version="5.0.0")
        logs_dir = Path("./benchmark-logs")
        docker_cache_db = Path("~/.cache/docker/buildkit/metadata.db").expanduser()
        buildah_cache_dir = Path("~/.local/share/containers/cache").expanduser()
        output_report = Path("./cache-analysis.json")

        if not logs_dir.exists():
            raise FileNotFoundError(f"Logs directory {logs_dir} not found. Run benchmarks first.")

        analyzer.run_analysis(logs_dir, docker_cache_db, buildah_cache_dir)
        analyzer.generate_report(output_report)
    except Exception as e:
        print(f"Analysis failed: {e}", file=sys.stderr)
        sys.exit(1)

// buildkit-inspector.go: Inspect Docker 25.0 BuildKit LLB cache internals
// Build: go build -o buildkit-inspector buildkit-inspector.go
// Dependencies: github.com/moby/buildkit/client v0.18.0, github.com/sirupsen/logrus v1.9.3
// Methodology: Connect to buildkitd v0.18.0, list all cache entries, calculate hit rates

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "os"
    "time"

    "github.com/moby/buildkit/client"
    "github.com/sirupsen/logrus"
)

const (
    buildkitdAddr = "unix:///run/buildkit/buildkitd.sock"
    timeout       = 30 * time.Second
)

type CacheEntry struct {
    ID        string `json:"id"`
    Digest    string `json:"digest"`
    Size      int64  `json:"size_bytes"`
    CreatedAt string `json:"created_at"`
    HitCount  int    `json:"hit_count"`
}

func main() {
    // Initialize logger
    logrus.SetFormatter(&logrus.JSONFormatter{})
    logrus.SetLevel(logrus.InfoLevel)

    // Validate buildkitd is running
    if err := validateBuildkitd(); err != nil {
        logrus.WithError(err).Fatal("buildkitd validation failed")
    }

    // Create BuildKit client
    ctx, cancel := context.WithTimeout(context.Background(), timeout)
    defer cancel()

    cli, err := client.New(ctx, buildkitdAddr)
    if err != nil {
        logrus.WithError(err).Fatal("failed to create BuildKit client")
    }
    defer cli.Close()

    // List all cache entries
    entries, err := listCacheEntries(ctx, cli)
    if err != nil {
        logrus.WithError(err).Fatal("failed to list cache entries")
    }

    // Calculate cache stats
    totalSize := int64(0)
    totalHits := 0
    for _, entry := range entries {
        totalSize += entry.Size
        totalHits += entry.HitCount
    }

    // Output report
    report := map[string]interface{}{
        "total_cache_entries": len(entries),
        "total_cache_size_bytes": totalSize,
        "total_cache_hits": totalHits,
        "average_hit_count": float64(totalHits) / float64(len(entries)),
        "entries": entries,
    }

    reportJSON, err := json.MarshalIndent(report, "", "  ")
    if err != nil {
        logrus.WithError(err).Fatal("failed to marshal report")
    }

    fmt.Println(string(reportJSON))
}

// validateBuildkitd checks if buildkitd is running and version matches 0.18.0
func validateBuildkitd() error {
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()

    cli, err := client.New(ctx, buildkitdAddr)
    if err != nil {
        return fmt.Errorf("buildkitd not reachable at %s: %w", buildkitdAddr, err)
    }
    defer cli.Close()

    // Get buildkitd version
    ver, err := cli.Version(ctx)
    if err != nil {
        return fmt.Errorf("failed to get buildkitd version: %w", err)
    }

    expectedVersion := "v0.18.0"
    if ver.Version != expectedVersion {
        return fmt.Errorf("buildkitd version mismatch: expected %s, got %s", expectedVersion, ver.Version)
    }

    logrus.WithField("version", ver.Version).Info("buildkitd validated")
    return nil
}

// listCacheEntries lists all LLB cache entries from BuildKit
func listCacheEntries(ctx context.Context, cli *client.Client) ([]CacheEntry, error) {
    // Use LLB to query cache (simplified for example; real implementation uses client's cache API)
    // Note: BuildKit's cache API is experimental, this uses the stable status API
    status, err := cli.Status(ctx)
    if err != nil {
        return nil, fmt.Errorf("failed to get buildkit status: %w", err)
    }

    var entries []CacheEntry
    // Parse status for cache info (simplified; actual cache listing requires experimental API)
    for _, s := range status {
        if s.ID == "cache" {
            // Mock cache entry for example; real implementation would parse cache manifest
            entries = append(entries, CacheEntry{
                ID:        "mock-cache-1",
                Digest:    "sha256:abc123",
                Size:      1024 * 1024, // 1MB
                CreatedAt: time.Now().Format(time.RFC3339),
                HitCount:  42,
            })
        }
    }

    logrus.WithField("count", len(entries)).Info("listed cache entries")
    return entries, nil
}

docker buildx build \
  --platform linux/amd64,linux/arm64 \
  --cache-from type=registry,ref=ghcr.io/myorg/build-cache:latest \
  --cache-to type=registry,ref=ghcr.io/myorg/build-cache:latest,mode=max \
  -t ghcr.io/myorg/myapp:$(git rev-parse HEAD) \
  -f Dockerfile \
  .

buildah bud \
  --platform linux/amd64 \
  --isolation chroot \
  --user 1000:1000 \
  --layers \
  -t ghcr.io/myorg/myapp:$(git rev-parse HEAD) \
  -f Dockerfile \
  .

# Run 50 warm cache builds for your production workload
BUILD_COUNT=50 ./build-benchmark.sh
# Generate report
python3 cache-analyzer.py