War Story: Debugging High Observability Overhead in Datadog 2025 and New Relic 2025 for Our 1k Container Cluster

// agent_overhead_bench.go measures CPU and memory overhead of Datadog 2025 and New Relic 2025 agents
// across simulated container workloads. Uses cgroups v2 to isolate agent resource consumption.
package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "os/exec"
    "path/filepath"
    "runtime"
    "strconv"
    "strings"
    "time"

    "github.com/shirou/gopsutil/v4/cpu"
    "github.com/shirou/gopsutil/v4/mem"
)

const (
    datadogAgentVersion  = "7.58.0-2025.04.1" // Datadog 2025.4.1 agent
    newRelicAgentVersion = "3.24.1-2025.03.0" // New Relic 2025.3.0 agent
    simulatedContainers  = 1024               // Match our production cluster size
    benchDuration        = 5 * time.Minute    // Run benchmark for 5 minutes per agent
)

// agentConfig holds configuration for a single observability agent
type agentConfig struct {
    name    string
    version string
    image   string // Docker image for agent
    cmd     []string // Start command for agent
}

func main() {
    ctx, cancel := context.WithTimeout(context.Background(), benchDuration*2) // Buffer for setup/teardown
    defer cancel()

    fmt.Printf("Starting observability agent overhead benchmark\n")
    fmt.Printf("Simulating %d containers, benchmark duration %v per agent\n", simulatedContainers, benchDuration)

    agents := []agentConfig{
        {
            name:    "Datadog 2025",
            version: datadogAgentVersion,
            image:   "gcr.io/datadog/agent:7.58.0-2025.04.1",
            cmd:     []string{"agent", "run", "--config=/etc/datadog-agent/datadog.yaml"},
        },
        {
            name:    "New Relic 2025",
            version: newRelicAgentVersion,
            image:   "gcr.io/newrelic/infrastructure:3.24.1-2025.03.0",
            cmd:     []string{"nri-agent", "--config=/etc/newrelic/nri-agent.yaml"},
        },
    }

    for _, agent := range agents {
        fmt.Printf("\n--- Benchmarking %s (v%s) ---\n", agent.name, agent.version)
        if err := runAgentBenchmark(ctx, agent); err != nil {
            log.Printf("Failed to benchmark %s: %v", agent.name, err)
            continue
        }
    }

    fmt.Println("\nBenchmark complete. Results written to overhead_results.csv")
}

// runAgentBenchmark starts the agent, simulates container workload, measures resource usage
func runAgentBenchmark(ctx context.Context, agent agentConfig) error {
    // Pull agent image
    pullCmd := exec.CommandContext(ctx, "docker", "pull", agent.image)
    pullCmd.Stdout = os.Stdout
    pullCmd.Stderr = os.Stderr
    if err := pullCmd.Run(); err != nil {
        return fmt.Errorf("failed to pull agent image: %w", err)
    }

    // Start agent container with cgroup limits to measure isolated usage
    startCmd := exec.CommandContext(ctx, "docker", "run", "-d",
        "--name", fmt.Sprintf("agent-bench-%s", strings.ReplaceAll(agent.name, " ", "-")),
        "--cgroup-parent", "/sys/fs/cgroup/benchmark",
        "-v", "/var/run/docker.sock:/var/run/docker.sock",
        "-v", "/proc:/host/proc:ro",
        "-v", "/sys/fs/cgroup:/host/sys/fs/cgroup:ro",
        agent.image,
        agent.cmd...,
    )
    agentContainerID, err := startCmd.Output()
    if err != nil {
        return fmt.Errorf("failed to start agent container: %w", err)
    }
    agentContainerID = strings.TrimSpace(string(agentContainerID))
    defer func() {
        // Cleanup agent container
        exec.Command("docker", "rm", "-f", agentContainerID).Run()
    }()

    // Wait for agent to initialize
    time.Sleep(30 * time.Second)

    // Simulate container workload: start 1024 mock containers generating telemetry
    simulateWorkload(ctx, simulatedContainers)

    // Measure CPU and memory usage for benchDuration
    cpuUsage, memUsage, err := measureAgentUsage(ctx, agentContainerID, benchDuration)
    if err != nil {
        return fmt.Errorf("failed to measure agent usage: %w", err)
    }

    // Log results
    fmt.Printf("%s Results:\n", agent.name)
    fmt.Printf("  Average CPU Usage: %.2f%%\n", cpuUsage)
    fmt.Printf("  Average Memory Usage: %.2f MB\n", memUsage)

    // Write to CSV
    writeResult(agent.name, agent.version, cpuUsage, memUsage)

    return nil
}

// simulateWorkload starts mock containers that generate minimal telemetry
func simulateWorkload(ctx context.Context, count int) {
    fmt.Printf("Simulating %d workload containers...\n", count)
    for i := 0; i < count; i++ {
        go func(id int) {
            cmd := exec.CommandContext(ctx, "docker", "run", "-d",
                "--name", fmt.Sprintf("workload-%d", id),
                "busybox", "sh", "-c", "while true; do echo 'telemetry payload'; sleep 1; done",
            )
            cmd.Run()
        }(i)
    }
    time.Sleep(10 * time.Second) // Wait for all containers to start
}

// measureAgentUsage uses cgroups to get accurate resource usage for the agent container
func measureAgentUsage(ctx context.Context, containerID string, duration time.Duration) (float64, float64, error) {
    // Get cgroup path for container
    cgroupPath, err := getContainerCgroupPath(containerID)
    if err != nil {
        return 0, 0, err
    }

    // Read CPU usage before
    cpuBefore, err := readCgroupCPUUsage(cgroupPath)
    if err != nil {
        return 0, 0, err
    }
    memUsage, err := readCgroupMemUsage(cgroupPath)
    if err != nil {
        return 0, 0, err
    }

    // Wait for duration
    select {
    case <-ctx.Done():
        return 0, 0, ctx.Err()
    case <-time.After(duration):
    }

    // Read CPU usage after
    cpuAfter, err := readCgroupCPUUsage(cgroupPath)
    if err != nil {
        return 0, 0, err
    }

    // Calculate CPU percentage (total CPU time / duration / number of cores)
    numCores, _ := cpu.Counts(true)
    cpuDelta := cpuAfter - cpuBefore
    cpuPercent := (float64(cpuDelta) / float64(duration.Nanoseconds())) * 100 * float64(numCores)

    return cpuPercent, float64(memUsage) / 1024 / 1024, nil // Convert mem to MB
}

// getContainerCgroupPath finds the cgroup path for a Docker container
func getContainerCgroupPath(containerID string) (string, error) {
    cmd := exec.Command("docker", "inspect", "-f", "{{.HostConfig.CgroupParent}}", containerID)
    output, err := cmd.Output()
    if err != nil {
        return "", err
    }
    return strings.TrimSpace(string(output)), nil
}

// readCgroupCPUUsage reads total CPU usage in nanoseconds from cgroup
func readCgroupCPUUsage(cgroupPath string) (uint64, error) {
    path := filepath.Join(cgroupPath, "cpu.stat")
    data, err := os.ReadFile(path)
    if err != nil {
        return 0, err
    }
    lines := strings.Split(string(data), "\\n")
    for _, line := range lines {
        if strings.HasPrefix(line, "usage_usec") {
            parts := strings.Fields(line)
            val, _ := strconv.ParseUint(parts[1], 10, 64)
            return val * 1000, nil // Convert usec to nsec
        }
    }
    return 0, fmt.Errorf("cpu usage not found in %s", path)
}

// readCgroupMemUsage reads memory usage in bytes from cgroup
func readCgroupMemUsage(cgroupPath string) (uint64, error) {
    path := filepath.Join(cgroupPath, "memory.current")
    data, err := os.ReadFile(path)
    if err != nil {
        return 0, err
    }
    val, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
    return val, err
}

// writeResult appends benchmark results to CSV
func writeResult(name, version string, cpu, mem float64) {
    f, err := os.OpenFile("overhead_results.csv", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
    if err != nil {
        log.Printf("Failed to write results: %v", err)
        return
    }
    defer f.Close()
    fmt.Fprintf(f, "%s,%s,%.2f,%.2f\\n", name, version, cpu, mem)
}

# optimize_datadog_config.py disables unused Datadog 2025 agent features to reduce overhead
# Generates a minimal datadog.yaml config for 1k container clusters
import os
import sys
import yaml
import argparse
import logging
from pathlib import Path
from typing import Dict, Any

# Default Datadog 2025.4.1 config path
DEFAULT_CONFIG_PATH = "/etc/datadog-agent/datadog.yaml"
# Features to disable that are unused in our cluster
UNUSED_FEATURES = [
    "process_config.enabled",
    "container_image.enabled",
    "security_agent.enabled",
    "runtime_security_config.enabled",
    "apm_config.enabled",  # We use New Relic for APM
    "dogstatsd_non_local_traffic",  # Only accept local traffic
    "logs_enabled",  # We use a separate log pipeline
]

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)

def load_config(config_path: str) -> Dict[str, Any]:
    """Load existing Datadog config, return empty dict if not found"""
    config_path = Path(config_path)
    if not config_path.exists():
        logger.warning(f"Config path {config_path} not found, starting with empty config")
        return {}
    try:
        with open(config_path, "r") as f:
            return yaml.safe_load(f) or {}
    except yaml.YAMLError as e:
        logger.error(f"Failed to parse config: {e}")
        raise
    except Exception as e:
        logger.error(f"Failed to load config: {e}")
        raise

def disable_unused_features(config: Dict[str, Any]) -> Dict[str, Any]:
    """Recursively disable unused features in config"""
    for feature_path in UNUSED_FEATURES:
        parts = feature_path.split(".")
        current = config
        for i, part in enumerate(parts[:-1]):
            if part not in current:
                current[part] = {}
            current = current[part]
        # Set the final key to False
        current[parts[-1]] = False
        logger.info(f"Disabled feature: {feature_path}")
    return config

def optimize_batching(config: Dict[str, Any]) -> Dict[str, Any]:
    """Optimize telemetry batching to reduce CPU overhead"""
    # Increase batch size and interval to reduce send frequency
    config.setdefault("telemetry", {})
    config["telemetry"]["max_batch_size"] = 5000  # Default is 1000
    config["telemetry"]["batch_interval"] = 30  # Default is 10s
    config["telemetry"]["compression"] = "zstd"  # Use zstd instead of gzip
    logger.info("Optimized telemetry batching: batch size 5000, interval 30s, zstd compression")

    # Optimize dogstatsd if enabled
    if config.get("dogstatsd_port"):
        config.setdefault("dogstatsd_config", {})
        config["dogstatsd_config"]["batch_size"] = 4096
        config["dogstatsd_config"]["batch_wait"] = 5  # Seconds
        logger.info("Optimized dogstatsd batching")

    return config

def write_config(config: Dict[str, Any], output_path: str) -> None:
    """Write optimized config to output path"""
    output_path = Path(output_path)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    try:
        with open(output_path, "w") as f:
            yaml.dump(config, f, default_flow_style=False, sort_keys=False)
        logger.info(f"Wrote optimized config to {output_path}")
    except Exception as e:
        logger.error(f"Failed to write config: {e}")
        raise

def validate_config(config: Dict[str, Any]) -> bool:
    """Validate config has required fields for agent to start"""
    required = ["api_key", "site"]
    missing = [field for field in required if field not in config]
    if missing:
        logger.error(f"Missing required config fields: {missing}")
        return False
    return True

def main():
    parser = argparse.ArgumentParser(description="Optimize Datadog 2025 agent config for overhead reduction")
    parser.add_argument("--input", default=DEFAULT_CONFIG_PATH, help="Path to existing datadog.yaml")
    parser.add_argument("--output", default="/etc/datadog-agent/datadog-optimized.yaml", help="Path to write optimized config")
    parser.add_argument("--dry-run", action="store_true", help="Print optimized config without writing")
    args = parser.parse_args()

    try:
        logger.info(f"Loading config from {args.input}")
        config = load_config(args.input)

        logger.info("Disabling unused features")
        config = disable_unused_features(config)

        logger.info("Optimizing telemetry batching")
        config = optimize_batching(config)

        if not validate_config(config):
            sys.exit(1)

        if args.dry_run:
            print(yaml.dump(config, default_flow_style=False, sort_keys=False))
            sys.exit(0)

        write_config(config, args.output)
        logger.info(f"Optimization complete. Restart agent with: systemctl restart datadog-agent")

    except Exception as e:
        logger.error(f"Optimization failed: {e}")
        sys.exit(1)

if __name__ == "__main__":
    main()

# k8s_agent_quotas.tf enforces CPU and memory quotas for Datadog and New Relic agents
# in Kubernetes to prevent observability overhead from consuming node resources
# Compatible with Kubernetes 1.29+ and Datadog 2025/New Relic 2025 agents

terraform {
  required_version = ">= 1.7.0"
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.30.0"
    }
  }
}

variable "cluster_name" {
  type        = string
  description = "Name of the target Kubernetes cluster"
  default     = "prod-1k-container-cluster"
}

variable "datadog_agent_namespace" {
  type        = string
  description = "Namespace where Datadog agent is deployed"
  default     = "datadog"
}

variable "newrelic_agent_namespace" {
  type        = string
  description = "Namespace where New Relic agent is deployed"
  default     = "newrelic"
}

variable "agent_cpu_limit" {
  type        = string
  description = "Maximum CPU limit for each observability agent pod (in millicores)"
  default     = "500m" # 0.5 CPU core per agent pod, max 500 cores for 1k nodes
}

variable "agent_memory_limit" {
  type        = string
  description = "Maximum memory limit for each observability agent pod"
  default     = "512Mi"
}

# Create LimitRange for Datadog agent namespace to enforce per-pod quotas
resource "kubernetes_limit_range" "datadog_agent_limits" {
  metadata {
    name      = "datadog-agent-limit-range"
    namespace = var.datadog_agent_namespace
  }

  spec {
    limit {
      type = "Container"
      default = {
        cpu    = var.agent_cpu_limit
        memory = var.agent_memory_limit
      }
      default_request = {
        cpu    = "100m"
        memory = "128Mi"
      }
      max = {
        cpu    = var.agent_cpu_limit
        memory = var.agent_memory_limit
      }
      min = {
        cpu    = "50m"
        memory = "64Mi"
      }
    }
  }
}

# Create LimitRange for New Relic agent namespace
resource "kubernetes_limit_range" "newrelic_agent_limits" {
  metadata {
    name      = "newrelic-agent-limit-range"
    namespace = var.newrelic_agent_namespace
  }

  spec {
    limit {
      type = "Container"
      default = {
        cpu    = var.agent_cpu_limit
        memory = var.agent_memory_limit
      }
      default_request = {
        cpu    = "100m"
        memory = "128Mi"
      }
      max = {
        cpu    = var.agent_cpu_limit
        memory = var.agent_memory_limit
      }
      min = {
        cpu    = "50m"
        memory = "64Mi"
      }
    }
  }
}

# Create ResourceQuota for Datadog namespace to enforce total cluster agent usage
resource "kubernetes_resource_quota" "datadog_cluster_quota" {
  metadata {
    name      = "datadog-cluster-agent-quota"
    namespace = var.datadog_agent_namespace
  }

  spec {
    hard = {
      "cpu"    = "500" # 500 cores total for all Datadog agents (1k pods * 0.5 core max)
      "memory" = "512Gi" # 512GB total memory for all Datadog agents
      "pods"   = "1024" # Max 1 agent pod per node
    }
  }
}

# Create ResourceQuota for New Relic namespace
resource "kubernetes_resource_quota" "newrelic_cluster_quota" {
  metadata {
    name      = "newrelic-cluster-agent-quota"
    namespace = var.newrelic_agent_namespace
  }

  spec {
    hard = {
      "cpu"    = "500" # 500 cores total for all New Relic agents
      "memory" = "512Gi"
      "pods"   = "1024"
    }
  }
}

# Output quota details for validation
output "datadog_agent_cpu_limit" {
  value       = var.agent_cpu_limit
  description = "CPU limit per Datadog agent pod"
}

output "newrelic_agent_cpu_limit" {
  value       = var.agent_cpu_limit
  description = "CPU limit per New Relic agent pod"
}

output "total_agent_cpu_quota" {
  value       = "1000 cores (500 Datadog + 500 New Relic)"
  description = "Total maximum CPU allocated to all observability agents"
}

// Snippet from agent_overhead_bench.go: Isolate agent CPU via cgroups
func measureAgentUsage(ctx context.Context, containerID string, duration time.Duration) (float64, float64, error) {
    cgroupPath, err := getContainerCgroupPath(containerID)
    if err != nil {
        return 0, 0, err
    }
    cpuBefore, _ := readCgroupCPUUsage(cgroupPath)
    time.Sleep(duration)
    cpuAfter, _ := readCgroupCPUUsage(cgroupPath)
    numCores, _ := cpu.Counts(true)
    cpuDelta := cpuAfter - cpuBefore
    cpuPercent := (float64(cpuDelta) / float64(duration.Nanoseconds())) * 100 * float64(numCores)
    return cpuPercent, 0, nil
}

# Snippet from optimize_datadog_config.py: Disable unused features
UNUSED_FEATURES = [
    "process_config.enabled",
    "container_image.enabled",
    "security_agent.enabled",
    "apm_config.enabled",
    "logs_enabled",
]

def disable_unused_features(config: Dict[str, Any]) -> Dict[str, Any]:
    for feature_path in UNUSED_FEATURES:
        parts = feature_path.split(".")
        current = config
        for i, part in enumerate(parts[:-1]):
            if part not in current:
                current[part] = {}
            current = current[part]
        current[parts[-1]] = False
    return config

# Snippet from k8s_agent_quotas.tf: Enforce per-pod CPU limit
resource "kubernetes_limit_range" "datadog_agent_limits" {
  metadata {
    name      = "datadog-agent-limit-range"
    namespace = var.datadog_agent_namespace
  }
  spec {
    limit {
      type = "Container"
      max = {
        cpu    = var.agent_cpu_limit # 500m
        memory = var.agent_memory_limit
      }
    }
  }
}