Hot Take: Most Companies Waste Money on APM Tools Like New Relic 2026

import os
import time
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
import uvicorn

# -----------------------------------------------------------------------
# Scenario: Compare New Relic vs OpenTelemetry instrumentation overhead
# for a production-grade FastAPI service processing 1k RPM
# -----------------------------------------------------------------------

# Configure logging for both agents
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Conditional agent loading to avoid conflicts in benchmark
USE_NEW_RELIC = os.getenv(\"USE_NEW_RELIC\", \"false\").lower() == \"true\"
USE_OTEL = os.getenv(\"USE_OTEL\", \"false\").lower() == \"true\"

if USE_NEW_RELIC:
    # New Relic Python agent v8.12.0 (latest as of 2026 Q1)
    import newrelic.agent
    newrelic.agent.initialize('newrelic.ini', 'production')
    logger.info(\"New Relic agent initialized\")
    # Wrap FastAPI with New Relic middleware
    from newrelic.agent import ASGIMiddleware
    nr_middleware = ASGIMiddleware

if USE_OTEL:
    # OpenTelemetry Python SDK v1.24.0 + FastAPI instrumentation
    from opentelemetry import trace
    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
    from opentelemetry.sdk.trace import TracerProvider
    from opentelemetry.sdk.trace.export import BatchSpanProcessor
    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
    from opentelemetry.sdk.resources import Resource

    # Define resource with service metadata
    resource = Resource.create({
        \"service.name\": \"fastapi-benchmark\",
        \"service.version\": \"1.0.0\",
        \"deployment.environment\": \"benchmark\"
    })
    trace.set_tracer_provider(TracerProvider(resource=resource))
    # Export to local OTEL collector for benchmarking
    otlp_exporter = OTLPSpanExporter(endpoint=\"localhost:4317\", insecure=True)
    span_processor = BatchSpanProcessor(otlp_exporter)
    trace.get_tracer_provider().add_span_processor(span_processor)
    logger.info(\"OpenTelemetry initialized\")

@asynccontextmanager
async def lifespan(app: FastAPI):
    # Startup logic
    logger.info(\"FastAPI service starting up\")
    yield
    # Shutdown logic
    if USE_NEW_RELIC:
        newrelic.agent.shutdown()
    if USE_OTEL:
        trace.get_tracer_provider().shutdown()
    logger.info(\"FastAPI service shut down\")

app = FastAPI(lifespan=lifespan)

if USE_OTEL:
    # Instrument FastAPI with OTel
    FastAPIInstrumentor.instrument_app(app)

if USE_NEW_RELIC:
    # Wrap app with New Relic ASGI middleware
    app = nr_middleware(app)

@app.get(\"/benchmark\")
async def benchmark_endpoint(request: Request):
    \"\"\"Simulate a typical CRUD endpoint with DB call and external API call\"\"\"
    start = time.perf_counter()
    try:
        # Simulate 10ms DB query
        time.sleep(0.01)
        # Simulate 20ms external API call
        time.sleep(0.02)
        # Simulate 5ms processing
        time.sleep(0.005)
        latency = (time.perf_counter() - start) * 1000
        return {\"status\": \"ok\", \"latency_ms\": round(latency, 2)}
    except Exception as e:
        logger.error(f\"Benchmark endpoint error: {str(e)}\")
        raise HTTPException(status_code=500, detail=\"Internal server error\")

if __name__ == \"__main__\":
    # Run with uvicorn, 4 workers for benchmark
    uvicorn.run(
        app,
        host=\"0.0.0.0\",
        port=8000,
        workers=4,
        log_level=\"info\"
    )

package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"time\"

    // New Relic Go agent v3.24.1 (latest as of 2026 Q1)
    newrelic \"github.com/newrelic/go-agent/v3/newrelic\"

    // OpenTelemetry Go SDK v1.28.0
    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    sdktrace \"go.opentelemetry.io/otel/sdk/trace\"
    \"go.opentelemetry.io/otel/trace\"
    \"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\"
)

const (
    serviceName    = \"go-benchmark-service\"
    serviceVersion = \"1.0.0\"
)

func main() {
    // Conditional agent initialization based on env vars
    useNewRelic := os.Getenv(\"USE_NEW_RELIC\") == \"true\"
    useOtel := os.Getenv(\"USE_OTEL\") == \"true\"

    var nrApp *newrelic.Application
    var tracer trace.Tracer

    // Initialize New Relic if enabled
    if useNewRelic {
        app, err := newrelic.NewApplication(
            newrelic.ConfigAppName(serviceName),
            newrelic.ConfigLicense(os.Getenv(\"NEW_RELIC_LICENSE_KEY\")),
            newrelic.ConfigDistributedTracingEnabled(true),
            newrelic.ConfigAppLogForwardingEnabled(true),
        )
        if err != nil {
            log.Fatalf(\"Failed to initialize New Relic: %v\", err)
        }
        // Wait for app to connect to New Relic
        if err := app.WaitForConnection(5 * time.Second); err != nil {
            log.Fatalf(\"New Relic connection timeout: %v\", err)
        }
        nrApp = app
        log.Println(\"New Relic Go agent initialized\")
    }

    // Initialize OpenTelemetry if enabled
    if useOtel {
        ctx := context.Background()
        // Configure OTLP exporter to send to local collector
        exporter, err := otlptracegrpc.New(ctx,
            otlptracegrpc.WithInsecure(),
            otlptracegrpc.WithEndpoint(\"localhost:4317\"),
        )
        if err != nil {
            log.Fatalf(\"Failed to create OTLP exporter: %v\", err)
        }

        // Define resource with service metadata
        res, err := resource.New(ctx,
            resource.WithAttributes(
                attribute.String(\"service.name\", serviceName),
                attribute.String(\"service.version\", serviceVersion),
                attribute.String(\"deployment.environment\", \"benchmark\"),
            ),
        )
        if err != nil {
            log.Fatalf(\"Failed to create resource: %v\", err)
        }

        // Configure trace provider with batch processor
        tp := sdktrace.NewTracerProvider(
            sdktrace.WithBatcher(exporter),
            sdktrace.WithResource(res),
        )
        otel.SetTracerProvider(tp)
        otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))

        // Shutdown trace provider on exit
        defer func() {
            if err := tp.Shutdown(ctx); err != nil {
                log.Printf(\"Error shutting down trace provider: %v\", err)
            }
        }()

        tracer = otel.Tracer(serviceName)
        log.Println(\"OpenTelemetry Go SDK initialized\")
    }

    // Define benchmark handler
    benchmarkHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        // Simulate 15ms DB query
        time.Sleep(15 * time.Millisecond)
        // Simulate 25ms external API call
        time.Sleep(25 * time.Millisecond)
        // Simulate 10ms processing
        time.Sleep(10 * time.Millisecond)

        latency := time.Since(start).Milliseconds()
        w.Header().Set(\"Content-Type\", \"application/json\")
        w.WriteHeader(http.StatusOK)
        fmt.Fprintf(w, `{\"status\": \"ok\", \"latency_ms\": %d}`, latency)
    })

    // Wrap handler with instrumentation
    var handler http.Handler = benchmarkHandler
    if useNewRelic {
        // Wrap with New Relic transaction
        handler = nrApp.WrapHandleFunc(benchmarkHandler)
    }
    if useOtel {
        // Wrap with OpenTelemetry instrumentation
        handler = otelhttp.NewHandler(benchmarkHandler, \"benchmark-endpoint\")
    }

    // Register routes
    http.Handle(\"/benchmark\", handler)
    http.HandleFunc(\"/health\", func(w http.ResponseWriter, r *http.Request) {
        w.WriteHeader(http.StatusOK)
        fmt.Fprint(w, \"healthy\")
    })

    // Start server
    port := os.Getenv(\"PORT\")
    if port == \"\" {
        port = \"8080\"
    }
    log.Printf(\"Starting server on port %s\", port)
    if err := http.ListenAndServe(fmt.Sprintf(\":%s\", port), nil); err != nil {
        log.Fatalf(\"Server failed: %v\", err)
    }
}

import os
import sys
import time
import json
import logging
import argparse
from locust import HttpUser, task, between, events
from locust.runners import STATE_STOPPING, STATE_STOPPED, STATE_CLEANUP, WorkerRunner
import statistics

# -----------------------------------------------------------------------
# Benchmark script to measure APM agent overhead for FastAPI/Go services
# Runs 10k requests across 50 users, measures p50/p95/p99 latency
# -----------------------------------------------------------------------

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Track latency per instrumentation type
latency_results = {
    \"baseline\": [],
    \"new_relic\": [],
    \"open_telemetry\": []
}

class BenchmarkUser(HttpUser):
    \"\"\"Locust user to simulate load against benchmark endpoints\"\"\"
    # Wait 100-500ms between requests to simulate realistic traffic
    wait_time = between(0.1, 0.5)
    host = \"http://localhost:8000\"  # Default to FastAPI port

    @task(1)
    def call_benchmark(self):
        \"\"\"Call the benchmark endpoint and record latency\"\"\"
        start = time.perf_counter()
        try:
            response = self.client.get(\"/benchmark\")
            if response.status_code != 200:
                logger.error(f\"Request failed with status {response.status_code}\")
                return
            latency = (time.perf_counter() - start) * 1000  # ms
            # Append to appropriate results bucket based on env
            instrument_type = os.getenv(\"INSTRUMENT_TYPE\", \"baseline\")
            if instrument_type in latency_results:
                latency_results[instrument_type].append(latency)
        except Exception as e:
            logger.error(f\"Request error: {str(e)}\")

    @task(0.1)
    def call_health(self):
        \"\"\"Call health endpoint to simulate mixed traffic\"\"\"
        self.client.get(\"/health\")

def calculate_stats(results):
    \"\"\"Calculate p50, p95, p99, mean from latency list\"\"\"
    if not results:
        return {}
    sorted_results = sorted(results)
    return {
        \"p50\": round(statistics.median(sorted_results[:int(len(sorted_results)*0.5)]), 2),
        \"p95\": round(statistics.quantiles(sorted_results, n=20)[18], 2),  # 95th percentile
        \"p99\": round(statistics.quantiles(sorted_results, n=100)[98], 2),  # 99th percentile
        \"mean\": round(statistics.mean(sorted_results), 2),
        \"samples\": len(results)
    }

@events.test_stop.add_listener
def on_test_stop(environment, **_kwargs):
    \"\"\"Print benchmark results when test stops\"\"\"
    if not isinstance(environment.runner, WorkerRunner):
        logger.info(\"\\n\" + \"=\"*50)
        logger.info(\"BENCHMARK RESULTS\")
        logger.info(\"=\"*50)
        for instrument_type, results in latency_results.items():
            stats = calculate_stats(results)
            if stats:
                logger.info(f\"\\nInstrumentation: {instrument_type.upper()}\")
                logger.info(f\"Samples: {stats['samples']}\")
                logger.info(f\"Mean Latency: {stats['mean']}ms\")
                logger.info(f\"P50 Latency: {stats['p50']}ms\")
                logger.info(f\"P95 Latency: {stats['p95']}ms\")
                logger.info(f\"P99 Latency: {stats['p99']}ms\")
        logger.info(\"=\"*50)

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"Run APM overhead benchmark\")
    parser.add_argument(\"--host\", type=str, default=\"http://localhost:8000\", help=\"Target service host\")
    parser.add_argument(\"--users\", type=int, default=50, help=\"Number of concurrent users\")
    parser.add_argument(\"--spawn-rate\", type=int, default=10, help=\"User spawn rate per second\")
    parser.add_argument(\"--run-time\", type=str, default=\"5m\", help=\"Test run time (e.g., 5m, 30s)\")
    parser.add_argument(\"--instrument-type\", type=str, choices=[\"baseline\", \"new_relic\", \"open_telemetry\"], default=\"baseline\", help=\"Instrumentation type being tested\")
    args = parser.parse_args()

    # Set env var for the user class
    os.environ[\"INSTRUMENT_TYPE\"] = args.instrument_type
    BenchmarkUser.host = args.host

    # Run locust programmatically (avoids needing command line)
    from locust.env import Environment
    from locust.stats import stats_printer, stats_history
    from locust.log import setup_logging

    setup_logging(\"INFO\", None)
    env = Environment(user_classes=[BenchmarkUser])
    env.create_local_runner()
    env.runner.start(args.users, spawn_rate=args.spawn_rate)
    logger.info(f\"Starting benchmark for {args.instrument_type} - {args.users} users, {args.run_time} run time\")

    # Stop test after run time
    time.sleep(int(args.run_time.replace(\"m\", \"\").replace(\"s\", \"\")) * (60 if \"m\" in args.run_time else 1))
    env.runner.quit()
    on_test_stop(env)

import os
import requests
import json

# New Relic API key (from https://one.newrelic.com/api-keys)
API_KEY = os.getenv(\"NEW_RELIC_API_KEY\")
ACCOUNT_ID = os.getenv(\"NEW_RELIC_ACCOUNT_ID\")

def get_feature_usage():
    \"\"\"Pull feature usage data from New Relic API v2\"\"\"
    url = f\"https://api.newrelic.com/v2/accounts/{ACCOUNT_ID}/usage/features.json\"
    headers = {
        \"X-Api-Key\": API_KEY,
        \"Content-Type\": \"application/json\"
    }
    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        raise Exception(f\"API request failed: {response.status_code} {response.text}\")
    return response.json()

if __name__ == \"__main__\":
    try:
        usage = get_feature_usage()
        print(json.dumps(usage, indent=2))
    except Exception as e:
        print(f\"Error: {str(e)}\")

# Auto-instrument FastAPI with OpenTelemetry in one line
opentelemetry-instrument --traces_exporter otlp \
  --metrics_exporter otlp \
  --service_name fastapi-service \
  uvicorn main:app --host 0.0.0.0 --port 8000

# Prometheus scrape config for Kubernetes pods
scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+):?(\\d+)?;(\\d+)
        target_label: __address__
        replacement: $1:$3