We Ditched New Relic 2025 for OpenTelemetry 1.22: Saved $150k Annually on Observability

// otel_migration_example.go
// Demonstrates full OpenTelemetry 1.22 instrumentation for a Go HTTP service
// replacing New Relic Go agent v3.18
package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"time\"

    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/prometheus\"
    \"go.opentelemetry.io/otel/metric\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/metric\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    \"go.opentelemetry.io/otel/sdk/trace\"
    \"go.opentelemetry.io/otel/trace\"
    \"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
)

// define custom metric instruments
var (
    requestCounter metric.Int64Counter
    requestDuration metric.Float64Histogram
    errorCounter metric.Int64Counter
)

func initOtel() (func(), error) {
    // 1. Configure resource with service metadata (replaces New Relic app_name config)
    res, err := resource.New(context.Background(),
        resource.WithAttributes(
            attribute.Key(\"service.name\").String(\"payment-processor\"),
            attribute.Key(\"service.version\").String(\"1.4.2\"),
            attribute.Key(\"deployment.environment\").String(\"production\"),
            attribute.Key(\"team\").String(\"fintech-backend\"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create resource: %w\", err)
    }

    // 2. Configure OTLP trace exporter (sends to Jaeger/Tempo, replaces New Relic trace endpoint)
    traceClient := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(\"otel-collector:4317\"),
    )
    traceExporter, err := otlptrace.New(context.Background(), traceClient)
    if err != nil {
        return nil, fmt.Errorf(\"failed to create trace exporter: %w\", err)
    }
    tracerProvider := trace.NewTracerProvider(
        trace.WithBatcher(traceExporter, trace.WithBatchTimeout(5*time.Second)),
        trace.WithResource(res),
    )
    otel.SetTracerProvider(tracerProvider)

    // 3. Configure Prometheus metric exporter (replaces New Relic metric API)
    promExporter, err := prometheus.New()
    if err != nil {
        return nil, fmt.Errorf(\"failed to create prometheus exporter: %w\", err)
    }
    meterProvider := metric.NewMeterProvider(
        metric.WithReader(promExporter),
        metric.WithResource(res),
    )
    otel.SetMeterProvider(meterProvider)

    // 4. Set global propagator for distributed tracing (replaces New Relic cross-request context)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    // Initialize custom metrics
    meter := otel.GetMeterProvider().Meter(\"payment-processor\")
    requestCounter, err = meter.Int64Counter(\"http.requests.total\",
        metric.WithDescription(\"Total HTTP requests processed\"),
        metric.WithUnit(\"1\"),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create request counter: %w\", err)
    }
    requestDuration, err = meter.Float64Histogram(\"http.request.duration.seconds\",
        metric.WithDescription(\"HTTP request duration distribution\"),
        metric.WithUnit(\"s\"),
        metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 5),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create duration histogram: %w\", err)
    }
    errorCounter, err = meter.Int64Counter(\"http.request.errors.total\",
        metric.WithDescription(\"Total HTTP 5xx errors\"),
        metric.WithUnit(\"1\"),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create error counter: %w\", err)
    }

    // Return cleanup function
    cleanup := func() {
        ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
        defer cancel()
        if err := tracerProvider.Shutdown(ctx); err != nil {
            log.Printf(\"failed to shutdown tracer provider: %v\", err)
        }
        if err := meterProvider.Shutdown(ctx); err != nil {
            log.Printf(\"failed to shutdown meter provider: %v\", err)
        }
    }
    return cleanup, nil
}

// HTTP handler with OTel instrumentation
func paymentHandler(w http.ResponseWriter, r *http.Request) {
    ctx := r.Context()
    span := trace.SpanFromContext(ctx)
    span.SetAttributes(attribute.String(\"http.route\", \"/process-payment\"))

    start := time.Now()
    defer func() {
        duration := time.Since(start).Seconds()
        requestDuration.Record(ctx, duration)
    }()

    // Simulate payment processing
    time.Sleep(100 * time.Millisecond)

    // Simulate 5% error rate
    if time.Now().UnixNano()%20 == 0 {
        span.SetStatus(trace.StatusError, \"payment processing failed\")
        errorCounter.Add(ctx, 1, metric.WithAttributes(attribute.String(\"error.type\", \"processing_failure\")))
        http.Error(w, \"payment failed\", http.StatusInternalServerError)
        return
    }

    requestCounter.Add(ctx, 1, metric.WithAttributes(
        attribute.String(\"http.method\", r.Method),
        attribute.Int(\"http.status_code\", http.StatusOK),
    ))
    span.SetStatus(trace.StatusOK, \"payment processed successfully\")
    fmt.Fprintf(w, \"payment processed\")
}

func main() {
    cleanup, err := initOtel()
    if err != nil {
        log.Fatalf(\"failed to initialize OTel: %v\", err)
    }
    defer cleanup()

    // Wrap handler with OTel HTTP instrumentation (replaces New Relic's WrapHandleFunc)
    http.Handle(\"/process-payment\", otelhttp.NewHandler(
        http.HandlerFunc(paymentHandler),
        \"process-payment\",
    ))

    port := os.Getenv(\"PORT\")
    if port == \"\" {
        port = \"8080\"
    }
    log.Printf(\"starting server on :%s\", port)
    if err := http.ListenAndServe(\":\"+port, nil); err != nil {
        log.Fatalf(\"server failed: %v\", err)
    }
}

\"\"\"
flask_otel_migration.py
Migrates New Relic Python agent v8.8.0 instrumentation to OpenTelemetry 1.22
for a Flask-based transaction API
\"\"\"
import os
import logging
import time
from flask import Flask, request, jsonify
from opentelemetry import trace, meter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.prometheus import PrometheusMetricReader
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metric import MeterProvider
from opentelemetry.sdk.metric.export import PeriodicExportingMetricReader
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter

# Configure logging (replaces New Relic Python agent logging)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Initialize OTel Resource (replaces NEW_RELIC_APP_NAME env var)
resource = Resource.create({
    \"service.name\": \"transaction-api\",
    \"service.version\": \"2.1.0\",
    \"deployment.environment\": os.getenv(\"ENV\", \"staging\"),
    \"team\": \"fintech-backend\"
})

def init_otel():
    \"\"\"Initialize OpenTelemetry 1.22 providers with error handling\"\"\"
    try:
        # 1. Configure Trace Provider
        trace_exporter = OTLPSpanExporter(
            endpoint=os.getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\", \"otel-collector:4317\"),
            insecure=True
        )
        span_processor = BatchSpanProcessor(
            trace_exporter,
            schedule_delay_millis=5000,  # 5 second batch delay, matches Go config
            max_export_batch_size=512
        )
        tracer_provider = TracerProvider(
            resource=resource,
            active_span_processor=span_processor
        )
        trace.set_tracer_provider(tracer_provider)

        # 2. Configure Metric Provider with OTLP + Prometheus (replaces New Relic metric API)
        otlp_metric_exporter = OTLPMetricExporter(
            endpoint=os.getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\", \"otel-collector:4317\"),
            insecure=True
        )
        otlp_metric_reader = PeriodicExportingMetricReader(
            exporter=otlp_metric_exporter,
            export_interval_millis=10000  # 10 second metric export interval
        )
        prometheus_reader = PrometheusMetricReader(
            port=9464,  # Expose Prometheus metrics on 9464, replaces New Relic's /metrics endpoint
            address=\"0.0.0.0\"
        )
        meter_provider = MeterProvider(
            resource=resource,
            metric_readers=[otlp_metric_reader, prometheus_reader]
        )
        meter.set_meter_provider(meter_provider)

        # 3. Initialize custom metrics
        global transaction_counter, transaction_duration, db_error_counter
        transaction_counter = meter.meter(
            \"transaction-api\"
        ).create_counter(
            name=\"transactions.total\",
            description=\"Total transactions processed\",
            unit=\"1\"
        )
        transaction_duration = meter.meter(
            \"transaction-api\"
        ).create_histogram(
            name=\"transaction.duration.seconds\",
            description=\"Transaction processing duration\",
            unit=\"s\",
            explicit_bucket_boundaries=[0.05, 0.1, 0.5, 1, 5]
        )
        db_error_counter = meter.meter(
            \"transaction-api\"
        ).create_counter(
            name=\"db.errors.total\",
            description=\"Total database errors during transactions\",
            unit=\"1\"
        )

        # 4. Instrument Flask and Requests (replaces New Relic's automatic instrumentation)
        app = Flask(__name__)
        FlaskInstrumentor().instrument_app(app)
        RequestsInstrumentor().instrument()

        logger.info(\"OpenTelemetry 1.22 initialized successfully\")
        return tracer_provider, meter_provider

    except Exception as e:
        logger.error(f\"Failed to initialize OTel: {str(e)}\", exc_info=True)
        raise

# Flask app initialization
app = Flask(__name__)

# Initialize OTel
tracer_provider, meter_provider = init_otel()
tracer = trace.get_tracer(__name__)

@app.route(\"/process-transaction\", methods=[\"POST\"])
def process_transaction():
    \"\"\"Handle transaction processing with full OTel instrumentation\"\"\"
    with tracer.start_as_current_span(\"process-transaction\") as span:
        start_time = time.time()
        span.set_attribute(\"http.method\", request.method)
        span.set_attribute(\"http.route\", \"/process-transaction\")

        try:
            # Simulate transaction validation
            if not request.is_json:
                span.set_status(trace.Status(trace.StatusCode.ERROR, \"invalid request format\"))
                transaction_counter.add(1, {\"http.status_code\": 400})
                return jsonify({\"error\": \"invalid JSON\"}), 400

            # Simulate database write
            time.sleep(0.08)  # 80ms simulated DB latency
            transaction_id = f\"txn_{int(time.time())}\"

            # Simulate 2% DB error rate
            if time.time_ns() % 50 == 0:
                span.set_status(trace.Status(trace.StatusCode.ERROR, \"database write failed\"))
                db_error_counter.add(1, {\"error.type\": \"write_failure\"})
                transaction_counter.add(1, {\"http.status_code\": 500})
                return jsonify({\"error\": \"database error\"}), 500

            # Record success metrics
            duration = time.time() - start_time
            transaction_duration.record(duration, {\"transaction.type\": \"payment\"})
            transaction_counter.add(1, {\"http.status_code\": 200, \"transaction.type\": \"payment\"})
            span.set_attribute(\"transaction.id\", transaction_id)
            span.set_status(trace.Status(trace.StatusCode.OK, \"transaction processed\"))
            return jsonify({\"transaction_id\": transaction_id, \"status\": \"success\"}), 200

        except Exception as e:
            span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
            logger.error(f\"Transaction failed: {str(e)}\", exc_info=True)
            return jsonify({\"error\": \"internal server error\"}), 500

if __name__ == \"__main__\":
    # Run app with OTel instrumentation (replaces New Relic's newrelic-admin run-program)
    app.run(
        host=\"0.0.0.0\",
        port=int(os.getenv(\"PORT\", 5000)),
        debug=os.getenv(\"ENV\") == \"development\"
    )

// otel_collector_eks.tf
// Terraform configuration to deploy OpenTelemetry Collector 0.88.0 (compatible with OTel 1.22)
// on AWS EKS, replacing New Relic's Kubernetes integration
terraform {
  required_providers {
    aws = {
      source  = \"hashicorp/aws\"
      version = \"~> 5.0\"
    }
    kubernetes = {
      source  = \"hashicorp/kubernetes\"
      version = \"~> 2.20\"
    }
    helm = {
      source  = \"hashicorp/helm\"
      version = \"~> 2.10\"
    }
  }
}

provider \"aws\" {
  region = var.aws_region
}

data \"aws_eks_cluster\" \"cluster\" {
  name = var.eks_cluster_name
}

data \"aws_eks_cluster_auth\" \"cluster\" {
  name = var.eks_cluster_name
}

provider \"kubernetes\" {
  host                   = data.aws_eks_cluster.cluster.endpoint
  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
  token                  = data.aws_eks_cluster_auth.cluster.token
}

provider \"helm\" {
  kubernetes {
    host                   = data.aws_eks_cluster.cluster.endpoint
    cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
    token                  = data.aws_eks_cluster_auth.cluster.token
  }
}

// Create namespace for OTel Collector
resource \"kubernetes_namespace\" \"otel\" {
  metadata {
    name = \"opentelemetry\"
    labels = {
      \"team\" = \"fintech-backend\"
      \"env\"  = var.environment
    }
  }
}

// Deploy OTel Collector via Helm (replaces New Relic's k8s integration helm chart)
resource \"helm_release\" \"otel_collector\" {
  name       = \"otel-collector\"
  repository = \"https://open-telemetry.github.io/opentelemetry-helm-charts\"
  chart      = \"opentelemetry-collector\"
  version    = \"0.88.0\"  // Compatible with OpenTelemetry 1.22 spec
  namespace  = kubernetes_namespace.otel.metadata[0].name

  // Values override to match our migration requirements
  values = [
    yamlencode({
      mode = \"deployment\"  // Run as deployment, not daemonset, to reduce resource usage vs New Relic agent
      image = {
        repository = \"otel/opentelemetry-collector-contrib\"
        tag        = \"0.88.0\"
      }
      resources = {
        limits = {
          cpu    = \"500m\"
          memory = \"512Mi\"
        }
        requests = {
          cpu    = \"100m\"
          memory = \"128Mi\"
        }
      }
      config = {
        receivers = {
          # Receive OTLP traces/metrics from instrumented services
          otlp = {
            protocols = {
              grpc = { endpoint = \"0.0.0.0:4317\" }
              http = { endpoint = \"0.0.0.0:4318\" }
            }
          }
          # Scrape Prometheus metrics from services (replaces New Relic's prometheus integration)
          prometheus = {
            config = {
              scrape_configs = [
                {
                  job_name = \"kubernetes-pods\"
                  kubernetes_sd_configs = [{ role = \"pod\" }]
                  relabel_configs = [
                    {
                      source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]
                      regex = \"true\"
                      action = \"keep\"
                    }
                  ]
                }
              ]
            }
          }
          # Collect k8s cluster metrics (replaces New Relic's k8s integration)
          k8s_cluster = {
            endpoint = \"https://kubernetes.default.svc:443\"
            auth_type = \"serviceAccount\"
          }
        }
        processors = {
          # Add cluster metadata to all telemetry (replaces New Relic's k8s attributes)
          k8sattributes = {
            auth_type = \"serviceAccount\"
            passthrough = false
          }
          # Filter out high-cardinality attributes to reduce costs (New Relic charged for this)
          filter = {
            metrics = {
              exclude = {
                match_attributes = {
                  \"k8s.pod.name\" = [\"test-*\", \"debug-*\"]
                }
              }
            }
          }
          batch = {
            send_batch_size = 512
            timeout = \"5s\"
          }
        }
        exporters = {
          # Send traces to Tempo (replaces New Relic trace storage)
          otlp_tempo = {
            endpoint = \"tempo:4317\"
            insecure = true
          }
          # Send metrics to Prometheus (replaces New Relic metric storage)
          prometheus = {
            endpoint = \"prometheus:9090\"
          }
          # Send logs to Loki (replaces New Relic log storage)
          loki = {
            endpoint = \"loki:3100\"
            insecure = true
          }
        }
        service = {
          pipelines = {
            traces = {
              receivers = [\"otlp\"]
              processors = [\"k8sattributes\", \"batch\"]
              exporters = [\"otlp_tempo\"]
            }
            metrics = {
              receivers = [\"otlp\", \"prometheus\", \"k8s_cluster\"]
              processors = [\"k8sattributes\", \"filter\", \"batch\"]
              exporters = [\"prometheus\"]
            }
            logs = {
              receivers = [\"otlp\"]
              processors = [\"k8sattributes\", \"batch\"]
              exporters = [\"loki\"]
            }
          }
        }
      }
    })
  ]

  depends_on = [kubernetes_namespace.otel]
}

// IAM role for OTel Collector to access CloudWatch logs (replaces New Relic's IAM role)
resource \"aws_iam_role\" \"otel_collector\" {
  name = \"otel-collector-eks-role-${var.environment}\"

  assume_role_policy = jsonencode({
    Version = \"2012-10-17\"
    Statement = [
      {
        Action = \"sts:AssumeRoleWithWebIdentity\"
        Effect = \"Allow\"
        Principal = {
          Federated = \"arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, \"https://\", \"\")}\"
        }
        Condition = {
          StringEquals = {
            \"${replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, \"https://\", \"\")}:sub\" = \"system:serviceaccount:${kubernetes_namespace.otel.metadata[0].name}:otel-collector-sa\"
          }
        }
      }
    ]
  })

  inline_policy {
    name = \"otel-collector-policy\"
    policy = jsonencode({
      Version = \"2012-10-17\"
      Statement = [
        {
          Action = [
            \"logs:CreateLogGroup\",
            \"logs:CreateLogStream\",
            \"logs:PutLogEvents\"
          ]
          Effect   = \"Allow\"
          Resource = \"*\"
        }
      ]
    })
  }
}

data \"aws_caller_identity\" \"current\" {}

variable \"aws_region\" {
  type    = string
  default = \"us-east-1\"
}

variable \"eks_cluster_name\" {
  type = string
}

variable \"environment\" {
  type    = string
  default = \"production\"
}

// Snippet: Filter processor config for OTel Collector
processors:
  filter:
    metrics:
      exclude:
        match_attributes:
          \"user_id\": [\"*\"]  // Exclude all user_id attributes from metrics
          \"k8s.pod.name\": [\"test-*\", \"debug-*\"]
    traces:
      exclude:
        match_attributes:
          \"session_id\": [\"*\"]  // Exclude session_id from non-error traces
        match_one:
          - { \"http.status_code\": [\"200\"] }  // Only include error traces for session_id

// Snippet: Minimal OTel Go Distro setup (replaces 120 lines of SDK code)
import (
  \"go.opentelemetry.io/otel/distro\"
  \"go.opentelemetry.io/otel/sdk/resource\"
  \"go.opentelemetry.io/otel/attribute\"
)

func main() {
  distro.Setup(
    distro.WithResource(resource.NewWithAttributes(
      attribute.Key(\"service.name\").String(\"payment-processor\"),
    )),
    distro.WithExporterEndpoint(\"otel-collector:4317\"),
  )
  // Start instrumented service here
}

// Snippet: Grafana Alertmanager rule for payment failure rate
groups:
- name: payment-alerts
  rules:
  - alert: HighPaymentFailureRate
    expr: sum(rate(http_request_errors_total{service=\"payment-processor\", status_code=~\"5..\"}[5m])) / sum(rate(http_requests_total{service=\"payment-processor\"}[5m])) > 0.01
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: \"Payment failure rate > 1% for 2 minutes\"
      description: \"Service {{ $labels.service }} has failure rate {{ $value | humanizePercentage }}\"