Step-by-Step Guide to Ditching Datadog 2025 for OpenTelemetry 1.20: Cost Savings Case Study

// Copyright 2025 Senior Engineer Article Examples
// SPDX-License-Identifier: MIT
package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "os"
    "time"

    // OpenTelemetry 1.20 core imports
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    "go.opentelemetry.io/otel/trace"
    "go.opentelemetry.io/otel/log"
    "go.opentelemetry.io/otel/log/global"
    "go.opentelemetry.io/otel/exporters/otlp/otlplog"
    "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
    "go.opentelemetry.io/otel/sdk/log"
    sdklog "go.opentelemetry.io/otel/sdk/log"
)

const (
    serviceName    = "payment-processor"
    serviceVersion = "1.2.0"
    otelEndpoint   = "otel-collector:4317" // OTLP gRPC endpoint
)

func initTracer(ctx context.Context) (*sdktrace.TracerProvider, error) {
    // Initialize OTLP trace exporter to send traces to OTel Collector
    traceExporter, err := otlptrace.New(ctx, otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(), // Use TLS in production, insecure for local dev
        otlptracegrpc.WithEndpoint(otelEndpoint),
    ))
    if err != nil {
        return nil, fmt.Errorf("failed to create trace exporter: %w", err)
    }

    // Define resource attributes (service name, version, environment)
    res, err := resource.New(ctx,
        resource.WithAttributes(
            attribute.String("service.name", serviceName),
            attribute.String("service.version", serviceVersion),
            attribute.String("deployment.environment", os.Getenv("ENVIRONMENT")),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // Configure trace provider with sampler (always sample for dev, 10% for prod)
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(traceExporter),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.AlwaysSample()), // Change to sdktrace.TraceIDRatioBased(0.1) for production
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
    return tp, nil
}

func initMeter(ctx context.Context) (metric.Meter, error) {
    // Initialize metric exporter (OTLP gRPC for metrics)
    // Note: OTel 1.20 uses separate exporters for traces, metrics, logs
    // For brevity, we reuse the same OTLP endpoint for all telemetry
    // In production, separate backends per telemetry type are recommended
    return nil, nil // Simplified for example, full meter init adds ~20 lines
}

func initLogger(ctx context.Context) (*sdklog.LoggerProvider, error) {
    logExporter, err := otlplog.New(ctx, otlploggrpc.NewClient(
        otlploggrpc.WithInsecure(),
        otlploggrpc.WithEndpoint(otelEndpoint),
    ))
    if err != nil {
        return nil, fmt.Errorf("failed to create log exporter: %w", err)
    }

    lp := sdklog.NewLoggerProvider(
        sdklog.WithBatcher(logExporter),
    )
    global.SetLoggerProvider(lp)
    return lp, nil
}

func paymentHandler(w http.ResponseWriter, r *http.Request) {
    ctx := r.Context()
    tracer := otel.Tracer("payment-handler")
    ctx, span := tracer.Start(ctx, "process-payment", trace.WithAttributes(
        attribute.String("payment.method", r.URL.Query().Get("method")),
        attribute.Float64("payment.amount", 99.99),
    ))
    defer span.End()

    // Simulate payment processing latency
    time.Sleep(100 * time.Millisecond)

    // Log payment attempt
    logger := global.GetLogger()
    logger.Emit(ctx, log.NewRecord(log.Info, "Processing payment",
        log.String("payment_id", "pay_123456"),
        log.String("user_id", "usr_789012"),
    ))

    // Record metric for payment success
    // meter := otel.GetMeter("payment-metrics")
    // counter, _ := meter.Int64Counter("payment.success.count")
    // counter.Add(ctx, 1)

    span.SetStatus(codes.Ok, "Payment processed successfully")
    w.WriteHeader(http.StatusOK)
    fmt.Fprintf(w, "Payment processed: %s", "pay_123456")
}

func main() {
    ctx := context.Background()

    // Initialize all telemetry providers
    tp, err := initTracer(ctx)
    if err != nil {
        log.Fatalf("Failed to initialize tracer: %v", err)
    }
    defer tp.Shutdown(ctx)

    lp, err := initLogger(ctx)
    if err != nil {
        log.Fatalf("Failed to initialize logger: %v", err)
    }
    defer lp.Shutdown(ctx)

    // Register HTTP handler with OTel instrumentation
    http.HandleFunc("/process-payment", paymentHandler)

    port := os.Getenv("PORT")
    if port == "" {
        port = "8080"
    }
    log.Printf("Starting %s v%s on port %s", serviceName, serviceVersion, port)
    if err := http.ListenAndServe(":"+port, nil); err != nil {
        log.Fatalf("HTTP server failed: %v", err)
    }
}

# OpenTelemetry Collector Contrib 0.91.0 Configuration
# Replaces Datadog Agent with 100% compatible telemetry ingestion
# Gradual migration: export to Datadog and self-hosted backends in parallel

receivers:
  # OTLP receiver for application telemetry (gRPC and HTTP)
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  # DogStatsD receiver for legacy Datadog SDK metrics
  dogstatsd:
    endpoint: 0.0.0.0:8125
    # Enable Datadog compatible tags and metric names
    namespace: "datadog."
    # Parse DogStatsD tags into OTel attributes
    tag_cardinality: high

processors:
  # Add common resource attributes to all telemetry
  resource:
    attributes:
      - key: cloud.provider
        value: "aws"
        action: insert
      - key: cloud.region
        value: "us-east-1"
        action: insert
  # Batch telemetry to reduce network calls
  batch:
    timeout: 5s
    send_batch_size: 1000
  # Filter out high-cardinality attributes to reduce costs
  filter:
    metrics:
      # Drop metrics with high cardinality tags (e.g., user_id)
      exclude:
        match_type: regexp
        metric_names: ["http.request.duration"]
        attribute_keys: ["user_id"]
  # Convert Datadog metric types to OTel equivalents
  datadog:
    # Enable Datadog to OTel metric name translation
    metric_name_to_telemetry: true

exporters:
  # Datadog exporter for gradual migration (send 10% of traffic to Datadog first)
  datadog:
    api:
      key: "${DATADOG_API_KEY}"
      site: "datadoghq.com"
    # Send metrics, traces, logs to Datadog
    metrics:
      enabled: true
      # Use Datadog's native metric types
      report_quantiles: true
    traces:
      enabled: true
    logs:
      enabled: true
    # Send only 10% of traffic to Datadog initially
    sampling:
      rate: 0.1
  # Self-hosted Prometheus for metrics
  prometheus:
    endpoint: 0.0.0.0:9090
    namespace: "otel"
    # Add resource attributes as Prometheus labels
    resource_to_telemetry_conversion:
      enabled: true
  # Self-hosted Tempo for traces
  otlp/otlp-tempo:
    endpoint: "tempo:4317"
    tls:
      insecure: true
  # Self-hosted Loki for logs
  loki:
    endpoint: "http://loki:3100/loki/api/v1/push"
    # Format logs as JSON for Loki compatibility
    format: json

service:
  pipelines:
    # Metrics pipeline: receive from OTLP/DogStatsD, process, export to Datadog and Prometheus
    metrics:
      receivers: [otlp, dogstatsd]
      processors: [resource, batch, filter, datadog]
      exporters: [datadog, prometheus]
    # Traces pipeline: receive from OTLP, process, export to Datadog and Tempo
    traces:
      receivers: [otlp]
      processors: [resource, batch]
      exporters: [datadog, otlp/otlp-tempo]
    # Logs pipeline: receive from OTLP, process, export to Datadog and Loki
    logs:
      receivers: [otlp]
      processors: [resource, batch]
      exporters: [datadog, loki]

  # Telemetry for the OTel Collector itself
  telemetry:
    logs:
      level: "info"
    metrics:
      address: "0.0.0.0:8888"

#!/usr/bin/env python3
# Copyright 2025 Senior Engineer Article Examples
# SPDX-License-Identifier: MIT
"""
Validation script to compare telemetry parity between Datadog and OpenTelemetry 1.20
during migration. Checks for missing traces, metrics, and logs.
"""

import os
import time
import json
from datadog_api_client import ApiClient, Configuration
from datadog_api_client.v2.api.metrics_api import MetricsApi
from datadog_api_client.v2.api.logs_api import LogsApi
from datadog_api_client.v2.api.traces_api import TracesApi
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# Configuration
DATADOG_API_KEY = os.getenv("DATADOG_API_KEY")
DATADOG_APP_KEY = os.getenv("DATADOG_APP_KEY")
OTEL_COLLECTOR_ENDPOINT = os.getenv("OTEL_ENDPOINT", "otel-collector:4317")
SERVICE_NAME = "payment-processor"
VALIDATION_WINDOW = 300  # 5 minutes in seconds

def init_datadog_client():
    """Initialize Datadog API client with error handling."""
    if not DATADOG_API_KEY or not DATADOG_APP_KEY:
        raise ValueError("DATADOG_API_KEY and DATADOG_APP_KEY must be set")
    config = Configuration()
    config.api_key["apiKeyAuth"] = DATADOG_API_KEY
    config.api_key["appKeyAuth"] = DATADOG_APP_KEY
    config.server_variables["site"] = "datadoghq.com"
    return ApiClient(config)

def init_otel_tracer():
    """Initialize OTel tracer to send test spans to OTel Collector."""
    resource = Resource.create({
        "service.name": SERVICE_NAME,
        "deployment.environment": "validation",
    })
    provider = TracerProvider(resource=resource)
    exporter = OTLPSpanExporter(endpoint=OTEL_COLLECTOR_ENDPOINT, insecure=True)
    processor = BatchSpanProcessor(exporter)
    provider.add_span_processor(processor)
    trace.set_tracer_provider(provider)
    return trace.get_tracer(__name__)

def get_datadog_traces(datadog_client, start_time, end_time):
    """Fetch traces from Datadog for the service in the time window."""
    api = TracesApi(datadog_client)
    try:
        response = api.list_traces(
            filter_query=f"service:{SERVICE_NAME}",
            filter_from=start_time.isoformat(),
            filter_to=end_time.isoformat(),
            page_limit=1000,
        )
        return response.data if response else []
    except Exception as e:
        print(f"Failed to fetch Datadog traces: {e}")
        return []

def get_otel_traces(start_time, end_time):
    """Fetch traces from OTel backend (Tempo) for validation."""
    # Note: This uses Tempo's API; adjust for your self-hosted backend
    import requests
    tempo_url = os.getenv("TEMPO_URL", "http://tempo:3200")
    try:
        response = requests.get(
            f"{tempo_url}/api/search",
            params={
                "service.name": SERVICE_NAME,
                "start": int(start_time.timestamp() * 1e9),  # Nanoseconds
                "end": int(end_time.timestamp() * 1e9),
                "limit": 1000,
            },
            timeout=10,
        )
        response.raise_for_status()
        return response.json().get("traces", [])
    except Exception as e:
        print(f"Failed to fetch OTel traces: {e}")
        return []

def compare_traces(datadog_traces, otel_traces):
    """Compare trace IDs between Datadog and OTel to check parity."""
    dd_trace_ids = {trace.id for trace in datadog_traces}
    otel_trace_ids = {trace["traceID"] for trace in otel_traces}
    missing_in_otel = dd_trace_ids - otel_trace_ids
    missing_in_dd = otel_trace_ids - dd_trace_ids
    return {
        "total_datadog": len(dd_trace_ids),
        "total_otel": len(otel_trace_ids),
        "missing_in_otel": len(missing_in_otel),
        "missing_in_dd": len(missing_in_dd),
        "parity_percentage": (1 - (len(missing_in_otel) + len(missing_in_dd)) / max(len(dd_trace_ids), 1)) * 100,
    }

def main():
    # Initialize clients
    try:
        datadog_client = init_datadog_client()
        tracer = init_otel_tracer()
    except Exception as e:
        print(f"Initialization failed: {e}")
        return

    # Generate test trace to send to both Datadog and OTel (during parallel export)
    with tracer.start_as_current_span("validation-test-span") as span:
        span.set_attribute("validation.run", True)
        span.set_attribute("test.id", "migrate-001")
        time.sleep(0.1)  # Simulate work

    # Wait for telemetry to propagate
    print("Waiting 30 seconds for telemetry propagation...")
    time.sleep(30)

    # Define validation window
    end_time = time.time()
    start_time = end_time - VALIDATION_WINDOW

    # Fetch traces from both platforms
    print("Fetching Datadog traces...")
    dd_traces = get_datadog_traces(datadog_client, start_time, end_time)
    print("Fetching OTel traces...")
    otel_traces = get_otel_traces(start_time, end_time)

    # Compare parity
    results = compare_traces(dd_traces, otel_traces)
    print("\n=== Telemetry Parity Results ===")
    print(json.dumps(results, indent=2))

    # Exit with error if parity is below 99%
    if results["parity_percentage"] < 99:
        print("ERROR: Telemetry parity below 99%")
        exit(1)
    else:
        print("SUCCESS: Telemetry parity above 99%")

if __name__ == "__main__":
    main()

processors:
  datadog:
    # Enable Datadog to OTel metric name translation
    metric_name_to_telemetry: true
    # Convert Datadog tags to OTel resource attributes
    tag_to_resource: true

exporters:
  datadog:
    sampling:
      # Start with 10% traffic to OTel backends
      rate: 0.1
    # Increase to 50% after 2 weeks of validation
    # rate: 0.5
    # 100% after 6 weeks
    # rate: 1.0

// Legacy Datadog metric
// statsd.Incr("myapp.payment.success", []string{"method:credit"}, 1)

// OTel semantic convention equivalent
meter := otel.GetMeter("payment")
counter, _ := meter.Int64Counter("payment.success.count")
counter.Add(ctx, 1, metric.WithAttributes(attribute.String("payment.method", "credit")))

otel-datadog-migration-2025/
├── app-examples/
│   ├── go-payment-service/       # Go service with OTel 1.20 SDK
│   │   ├── main.go               # Full application code from Section 3
│   │   ├── go.mod                # OTel 1.20 dependencies
│   │   └── Dockerfile            # Container image with OTel SDK
│   └── python-validation-script/ # Parity validation script from Section 3
│       ├── validate_parity.py    # Full validation code
│       └── requirements.txt      # Datadog + OTel Python dependencies
├── otel-collector/
│   ├── collector-config.yaml     # Full OTel Collector config from Section 3
│   ├── k8s-daemonset.yaml        # Kubernetes daemonset deployment
│   └── k8s-sidecar.yaml          # Sidecar deployment for legacy services
├── case-study/
│   ├── cost-breakdown.xlsx       # Detailed cost comparison spreadsheet
│   └── migration-timeline.xlsx   # 6-week migration timeline template
├── grafana-dashboards/
│   ├── service-overview.json     # Replacement for Datadog service dashboard
│   └── trace-explorer.json       # Replacement for Datadog trace view
└── README.md                     # Full migration guide with links