Opinion: Why We Ditched OpenTelemetry 1.20 for Datadog 7 – Compliance and Support Matter More

// otel_grpc_service.go
// OpenTelemetry 1.20 instrumentation for a Go gRPC service
// Demonstrates the boilerplate and overhead required for OTel 1.20
package main

import (
    "context"
    "fmt"
    "log"
    "net"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    "go.opentelemetry.io/otel/semconv/v1.20.0"
    "go.opentelemetry.io/otel/trace"
    "go.opentelemetry.io/otel/instrumentation/grpc/otelgrpc"
    "google.golang.org/grpc"
    pb "google.golang.org/grpc/examples/helloworld/helloworld"
)

const (
    port           = ":50051"
    otelCollectorAddr = "otel-collector:4317"
)

// helloServer implements the helloworld.GreeterServer interface
type helloServer struct {
    pb.UnimplementedGreeterServer
    tracer trace.Tracer
}

// SayHello handles gRPC requests with OTel 1.20 tracing
func (s *helloServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
    // Start a new span for this request
    ctx, span := s.tracer.Start(ctx, "SayHello", trace.WithAttributes(
        attribute.String("request.name", in.GetName()),
    ))
    defer span.End()

    // Simulate business logic latency
    time.Sleep(100 * time.Millisecond)

    // Log request details to the span
    span.AddEvent("processing_request", trace.WithAttributes(
        attribute.Int64("timestamp", time.Now().UnixNano()),
    ))

    return &pb.HelloReply{Message: fmt.Sprintf("Hello %s", in.GetName())}, nil
}

// initOTel sets up OpenTelemetry 1.20 tracing with OTLP exporter
func initOTel() (*sdktrace.TracerProvider, error) {
    // Create OTLP gRPC client for OTel collector
    client := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(otelCollectorAddr),
    )
    // Create trace exporter
    exporter, err := otlptrace.New(context.Background(), client)
    if err != nil {
        return nil, fmt.Errorf("failed to create OTLP exporter: %w", err)
    }

    // Define resource attributes for the service
    res, err := resource.New(context.Background(),
        resource.WithAttributes(
            semconv.ServiceNameKey.String("otel-grpc-demo"),
            semconv.ServiceVersionKey.String("1.20.0"),
            attribute.String("environment", "production"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // Configure tracer provider with batch span processor
    tracerProvider := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter,
            sdktrace.WithMaxExportBatchSize(100),
            sdktrace.WithBatchTimeout(5*time.Second),
        ),
        sdktrace.WithResource(res),
    )

    // Set global tracer provider and propagator
    otel.SetTracerProvider(tracerProvider)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return tracerProvider, nil
}

func main() {
    // Initialize OpenTelemetry 1.20
    tp, err := initOTel()
    if err != nil {
        log.Fatalf("Failed to initialize OTel: %v", err)
    }
    defer func() {
        if err := tp.Shutdown(context.Background()); err != nil {
            log.Printf("Error shutting down OTel tracer provider: %v", err)
        }
    }()

    // Create gRPC server with OTel instrumentation
    lis, err := net.Listen("tcp", port)
    if err != nil {
        log.Fatalf("Failed to listen: %v", err)
    }
    s := grpc.NewServer(
        grpc.UnaryInterceptor(otelgrpc.UnaryServerInterceptor()),
    )
    pb.RegisterGreeterServer(s, &helloServer{
        tracer: otel.Tracer("otel-grpc-demo"),
    })

    log.Printf("OTel 1.20 gRPC server listening on %s", port)
    if err := s.Serve(lis); err != nil {
        log.Fatalf("Failed to serve: %v", err)
    }
}

// datadog_grpc_service.go
// Datadog 7 instrumentation for the same Go gRPC service
// Demonstrates reduced boilerplate and lower overhead vs OTel 1.20
package main

import (
    "context"
    "fmt"
    "log"
    "net"
    "time"

    "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
    "gopkg.in/DataDog/dd-trace-go.v1/instrumentation/google.golang.org/grpc/grpcserver"
    pb "google.golang.org/grpc/examples/helloworld/helloworld"
    "google.golang.org/grpc"
)

const (
    port = ":50051"
)

// helloServer implements the helloworld.GreeterServer interface
type helloServer struct {
    pb.UnimplementedGreeterServer
}

// SayHello handles gRPC requests with Datadog 7 tracing
func (s *helloServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
    // Datadog 7 automatically creates spans for gRPC methods, no manual start required
    // Add custom tags to the active span
    span := tracer.SpanFromContext(ctx)
    span.SetTag("request.name", in.GetName())
    span.SetTag("environment", "production")

    // Simulate business logic latency
    time.Sleep(100 * time.Millisecond)

    // Add event to span
    span.LogFields(
        tracer.LogString("event", "processing_request"),
        tracer.LogInt64("timestamp", time.Now().UnixNano()),
    )

    return &pb.HelloReply{Message: fmt.Sprintf("Hello %s", in.GetName())}, nil
}

func main() {
    // Initialize Datadog 7 tracer with minimal config
    tracer.Start(
        tracer.WithService("datadog-grpc-demo"),
        tracer.WithServiceVersion("7.48.0"),
        tracer.WithEnv("production"),
        tracer.WithAgentAddr("datadog-agent:8126"),
    )
    defer tracer.Stop()

    // Create gRPC server with Datadog 7 instrumentation
    lis, err := net.Listen("tcp", port)
    if err != nil {
        log.Fatalf("Failed to listen: %v", err)
    }
    s := grpc.NewServer(
        grpcserver.UnaryServerInterceptor(),
    )
    pb.RegisterGreeterServer(s, &helloServer{})

    log.Printf("Datadog 7 gRPC server listening on %s", port)
    if err := s.Serve(lis); err != nil {
        log.Fatalf("Failed to serve: %v", err)
    }
}

# otel_to_datadog_migration.py
# Python script to migrate OpenTelemetry 1.20 traces to Datadog 7
# Uses Datadog API v2 to batch import historical traces with error handling
import os
import time
import json
from datadog_api_client import ApiClient, Configuration
from datadog_api_client.v2.api.spans_api import SpansApi
from datadog_api_client.v2.models import SpanCreateRequest, SpanCreateRequestAttributes, SpanCreateRequestData, SpanCreateRequestType

# Configuration from environment variables
DATADOG_API_KEY = os.getenv("DATADOG_API_KEY")
DATADOG_SITE = os.getenv("DATADOG_SITE", "datadoghq.com")
OTEL_TRACE_DIR = os.getenv("OTEL_TRACE_DIR", "/var/log/otel/traces")
BATCH_SIZE = 1000

def load_otel_traces(trace_dir: str) -> list:
    """Load OTel 1.20 trace JSON files from disk."""
    traces = []
    for filename in os.listdir(trace_dir):
        if not filename.endswith(".json"):
            continue
        filepath = os.path.join(trace_dir, filename)
        try:
            with open(filepath, "r") as f:
                trace_data = json.load(f)
                traces.append(trace_data)
        except json.JSONDecodeError as e:
            print(f"Failed to parse {filename}: {e}")
        except Exception as e:
            print(f"Error reading {filepath}: {e}")
    return traces

def convert_otel_to_datadog(otel_trace: dict) -> SpanCreateRequestData:
    """Convert OTel 1.20 span format to Datadog 7 span format."""
    attributes = otel_trace.get("attributes", {})
    return SpanCreateRequestData(
        type=SpanCreateRequestType.SPAN,
        attributes=SpanCreateRequestAttributes(
            trace_id=otel_trace.get("traceId"),
            span_id=otel_trace.get("spanId"),
            parent_id=otel_trace.get("parentSpanId", "0000000000000000"),
            name=otel_trace.get("name"),
            service=attributes.get("service.name", "unknown"),
            resource=otel_trace.get("name"),
            start=int(otel_trace.get("startTimeUnixNano", 0) / 1e6),  # Convert ns to ms
            duration=int(otel_trace.get("endTimeUnixNano", 0) - otel_trace.get("startTimeUnixNano", 0)) / 1e6,
            meta={k: str(v) for k, v in attributes.items()},
            metrics={},
            error=1 if otel_trace.get("status", {}).get("code") == "ERROR" else 0,
        ),
    )

def batch_import_traces(traces: list, batch_size: int = BATCH_SIZE):
    """Batch import OTel traces to Datadog 7 via API."""
    configuration = Configuration()
    configuration.api_key["apiKeyAuth"] = DATADOG_API_KEY
    configuration.server_variables["site"] = DATADOG_SITE

    with ApiClient(configuration) as api_client:
        spans_api = SpansApi(api_client)
        for i in range(0, len(traces), batch_size):
            batch = traces[i:i + batch_size]
            datadog_spans = []
            for otel_span in batch:
                try:
                    datadog_span = convert_otel_to_datadog(otel_span)
                    datadog_spans.append(datadog_span)
                except Exception as e:
                    print(f"Failed to convert span {otel_span.get('spanId')}: {e}")

            if not datadog_spans:
                continue

            try:
                request = SpanCreateRequest(data=datadog_spans)
                response = spans_api.create_spans(request)
                print(f"Imported {len(datadog_spans)} spans (batch {i//batch_size + 1})")
                time.sleep(1)  # Rate limit to avoid API throttling
            except Exception as e:
                print(f"Failed to import batch {i//batch_size + 1}: {e}")
                # Retry once on failure
                time.sleep(5)
                try:
                    response = spans_api.create_spans(request)
                    print(f"Retried and imported {len(datadog_spans)} spans")
                except Exception as retry_e:
                    print(f"Retry failed for batch {i//batch_size + 1}: {retry_e}")

if __name__ == "__main__":
    if not DATADOG_API_KEY:
        raise ValueError("DATADOG_API_KEY environment variable is required")

    print("Loading OpenTelemetry 1.20 traces...")
    otel_traces = load_otel_traces(OTEL_TRACE_DIR)
    print(f"Loaded {len(otel_traces)} OTel traces")

    print("Migrating traces to Datadog 7...")
    batch_import_traces(otel_traces)
    print("Migration complete")

resource "datadog_dashboard" "pci_dss" {
  title       = "PCI-DSS Compliance Dashboard"
  description = "Pre-built PCI-DSS trace lineage dashboard"
  layout_type = "ordered"
  widget {
    query_value_definition {
      title   = "Payment Trace Lineage Coverage"
      query   = "avg:trace.pci_dss.payment_lineage.coverage{service:payment-api}"
      unit    = "percent"
    }
  }
}

func BenchmarkOTelOverhead(b *testing.B) {
  // Initialize OTel 1.20 tracer
  tp, _ := initOTel()
  defer tp.Shutdown(context.Background())

  b.ResetTimer()
  for i := 0; i < b.N; i++ {
    ctx, span := otel.Tracer("bench").Start(context.Background(), "bench-span")
    span.End()
  }
}

func BenchmarkDatadogOverhead(b *testing.B) {
  // Initialize Datadog 7 tracer
  tracer.Start()
  defer tracer.Stop()

  b.ResetTimer()
  for i := 0; i < b.N; i++ {
    span := tracer.StartSpan("bench-span")
    span.Finish()
  }
}

curl -X POST "https://api.datadoghq.com/api/v2/support/tickets" \
  -H "Content-Type: application/json" \
  -H "DD-API-KEY: ${DATADOG_API_KEY}" \
  -d '{
    "data": {
      "type": "support_ticket",
      "attributes": {
        "subject": "Critical: Payment API trace ingestion delay",
        "description": "Trace ingestion for payment-api is delayed by 10 minutes",
        "priority": "urgent",
        "component": "apm"
      }
    }
  }'