Opinion: Why OpenTelemetry 1.30 Is Still Not Ready for Production for Small Teams Compared to Datadog

// otel_gin_setup.go
// Benchmarked setup for OpenTelemetry 1.30 Gin instrumentation
// Requires: go 1.21+, github.com/gin-gonic/gin v1.9.1, go.opentelemetry.io/otel v1.30.0
package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"time\"

    \"github.com/gin-gonic/gin\"
    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    sdktrace \"go.opentelemetry.io/otel/sdk/trace\"
    semconv \"go.opentelemetry.io/otel/semconv/v1.30.0\"
    \"go.opentelemetry.io/otel/trace\"
    \"google.golang.org/grpc\"
    \"google.golang.org/grpc/credentials/insecure\"
)

// initTracer sets up OTel 1.30 tracer provider with OTLP gRPC exporter
// Returns error if exporter or provider initialization fails
func initTracer(ctx context.Context) (*sdktrace.TracerProvider, error) {
    // Configure OTLP gRPC client to send traces to OTel Collector
    client := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(\"otel-collector:4317\"),
        otlptracegrpc.WithDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())),
    )

    // Create OTLP trace exporter
    exporter, err := otlptrace.New(ctx, client)
    if err != nil {
        return nil, fmt.Errorf(\"failed to create OTLP trace exporter: %w\", err)
    }

    // Define resource attributes for the service
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName(\"gin-inventory-api\"),
            semconv.ServiceVersion(\"1.0.2\"),
            attribute.String(\"team\", \"backend-small\"),
            attribute.Int(\"team_size\", 4),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create resource: %w\", err)
    }

    // Configure tracer provider with sampler, resource, and exporter
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithSampler(sdktrace.AlwaysSample()), // For small teams: full sampling to avoid missing issues
        sdktrace.WithResource(res),
        sdktrace.WithBatcher(exporter,
            sdktrace.WithBatchTimeout(5*time.Second), // Small teams: shorter batch timeout to reduce memory
            sdktrace.WithMaxQueueSize(100),
        ),
    )

    // Set global tracer provider and propagator
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return tp, nil
}

func main() {
    ctx := context.Background()

    // Initialize OTel tracer
    tp, err := initTracer(ctx)
    if err != nil {
        log.Fatalf(\"Failed to initialize OTel tracer: %v\", err)
    }
    defer func() {
        if err := tp.Shutdown(ctx); err != nil {
            log.Printf(\"Failed to shutdown tracer provider: %v\", err)
        }
    }()

    // Initialize Gin router with OTel instrumentation
    r := gin.Default()

    // Manually add OTel middleware (OTel 1.30 does not have first-party Gin middleware)
    // This is a custom implementation that mimics Datadog's Gin integration
    r.Use(func(c *gin.Context) {
        // Start span for incoming request
        ctx, span := otel.Tracer(\"gin-inventory-api\").Start(c.Request.Context(), c.FullPath(),
            trace.WithAttributes(
                attribute.String(\"http.method\", c.Request.Method),
                attribute.String(\"http.url\", c.Request.URL.String()),
            ),
        )
        defer span.End()

        // Propagate context to request
        c.Request = c.Request.WithContext(ctx)
        c.Next()

        // Add response attributes to span
        span.SetAttributes(attribute.Int(\"http.status_code\", c.Writer.Status()))
        if len(c.Errors) > 0 {
            span.SetAttributes(attribute.String(\"error.message\", c.Errors.Last().Error()))
        }
    })

    // Health check endpoint
    r.GET(\"/health\", func(c *gin.Context) {
        c.JSON(http.StatusOK, gin.H{\"status\": \"healthy\"})
    })

    // Inventory endpoint with simulated latency
    r.GET(\"/inventory/:id\", func(c *gin.Context) {
        id := c.Param(\"id\")
        // Simulate DB call latency
        time.Sleep(100 * time.Millisecond)

        c.JSON(http.StatusOK, gin.H{\"id\": id, \"quantity\": 42})
    })

    // Start server
    log.Println(\"Starting Gin server on :8080\")
    if err := r.Run(\":8080\"); err != nil {
        log.Fatalf(\"Failed to start server: %v\", err)
    }
}

// datadog_gin_setup.go
// Equivalent Datadog instrumentation for Gin service
// Requires: go 1.21+, github.com/gin-gonic/gin v1.9.1, github.com/DataDog/dd-trace-go v1.62.0
package main

import (
    \"fmt\"
    \"log\"
    \"net/http\"
    \"time\"

    \"github.com/gin-gonic/gin\"
    \"github.com/DataDog/dd-trace-go/v2/ddtrace/tracer\"
    \"github.com/DataDog/dd-trace-go/v2/instrumentation/github.com/gin-gonic/gin\"
)

func main() {
    // Initialize Datadog tracer with small team-optimized config
    tracer.Start(
        tracer.WithService(\"gin-inventory-api\"),
        tracer.WithServiceVersion(\"1.0.2\"),
        tracer.WithEnv(\"staging\"),
        tracer.WithTags(map[string]string{
            \"team\":       \"backend-small\",
            \"team_size\":  \"4\",
        }),
        tracer.WithSampleRate(1.0), // Full sampling for small teams
        tracer.WithRuntimeMetrics(), // Enable runtime metrics by default
    )
    defer tracer.Stop()

    // Initialize Gin router with Datadog's first-party middleware
    // One line vs 40+ lines for OTel custom middleware
    r := gin.Default()
    r.Use(ddotgin.Middleware(ddotgin.WithTracer(tracer.Default())))

    // Health check endpoint (no extra instrumentation needed)
    r.GET(\"/health\", func(c *gin.Context) {
        c.JSON(http.StatusOK, gin.H{\"status\": \"healthy\"})
    })

    // Inventory endpoint with simulated latency
    r.GET(\"/inventory/:id\", func(c *gin.Context) {
        id := c.Param(\"id\")
        // Simulate DB call latency
        time.Sleep(100 * time.Millisecond)

        // Datadog automatically traces this handler, no manual span creation
        c.JSON(http.StatusOK, gin.H{\"id\": id, \"quantity\": 42})
    })

    // Start server
    log.Println(\"Starting Gin server on :8080\")
    if err := r.Run(\":8080\"); err != nil {
        log.Fatalf(\"Failed to start server: %v\", err)
    }
}

# otel_fastapi_metrics.py
# OpenTelemetry 1.30 FastAPI metrics setup (Python 3.11+)
# Requires: fastapi==0.104.1, uvicorn==0.24.0, opentelemetry-api==1.30.0, opentelemetry-sdk==1.30.0,
# opentelemetry-instrumentation-fastapi==0.51b0, opentelemetry-exporter-otlp-proto-grpc==1.30.0
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
import time
from opentelemetry import metrics
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.semconv.resource import ResourceAttributes
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor

# Initialize OpenTelemetry metrics for FastAPI
def init_otel_metrics():
    try:
        # Define resource attributes
        resource = Resource.create({
            ResourceAttributes.SERVICE_NAME: \"fastapi-checkout-api\",
            ResourceAttributes.SERVICE_VERSION: \"2.1.0\",
            \"team\": \"backend-small\",
            \"team_size\": 5,
        })

        # Configure OTLP gRPC metric exporter
        exporter = OTLPMetricExporter(
            endpoint=\"otel-collector:4317\",
            insecure=True,
        )

        # Set up metric reader with 10s export interval (small team friendly)
        reader = PeriodicExportingMetricReader(
            exporter=exporter,
            export_interval_millis=10000,
        )

        # Create meter provider
        provider = MeterProvider(
            resource=resource,
            metric_readers=[reader],
        )
        metrics.set_meter_provider(provider)

        # Create custom counter for checkout events
        meter = metrics.get_meter(\"fastapi-checkout-api\")
        checkout_counter = meter.create_counter(
            name=\"checkout.events\",
            description=\"Number of checkout events processed\",
            unit=\"1\",
        )

        return provider, checkout_counter
    except Exception as e:
        raise RuntimeError(f\"Failed to initialize OTel metrics: {str(e)}\")

app = FastAPI(title=\"Checkout API\")

# Initialize OTel metrics and instrument FastAPI
try:
    metric_provider, checkout_counter = init_otel_metrics()
    FastAPIInstrumentor.instrument_app(app)
except Exception as e:
    print(f\"Warning: Failed to initialize OTel metrics: {str(e)}\")
    metric_provider = None
    checkout_counter = None

@app.get(\"/health\")
async def health_check():
    return {\"status\": \"healthy\"}

@app.post(\"/checkout\")
async def create_checkout(request: Request, items: list[dict]):
    start_time = time.time()
    try:
        # Simulate checkout processing
        time.sleep(0.2)

        # Increment custom checkout counter if OTel is initialized
        if checkout_counter:
            checkout_counter.add(1, {\"status\": \"success\"})

        return JSONResponse(
            status_code=200,
            content={\"checkout_id\": \"chk_123\", \"item_count\": len(items)},
        )
    except Exception as e:
        if checkout_counter:
            checkout_counter.add(1, {\"status\": \"error\"})
        return JSONResponse(
            status_code=500,
            content={\"error\": str(e)},
        )
    finally:
        # Record request duration (OTel auto-instruments, but custom metric for demo)
        duration = time.time() - start_time
        if metric_provider:
            meter = metrics.get_meter(\"fastapi-checkout-api\")
            duration_histogram = meter.create_histogram(
                name=\"checkout.duration\",
                description=\"Checkout processing duration\",
                unit=\"ms\",
            )
            duration_histogram.record(duration * 1000, {\"endpoint\": \"/checkout\"})

if __name__ == \"__main__\":
    import uvicorn
    uvicorn.run(app, host=\"0.0.0.0\", port=8000)

// Check OTel Go SDK version and supported frameworks
import (
    \"fmt\"
    \"runtime/debug\"
    \"go.opentelemetry.io/otel\"
)

func printOTelInfo() {
    info := debug.ReadBuildInfo()
    for _, dep := range info.Deps {
        if dep.Path == \"go.opentelemetry.io/otel\" {
            fmt.Printf(\"OTel SDK Version: %s\\n\", dep.Version)
        }
    }
    fmt.Printf(\"OTel Tracer Provider: %T\\n\", otel.GetTracerProvider())
}

# Estimate monthly OTel backend storage cost (AWS us-east-1 pricing)
PROMETHEUS_STORAGE_GB = 100  # Average for <100 hosts
JAEGER_STORAGE_GB = 50       # Average for <100 hosts
GB_COST = 0.10               # $0.10/GB/month for GP2 EBS

monthly_cost = (PROMETHEUS_STORAGE_GB + JAEGER_STORAGE_GB) * GB_COST
print(f\"Estimated monthly OTel backend storage cost: ${monthly_cost}\")

// Configure Datadog tracer for free tier (5 hosts max, 1-day retention)
import (
    \"github.com/DataDog/dd-trace-go/v2/ddtrace/tracer\"
)

func initDatadogFreeTier() {
    tracer.Start(
        tracer.WithService(\"my-mvp-api\"),
        tracer.WithSampleRate(1.0), // Free tier supports full sampling for <5 hosts
        tracer.WithLogsInjection(true), // Log correlation included in free tier
    )
    defer tracer.Stop()
}