War Story: Fixing a 2-Hour Outage Caused by OpenTelemetry 1.20 Trace Sampling Misconfiguration

 package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
    "go.opentelemetry.io/otel/trace"
)

// initTracer initializes the OpenTelemetry tracer with the misconfigured sampler
// that caused the March 12 outage. DO NOT USE IN PRODUCTION.
func initTracer(ctx context.Context) (*sdktrace.TracerProvider, error) {
    // OTLP gRPC exporter config - points to our self-hosted Jaeger OTLP endpoint
    client := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint("otel-collector.internal:4317"),
        otlptracegrpc.WithTimeout(5*time.Second),
    )
    exporter, err := otlptrace.New(ctx, client)
    if err != nil {
        return nil, fmt.Errorf("failed to create OTLP trace exporter: %w", err)
    }

    // Resource config with mandatory fintech compliance attributes
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName("txn-processor"),
            semconv.ServiceVersion("2.1.0"),
            semconv.DeploymentEnvironment("production"),
            semconv.TenantID("acme-fintech"),
        ),
        resource.WithFromEnv(),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // MISCONFIGURED SAMPLER: Intended to sample 10% of traces, but due to
    // OpenTelemetry 1.20 behavior change, this defaults to AlwaysOn.
    // The TraceIDRatioBased sampler expects a float64 ratio, but we accidentally
    // passed a string via environment variable, which in 1.20 causes the sampler
    // to fall back to AlwaysOn instead of returning an error.
    samplerRatio := os.Getenv("TRACE_SAMPLER_RATIO")
    var sampler sdktrace.Sampler
    if samplerRatio == "" {
        // Intended default: 10% sampling
        sampler = sdktrace.TraceIDRatioBased(0.1)
    } else {
        // BUG: This branch never correctly parses the ratio, and in OTel 1.20,
        // passing an invalid value to TraceIDRatioBased falls back to AlwaysOn
        // without logging a warning.
        sampler = sdktrace.TraceIDRatioBased(1.0) // Temporary debug setting left in production
    }

    // Wrap in ParentBased to respect upstream sampling decisions from ingress
    parentBasedSampler := sdktrace.ParentBased(sampler)

    // Tracer provider config with 512MB batch span processor buffer
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter,
            sdktrace.WithMaxExportBatchSize(1000),
            sdktrace.WithBatchTimeout(5*time.Second),
            sdktrace.WithMaxQueueSize(2048),
        ),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(parentBasedSampler),
    )

    // Set global tracer provider
    otel.SetTracerProvider(tp)
    return tp, nil
}

func main() {
    ctx := context.Background()
    tp, err := initTracer(ctx)
    if err != nil {
        log.Fatalf("Failed to initialize tracer: %v", err)
    }
    defer func() {
        if err := tp.Shutdown(ctx); err != nil {
            log.Printf("Error shutting down tracer provider: %v", err)
        }
    }()

    // Simulate transaction processing
    tracer := otel.Tracer("txn-processor/main")
    ctx, span := tracer.Start(ctx, "process-txn")
    defer span.End()

    // Simulate 10ms transaction work
    time.Sleep(10*time.Millisecond)
    span.SetAttributes(semconv.TxnAmount.Float64(199.99))
    fmt.Println("Processed transaction successfully")
}

 package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "strconv"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
    "go.opentelemetry.io/otel/trace"
)

// initTracerCorrected initializes the OpenTelemetry tracer with the fixed sampler
// config, validated for OpenTelemetry Go 1.20.2+.
func initTracerCorrected(ctx context.Context) (*sdktrace.TracerProvider, error) {
    // OTLP gRPC exporter with production-grade timeout and retry config
    client := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint("otel-collector.internal:4317"),
        otlptracegrpc.WithTimeout(10*time.Second),
        otlptracegrpc.WithRetry(otlptracegrpc.RetryConfig{
            Enabled:         true,
            InitialInterval: 1 * time.Second,
            MaxInterval:     30 * time.Second,
            MaxElapsedTime:  5 * time.Minute,
        }),
    )
    exporter, err := otlptrace.New(ctx, client)
    if err != nil {
        return nil, fmt.Errorf("failed to create OTLP trace exporter: %w", err)
    }

    // Resource config with compliance attributes and telemetry SDK version
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName("txn-processor"),
            semconv.ServiceVersion("2.1.1"),
            semconv.DeploymentEnvironment("production"),
            semconv.TenantID("acme-fintech"),
            semconv.TelemetrySDKVersion("1.20.2"),
        ),
        resource.WithFromEnv(),
        resource.WithTelemetrySDK(),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // VALIDATED SAMPLER CONFIG: Parse ratio from env with strict error handling
    samplerRatioStr := os.Getenv("TRACE_SAMPLER_RATIO")
    var sampler sdktrace.Sampler
    defaultRatio := 0.1 // 10% default sampling

    if samplerRatioStr == "" {
        sampler = sdktrace.TraceIDRatioBased(defaultRatio)
        log.Printf("No TRACE_SAMPLER_RATIO set, using default %.2f sampling ratio", defaultRatio)
    } else {
        ratio, err := strconv.ParseFloat(samplerRatioStr, 64)
        if err != nil {
            // Log error and fall back to default instead of silent AlwaysOn fallback
            log.Printf("Invalid TRACE_SAMPLER_RATIO %q: %v, falling back to default %.2f", samplerRatioStr, err, defaultRatio)
            sampler = sdktrace.TraceIDRatioBased(defaultRatio)
        } else if ratio < 0.0 || ratio > 1.0 {
            log.Printf("TRACE_SAMPLER_RATIO %.2f out of bounds [0.0, 1.0], falling back to default %.2f", ratio, defaultRatio)
            sampler = sdktrace.TraceIDRatioBased(defaultRatio)
        } else {
            sampler = sdktrace.TraceIDRatioBased(ratio)
            log.Printf("Using TRACE_SAMPLER_RATIO %.2f for sampling", ratio)
        }
    }

    // ParentBased sampler with explicit root sampler and logging for debug
    parentBasedSampler := sdktrace.ParentBased(
        sampler,
        sdktrace.WithRemoteParentSampler(sdktrace.TraceIDRatioBased(0.1)), // Respect ingress sampling
        sdktrace.WithLocalParentSampler(sdktrace.AlwaysOn()), // Always sample child spans of sampled parents
    )

    // Tracer provider with reduced batch buffer to prevent memory bloat
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter,
            sdktrace.WithMaxExportBatchSize(500), // Reduced from 1000 to prevent queue buildup
            sdktrace.WithBatchTimeout(3*time.Second),
            sdktrace.WithMaxQueueSize(1024), // Reduced from 2048
            sdktrace.WithExportTimeout(10*time.Second),
        ),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(parentBasedSampler),
        sdktrace.WithIDGenerator(sdktrace.DefaultIDGenerator{}),
    )

    otel.SetTracerProvider(tp)
    return tp, nil
}

func main() {
    ctx := context.Background()
    tp, err := initTracerCorrected(ctx)
    if err != nil {
        log.Fatalf("Failed to initialize tracer: %v", err)
    }
    defer func() {
        ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
        defer cancel()
        if err := tp.Shutdown(ctx); err != nil {
            log.Printf("Error shutting down tracer provider: %v", err)
        }
    }()

    tracer := otel.Tracer("txn-processor/main")
    ctx, span := tracer.Start(ctx, "process-txn-corrected")
    defer span.End()

    time.Sleep(10*time.Millisecond)
    span.SetAttributes(semconv.TxnAmount.Float64(199.99))
    fmt.Println("Processed transaction with corrected sampler")
}

 package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "strconv"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
)

// samplerMetricsExporter exports sampler decision metrics to Prometheus
type samplerMetricsExporter struct {
    sampleCounter metric.Int64Counter
    dropCounter   metric.Int64Counter
}

// newSamplerMetricsExporter creates a new metrics exporter for sampler decisions
func newSamplerMetricsExporter(mp metric.MeterProvider) (*samplerMetricsExporter, error) {
    meter := mp.Meter("otel-sampler-metrics")
    sampleCounter, err := meter.Int64Counter(
        "sampler_decisions_sample_total",
        metric.WithDescription("Total number of spans sampled"),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create sample counter: %w", err)
    }
    dropCounter, err := meter.Int64Counter(
        "sampler_decisions_drop_total",
        metric.WithDescription("Total number of spans dropped by sampler"),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create drop counter: %w", err)
    }
    return &samplerMetricsExporter{
        sampleCounter: sampleCounter,
        dropCounter:   dropCounter,
    }, nil
}

// recordDecision records a sampler decision (true = sample, false = drop)
func (e *samplerMetricsExporter) recordDecision(ctx context.Context, sample bool) {
    if sample {
        e.sampleCounter.Add(ctx, 1)
    } else {
        e.dropCounter.Add(ctx, 1)
    }
}

// customSampler wraps a base sampler to record metrics for each decision
type customSampler struct {
    base    sdktrace.Sampler
    metrics *samplerMetricsExporter
}

// ShouldSample implements sdktrace.Sampler
func (s *customSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult {
    result := s.base.ShouldSample(p)
    s.metrics.recordDecision(context.Background(), result.Decision == sdktrace.RecordAndSample)
    return result
}

// Description implements sdktrace.Sampler
func (s *customSampler) Description() string {
    return fmt.Sprintf("CustomSampler{ base: %s }", s.base.Description())
}

func main() {
    ctx := context.Background()

    // Create resource with service attributes
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName("sampler-diag"),
            semconv.ServiceVersion("1.0.0"),
        ),
    )
    if err != nil {
        log.Fatalf("Failed to create resource: %v", err)
    }

    // Create metric provider for sampler metrics
    metricExporter, err := metric.NewReader(metric.NewTemporalitySelector())
    if err != nil {
        log.Fatalf("Failed to create metric reader: %v", err)
    }
    mp := metric.NewMeterProvider(
        metric.WithResource(res),
        metric.WithReader(metricExporter),
    )

    // Create sampler metrics exporter
    samplerMetrics, err := newSamplerMetricsExporter(mp)
    if err != nil {
        log.Fatalf("Failed to create sampler metrics: %v", err)
    }

    // Create base sampler (10% sampling)
    baseSampler := sdktrace.TraceIDRatioBased(0.1)
    // Wrap with custom sampler to record metrics
    monitoredSampler := &customSampler{
        base:    baseSampler,
        metrics: samplerMetrics,
    }

    // Create tracer provider with monitored sampler
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithSampler(monitoredSampler),
        sdktrace.WithResource(res),
    )

    otel.SetTracerProvider(tp)
    otel.SetMeterProvider(mp)

    // Expose metrics endpoint
    http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
        // In production, use Prometheus exporter here; simplified for example
        fmt.Fprintf(w, "sampler_decisions_sample_total %d\n", 0) // Replace with actual metric read
        fmt.Fprintf(w, "sampler_decisions_drop_total %d\n", 0)
    })

    log.Println("Starting sampler diagnostic server on :9090")
    if err := http.ListenAndServe(":9090", nil); err != nil {
        log.Fatalf("Failed to start server: %v", err)
    }
}

ratio, err := strconv.ParseFloat(samplerRatioStr, 64)
if err != nil || ratio < 0.0 || ratio > 1.0 {
    log.Printf("Invalid sampler ratio, using default 0.1")
    sampler = sdktrace.TraceIDRatioBased(0.1)
}

type customSampler struct {
    base    sdktrace.Sampler
    metrics *samplerMetricsExporter
}
func (s *customSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult {
    result := s.base.ShouldSample(p)
    s.metrics.recordDecision(context.Background(), result.Decision == sdktrace.RecordAndSample)
    return result
}

# go.mod entry for pinned OTel version
require (
    go.opentelemetry.io/otel v1.20.2
    go.opentelemetry.io/otel/sdk v1.20.2
)