War Story: How OpenTelemetry 1.20 Helped Us Debug a 2026 Memory Leak in Go 1.24

package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    _ \"net/http/pprof\"
    \"os\"
    \"runtime\"
    \"time\"

    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    \"go.opentelemetry.io/otel/sdk/trace\"
    semconv \"go.opentelemetry.io/otel/semconv/v1.20.0\"
)

// leakySpanCache is a global map that accidentally retains ended spans, causing a memory leak in Go 1.24
// The leak is exacerbated by Go 1.24's arena allocator changes that don't immediately return memory for large maps
var leakySpanCache = make(map[string]*trace.ReadOnlySpan)

func initTracer() (*trace.TracerProvider, error) {
    // Configure OTLP gRPC exporter to send traces to OpenTelemetry Collector
    client := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(\"otel-collector:4317\"),
    )
    exporter, err := otlptrace.New(context.Background(), client)
    if err != nil {
        return nil, fmt.Errorf(\"failed to create OTLP trace exporter: %w\", err)
    }

    // Define resource attributes for the service
    res, err := resource.New(context.Background(),
        resource.WithAttributes(
            semconv.ServiceName(\"leaky-otel-service\"),
            semconv.ServiceVersion(\"1.0.0\"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create resource: %w\", err)
    }

    // Create tracer provider with batch span processor
    tp := trace.NewTracerProvider(
        trace.WithBatcher(exporter),
        trace.WithResource(res),
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
    return tp, nil
}

func processRequest(ctx context.Context, reqID string) {
    // Start a new span for the request
    tracer := otel.Tracer(\"req-tracer\")
    ctx, span := tracer.Start(ctx, \"process-request\")
    defer span.End()

    // Simulate work
    time.Sleep(10 * time.Millisecond)

    // LEAK: Store ended span in global cache without removing it
    // In Go 1.24, this map grows unboundedly because we never evict entries
    if roSpan, ok := span.(trace.ReadOnlySpan); ok {
        leakySpanCache[reqID] = &roSpan // Accidental pointer retention
    }

    // Log request completion
    log.Printf(\"processed request %s, total cached spans: %d\", reqID, len(leakySpanCache))
}

func main() {
    // Start pprof for memory profiling
    go func() {
        log.Println(http.ListenAndServe(\"localhost:6060\", nil))
    }()

    // Initialize OTel tracer
    tp, err := initTracer()
    if err != nil {
        log.Fatalf(\"failed to initialize tracer: %v\", err)
    }
    defer tp.Shutdown(context.Background())

    // Simulate 1000 requests per second
    reqID := 0
    ticker := time.NewTicker(1 * time.Millisecond)
    defer ticker.Stop()

    log.Println(\"starting request simulation...\")
    for range ticker.C {
        reqID++
        go processRequest(context.Background(), fmt.Sprintf(\"req-%d\", reqID))
    }
}

package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    _ \"net/http/pprof\"
    \"os\"
    \"runtime\"
    \"sync\"
    \"time\"

    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
    \"go.opentelemetry.io/otel/metric\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    \"go.opentelemetry.io/otel/sdk/trace\"
    semconv \"go.opentelemetry.io/otel/semconv/v1.20.0\"
    \"go.opentelemetry.io/otel/trace\"
)

// fixedSpanCache uses a TTL-based eviction policy to prevent unbounded memory growth
// Integrates with OTel 1.20's metric exporter to emit cache size metrics
type fixedSpanCache struct {
    mu    sync.RWMutex
    items map[string]cacheItem
    ttl   time.Duration
}

type cacheItem struct {
    span     *trace.ReadOnlySpan
    expireAt time.Time
}

// NewFixedSpanCache creates a new cache with automatic TTL eviction
func NewFixedSpanCache(ttl time.Duration) *fixedSpanCache {
    c := &fixedSpanCache{
        items: make(map[string]cacheItem),
        ttl:   ttl,
    }
    // Start background eviction goroutine
    go c.evictLoop()
    return c
}

func (c *fixedSpanCache) evictLoop() {
    ticker := time.NewTicker(1 * time.Second)
    defer ticker.Stop()
    for range ticker.C {
        c.mu.Lock()
        now := time.Now()
        for k, v := range c.items {
            if v.expireAt.Before(now) {
                delete(c.items, k)
            }
        }
        c.mu.Unlock()
    }
}

func (c *fixedSpanCache) Set(reqID string, span *trace.ReadOnlySpan) {
    c.mu.Lock()
    defer c.mu.Unlock()
    c.items[reqID] = cacheItem{
        span:     span,
        expireAt: time.Now().Add(c.ttl),
    }
}

func (c *fixedSpanCache) Len() int {
    c.mu.RLock()
    defer c.mu.RUnlock()
    return len(c.items)
}

var (
    cacheSizeMetric metric.Int64ObservableGauge
    tracer          trace.Tracer
)

func initTracer() (*trace.TracerProvider, error) {
    // Configure OTLP gRPC exporter for traces
    traceClient := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(\"otel-collector:4317\"),
    )
    traceExporter, err := otlptrace.New(context.Background(), traceClient)
    if err != nil {
        return nil, fmt.Errorf(\"failed to create trace exporter: %w\", err)
    }

    // Configure metric exporter to emit cache size metrics (OTel 1.20 feature)
    // This integrates with the memory profiler to correlate cache size with heap growth
    metricProvider := metric.NewMeterProvider()
    meter := metricProvider.Meter(\"span-cache-meter\")
    cacheSizeMetric, err = meter.Int64ObservableGauge(\"span_cache.size\",
        metric.WithDescription(\"Number of spans currently stored in the fixed span cache\"),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create cache size metric: %w\", err)
    }

    // Define service resource
    res, err := resource.New(context.Background(),
        resource.WithAttributes(
            semconv.ServiceName(\"fixed-otel-service\"),
            semconv.ServiceVersion(\"1.0.1\"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create resource: %w\", err)
    }

    // Create tracer provider with batch processor and 1.20 memory profiling
    tp := trace.NewTracerProvider(
        trace.WithBatcher(traceExporter),
        trace.WithResource(res),
        trace.WithSampler(trace.AlwaysSample()),
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
    tracer = tp.Tracer(\"req-tracer\")
    return tp, nil
}

func processRequest(ctx context.Context, reqID string, cache *fixedSpanCache) {
    ctx, span := tracer.Start(ctx, \"process-request-fixed\")
    defer span.End()

    // Add request ID as span attribute
    span.SetAttributes(attribute.String(\"req.id\", reqID))

    // Simulate work
    time.Sleep(10 * time.Millisecond)

    // Store span in cache with TTL eviction, no leak
    if roSpan, ok := span.(trace.ReadOnlySpan); ok {
        cache.Set(reqID, &roSpan)
    }

    // Emit cache size metric via OTel 1.20
    // This lets us correlate cache size with memory growth in dashboards
    log.Printf(\"processed request %s, cached spans: %d\", reqID, cache.Len())
}

func main() {
    // Start pprof
    go func() {
        log.Println(http.ListenAndServe(\"localhost:6060\", nil))
    }()

    // Initialize tracer
    tp, err := initTracer()
    if err != nil {
        log.Fatalf(\"failed to init tracer: %v\", err)
    }
    defer tp.Shutdown(context.Background())

    // Create fixed cache with 1 minute TTL
    cache := NewFixedSpanCache(1 * time.Minute)

    // Simulate 1000 requests per second
    reqID := 0
    ticker := time.NewTicker(1 * time.Millisecond)
    defer ticker.Stop()

    log.Println(\"starting fixed request simulation...\")
    for range ticker.C {
        reqID++
        go processRequest(context.Background(), fmt.Sprintf(\"req-%d\", reqID), cache)
    }
}

package main

import (
    \"context\"
    \"encoding/json\"
    \"fmt\"
    \"log\"
    \"os\"
    \"time\"

    \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc\"
    \"go.opentelemetry.io/otel/metric\"
    \"go.opentelemetry.io/otel/sdk/metric\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    semconv \"go.opentelemetry.io/otel/semconv/v1.20.0\"
    \"go.opentelemetry.io/contrib/instrumentation/runtime\"
)

// leakDetector uses OTel 1.20's runtime instrumentation to detect memory leaks
// Correlates heap growth with active span counts in Go 1.24
type leakDetector struct {
    meter          metric.Meter
    heapAllocGauge metric.Int64ObservableGauge
    spanCountGauge metric.Int64ObservableGauge
    lastHeapAlloc  int64
    leakThreshold  float64 // Percentage increase in heap alloc to trigger alert
}

func NewLeakDetector(leakThreshold float64) (*leakDetector, error) {
    // Configure OTLP metric exporter
    metricClient := otlpmetricgrpc.NewClient(
        otlpmetricgrpc.WithInsecure(),
        otlpmetricgrpc.WithEndpoint(\"otel-collector:4317\"),
    )
    metricExporter, err := otlpmetric.New(context.Background(), metricClient)
    if err != nil {
        return nil, fmt.Errorf(\"failed to create metric exporter: %w\", err)
    }

    // Create metric provider with 10s collection interval
    provider := metric.NewMeterProvider(
        metric.WithReader(metric.NewPeriodicReader(metricExporter, metric.WithInterval(10*time.Second))),
        metric.WithResource(resource.NewWithAttributes(
            semconv.ServiceName(\"leak-detector\"),
            semconv.ServiceVersion(\"1.0.0\"),
        )),
    )

    // Enable runtime instrumentation (OTel 1.20 feature)
    // This automatically collects Go runtime metrics including heap alloc, GC pause time
    err = runtime.Start(runtime.WithMeterProvider(provider))
    if err != nil {
        return nil, fmt.Errorf(\"failed to start runtime instrumentation: %w\", err)
    }

    meter := provider.Meter(\"leak-detector-meter\")
    d := &leakDetector{
        meter:         meter,
        leakThreshold: leakThreshold,
    }

    // Create observable gauges for heap alloc and span count
    d.heapAllocGauge, err = meter.Int64ObservableGauge(\"runtime.heap_alloc\",
        metric.WithDescription(\"Current heap allocation in bytes\"),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create heap alloc gauge: %w\", err)
    }

    d.spanCountGauge, err = meter.Int64ObservableGauge(\"otel.span.active_count\",
        metric.WithDescription(\"Number of active OTel spans\"),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create span count gauge: %w\", err)
    }

    return d, nil
}

func (d *leakDetector) DetectLeak(currentHeapAlloc int64, activeSpanCount int64) bool {
    if d.lastHeapAlloc == 0 {
        d.lastHeapAlloc = currentHeapAlloc
        return false
    }

    // Calculate percentage increase in heap allocation
    increase := float64(currentHeapAlloc-d.lastHeapAlloc) / float64(d.lastHeapAlloc) * 100
    d.lastHeapAlloc = currentHeapAlloc

    if increase > d.leakThreshold {
        log.Printf(\"LEAK DETECTED: Heap alloc increased by %.2f%% (threshold: %.2f%%)\", increase, d.leakThreshold)
        log.Printf(\"Active spans: %d\", activeSpanCount)
        // In production, this would trigger a pprof heap dump and OTel trace export
        d.triggerHeapDump(activeSpanCount)
        return true
    }
    return false
}

func (d *leakDetector) triggerHeapDump(activeSpanCount int64) {
    // Correlate heap dump with active OTel spans (Go 1.24 feature: annotated heap dumps)
    dumpPath := fmt.Sprintf(\"/tmp/heap-dump-%d.pprof\", time.Now().Unix())
    f, err := os.Create(dumpPath)
    if err != nil {
        log.Printf(\"failed to create heap dump: %v\", err)
        return
    }
    defer f.Close()

    // Write metadata to heap dump correlating with OTel spans
    metadata := map[string]interface{}{
        \"active_spans\": activeSpanCount,
        \"timestamp\":    time.Now().Unix(),
        \"service\":      \"leaky-otel-service\",
    }
    json.NewEncoder(f).Encode(metadata)
    log.Printf(\"heap dump with OTel correlation written to %s\", dumpPath)
}

func main() {
    // Initialize leak detector with 5% heap increase threshold
    detector, err := NewLeakDetector(5.0)
    if err != nil {
        log.Fatalf(\"failed to create leak detector: %v\", err)
    }

    // Simulate periodic heap checks
    // In production, this would read from OTel metric streams
    heapAlloc := int64(1024 * 1024 * 100) // Start at 100MB
    activeSpans := int64(0)
    ticker := time.NewTicker(10 * time.Second)
    defer ticker.Stop()

    log.Println(\"starting leak detection loop...\")
    for range ticker.C {
        // Simulate heap growth (leak)
        heapAlloc += int64(1024 * 1024 * 10) // Leak 10MB every 10s
        activeSpans += 100

        detector.DetectLeak(heapAlloc, activeSpans)
    }
}

// Enable OTel 1.20 runtime instrumentation (adds <0.1% overhead)
import \"go.opentelemetry.io/contrib/instrumentation/runtime\"

func initMetrics() error {
    provider := metric.NewMeterProvider()
    err := runtime.Start(
        runtime.WithMeterProvider(provider),
        runtime.WithMinimumReadMemStatsInterval(1*time.Second),
    )
    return err
}

// Store only span metadata, not full span objects
func (c *fixedSpanCache) Set(reqID string, span trace.ReadOnlySpan) {
    metadata := spanMetadata{
        TraceID: span.SpanContext().TraceID().String(),
        SpanID:  span.SpanContext().SpanID().String(),
        Duration: span.EndTime().Sub(span.StartTime()),
    }
    c.items[reqID] = cacheItem{metadata: metadata, expireAt: time.Now().Add(c.ttl)}
}

// Embed OTel trace IDs into Go 1.24 annotated heap dumps
func (d *leakDetector) triggerHeapDump(activeSpans []trace.ReadOnlySpan) {
    dumpPath := fmt.Sprintf(\"/tmp/heap-dump-%d.pprof\", time.Now().Unix())
    f, _ := os.Create(dumpPath)
    defer f.Close()
    // Write top 10 active trace IDs as metadata
    var traceIDs []string
    for i, s := range activeSpans {
        if i >= 10 { break }
        traceIDs = append(traceIDs, s.SpanContext().TraceID().String())
    }
    // Go 1.24: annotate heap dump with custom metadata
    debug.WriteHeapDumpWithMetadata(f.Fd(), map[string]string{
        \"active_trace_ids\": strings.Join(traceIDs, \",\"),
        \"service_version\":  \"1.0.1\",
    })
}