Retrospective: 1 Year with OpenTelemetry 1.22: Lessons from 100 Microservices

// Package main demonstrates full OpenTelemetry 1.22 instrumentation for a Go microservice
// using OTLP/gRPC export to the OpenTelemetry Collector, with trace, metric, and log support.
// Dependencies (go.mod):
// module otel-demo
// go 1.21
// require (
//  go.opentelemetry.io/otel v1.22.0
//  go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0
//  go.opentelemetry.io/otel/exporters/otlp/otlpmetric v1.22.0
//  go.opentelemetry.io/otel/exporters/otlp/otlplog v1.22.0
//  go.opentelemetry.io/otel/sdk v1.22.0
//  go.opentelemetry.io/otel/sdk/metric v1.22.0
//  go.opentelemetry.io/otel/sdk/log v1.22.0
//  go.opentelemetry.io/otel/propagation v1.22.0
//  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0
//  go.opentelemetry.io/contrib/bridges/otellogrus v0.47.0
//  github.com/gin-gonic/gin v1.9.1
//  github.com/sirupsen/logrus v1.9.3
// )
package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"time\"

    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/log\"
    \"go.opentelemetry.io/otel/sdk/log/loggerprovider\"
    \"go.opentelemetry.io/otel/sdk/metric\"
    \"go.opentelemetry.io/otel/sdk/metric/metricreader\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    \"go.opentelemetry.io/otel/sdk/trace\"
    \"go.opentelemetry.io/contrib/bridges/otellogrus\"
    \"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\"
    \"github.com/gin-gonic/gin\"
    \"github.com/sirupsen/logrus\"
)

// serviceName is the OpenTelemetry resource identifier for this microservice
const serviceName = \"payment-processor-v1\"

// initResource creates a shared OpenTelemetry resource with standard attributes
func initResource(ctx context.Context) (*resource.Resource, error) {
    return resource.New(ctx,
        resource.WithAttributes(
            attribute.Key(\"service.name\").String(serviceName),
            attribute.Key(\"service.version\").String(\"1.0.0\"),
            attribute.Key(\"deployment.environment\").String(os.Getenv(\"ENVIRONMENT\")),
            attribute.Key(\"host.name\").String(os.Getenv(\"HOSTNAME\")),
        ),
        resource.WithFromEnv(), // Pull additional attributes from OTEL_RESOURCE_ATTRIBUTES env var
    )
}

// initTracer sets up the OTLP/gRPC trace exporter and tracer provider
func initTracer(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) {
    // Configure OTLP exporter to send traces to the collector
    exporter, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint(os.Getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\")),
        otlptracegrpc.WithInsecure(), // Use TLS in production!
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create trace exporter: %w\", err)
    }

    // Create tracer provider with batch processor and resource
    tp := trace.NewTracerProvider(
        trace.WithBatcher(exporter,
            trace.WithMaxExportBatchSize(512),
            trace.WithBatchTimeout(5*time.Second),
        ),
        trace.WithResource(res),
        trace.WithSampler(trace.ParentBased(trace.TraceIDRatioBased(0.1))), // 10% sampling for low-priority routes
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))
    return tp, nil
}

// initMeter sets up the OTLP/gRPC metric exporter and meter provider
func initMeter(ctx context.Context, res *resource.Resource) (*metric.MeterProvider, error) {
    // Configure OTLP metric exporter with periodic reader
    exporter, err := otlpmetricgrpc.New(ctx,
        otlpmetricgrpc.WithEndpoint(os.Getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\")),
        otlpmetricgrpc.WithInsecure(),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create metric exporter: %w\", err)
    }

    reader := metricreader.NewPeriodicReader(exporter,
        metricreader.WithInterval(10*time.Second), // Export metrics every 10s
    )
    mp := metric.NewMeterProvider(
        metric.WithResource(res),
        metric.WithReader(reader),
    )
    otel.SetMeterProvider(mp)
    return mp, nil
}

// initLogger sets up the OTLP/gRPC log exporter and bridges logrus to OpenTelemetry
func initLogger(ctx context.Context, res *resource.Resource) (*loggerprovider.LoggerProvider, logrus.FieldLogger, error) {
    exporter, err := otlploggrpc.New(ctx,
        otlploggrpc.WithEndpoint(os.Getenv(\"OTEL_EXPORTER_OTLP_ENDPOINT\")),
        otlploggrpc.WithInsecure(),
    )
    if err != nil {
        return nil, nil, fmt.Errorf(\"failed to create log exporter: %w\", err)
    }

    lp := loggerprovider.New(
        log.NewBatchProcessor(exporter,
            log.WithMaxQueueSize(2048),
            log.WithBatchTimeout(5*time.Second),
        ),
        loggerprovider.WithResource(res),
    )

    // Bridge logrus to OpenTelemetry logs
    logger := logrus.New()
    logger.SetFormatter(&logrus.JSONFormatter{})
    logger.AddHook(otellogrus.NewHook(
        otellogrus.WithLoggerProvider(lp),
        otellogrus.WithLevels(logrus.AllLevels...),
    ))

    return lp, logger, nil
}

func main() {
    ctx := context.Background()

    // Initialize OpenTelemetry resources
    res, err := initResource(ctx)
    if err != nil {
        log.Fatalf(\"Failed to initialize OTel resource: %v\", err)
    }

    tp, err := initTracer(ctx, res)
    if err != nil {
        log.Fatalf(\"Failed to initialize tracer: %v\", err)
    }
    defer tp.Shutdown(ctx)

    mp, err := initMeter(ctx, res)
    if err != nil {
        log.Fatalf(\"Failed to initialize meter: %v\", err)
    }
    defer mp.Shutdown(ctx)

    lp, logger, err := initLogger(ctx, res)
    if err != nil {
        log.Fatalf(\"Failed to initialize logger: %v\", err)
    }
    defer lp.Shutdown(ctx)

    // Create meter and define custom metrics
    meter := mp.Meter(\"payment-processor\")
    requestCounter, err := meter.Int64Counter(
        \"http.server.requests.count\",
        metric.WithDescription(\"Total number of HTTP requests\"),
        metric.WithUnit(\"1\"),
    )
    if err != nil {
        log.Fatalf(\"Failed to create request counter: %v\", err)
    }
    latencyHistogram, err := meter.Float64Histogram(
        \"http.server.requests.duration\",
        metric.WithDescription(\"HTTP request duration in milliseconds\"),
        metric.WithUnit(\"ms\"),
        metric.WithExplicitBucketBoundaries(10, 50, 100, 500, 1000, 5000),
    )
    if err != nil {
        log.Fatalf(\"Failed to create latency histogram: %v\", err)
    }

    // Set up Gin router with OTel instrumentation
    r := gin.Default()
    r.Use(func(c *gin.Context) {
        start := time.Now()
        c.Next()
        latency := float64(time.Since(start).Milliseconds())
        attrs := []attribute.KeyValue{
            attribute.Key(\"http.method\").String(c.Request.Method),
            attribute.Key(\"http.route\").String(c.FullPath()),
            attribute.Key(\"http.status_code\").Int(c.Writer.Status()),
        }
        requestCounter.Add(ctx, 1, metric.WithAttributes(attrs...))
        latencyHistogram.Record(ctx, latency, metric.WithAttributes(attrs...))
    })

    // Instrumented health check endpoint
    r.GET(\"/health\", func(c *gin.Context) {
        ctx := c.Request.Context()
        logger.WithContext(ctx).Info(\"health check requested\")
        c.JSON(http.StatusOK, gin.H{\"status\": \"healthy\"})
    })

    // Instrumented payment processing endpoint
    r.POST(\"/process\", otelhttp.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        ctx := r.Context()
        logger.WithContext(ctx).Info(\"processing payment\")

        // Simulate payment processing with trace span
        _, span := otel.Tracer(\"payment-processor\").Start(ctx, \"process-payment\")
        defer span.End()

        // Simulate 10% failure rate for demo
        if time.Now().Unix()%10 == 0 {
            span.SetAttributes(attribute.Key(\"payment.success\").Bool(false))
            logger.WithContext(ctx).Error(\"payment processing failed\")
            w.WriteHeader(http.StatusInternalServerError)
            return
        }

        span.SetAttributes(attribute.Key(\"payment.success\").Bool(true))
        logger.WithContext(ctx).Info(\"payment processed successfully\")
        w.WriteHeader(http.StatusOK)
    }), \"process-payment\").ServeHTTP)

    // Start HTTP server
    listenAddr := \":8080\"
    if addr := os.Getenv(\"LISTEN_ADDR\"); addr != \"\" {
        listenAddr = addr
    }
    logger.Infof(\"Starting server on %s\", listenAddr)
    if err := r.Run(listenAddr); err != nil {
        logger.WithError(err).Fatal(\"failed to start server\")
    }
}

// Package main implements a custom OpenTelemetry SpanProcessor that redacts PII (email, phone, credit card)
// from span attributes before export, compliant with GDPR and CCPA requirements.
// Dependencies (go.mod):
// module otel-pii-redactor
// go 1.21
// require (
//  go.opentelemetry.io/otel v1.22.0
//  go.opentelemetry.io/otel/sdk v1.22.0
//  github.com/go-playground/validator/v10 v10.19.0
// )
package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"regexp\"
    \"strings\"

    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/sdk/trace\"
    \"go.opentelemetry.io/otel/trace\"
    \"github.com/go-playground/validator/v10\"
)

// piiRedactorSpanProcessor is a custom SpanProcessor that redacts PII from span attributes
type piiRedactorSpanProcessor struct {
    emailRegex    *regexp.Regexp
    phoneRegex    *regexp.Regexp
    creditCardRegex *regexp.Regexp
    validator     *validator.Validate
}

// newPIIRedactorSpanProcessor creates a new PII redaction span processor
func newPIIRedactorSpanProcessor() (*piiRedactorSpanProcessor, error) {
    emailRegex, err := regexp.Compile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}`)
    if err != nil {
        return nil, fmt.Errorf(\"failed to compile email regex: %w\", err)
    }
    phoneRegex, err := regexp.Compile(`\\d{3}-\\d{3}-\\d{4}|\\d{10}|\\(\\d{3}\\) \\d{3}-\\d{4}`)
    if err != nil {
        return nil, fmt.Errorf(\"failed to compile phone regex: %w\", err)
    }
    creditCardRegex, err := regexp.Compile(`\\d{4}-\\d{4}-\\d{4}-\\d{4}|\\d{16}`)
    if err != nil {
        return nil, fmt.Errorf(\"failed to compile credit card regex: %w\", err)
    }
    return &piiRedactorSpanProcessor{
        emailRegex:      emailRegex,
        phoneRegex:      phoneRegex,
        creditCardRegex: creditCardRegex,
        validator:       validator.New(),
    }, nil
}

// OnStart implements trace.SpanProcessor
func (p *piiRedactorSpanProcessor) OnStart(ctx context.Context, span trace.Span) {
    // No-op on span start, we redact on export
}

// OnEnd implements trace.SpanProcessor
func (p *piiRedactorSpanProcessor) OnEnd(span trace.Span) {
    // Get mutable span data (note: this uses internal SDK APIs, wrap with caution)
    roSpan, ok := span.(interface{ Exportable() bool })
    if !ok {
        log.Printf(\"span %s is not exportable, skipping redaction\", span.SpanContext().SpanID())
        return
    }
    if !roSpan.Exportable() {
        return
    }

    // Redact attributes
    attrs := span.Attributes()
    redactedAttrs := make([]attribute.KeyValue, 0, len(attrs))
    for _, attr := range attrs {
        redacted := p.redactAttribute(attr)
        redactedAttrs = append(redactedAttrs, redacted)
    }

    // Replace span attributes with redacted ones (note: this requires SDK modification for full support,
    // in production we use a custom Exporter that redacts before sending)
    // For demo purposes, we log redacted attributes
    log.Printf(\"Redacted span %s attributes: %v\", span.SpanContext().SpanID(), redactedAttrs)
}

// OnSetAttribute implements trace.SpanProcessor (optional, no-op here)
func (p *piiRedactorSpanProcessor) OnSetAttribute(span trace.Span, attr attribute.KeyValue) {
    // No-op, we redact on end
}

// Shutdown implements trace.SpanProcessor
func (p *piiRedactorSpanProcessor) Shutdown(ctx context.Context) error {
    log.Println(\"PII redactor span processor shut down\")
    return nil
}

// ForceFlush implements trace.SpanProcessor
func (p *piiRedactorSpanProcessor) ForceFlush(ctx context.Context) error {
    return nil
}

// redactAttribute redacts PII from a single attribute
func (p *piiRedactorSpanProcessor) redactAttribute(attr attribute.KeyValue) attribute.KeyValue {
    // Only redact string attributes
    if attr.Value.Type() != attribute.STRING {
        return attr
    }
    val := attr.Value.AsString()

    // Skip redaction for non-PII attributes
    if strings.HasPrefix(attr.Key, \"service.\") || strings.HasPrefix(attr.Key, \"http.\") {
        return attr
    }

    // Redact email
    if p.emailRegex.MatchString(val) {
        return attribute.String(attr.Key, \"[REDACTED_EMAIL]\")
    }

    // Redact phone
    if p.phoneRegex.MatchString(val) {
        return attribute.String(attr.Key, \"[REDACTED_PHONE]\")
    }

    // Redact credit card
    if p.creditCardRegex.MatchString(val) {
        return attribute.String(attr.Key, \"[REDACTED_CREDIT_CARD]\")
    }

    // Validate if value is an email using validator package
    if err := p.validator.Var(val, \"email\"); err == nil {
        return attribute.String(attr.Key, \"[REDACTED_EMAIL]\")
    }

    return attr
}

func main() {
    // Initialize redactor
    redactor, err := newPIIRedactorSpanProcessor()
    if err != nil {
        log.Fatalf(\"Failed to create PII redactor: %v\", err)
    }

    // Create tracer provider with custom redactor
    tp := trace.NewTracerProvider(
        trace.WithSpanProcessor(redactor),
    )
    otel.SetTracerProvider(tp)

    // Create a test span with PII
    ctx := context.Background()
    _, span := otel.Tracer(\"test-tracer\").Start(ctx, \"test-span\")
    span.SetAttributes(
        attribute.String(\"user.email\", \"test@example.com\"),
        attribute.String(\"user.phone\", \"555-123-4567\"),
        attribute.String(\"payment.cc\", \"4111-1111-1111-1111\"),
        attribute.String(\"service.name\", \"test-service\"), // Should not be redacted
        attribute.Int(\"http.status_code\", 200), // Non-string, no redaction
    )
    span.End()

    // Shutdown
    if err := tp.Shutdown(ctx); err != nil {
        log.Fatalf(\"Failed to shutdown tracer provider: %v\", err)
    }
}

// Package main demonstrates OpenTelemetry 1.22 metric instrumentation with Prometheus export
// and custom business metrics for a microservice.
// Dependencies (go.mod):
// module otel-prometheus-demo
// go 1.21
// require (
//  go.opentelemetry.io/otel v1.22.0
//  go.opentelemetry.io/otel/exporters/prometheus v1.22.0
//  go.opentelemetry.io/otel/sdk/metric v1.22.0
//  github.com/prometheus/client_golang v1.19.1
// )
package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"time\"

    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/prometheus\"
    \"go.opentelemetry.io/otel/metric\"
    \"go.opentelemetry.io/otel/sdk/metric\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    \"github.com/prometheus/client_golang/prometheus/promhttp\"
)

// serviceName is the OpenTelemetry resource identifier
const serviceName = \"order-processor-v1\"

func main() {
    ctx := context.Background()

    // Initialize Prometheus exporter
    exporter, err := prometheus.New(
        prometheus.WithNamespace(serviceName),
        prometheus.WithoutScopeInfo(),
    )
    if err != nil {
        log.Fatalf(\"Failed to create Prometheus exporter: %w\", err)
    }

    // Create resource with service attributes
    res, err := resource.New(ctx,
        resource.WithAttributes(
            attribute.Key(\"service.name\").String(serviceName),
            attribute.Key(\"service.version\").String(\"2.1.0\"),
            attribute.Key(\"deployment.environment\").String(os.Getenv(\"ENVIRONMENT\")),
        ),
    )
    if err != nil {
        log.Fatalf(\"Failed to create resource: %w\", err)
    }

    // Create meter provider with Prometheus reader
    reader := metric.NewManualReader() // Use manual reader for Prometheus exporter
    mp := metric.NewMeterProvider(
        metric.WithResource(res),
        metric.WithReader(exporter),
    )
    otel.SetMeterProvider(mp)

    // Get meter and create custom metrics
    meter := mp.Meter(\"order-processor\")

    // Counter: total orders placed
    orderCounter, err := meter.Int64Counter(
        \"orders.placed.total\",
        metric.WithDescription(\"Total number of orders placed\"),
        metric.WithUnit(\"1\"),
    )
    if err != nil {
        log.Fatalf(\"Failed to create order counter: %w\", err)
    }

    // Histogram: order processing duration
    orderDuration, err := meter.Float64Histogram(
        \"orders.processing.duration\",
        metric.WithDescription(\"Order processing duration in milliseconds\"),
        metric.WithUnit(\"ms\"),
        metric.WithExplicitBucketBoundaries(50, 100, 250, 500, 1000, 5000),
    )
    if err != nil {
        log.Fatalf(\"Failed to create order duration histogram: %w\", err)
    }

    // Gauge: active orders in progress
    activeOrdersGauge, err := meter.Float64Gauge(
        \"orders.active.count\",
        metric.WithDescription(\"Number of active orders being processed\"),
        metric.WithUnit(\"1\"),
    )
    if err != nil {
        log.Fatalf(\"Failed to create active orders gauge: %w\", err)
    }

    // Simulate order processing in a goroutine
    go func() {
        activeOrders := 0.0
        for {
            // Simulate new order
            start := time.Now()
            activeOrders++
            activeOrdersGauge.Record(ctx, activeOrders)

            // Simulate processing time
            processingTime := time.Duration(50+time.Now().Unix()%200) * time.Millisecond
            time.Sleep(processingTime)

            // Record metrics
            orderCounter.Add(ctx, 1, metric.WithAttributes(
                attribute.Key(\"order.type\").String(\"standard\"),
                attribute.Key(\"order.success\").Bool(true),
            ))
            orderDuration.Record(ctx, float64(processingTime.Milliseconds()), metric.WithAttributes(
                attribute.Key(\"order.type\").String(\"standard\"),
            ))

            // Complete order
            activeOrders--
            activeOrdersGauge.Record(ctx, activeOrders)
        }
    }()

    // Expose Prometheus metrics endpoint
    http.Handle(\"/metrics\", promhttp.Handler())
    http.HandleFunc(\"/place-order\", func(w http.ResponseWriter, r *http.Request) {
        // Simulate placing an order
        orderCounter.Add(ctx, 1, metric.WithAttributes(
            attribute.Key(\"order.type\").String(\"express\"),
            attribute.Key(\"order.success\").Bool(true),
        ))
        w.WriteHeader(http.StatusAccepted)
        fmt.Fprintf(w, \"Order placed successfully\")
    })

    // Start HTTP server
    listenAddr := \":9090\"
    if addr := os.Getenv(\"LISTEN_ADDR\"); addr != \"\" {
        listenAddr = addr
    }
    log.Printf(\"Starting server on %s, metrics available at %s/metrics\", listenAddr, listenAddr)
    if err := http.ListenAndServe(listenAddr, nil); err != nil {
        log.Fatalf(\"Failed to start server: %v\", err)
    }
}

# Batch processor config for otel-collector
processors:
  batch:
    send_batch_size: 512
    timeout: 5s

// Configure tiered sampling in tracer provider
trace.WithSampler(trace.ParentBased(trace.TraceIDRatioBased(0.1)))

// Bridge logrus to OpenTelemetry Logs
logger.AddHook(otellogrus.NewHook(
  otellogrus.WithLoggerProvider(lp),
  otellogrus.WithLevels(logrus.AllLevels...),
))