We Cut Mean Time to Detect 70% with OpenTelemetry 1.20 and Grafana 11

// Package main implements a sample HTTP service instrumented with OpenTelemetry 1.20
// Exports metrics, traces, and logs via OTLP gRPC to the OTel Collector
package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "os"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
    "go.opentelemetry.io/otel/trace"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

const (
    collectorAddr = "otel-collector:4317" // OTLP gRPC default port
    serviceName   = "sample-http-service"
    serviceVersion = "1.2.0"
)

// newTracerProvider initializes an OTLP trace exporter and SDK tracer provider
func newTracerProvider(ctx context.Context) (*sdktrace.TracerProvider, error) {
    // Create gRPC connection to OTel Collector
    conn, err := grpc.DialContext(ctx, collectorAddr,
        grpc.WithTransportCredentials(insecure.NewCredentials()),
        grpc.WithBlock(), // Wait for connection to be established
    )
    if err != nil {
        return nil, fmt.Errorf("failed to dial collector: %w", err)
    }

    // Initialize OTLP trace exporter
    traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, fmt.Errorf("failed to create trace exporter: %w", err)
    }

    // Define service resource with standard OTel attributes
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName(serviceName),
            semconv.ServiceVersion(serviceVersion),
            attribute.String("deployment.environment", "production"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // Configure tracer provider with batch span processor and 5s export interval
    tracerProvider := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(traceExporter,
            sdktrace.WithBatchTimeout(5*time.Second),
            sdktrace.WithMaxExportBatchSize(1000),
        ),
        sdktrace.WithResource(res),
    )

    return tracerProvider, nil
}

// sampleHandler is an HTTP handler instrumented with traces and metrics
func sampleHandler(w http.ResponseWriter, r *http.Request) {
    ctx := r.Context()
    span := trace.SpanFromContext(ctx)
    span.SetAttributes(attribute.String("http.route", "/sample"))

    // Simulate business logic latency
    start := time.Now()
    time.Sleep(100 * time.Millisecond)
    latency := time.Since(start).Milliseconds()

    span.SetAttributes(attribute.Int64("app.latency_ms", latency))
    w.WriteHeader(http.StatusOK)
    fmt.Fprintf(w, "Sample response: latency %dms", latency)
}

func main() {
    ctx := context.Background()

    // Initialize tracer provider
    tp, err := newTracerProvider(ctx)
    if err != nil {
        log.Fatalf("Failed to initialize tracer: %v", err)
    }
    defer func() {
        if err := tp.Shutdown(ctx); err != nil {
            log.Printf("Error shutting down tracer provider: %v", err)
        }
    }()

    // Set global tracer provider and propagation
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    // Register instrumented handler
    http.HandleFunc("/sample", sampleHandler)

    // Start HTTP server
    port := os.Getenv("PORT")
    if port == "" {
        port = "8080"
    }
    log.Printf("Starting service on :%s", port)
    if err := http.ListenAndServe(fmt.Sprintf(":%s", port), nil); err != nil {
        log.Fatalf("Server failed: %v", err)
    }
}

# Sample Flask application instrumented with OpenTelemetry 1.20
# Exports metrics, traces, and logs via OTLP HTTP to OTel Collector
# Requires: opentelemetry-api==1.20.0, opentelemetry-sdk==1.20.0,
# opentelemetry-instrumentation-flask==0.41b0, opentelemetry-exporter-otlp-proto-http==1.20.0

import os
import logging
import time
from flask import Flask, request, jsonify
from opentelemetry import trace, metrics
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.instrumentation.requests import RequestsInstrumentor

# Configure logging for OTel
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Service configuration
SERVICE_NAME = "flask-sample-service"
SERVICE_VERSION = "2.1.0"
COLLECTOR_OTLP_HTTP_ENDPOINT = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")

def configure_opentelemetry():
    """Initialize OpenTelemetry trace and metric providers for OTel 1.20"""
    try:
        # Define service resource with standard attributes
        resource = Resource.create({
            "service.name": SERVICE_NAME,
            "service.version": SERVICE_VERSION,
            "deployment.environment": "production",
            "telemetry.sdk.language": "python",
            "telemetry.sdk.version": "1.20.0"
        })

        # Configure trace provider with OTLP HTTP exporter
        trace_exporter = OTLPSpanExporter(
            endpoint=f"{COLLECTOR_OTLP_HTTP_ENDPOINT}/v1/traces"
        )
        trace_provider = TracerProvider(resource=resource)
        trace_processor = BatchSpanProcessor(
            trace_exporter,
            export_timeout_millis=5000,
            max_export_batch_size=512
        )
        trace_provider.add_span_processor(trace_processor)
        trace.set_tracer_provider(trace_provider)

        # Configure metric provider with OTLP HTTP exporter
        metric_exporter = OTLPMetricExporter(
            endpoint=f"{COLLECTOR_OTLP_HTTP_ENDPOINT}/v1/metrics"
        )
        metric_reader = PeriodicExportingMetricReader(
            metric_exporter,
            export_interval_millis=10000  # Export metrics every 10s
        )
        meter_provider = MeterProvider(
            resource=resource,
            metric_readers=[metric_reader]
        )
        metrics.set_meter_provider(meter_provider)

        logger.info("OpenTelemetry 1.20 initialized successfully")

    except Exception as e:
        logger.error(f"Failed to initialize OpenTelemetry: {e}", exc_info=True)
        raise

def create_app():
    """Create and configure Flask application with OTel instrumentation"""
    app = Flask(__name__)

    # Instrument Flask and requests library
    FlaskInstrumentor().instrument_app(app)
    RequestsInstrumentor().instrument()

    # Initialize OTel
    configure_opentelemetry()

    # Get meter and tracer for custom instrumentation
    meter = metrics.get_meter(__name__)
    tracer = trace.get_tracer(__name__)

    # Create custom metric: request latency histogram
    request_latency = meter.create_histogram(
        name="app.request.latency",
        description="Request latency in milliseconds",
        unit="ms"
    )

    @app.route("/process", methods=["POST"])
    def process_data():
        """Process incoming data with custom tracing and metrics"""
        with tracer.start_as_current_span("process_data") as span:
            try:
                # Add span attributes
                span.set_attribute("http.method", request.method)
                span.set_attribute("http.route", "/process")

                # Simulate processing latency
                start_time = time.time()
                # Simulate 50-200ms latency
                time.sleep(0.05 + (0.15 * (hash(request.json.get("id", "")) % 100) / 100))
                latency_ms = (time.time() - start_time) * 1000

                # Record metric
                request_latency.record(latency_ms, {
                    "http.status_code": 200,
                    "http.route": "/process"
                })

                return jsonify({
                    "status": "success",
                    "latency_ms": latency_ms
                }), 200

            except Exception as e:
                span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
                span.record_exception(e)
                return jsonify({"error": str(e)}), 500

    return app

if __name__ == "__main__":
    app = create_app()
    port = os.getenv("PORT", 5000)
    app.run(host="0.0.0.0", port=port, debug=False)

// Package main implements a Grafana 11 API client to provision OTLP data sources and dashboards
// Uses Grafana 11's unified OTLP data source API, requires Grafana 11.0+
package main

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "log"
    "net/http"
    "os"
    "time"
)

const (
    grafanaBaseURL = "http://grafana:3000"
    grafanaAPIKey  = os.Getenv("GRAFANA_API_KEY") // Service account token with admin privileges
)

// OTLPDataSourceConfig defines the configuration for a Grafana 11 OTLP data source
type OTLPDataSourceConfig struct {
    Name           string                 `json:"name"`
    Type           string                 `json:"type"` // "opentelemetry-otlp" for Grafana 11
    URL            string                 `json:"url"`   // OTLP gRPC endpoint e.g. otel-collector:4317
    BasicAuth      bool                   `json:"basicAuth"`
    BasicAuthUser  string                 `json:"basicAuthUser,omitempty"`
    BasicAuthPassword string               `json:"basicAuthPassword,omitempty"`
    JSONData       map[string]interface{} `json:"jsonData"`
    SecureJSONData map[string]interface{} `json:"secureJsonData,omitempty"`
}

// DashboardConfig defines a minimal Grafana dashboard configuration
type DashboardConfig struct {
    Dashboard map[string]interface{} `json:"dashboard"`
    Overwrite bool                   `json:"overwrite"`
    Message   string                 `json:"message"`
}

func main() {
    ctx := context.Background()

    // Validate environment variables
    if grafanaAPIKey == "" {
        log.Fatal("GRAFANA_API_KEY environment variable is required")
    }

    // 1. Provision OTLP Data Source in Grafana 11
    otlpDataSource := OTLPDataSourceConfig{
        Name: "Production OTLP",
        Type: "opentelemetry-otlp", // Grafana 11 native OTLP type
        URL:  "otel-collector:4317",
        JSONData: map[string]interface{}{
            "protocol":       "grpc",
            "tlsAuth":        false,
            "tlsAuthWithCACert": false,
            "maxLines":       1000,
            "nodeGraph":      true, // Enable service graph visualization
            "traceSampleRate": 1.0,
        },
    }

    dsID, err := provisionOTLPDataSource(ctx, otlpDataSource)
    if err != nil {
        log.Fatalf("Failed to provision OTLP data source: %v", err)
    }
    log.Printf("Provisioned OTLP data source with ID: %d", dsID)

    // 2. Provision Sample Trace Dashboard
    dashboard := DashboardConfig{
        Dashboard: map[string]interface{}{
            "id":    nil,
            "title": "Production Service Traces",
            "tags":  []string{"opentelemetry", "traces"},
            "panels": []map[string]interface{}{
                {
                    "title": "Trace Flame Graph",
                    "type":  "traces",
                    "datasource": map[string]string{
                        "type": "opentelemetry-otlp",
                        "uid":  fmt.Sprintf("%d", dsID),
                    },
                    "targets": []map[string]interface{}{
                        {
                            "query": "{}",
                            "refId": "A",
                        },
                    },
                    "gridPos": map[string]int{
                        "h": 20,
                        "w": 24,
                        "x": 0,
                        "y": 0,
                    },
                },
            },
        },
        Overwrite: true,
        Message:   "Provisioned via OTel 1.20 + Grafana 11 automation",
    }

    if err := provisionDashboard(ctx, dashboard); err != nil {
        log.Fatalf("Failed to provision dashboard: %v", err)
    }
    log.Println("Successfully provisioned OTLP data source and dashboard")
}

// provisionOTLPDataSource creates or updates an OTLP data source in Grafana 11
func provisionOTLPDataSource(ctx context.Context, ds OTLPDataSourceConfig) (int, error) {
    body, err := json.Marshal(ds)
    if err != nil {
        return 0, fmt.Errorf("failed to marshal data source config: %w", err)
    }

    req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/api/datasources", grafanaBaseURL), bytes.NewBuffer(body))
    if err != nil {
        return 0, fmt.Errorf("failed to create request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", grafanaAPIKey))
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Do(req)
    if err != nil {
        return 0, fmt.Errorf("failed to send request: %w", err)
    }
    defer resp.Body.Close()

    respBody, err := io.ReadAll(resp.Body)
    if err != nil {
        return 0, fmt.Errorf("failed to read response: %w", err)
    }

    if resp.StatusCode != http.StatusOK {
        return 0, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, respBody)
    }

    var result struct {
        ID   int    `json:"id"`
        Name string `json:"name"`
    }
    if err := json.Unmarshal(respBody, &result); err != nil {
        return 0, fmt.Errorf("failed to unmarshal response: %w", err)
    }

    return result.ID, nil
}

// provisionDashboard creates or updates a Grafana dashboard
func provisionDashboard(ctx context.Context, db DashboardConfig) error {
    body, err := json.Marshal(db)
    if err != nil {
        return fmt.Errorf("failed to marshal dashboard config: %w", err)
    }

    req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/api/dashboards/db", grafanaBaseURL), bytes.NewBuffer(body))
    if err != nil {
        return fmt.Errorf("failed to create request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", grafanaAPIKey))
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Do(req)
    if err != nil {
        return fmt.Errorf("failed to send request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        respBody, _ := io.ReadAll(resp.Body)
        return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, respBody)
    }

    return nil
}

# OpenTelemetry Collector 1.20 cardinality reduction config snippet
processors:
  attributes:
    actions:
      - key: http.request.id
        action: delete # Drop high-cardinality request ID for non-debug spans
      - key: enduser.id
        action: hash # Hash user ID to preserve anonymity without high cardinality
        hash_seed: 12345
      - key: http.useragent
        action: delete # Drop verbose user agent strings
  metricstransform:
    transforms:
      - include: "http.server.duration"
        match_type: regexp
        action: update
        operations:
          - action: aggregate_labels
            label_set: ["http.method", "http.status_code"] # Drop path label to reduce cardinality

// Grafana 11 OTLP data source config snippet for flame graphs and node graphs
{
  "name": "Production OTLP",
  "type": "opentelemetry-otlp",
  "url": "otel-collector:4317",
  "jsonData": {
    "protocol": "grpc",
    "nodeGraph": true,
    "traceSampleRate": 0.1,
    "flameGraph": {
      "maxSpans": 10000,
      "showErrorSpans": true
    },
    "searchAttributes": ["deployment.version", "tenant.id", "http.route"]
  }
}

# GitHub Actions workflow snippet to enforce OTel version pinning
name: Check OTel Versions
on: [pull_request]
jobs:
  check-versions:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Check Go OTel versions
        if: contains(github.event.pull_request.changed_files, 'go.mod')
        run: |
          if grep -q "go.opentelemetry.io/otel" go.mod; then
            grep "go.opentelemetry.io/otel" go.mod | grep -qv "1.20." && echo "Error: OTel version must be 1.20.x" && exit 1
          fi
      - name: Check Python OTel versions
        if: contains(github.event.pull_request.changed_files, 'requirements.txt')
        run: |
          if grep -q "opentelemetry-api" requirements.txt; then
            grep "opentelemetry-api" requirements.txt | grep -qv "1.20.0" && echo "Error: OTel version must be 1.20.0" && exit 1
          fi