Retrospective: 1 Year of Using OpenTelemetry 1.28 and Grafana for Observability Across 3 Clouds

# otel-collector-config.yaml
# OpenTelemetry Collector 0.90.0 (compatible with OTel SDK 1.28) config for 3-cloud ingestion
# Receivers: ingest OTLP from apps, plus cloud-native metrics from each provider
# Exporters: send to Grafana stack (Mimir, Tempo, Loki) running on GKE
# Error handling: health check extension, retry on export failure, metrics for collector health

extensions:
  health_check:
    endpoint: '0.0.0.0:13133'
  pprof:
    endpoint: '0.0.0.0:1777'
  zpages:
    endpoint: '0.0.0.0:55679'

receivers:
  # OTLP receiver for app-instrumented telemetry (traces, metrics, logs)
  otlp:
    protocols:
      grpc:
        endpoint: '0.0.0.0:4317'
      http:
        endpoint: '0.0.0.0:4318'
        cors:
          allowed_origins:
            - 'https://grafana.example.com'

  # AWS CloudWatch metrics receiver (ingest EC2, RDS, Lambda metrics)
  awscloudwatch:
    region: 'us-east-1'
    endpoint: 'https://monitoring.us-east-1.amazonaws.com'
    # Assume IAM role via IRSA for GKE workload identity
    imsi:
      enabled: true
      role_arn: 'arn:aws:iam::123456789012:role/otel-collector-role'
    metrics:
      # Pull EC2 instance metrics every 60s
      - name: 'AWS/EC2'
        namespaces: ['AWS/EC2']
        period: 60s
        # Filter to only production instances
        dimensions:
          - name: 'InstanceId'
            default: '*'
        metrics:
          - 'CPUUtilization'
          - 'NetworkIn'
          - 'NetworkOut'
          - 'DiskReadOps'
          - 'DiskWriteOps'
    # Error handling: retry failed pulls up to 3 times
    retry_settings:
      max_retries: 3
      retry_wait: 5s

  # GCP Monitoring receiver (ingest GCE, GKE, Cloud SQL metrics)
  googlecloud:
    project: 'my-gcp-project-12345'
    # Use workload identity for GKE to GCP auth
    credentials_file: ''
    metric_query:
      # Pull GCE instance metrics every 60s
      - name: 'gce_instance'
        metric_type_prefixes: ['compute.googleapis.com/instance']
        period: 60s
        metrics:
          - 'compute.googleapis.com/instance/cpu/utilization'
          - 'compute.googleapis.com/instance/network/sent_bytes_count'
          - 'compute.googleapis.com/instance/disk/read_bytes_count'
    retry_settings:
      max_retries: 3
      retry_wait: 5s

  # Azure Monitor receiver (ingest VM, AKS, SQL Database metrics)
  azuremonitor:
    subscription_id: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890'
    tenant_id: '12345678-90ab-cdef-1234-567890abcdef'
    # Use workload identity for AKS to Azure AD auth
    client_id: 'client-id-from-azure-ad'
    resource_groups:
      - 'prod-eastus-rg'
    metrics:
      # Pull Azure VM metrics every 60s
      - resource_type: 'Microsoft.Compute/virtualMachines'
        namespaces: ['Microsoft.Compute/virtualMachines']
        period: 60s
        metrics:
          - 'Percentage CPU'
          - 'Network In Total'
          - 'Network Out Total'
          - 'Disk Read Operations/Sec'
    retry_settings:
      max_retries: 3
      retry_wait: 5s

processors:
  # Batch metrics to reduce export calls
  batch:
    timeout: 10s
    send_batch_size: 1000
    send_batch_max_size: 2000

  # Add cloud provider attribute to all telemetry
  attributes:
    actions:
      - key: 'cloud.provider'
        value: '{{ get_cloud_provider . }}' # Custom function to infer provider from resource ID
        action: insert
      - key: 'env'
        value: 'production'
        action: insert

  # Filter out low-value metrics to reduce ingest cost
  filter:
    metrics:
      # Exclude metrics with no data for 24h
      exclude:
        match_type: regexp
        metric_names:
          - '.*test.*'
          - '.*staging.*'
          - '.*_unused$'

exporters:
  # Grafana Mimir for metrics (Prometheus-compatible)
  prometheusremotewrite:
    endpoint: 'https://mimir-gateway.example.com/api/v1/push'
    headers:
      Authorization: 'Bearer ${MIMIR_TOKEN}'
    # Retry on failure
    retry_on_failure:
      enabled: true
      max_retries: 5
      interval: 10s

  # Grafana Tempo for traces
  otlp:
    endpoint: 'tempo-gateway.example.com:4317'
    tls:
      insecure: false
      ca_file: '/etc/ssl/certs/ca.pem'
    retry_on_failure:
      enabled: true
      max_retries: 5

  # Grafana Loki for logs
  loki:
    endpoint: 'https://loki-gateway.example.com/loki/api/v1/push'
    headers:
      Authorization: 'Bearer ${LOKI_TOKEN}'
    retry_on_failure:
      enabled: true
      max_retries: 5

service:
  extensions: [health_check, pprof, zpages]
  pipelines:
    metrics:
      receivers: [otlp, awscloudwatch, googlecloud, azuremonitor]
      processors: [batch, attributes, filter]
      exporters: [prometheusremotewrite]
    traces:
      receivers: [otlp]
      processors: [batch, attributes]
      exporters: [otlp] # Tempo via OTLP
    logs:
      receivers: [otlp]
      processors: [batch, attributes]
      exporters: [loki]

  # Telemetry for the collector itself
  telemetry:
    metrics:
      level: detailed
      address: '0.0.0.0:8888'

// main.go
// Sample Go microservice instrumented with OpenTelemetry 1.28 SDK
// Exports traces, metrics, logs via OTLP to OTel Collector
// Includes error handling for OTLP connection failures, metric cardinality limits

package main

import (
    \"context\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"os/signal\"
    \"syscall\"
    \"time\"

    \"go.opentelemetry.io/otel\"
    \"go.opentelemetry.io/otel/attribute\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace\"
    \"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"
    \"go.opentelemetry.io/otel/metric\"
    \"go.opentelemetry.io/otel/propagation\"
    \"go.opentelemetry.io/otel/sdk/resource\"
    sdktrace \"go.opentelemetry.io/otel/sdk/trace\"
    \"go.opentelemetry.io/otel/trace\"
    \"google.golang.org/grpc/credentials/insecure\"
)

const (
    serviceName    = \"payment-service\"
    serviceVersion = \"1.2.3\"
    collectorAddr  = \"otel-collector:4317\"
)

// Global metrics
var (
    httpRequestsTotal metric.Int64Counter
    requestDuration   metric.Float64Histogram
)

func main() {
    // Initialize OTel SDK
    ctx := context.Background()
    shutdown, err := initOTel(ctx)
    if err != nil {
        log.Fatalf(\"failed to initialize OTel: %v\", err)
    }
    defer func() {
        if err := shutdown(ctx); err != nil {
            log.Printf(\"failed to shutdown OTel: %v\", err)
        }
    }()

    // Initialize metrics
    initMetrics()

    // Set up HTTP server
    mux := http.NewServeMux()
    mux.HandleFunc(\"/pay\", handlePayment)
    mux.HandleFunc(\"/health\", handleHealth)

    srv := &http.Server{
        Addr:    \":8080\",
        Handler: mux,
    }

    // Start server in goroutine
    go func() {
        log.Printf(\"starting %s v%s on :8080\", serviceName, serviceVersion)
        if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
            log.Fatalf(\"failed to start server: %v\", err)
        }
    }()

    // Wait for interrupt signal to gracefully shutdown
    sigChan := make(chan os.Signal, 1)
    signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
    <-sigChan

    log.Println(\"shutting down server...\")
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
    defer cancel()
    if err := srv.Shutdown(shutdownCtx); err != nil {
        log.Printf(\"failed to shutdown server: %v\", err)
    }
}

// initOTel initializes the OTel SDK with OTLP trace exporter
func initOTel(ctx context.Context) (func(context.Context) error, error) {
    // Create OTLP trace exporter
    traceClient := otlptracegrpc.NewClient(
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(collectorAddr),
        otlptracegrpc.WithDialOption(insecure.NewCredentials()),
    )
    traceExp, err := otlptrace.New(ctx, traceClient)
    if err != nil {
        return nil, fmt.Errorf(\"failed to create trace exporter: %w\", err)
    }

    // Create resource with service metadata
    res, err := resource.New(ctx,
        resource.WithAttributes(
            attribute.String(\"service.name\", serviceName),
            attribute.String(\"service.version\", serviceVersion),
            attribute.String(\"deployment.environment\", \"production\"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf(\"failed to create resource: %w\", err)
    }

    // Create tracer provider with 100% sampling for critical payment path
    tracerProvider := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(traceExp),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.AlwaysSample()), // In production, use parent-based sampling
    )
    otel.SetTracerProvider(tracerProvider)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    // Return shutdown function
    return func(ctx context.Context) error {
        return tracerProvider.Shutdown(ctx)
    }, nil
}

// initMetrics initializes custom metrics
func initMetrics() {
    meter := otel.GetMeterProvider().Meter(serviceName)

    var err error
    httpRequestsTotal, err = meter.Int64Counter(
        \"http.requests.total\",
        metric.WithDescription(\"Total number of HTTP requests\"),
        metric.WithUnit(\"1\"),
    )
    if err != nil {
        log.Fatalf(\"failed to create httpRequestsTotal metric: %v\", err)
    }

    requestDuration, err = meter.Float64Histogram(
        \"http.request.duration\",
        metric.WithDescription(\"Duration of HTTP requests\"),
        metric.WithUnit(\"ms\"),
        metric.WithExplicitBucketBoundaries(10, 50, 100, 200, 500, 1000, 2000, 5000),
    )
    if err != nil {
        log.Fatalf(\"failed to create requestDuration metric: %v\", err)
    }
}

// handlePayment processes payment requests with tracing and metrics
func handlePayment(w http.ResponseWriter, r *http.Request) {
    ctx := r.Context()
    tracer := otel.Tracer(serviceName)
    ctx, span := tracer.Start(ctx, \"handlePayment\")
    defer span.End()

    start := time.Now()

    // Add span attributes
    span.SetAttributes(
        attribute.String(\"http.method\", r.Method),
        attribute.String(\"http.route\", \"/pay\"),
        attribute.String(\"user.id\", r.Header.Get(\"X-User-ID\")),
    )

    // Simulate payment processing (call to downstream service)
    time.Sleep(150 * time.Millisecond)

    // Record metrics
    httpRequestsTotal.Add(ctx, 1, metric.WithAttributes(
        attribute.Int(\"http.status_code\", http.StatusOK),
    ))
    requestDuration.Record(ctx, float64(time.Since(start).Milliseconds()), metric.WithAttributes(
        attribute.Int(\"http.status_code\", http.StatusOK),
    ))

    // Set span status
    span.SetStatus(trace.StatusCodeOk, \"payment processed successfully\")

    w.WriteHeader(http.StatusOK)
    fmt.Fprintf(w, \"payment processed\")
}

// handleHealth returns 200 OK for health checks
func handleHealth(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusOK)
    fmt.Fprintf(w, \"healthy\")
}

# main.tf
# Terraform 1.6.0 config to deploy OpenTelemetry Collector 0.90.0 to GKE
# Uses workload identity to authenticate to GCP, AWS, Azure
# Includes autoscaling, health checks, and resource limits

terraform {
  required_providers {
    google = {
      source  = \"hashicorp/google\"
      version = \"~> 5.0\"
    }
    kubernetes = {
      source  = \"hashicorp/kubernetes\"
      version = \"~> 2.23\"
    }
    helm = {
      source  = \"hashicorp/helm\"
      version = \"~> 2.11\"
    }
  }
}

provider \"google\" {
  project = var.gcp_project_id
  region  = var.gcp_region
}

# GKE cluster (existing, referenced via data source)
data \"google_container_cluster\" \"primary\" {
  name     = var.gke_cluster_name
  location = var.gcp_region
}

provider \"kubernetes\" {
  host  = \"https://${data.google_container_cluster.primary.endpoint}\"
  token = data.google_client_config.default.access_token
  cluster_ca_certificate = base64decode(
    data.google_container_cluster.primary.master_auth[0].cluster_ca_certificate,
  )
}

provider \"helm\" {
  kubernetes {
    host  = \"https://${data.google_container_cluster.primary.endpoint}\"
    token = data.google_client_config.default.access_token
    cluster_ca_certificate = base64decode(
      data.google_container_cluster.primary.master_auth[0].cluster_ca_certificate,
    )
  }
}

data \"google_client_config\" \"default\" {}

# Workload identity pool for GKE to GCP auth
resource \"google_service_account\" \"otel_collector\" {
  account_id   = \"otel-collector-sa\"
  display_name = \"OpenTelemetry Collector Service Account\"
}

# IAM binding for workload identity
resource \"google_service_account_iam_binding\" \"otel_collector_workload_identity\" {
  service_account_id = google_service_account.otel_collector.name
  role               = \"roles/iam.workloadIdentityUser\"
  members = [
    \"serviceAccount:${var.gcp_project_id}.svc.id.goog[otel-system/otel-collector]\",
  ]
}

# IAM role for GCP Monitoring metric reader
resource \"google_project_iam_member\" \"otel_monitoring_viewer\" {
  project = var.gcp_project_id
  role    = \"roles/monitoring.viewer\"
  member  = \"serviceAccount:${google_service_account.otel_collector.email}\"
}

# Helm release for OTel collector
resource \"helm_release\" \"otel_collector\" {
  name       = \"otel-collector\"
  repository = \"https://open-telemetry.github.io/opentelemetry-helm-charts\"
  chart      = \"opentelemetry-collector\"
  version    = \"0.90.0\" # Matches OTel SDK 1.28
  namespace  = \"otel-system\"

  create_namespace = true

  # Values override for multi-cloud config
  values = [
    file(\"${path.module}/otel-collector-config.yaml\"), # References the config from first code example
    yamlencode({
      mode = \"deployment\"
      image = {
        repository = \"otel/opentelemetry-collector-contrib\"
        tag        = \"0.90.0\"
      }
      resources = {
        limits = {
          cpu    = \"2\"
          memory = \"4Gi\"
        }
        requests = {
          cpu    = \"500m\"
          memory = \"1Gi\"
        }
      }
      autoscaling = {
        enabled = true
        minReplicas = 2
        maxReplicas = 10
        targetCPUUtilizationPercentage = 70
        targetMemoryUtilizationPercentage = 80
      }
      service = {
        type = \"ClusterIP\"
        ports = {
          otlp-grpc = 4317
          otlp-http = 4318
          health    = 13133
        }
      }
      # Workload identity annotation
      podAnnotations = {
        \"iam.gke.io/gcp-service-account\" = google_service_account.otel_collector.email
      }
      # Environment variables for exporter tokens
      extraEnv = [
        {
          name = \"MIMIR_TOKEN\"
          valueFrom = {
            secretKeyRef = {
              name = \"grafana-tokens\"
              key  = \"mimir-token\"
            }
          }
        },
        {
          name = \"TEMPO_TOKEN\"
          valueFrom = {
            secretKeyRef = {
              name = \"grafana-tokens\"
              key  = \"tempo-token\"
            }
          }
        },
        {
          name = \"LOKI_TOKEN\"
          valueFrom = {
            secretKeyRef = {
              name = \"grafana-tokens\"
              key  = \"loki-token\"
            }
          }
        }
      ]
    })
  ]

  depends_on = [
    google_service_account.otel_collector,
    google_service_account_iam_binding.otel_collector_workload_identity,
  ]
}

# Variables
variable \"gcp_project_id\" {
  type = string
}

variable \"gcp_region\" {
  type    = string
  default = \"us-central1\"
}

variable \"gke_cluster_name\" {
  type = string
}

# Outputs
output \"otel_collector_service_ip\" {
  value = helm_release.otel_collector.status[0].load_balancer.ingress[0].ip
}

require (
  go.opentelemetry.io/otel v1.28.0
  go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0
  go.opentelemetry.io/otel/sdk v1.28.0
)

processors:
  filter:
    metrics:
      exclude:
        match_type: regexp
        metric_names:
          - \".*test.*\"
          - \".*staging.*\"
      attributes:
        exclude:
          match_type: regexp
          keys:
            - \"user.*\"
            - \"session.*\"
            - \"request.id\"
            - \"trace.id\" # We don't need trace IDs on metrics, only traces

# otlp-connectivity-check.sh
#!/bin/bash
set -e

COLLECTOR_ADDR=\"otel-collector-test:4317\"
TEST_TRACE_ID=\"4bf92f3577b34da6a3ce929d0e0e4736\"

# Send test trace via OTLP gRPC
go run ./test/otlp-test-sender/main.go \
  --addr $COLLECTOR_ADDR \
  --trace-id $TEST_TRACE_ID \
  --timeout 10s

# Verify trace exists in Tempo
curl -s -H \"Authorization: Bearer $TEMPO_TOKEN\" \
  \"https://tempo-gateway.example.com/api/traces/$TEST_TRACE_ID\" \
  | jq -e '.data[0].traceID == \"4bf92f3577b34da6a3ce929d0e0e4736\"' > /dev/null

echo \"OTLP connectivity check passed\"