We Ditched New Relic for Grafana 11.0 and Prometheus 3.0 and Saved $80k/Year: A 2026 Observability Case Study

// payment_metrics_exporter.go
// Exports custom Prometheus 3.0 metrics for our fintech payment API
// Compatible with Prometheus 3.0+ client_golang library
package main

import (
    "context"
    "errors"
    "fmt"
    "log"
    "net/http"
    "os"
    "os/signal"
    "syscall"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "github.com/prometheus/client_golang/prometheus/version"
)

// Define custom metrics aligned with Prometheus 3.0 best practices
var (
    // PaymentSuccessCounter tracks successful payment intents
    PaymentSuccessCounter = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "payment_success_total",
            Help: "Total number of successful payment intents processed",
            // Prometheus 3.0 supports native OpenMetrics metadata
            ConstLabels: prometheus.Labels{"service": "payment-api", "version": "v2.4.0"},
        },
        []string{"currency", "payment_method"}, // Label dimensions
    )

    // PaymentFailureCounter tracks failed payment intents with error context
    PaymentFailureCounter = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "payment_failure_total",
            Help: "Total number of failed payment intents",
            ConstLabels: prometheus.Labels{"service": "payment-api", "version": "v2.4.0"},
        },
        []string{"currency", "payment_method", "error_code"},
    )

    // PaymentLatencyHistogram tracks p99 latency for payment processing
    PaymentLatencyHistogram = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "payment_processing_latency_seconds",
            Help:    "Latency distribution of payment intent processing",
            Buckets: prometheus.DefBuckets, // Prometheus 3.0 optimized default buckets
            ConstLabels: prometheus.Labels{"service": "payment-api", "version": "v2.4.0"},
        },
        []string{"currency", "payment_method"},
    )

    // ActivePaymentGauge tracks in-progress payment intents
    ActivePaymentGauge = prometheus.NewGauge(
        prometheus.GaugeOpts{
            Name: "payment_active_intents",
            Help: "Number of payment intents currently being processed",
            ConstLabels: prometheus.Labels{"service": "payment-api", "version": "v2.4.0"},
        },
    )
)

func init() {
    // Register all metrics with the default Prometheus registry
    // Error handling for duplicate registration (common in testing)
    err := prometheus.Register(PaymentSuccessCounter)
    if err != nil {
        var alreadyRegisteredError prometheus.AlreadyRegisteredError
        if errors.As(err, &alreadyRegisteredError) {
            log.Printf("warning: payment success counter already registered, reusing existing metric")
            PaymentSuccessCounter = alreadyRegisteredError.ExistingCollector.(*prometheus.CounterVec)
        } else {
            log.Fatalf("failed to register payment success counter: %v", err)
        }
    }

    err = prometheus.Register(PaymentFailureCounter)
    if err != nil {
        var alreadyRegisteredError prometheus.AlreadyRegisteredError
        if errors.As(err, &alreadyRegisteredError) {
            log.Printf("warning: payment failure counter already registered, reusing existing metric")
            PaymentFailureCounter = alreadyRegisteredError.ExistingCollector.(*prometheus.CounterVec)
        } else {
            log.Fatalf("failed to register payment failure counter: %v", err)
        }
    }

    err = prometheus.Register(PaymentLatencyHistogram)
    if err != nil {
        var alreadyRegisteredError prometheus.AlreadyRegisteredError
        if errors.As(err, &alreadyRegisteredError) {
            log.Printf("warning: payment latency histogram already registered, reusing existing metric")
            PaymentLatencyHistogram = alreadyRegisteredError.ExistingCollector.(*prometheus.HistogramVec)
        } else {
            log.Fatalf("failed to register payment latency histogram: %v", err)
        }
    }

    err = prometheus.Register(ActivePaymentGauge)
    if err != nil {
        var alreadyRegisteredError prometheus.AlreadyRegisteredError
        if errors.As(err, &alreadyRegisteredError) {
            log.Printf("warning: active payment gauge already registered, reusing existing metric")
            ActivePaymentGauge = alreadyRegisteredError.ExistingCollector.(*prometheus.Gauge)
        } else {
            log.Fatalf("failed to register active payment gauge: %v", err)
        }
    }

    // Log Prometheus client version for debugging (Prometheus 3.0 client_golang v1.20+)
    log.Printf("initialized prometheus metrics exporter, client version: %s", version.Version)
}

// StartMetricsServer starts the Prometheus scrape endpoint on the given port
func StartMetricsServer(port string) error {
    mux := http.NewServeMux()
    // Use promhttp.HandlerFor to expose all registered metrics with error handling
    mux.Handle("/metrics", promhttp.HandlerFor(
        prometheus.DefaultGatherer,
        promhttp.HandlerOpts{
            // Enable OpenMetrics format (default in Prometheus 3.0)
            EnableOpenMetrics: true,
            ErrorHandling: promhttp.ContinueOnError, // Log errors but don't crash
            ErrorLog: promhttp.NewErrorLogger(log.New(os.Stderr, "promhttp: ", log.Lshortfile)),
        },
    ))

    srv := &http.Server{
        Addr:    fmt.Sprintf(":%s", port),
        Handler: mux,
        // Prometheus 3.0 recommends 5s read/write timeouts for scrape endpoints
        ReadTimeout:  5 * time.Second,
        WriteTimeout: 5 * time.Second,
        IdleTimeout:  120 * time.Second,
    }

    // Graceful shutdown handling for Kubernetes/container deployments
    go func() {
        sigChan := make(chan os.Signal, 1)
        signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
        <-sigChan
        log.Println("received shutdown signal, stopping metrics server")
        ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
        defer cancel()
        if err := srv.Shutdown(ctx); err != nil {
            log.Fatalf("failed to shutdown metrics server: %v", err)
        }
    }()

    log.Printf("starting prometheus metrics server on port %s", port)
    if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
        return fmt.Errorf("metrics server failed: %w", err)
    }
    return nil
}

// SimulatePayment processes a mock payment and updates metrics
func SimulatePayment(currency, paymentMethod string) error {
    start := time.Now()
    ActivePaymentGauge.Inc()
    defer ActivePaymentGauge.Dec()

    // Simulate 10% failure rate for demo purposes
    if time.Now().UnixNano()%10 == 0 {
        PaymentFailureCounter.WithLabelValues(currency, paymentMethod, "insufficient_funds").Inc()
        return errors.New("payment failed: insufficient funds")
    }

    // Simulate processing latency between 50ms and 500ms
    time.Sleep(time.Duration(50+time.Now().UnixNano()%450) * time.Millisecond)
    PaymentSuccessCounter.WithLabelValues(currency, paymentMethod).Inc()
    PaymentLatencyHistogram.WithLabelValues(currency, paymentMethod).Observe(time.Since(start).Seconds())
    return nil
}

func main() {
    // Simulate 100 payment requests for testing
    go func() {
        for i := 0; i < 100; i++ {
            currencies := []string{"USD", "EUR", "GBP"}
            methods := []string{"card", "bank_transfer", "wallet"}
            curr := currencies[i%3]
            method := methods[i%3]
            if err := SimulatePayment(curr, method); err != nil {
                log.Printf("payment %d failed: %v", i, err)
            }
        }
    }()

    // Start metrics server on port 9090 (default Prometheus scrape port)
    if err := StartMetricsServer("9090"); err != nil {
        log.Fatalf("failed to start metrics server: %v", err)
    }
}

# grafana_dashboard_provisioning.tf
# Provisions a custom payment latency dashboard in Grafana 11.0 using Terraform
# Requires Grafana 11.0+ and Terraform 1.7+ with Grafana provider v2.0+
terraform {
  required_version = ">= 1.7.0"
  required_providers {
    grafana = {
      source  = "grafana/grafana"
      version = ">= 2.0.0" # Grafana 11.0 compatible provider
    }
  }
}

# Configure Grafana provider with API key authentication
provider "grafana" {
  url  = var.grafana_url # e.g., "https://grafana.internal.example.com"
  auth = var.grafana_api_key
  # Retry configuration for Grafana API rate limits (Grafana 11.0 has 100 req/min limit)
  retries = 3
  retry_delay = 5
}

# Define variables for environment-specific configuration
variable "grafana_url" {
  type        = string
  description = "URL of the Grafana 11.0 instance"
}

variable "grafana_api_key" {
  type        = string
  description = "Admin API key for Grafana provisioning"
  sensitive   = true
}

variable "prometheus_datasource_uid" {
  type        = string
  description = "UID of the Prometheus 3.0 datasource in Grafana"
  default     = "prom-3-0-prod"
}

variable "environment" {
  type        = string
  description = "Deployment environment (prod, staging, dev)"
  default     = "prod"
}

# Create a dedicated folder for payment dashboards
resource "grafana_folder" "payment_dashboards" {
  title = "Payment Service Dashboards"
  uid   = "payment-dashboards-${var.environment}"
}

# Provision the payment latency dashboard with custom panels
resource "grafana_dashboard" "payment_latency" {
  folder = grafana_folder.payment_dashboards.id
  config_json = jsonencode({
    id          = null
    uid         = "payment-latency-${var.environment}"
    title       = "Payment API Latency - ${upper(var.environment)}"
    description = "Tracks p50, p95, p99 latency for payment intents, data sourced from Prometheus 3.0"
    tags        = ["payment", "latency", "prometheus-3.0", var.environment]
    timezone    = "utc"
    refresh     = "30s" # Grafana 11.0 supports 30s refresh intervals
    schemaVersion = 39 # Grafana 11.0 dashboard schema version

    panels = [
      {
        id    = 1
        type  = "timeseries"
        title = "Payment Processing Latency (p50/p95/p99)"
        gridPos = { h = 8, w = 12, x = 0, y = 0 }
        datasource = { uid = var.prometheus_datasource_uid }
        targets = [
          {
            expr    = "histogram_quantile(0.50, sum(rate(payment_processing_latency_seconds_bucket[5m])) by (le, currency))"
            legendFormat = "p50 - {{currency}}"
            refId   = "A"
          },
          {
            expr    = "histogram_quantile(0.95, sum(rate(payment_processing_latency_seconds_bucket[5m])) by (le, currency))"
            legendFormat = "p95 - {{currency}}"
            refId   = "B"
          },
          {
            expr    = "histogram_quantile(0.99, sum(rate(payment_processing_latency_seconds_bucket[5m])) by (le, currency))"
            legendFormat = "p99 - {{currency}}"
            refId   = "C"
          }
        ]
        fieldConfig = {
          defaults = {
            unit = "s" # Seconds unit for latency
            thresholds = {
              steps = [
                { color = "green", value = 0 },
                { color = "yellow", value = 0.2 }, # 200ms threshold
                { color = "red", value = 0.5 } # 500ms threshold
              ]
            }
          }
        }
      },
      {
        id    = 2
        type  = "stat"
        title = "Active Payment Intents"
        gridPos = { h = 4, w = 6, x = 12, y = 0 }
        datasource = { uid = var.prometheus_datasource_uid }
        targets = [
          {
            expr    = "payment_active_intents"
            legendFormat = "Active Intents"
            refId   = "A"
          }
        ]
        fieldConfig = {
          defaults = {
            mappings = [
              { type = "value", options = { 0 = { text = "No Active Intents" } } }
            ]
          }
        }
      },
      {
        id    = 3
        type  = "bargauge"
        title = "Payment Success/Failure Rate (Last 1h)"
        gridPos = { h = 8, w = 12, x = 0, y = 8 }
        datasource = { uid = var.prometheus_datasource_uid }
        targets = [
          {
            expr    = "sum(rate(payment_success_total[1h])) by (currency)"
            legendFormat = "Success - {{currency}}"
            refId   = "A"
          },
          {
            expr    = "sum(rate(payment_failure_total[1h])) by (currency)"
            legendFormat = "Failure - {{currency}}"
            refId   = "B"
          }
        ]
        fieldConfig = {
          defaults = {
            unit = "ops"
            thresholds = {
              steps = [
                { color = "green", value = 0 },
                { color = "red", value = 10 } # Alert if failure rate exceeds 10 ops
              ]
            }
          }
        }
      }
    ]

    # Grafana 11.0 time picker configuration
    time = {
      from = "now-1h"
      to   = "now"
    }
  })

  # Error handling: validate dashboard JSON before applying
  lifecycle {
    precondition {
      condition     = can(jsondecode(self.config_json))
      error_message = "Dashboard configuration is not valid JSON."
    }
    precondition {
      condition     = length(jsondecode(self.config_json).panels) > 0
      error_message = "Dashboard must contain at least one panel."
    }
  }
}

# Output dashboard URL for easy access
output "payment_latency_dashboard_url" {
  value = "${var.grafana_url}/d/${grafana_dashboard.payment_latency.uid}/payment-api-latency-${lower(var.environment)}"
  description = "URL of the provisioned payment latency dashboard"
}

# prometheus-3.0-config.yaml
# Prometheus 3.0 configuration for production payment service metrics scraping
# Compatible with Prometheus 3.0.0+ (https://github.com/prometheus/prometheus/releases/tag/v3.0.0)
global:
  scrape_interval: 30s # Default scrape interval for all jobs
  evaluation_interval: 30s # Rule evaluation interval
  external_labels:
    cluster: 'prod-eks-us-east-1'
    environment: 'production'
    monitor: 'prometheus-3-0'

# Rule files for alerting and recording rules
rule_files:
  - "rules/alerts.yaml"
  - "rules/recording.yaml"

# Scrape configurations for all services
scrape_configs:
  # Scrape Prometheus self-metrics
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
    scrape_interval: 15s
    metrics_path: '/metrics'
    # Prometheus 3.0 supports native OpenMetrics scraping
    params:
      format: ['openmetrics']

  # Scrape payment API metrics using Kubernetes service discovery (EKS)
  - job_name: 'payment-api'
    kubernetes_sd_configs:
      - role: pod
        api_server: 'https://eks-api.us-east-1.amazonaws.com'
        # Use IRSA for secure access to EKS API (Prometheus 3.0 supports IRSA natively)
        bearer_token_file: '/var/run/secrets/eks.amazonaws.com/serviceaccount/token'
        tls_config:
          ca_file: '/var/run/secrets/eks.amazonaws.com/serviceaccount/ca.crt'
    # Filter pods with the payment-api label
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: 'payment-api'
        action: keep
      - source_labels: [__meta_kubernetes_pod_ip]
        target_label: __address__
        regex: '(.*)'
        replacement: '${1}:9090' # Payment service metrics port
      - source_labels: [__meta_kubernetes_pod_label_version]
        target_label: version
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
    scrape_interval: 30s
    metrics_path: '/metrics'
    params:
      format: ['openmetrics']
    # Error handling: skip pods that don't respond within 5s
    scrape_timeout: 5s
    # Prometheus 3.0 supports sample limit to prevent OOM
    sample_limit: 10000
    # Label limit to prevent metric cardinality explosion
    label_limit: 30

  # Scrape node exporter metrics for infrastructure monitoring
  - job_name: 'node-exporter'
    kubernetes_sd_configs:
      - role: node
    relabel_configs:
      - source_labels: [__meta_kubernetes_node_label_node_role]
        regex: 'worker'
        action: keep
      - source_labels: [__address__]
        target_label: __address__
        regex: '(.*):10250'
        replacement: '${1}:9100' # Node exporter port
    scrape_interval: 60s
    metrics_path: '/metrics'

# Remote write to long-term storage (S3-compatible storage using Thanos)
remote_write:
  - url: 'https://thanos-receive.internal.example.com/api/v1/receive'
    queue_config:
      capacity: 10000
      max_shards: 10
      min_shards: 1
      max_samples_per_send: 2000
      batch_send_deadline: 5s
      min_backoff: 30ms
      max_backoff: 5s
    metadata_config:
      send: true
      send_interval: 1m
    # Prometheus 3.0 supports remote write retry with exponential backoff
    retry_config:
      base_delay: 1s
      max_delay: 30s
      retries: 5

# Local storage configuration with Prometheus 3.0 TSDB compression
storage:
  tsdb:
    path: '/prometheus-data'
    # Retention time for raw metrics (180 days)
    retention_time: 180d
    # Prometheus 3.0 new block compression (saves 78% storage)
    compression: 'zstd'
    # Maximum number of bytes for local storage (1TB)
    max_bytes: 1099511627776
    # Enable WAL compression (Prometheus 3.0 default)
    wal_compression: true
    # Scrape high availability: keep 2 replicas of WAL
    wal_segment_size: 268435456 # 256MB segments

# Alertmanager configuration for sending alerts
alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']
      # Prometheus 3.0 supports Alertmanager API v2
      api_version: 'v2'
      timeout: 10s

# Prometheus 3.0 web configuration
web:
  listen_address: '0.0.0.0:9090'
  enable_admin_api: false # Disable admin API in production
  # Enable CORS for Grafana 11.0 access
  cors:
    allowed_origins: ['https://grafana.internal.example.com']
    allowed_methods: ['GET', 'POST', 'OPTIONS']
    allowed_headers: ['Authorization', 'Content-Type']
  # Enable TLS for secure access
  tls_server_config:
    cert_file: '/etc/prometheus/tls/cert.pem'
    key_file: '/etc/prometheus/tls/key.pem'
  # Prometheus 3.0 supports rate limiting for scrape endpoints
  rate_limit:
    qps: 100 # 100 requests per second
    burst: 200

# Short snippet: Parallel metric validation script (Python)
import requests
import pandas as pd
from scipy import stats

# Fetch p99 latency from New Relic (NRQL)
nr_query = "SELECT p99(payment_processing_latency) FROM Transaction WHERE appName='payment-api' SINCE 1 hour ago"
nr_response = requests.get(
    "https://api.newrelic.com/v2/applications/12345/metrics/data.json",
    headers={"X-Api-Key": "NRII-XXXX"},
    params={"query": nr_query}
)

# Fetch p99 latency from Prometheus
prom_query = "histogram_quantile(0.99, sum(rate(payment_processing_latency_seconds_bucket[1h])) by (le))"
prom_response = requests.get(
    "https://prometheus.internal.example.com/api/v1/query",
    params={"query": prom_query}
)

# Compare with t-test
nr_values = nr_response.json()['metrics'][0]['values']
prom_values = prom_response.json()['data']['result'][0]['values']
t_stat, p_value = stats.ttest_ind(nr_values, prom_values)
print(f"T-test p-value: {p_value} (p < 0.05 means significant difference)")

# Short snippet: Prometheus recording rule for payment latency
groups:
  - name: payment_recording_rules
    rules:
      - record: payment:p99_latency_seconds
        expr: histogram_quantile(0.99, sum(rate(payment_processing_latency_seconds_bucket[5m])) by (le, currency))
        labels:
          team: "payment"
          environment: "production"
      - record: payment:success_rate_1h
        expr: sum(rate(payment_success_total[1h])) / (sum(rate(payment_success_total[1h])) + sum(rate(payment_failure_total[1h])))

# Short snippet: Promtool command to analyze TSDB cardinality
docker run -it --rm \
  -v /prometheus-data:/prometheus-data \
  prom/prometheus:v3.0.1 \
  promtool tsdb analyze /prometheus-data --max-label-names=20 --max-label-values=10