Opinion: Commercial Observability Tools Are a Scam – Open Source Grafana 11 Is All You Need

// grafana_provisioner.go
// Provisions Grafana 11 instances with production-ready dashboards, alerts, and data sources
// Requires GRAFANA_URL and GRAFANA_API_KEY environment variables
package main

import (
    "bytes",
    "encoding/json",
    "fmt",
    "io",
    "net/http",
    "os",
    "time",
)

// DataSource represents a Grafana Prometheus data source configuration
type DataSource struct {
    Name      string `json:"name"`
    Type      string `json:"type"`
    URL       string `json:"url"`
    Access    string `json:"access"`
    IsDefault bool   `json:"isDefault"`
}

// Dashboard represents a minimal Grafana dashboard payload
type Dashboard struct {
    Dashboard map[string]interface{} `json:"dashboard"`
    Overwrite bool                   `json:"overwrite"`
}

func main() {
    grafanaURL := os.Getenv("GRAFANA_URL")
    apiKey := os.Getenv("GRAFANA_API_KEY")
    if grafanaURL == "" || apiKey == "" {
        fmt.Fprintf(os.Stderr, "FATAL: GRAFANA_URL and GRAFANA_API_KEY must be set\n")
        os.Exit(1)
    }

    // 1. Provision Prometheus data source
    ds := DataSource{
        Name:      "prometheus-prod",
        Type:      "prometheus",
        URL:       "http://prometheus:9090",
        Access:    "proxy",
        IsDefault: true,
    }
    if err := createDataSource(grafanaURL, apiKey, ds); err != nil {
        fmt.Fprintf(os.Stderr, "Failed to create data source: %v\n", err)
        os.Exit(1)
    }
    fmt.Println("Provisioned Prometheus data source")

    // 2. Provision Node Exporter dashboard from embedded JSON
    dashboardJSON := `{
        "annotations": { "list": [] },
        "editable": true,
        "fiscalYearStartMonth": 0,
        "graphTooltip": 0,
        "links": [],
        "panels": [
            {
                "title": "Node CPU Usage",
                "type": "timeseries",
                "targets": [
                    {
                        "expr": "rate(node_cpu_seconds_total{mode!=\"idle\"}[5m])",
                        "legendFormat": "{{instance}} - {{mode}}"
                    }
                ]
            }
        ],
        "schemaVersion": 39,
        "tags": ["node-exporter", "prod"],
        "templating": { "list": [] },
        "time": { "from": "now-1h", "to": "now" },
        "title": "Node Exporter Overview",
        "uid": "node-exporter-prod"
    }`
    var dashboardMap map[string]interface{}
    if err := json.Unmarshal([]byte(dashboardJSON), &dashboardMap); err != nil {
        fmt.Fprintf(os.Stderr, "Failed to parse dashboard JSON: %v\n", err)
        os.Exit(1)
    }
    dash := Dashboard{Dashboard: dashboardMap, Overwrite: true}
    if err := createDashboard(grafanaURL, apiKey, dash); err != nil {
        fmt.Fprintf(os.Stderr, "Failed to create dashboard: %v\n", err)
        os.Exit(1)
    }
    fmt.Println("Provisioned Node Exporter dashboard")

    // 3. Configure unified alerting rule
    alertRule := `{
        "alert": "High CPU Usage",
        "expr": "rate(node_cpu_seconds_total{mode!=\"idle\"}[5m]) > 0.8",
        "for": "5m",
        "labels": { "severity": "critical" },
        "annotations": { "summary": "CPU usage above 80% for 5 minutes" }
    }`
    if err := createAlertRule(grafanaURL, apiKey, alertRule); err != nil {
        fmt.Fprintf(os.Stderr, "Failed to create alert rule: %v\n", err)
        os.Exit(1)
    }
    fmt.Println("Provisioned unified alerting rule")
}

func createDataSource(url, apiKey string, ds DataSource) error {
    body, err := json.Marshal(ds)
    if err != nil {
        return fmt.Errorf("marshal data source: %w", err)
    }
    req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/datasources", url), bytes.NewBuffer(body))
    if err != nil {
        return fmt.Errorf("create request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
    req.Header.Set("Content-Type", "application/json")
    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Do(req)
    if err != nil {
        return fmt.Errorf("send request: %w", err)
    }
    defer resp.Body.Close()
    if resp.StatusCode != http.StatusOK {
        respBody, _ := io.ReadAll(resp.Body)
        return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody))
    }
    return nil
}

func createDashboard(url, apiKey string, dash Dashboard) error {
    body, err := json.Marshal(dash)
    if err != nil {
        return fmt.Errorf("marshal dashboard: %w", err)
    }
    req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/dashboards/db", url), bytes.NewBuffer(body))
    if err != nil {
        return fmt.Errorf("create request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
    req.Header.Set("Content-Type", "application/json")
    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Do(req)
    if err != nil {
        return fmt.Errorf("send request: %w", err)
    }
    defer resp.Body.Close()
    if resp.StatusCode != http.StatusOK {
        respBody, _ := io.ReadAll(resp.Body)
        return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody))
    }
    return nil
}

func createAlertRule(url, apiKey string, rule string) error {
    req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/v1/alerts", url), bytes.NewBufferString(rule))
    if err != nil {
        return fmt.Errorf("create request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
    req.Header.Set("Content-Type", "application/json")
    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Do(req)
    if err != nil {
        return fmt.Errorf("send request: %w", err)
    }
    defer resp.Body.Close()
    if resp.StatusCode != http.StatusOK {
        respBody, _ := io.ReadAll(resp.Body)
        return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody))
    }
    return nil
}

"""
datadog_to_grafana_migrator.py
Migrates Datadog monitors to Grafana 11 unified alerting rules
Requires DATADOG_API_KEY, DATADOG_APP_KEY, GRAFANA_URL, GRAFANA_API_KEY env vars
"""

import os
import json
import requests
from typing import Dict, List, Any

DATADOG_API_BASE = "https://api.datadoghq.com/api/v1"
GRAFANA_API_BASE = os.getenv("GRAFANA_URL", "http://localhost:3000")

class DatadogClient:
    def __init__(self):
        self.api_key = os.getenv("DATADOG_API_KEY")
        self.app_key = os.getenv("DATADOG_APP_KEY")
        if not all([self.api_key, self.app_key]):
            raise ValueError("DATADOG_API_KEY and DATADOG_APP_KEY must be set")
        self.session = requests.Session()
        self.session.headers.update({
            "DD-API-KEY": self.api_key,
            "DD-APPLICATION-KEY": self.app_key,
            "Content-Type": "application/json"
        })

    def get_monitors(self, query: str = "status:all") -> List[Dict[str, Any]]:
        """Fetch all Datadog monitors matching the query"""
        monitors = []
        page = 0
        per_page = 100
        while True:
            resp = self.session.get(
                f"{DATADOG_API_BASE}/monitor",
                params={"query": query, "page": page, "per_page": per_page}
            )
            resp.raise_for_status()
            batch = resp.json()
            if not batch:
                break
            monitors.extend(batch)
            page += 1
        return monitors

class GrafanaClient:
    def __init__(self):
        self.api_key = os.getenv("GRAFANA_API_KEY")
        if not self.api_key:
            raise ValueError("GRAFANA_API_KEY must be set")
        self.session = requests.Session()
        self.session.headers.update({
            "Authorization": f"Bearer {self.api_key}",
            "Content-Type": "application/json"
        })
        self.base_url = GRAFANA_API_BASE

    def create_alert_rule(self, rule: Dict[str, Any]) -> Dict[str, Any]:
        """Create a Grafana 11 unified alert rule"""
        resp = self.session.post(
            f"{self.base_url}/api/v1/alerts",
            json=rule
        )
        resp.raise_for_status()
        return resp.json()

def map_datadog_to_grafana_monitor(dd_monitor: Dict[str, Any]) -> Dict[str, Any]:
    """Map Datadog monitor fields to Grafana alert rule format"""
    # Extract query from Datadog monitor (simplified for demo)
    query = dd_monitor.get("query", "")
    # Convert Datadog threshold to Grafana expr
    thresholds = dd_monitor.get("options", {}).get("thresholds", {})
    critical = thresholds.get("critical")
    alert_expr = query
    if critical:
        # Simplistic conversion: assume query is a PromQL-like expression
        alert_expr = f"{query} > {critical}"

    return {
        "alert": dd_monitor.get("name", "Migrated Alert"),
        "expr": alert_expr,
        "for": "5m",
        "labels": {
            "severity": "critical" if dd_monitor.get("priority") == "P1" else "warning",
            "datadog_id": str(dd_monitor.get("id"))
        },
        "annotations": {
            "summary": dd_monitor.get("message", "Migrated from Datadog"),
            "description": f"Original Datadog monitor: {dd_monitor.get('url', '')}"
        }
    }

def main():
    try:
        dd_client = DatadogClient()
        grafana_client = GrafanaClient()
    except ValueError as e:
        print(f"FATAL: {e}")
        return 1

    print("Fetching Datadog monitors...")
    try:
        monitors = dd_client.get_monitors()
    except requests.exceptions.RequestException as e:
        print(f"Failed to fetch Datadog monitors: {e}")
        return 1
    print(f"Found {len(monitors)} Datadog monitors")

    success = 0
    failed = 0
    for monitor in monitors:
        try:
            grafana_rule = map_datadog_to_grafana_monitor(monitor)
            grafana_client.create_alert_rule(grafana_rule)
            success += 1
            print(f"Migrated monitor {monitor.get('id')}: {monitor.get('name')}")
        except requests.exceptions.RequestException as e:
            failed += 1
            print(f"Failed to migrate monitor {monitor.get('id')}: {e}")
        except Exception as e:
            failed += 1
            print(f"Unexpected error migrating monitor {monitor.get('id')}: {e}")

    print(f"\nMigration complete: {success} succeeded, {failed} failed")
    return 0 if failed == 0 else 1

if __name__ == "__main__":
    exit(main())

/**
 * grafana_tempo_otel_setup.ts
 * Sets up Grafana 11 Tempo for distributed tracing with OpenTelemetry
 * Requires @opentelemetry/sdk-node, @opentelemetry/exporter-trace-otlp-http, express
 */

import { NodeSDK } from '@opentelemetry/sdk-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import express, { Request, Response, NextFunction } from 'express';
import { trace, SpanStatusCode } from '@opentelemetry/api';
import axios from 'axios';

// 1. Configure OpenTelemetry SDK to export to Grafana Tempo
const sdk = new NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'grafana-tempo-demo',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://tempo:4318/v1/traces', // Tempo OTLP HTTP endpoint
    headers: {}, // Add auth headers if Tempo is configured with auth
  }),
});

// Start the SDK
try {
  sdk.start();
  console.log('OpenTelemetry SDK started, exporting traces to Tempo');
} catch (err) {
  console.error('Failed to start OpenTelemetry SDK:', err);
  process.exit(1);
}

// Handle graceful shutdown
process.on('SIGTERM', () => {
  sdk.shutdown()
    .then(() => console.log('OpenTelemetry SDK shut down'))
    .catch((err) => console.error('Error shutting down SDK:', err))
    .finally(() => process.exit(0));
});

// 2. Create instrumented Express app
const app = express();
const port = process.env.PORT || 3000;

// Middleware to create root span for each request
app.use((req: Request, res: Response, next: NextFunction) => {
  const span = trace.getTracer('express-middleware').startSpan(`${req.method} ${req.path}`);
  span.setAttribute('http.method', req.method);
  span.setAttribute('http.url', req.url);

  res.on('finish', () => {
    span.setAttribute('http.status_code', res.statusCode);
    if (res.statusCode >= 400) {
      span.setStatus({ code: SpanStatusCode.ERROR });
    }
    span.end();
  });

  next();
});

// Sample endpoint that calls a downstream service
app.get('/api/users', async (req: Request, res: Response) => {
  const tracer = trace.getTracer('user-service');
  const span = tracer.startSpan('fetch-downstream-users');

  try {
    // Call downstream service (instrumented with OTel)
    const response = await axios.get('http://downstream-service:4000/users', {
      headers: {
        // Propagate trace context
        ...trace.getSpanContext(span),
      },
    });

    span.setAttribute('downstream.status', response.status);
    res.json(response.data);
  } catch (err) {
    span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
    res.status(500).json({ error: 'Failed to fetch users' });
  } finally {
    span.end();
  }
});

// Health check endpoint
app.get('/health', (req: Request, res: Response) => {
  res.json({ status: 'healthy' });
});

// Global error handler
app.use((err: Error, req: Request, res: Response, next: NextFunction) => {
  console.error('Unhandled error:', err);
  res.status(500).json({ error: 'Internal server error' });
});

// Start the server
app.listen(port, () => {
  console.log(`Instrumented Express app listening on port ${port}`);
  console.log('Traces will be sent to Grafana Tempo and viewable in Grafana 11');
});

curl -X POST "https://grafana.example.com/api/v1/alerts" \
  -H "Authorization: Bearer $GRAFANA_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "alert": "High API Latency",
    "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{job=\"api\"}[5m])) > 0.5",
    "for": "2m",
    "labels": { "severity": "critical", "service": "api" },
    "annotations": { "summary": "p99 API latency above 500ms for 2 minutes" }
  }'

import "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
import "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"

exporter, err := otlptrace.New(
  context.Background(),
  otlptracehttp.NewClient(
    otlptracehttp.WithEndpoint("tempo:4318"),
    otlptracehttp.WithURLPath("/v1/traces"),
    otlptracehttp.WithInsecure(), // Remove for production with TLS
  ),
)

resource "grafana_dashboard" "node_exporter" {
  config_json = file("${path.module}/dashboards/node-exporter.json")
  overwrite = true
  folder = grafana_folder.observability.id
}