We Ditched New Relic for Honeycomb 2.5: Our MTTR Dropped 48% for Production Outages

package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

// HoneycombConfig holds migration configuration for Honeycomb 2.5
type HoneycombConfig struct {
    APIKey     string
    Dataset    string
    Endpoint   string
    SampleRate float64
}

// NewRelicConfig holds legacy New Relic configuration for deprecation
type NewRelicConfig struct {
    LicenseKey string
    AppName    string
    Endpoint   string
}

// migrateTracing migrates existing New Relic tracing instrumentation to Honeycomb via OTLP
// Returns a shutdown function to flush traces on application exit
func migrateTracing(ctx context.Context, honeycomb HoneycombConfig, nr NewRelicConfig) (func(context.Context) error, error) {
    // Validate required configuration
    if honeycomb.APIKey == "" {
        return nil, fmt.Errorf("honeycomb API key is required")
    }
    if honeycomb.Dataset == "" {
        return nil, fmt.Errorf("honeycomb dataset is required")
    }

    // Initialize OTLP gRPC client for Honeycomb (Honeycomb 2.5 uses OTLP 1.0+)
    client := otlptracegrpc.NewClient(
        otlptracegrpc.WithEndpoint(honeycomb.Endpoint),
        otlptracegrpc.WithDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())),
        otlptracegrpc.WithHeaders(map[string]string{
            "x-honeycomb-team": honeycomb.APIKey,
            "x-honeycomb-dataset": honeycomb.Dataset,
        }),
    )

    // Create OTLP trace exporter
    exporter, err := otlptrace.New(ctx, client)
    if err != nil {
        return nil, fmt.Errorf("failed to create honeycomb exporter: %w", err)
    }

    // Define resource attributes matching Honeycomb 2.5 schema requirements
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceNameKey.String(os.Getenv("SERVICE_NAME")),
            semconv.ServiceVersionKey.String(os.Getenv("SERVICE_VERSION")),
            semconv.DeploymentEnvironmentKey.String(os.Getenv("DEPLOY_ENV")),
            // Custom attribute for migration tracking
            semconv.AttributeKey("migration.source").String("new-relic"),
            semconv.AttributeKey("migration.version").String("1.0.0"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // Configure sampler: use dynamic sampling matching Honeycomb 2.5 recommendations
    sampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(honeycomb.SampleRate))

    // Initialize tracer provider with Honeycomb exporter
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter, sdktrace.WithBatchTimeout(5*time.Second)),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sampler),
    )

    // Set global tracer provider and propagator
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    // Log migration status
    log.Printf("✅ Migrated tracing from New Relic to Honeycomb 2.5 (dataset: %s, sample rate: %.2f)", honeycomb.Dataset, honeycomb.SampleRate)

    // Return shutdown function to flush traces on exit
    return tp.Shutdown, nil
}

func main() {
    ctx := context.Background()

    // Load configuration from environment (matching our production setup)
    honeycombCfg := HoneycombConfig{
        APIKey:     os.Getenv("HONEYCOMB_API_KEY"),
        Dataset:    os.Getenv("HONEYCOMB_DATASET"),
        Endpoint:   os.Getenv("HONEYCOMB_ENDPOINT"), // Default: api.honeycomb.io:443
        SampleRate: 0.1, // 10% sampling as recommended for high-volume services
    }

    nrCfg := NewRelicConfig{
        LicenseKey: os.Getenv("NEW_RELIC_LICENSE_KEY"),
        AppName:    os.Getenv("NEW_RELIC_APP_NAME"),
        Endpoint:   os.Getenv("NEW_RELIC_ENDPOINT"),
    }

    // Execute migration
    shutdown, err := migrateTracing(ctx, honeycombCfg, nrCfg)
    if err != nil {
        log.Fatalf("❌ Migration failed: %v", err)
    }

    // Simulate application runtime
    time.Sleep(10 * time.Second)

    // Flush traces on exit
    if err := shutdown(ctx); err != nil {
        log.Fatalf("❌ Failed to shutdown tracer provider: %v", err)
    }
}

#!/usr/bin/env python3
"""
Honeycomb 2.5 MTTR Calculator
Queries Honeycomb and New Relic APIs to compare outage resolution times
Requires: requests, python-dateutil, pandas
"""

import os
import json
import time
from datetime import datetime, timedelta
from typing import Dict, List, Optional

import requests
from dateutil import parser as date_parser

# Configuration from environment variables
HONEYCOMB_API_KEY = os.getenv("HONEYCOMB_API_KEY")
HONEYCOMB_DATASET = os.getenv("HONEYCOMB_DATASET")
HONEYCOMB_ENDPOINT = os.getenv("HONEYCOMB_ENDPOINT", "https://api.honeycomb.io")
NEW_RELIC_API_KEY = os.getenv("NEW_RELIC_API_KEY")
NEW_RELIC_ACCOUNT_ID = os.getenv("NEW_RELIC_ACCOUNT_ID")
NEW_RELIC_ENDPOINT = os.getenv("NEW_RELIC_ENDPOINT", "https://api.newrelic.com/v2")

class ObservabilityClient:
    """Base client for observability platform queries"""
    def __init__(self, timeout: int = 30):
        self.timeout = timeout
        self.session = requests.Session()

    def _handle_response(self, response: requests.Response) -> Dict:
        """Handle API response, raise for status, parse JSON"""
        try:
            response.raise_for_status()
            return response.json()
        except requests.exceptions.HTTPError as e:
            raise RuntimeError(f"API request failed: {e} - Response: {response.text}")
        except json.JSONDecodeError as e:
            raise RuntimeError(f"Failed to parse JSON response: {e} - Response: {response.text}")

class HoneycombClient(ObservabilityClient):
    """Client for Honeycomb 2.5 Query API"""
    def __init__(self):
        super().__init__()
        if not HONEYCOMB_API_KEY:
            raise ValueError("HONEYCOMB_API_KEY environment variable is required")
        if not HONEYCOMB_DATASET:
            raise ValueError("HONEYCOMB_DATASET environment variable is required")
        self.session.headers.update({
            "X-Honeycomb-Team": HONEYCOMB_API_KEY,
            "Content-Type": "application/json",
        })
        self.base_url = f"{HONEYCOMB_ENDPOINT}/1/query/{HONEYCOMB_DATASET}"

    def query_mttr(self, start_time: datetime, end_time: datetime) -> float:
        """
        Calculate MTTR for outages in the given time range using Honeycomb 2.5 Bubbleup
        Returns MTTR in minutes
        """
        # Honeycomb query to find outage start/end times
        # Matches our outage definition: 5xx error rate > 1% for > 1 minute
        query = {
            "query": {
                "calculations": [
                    {"op": "COUNT", "alias": "outage_count"},
                    {"op": "AVG", "column": "duration_ms", "alias": "avg_outage_duration_ms"},
                ],
                "filters": [
                    {"column": "http.status_code", "op": ">=", "value": 500},
                    {"column": "http.status_code", "op": "<=", "value": 599},
                    {"column": "is_outage", "op": "=", "value": True},
                ],
                "time_range": int(end_time.timestamp() - start_time.timestamp()),
                "start_time": int(start_time.timestamp()),
                "end_time": int(end_time.timestamp()),
            }
        }

        response = self.session.post(f"{self.base_url}/query", json=query, timeout=self.timeout)
        data = self._handle_response(response)

        # Parse results: Honeycomb 2.5 returns results in data.series
        if not data.get("results"):
            return 0.0

        total_outages = 0
        total_duration_ms = 0
        for series in data["results"]:
            total_outages += series.get("outage_count", 0)
            total_duration_ms += series.get("avg_outage_duration_ms", 0) * series.get("outage_count", 0)

        if total_outages == 0:
            return 0.0

        # Convert to minutes
        return (total_duration_ms / total_outages) / (1000 * 60)

class NewRelicClient(ObservabilityClient):
    """Client for New Relic Insights API (legacy)"""
    def __init__(self):
        super().__init__()
        if not NEW_RELIC_API_KEY:
            raise ValueError("NEW_RELIC_API_KEY environment variable is required")
        if not NEW_RELIC_ACCOUNT_ID:
            raise ValueError("NEW_RELIC_ACCOUNT_ID environment variable is required")
        self.session.headers.update({
            "X-Query-Key": NEW_RELIC_API_KEY,
            "Content-Type": "application/json",
        })
        self.base_url = f"{NEW_RELIC_ENDPOINT}/accounts/{NEW_RELIC_ACCOUNT_ID}/query"

    def query_mttr(self, start_time: datetime, end_time: datetime) -> float:
        """Calculate MTTR from New Relic NRQL (legacy)"""
        # NRQL query to match Honeycomb outage definition
        nrql = f"""
        SELECT average(duration) FROM Transaction 
        WHERE http.statusCode >= 500 AND is_outage = true 
        SINCE '{start_time.isoformat()}' UNTIL '{end_time.isoformat()}'
        """
        query = {"nrql": nrql.strip()}

        response = self.session.post(self.base_url, json=query, timeout=self.timeout)
        data = self._handle_response(response)

        # Parse New Relic response format
        if not data.get("results"):
            return 0.0

        total_duration = 0.0
        total_outages = 0
        for result in data["results"]:
            total_duration += result.get("average.duration", 0) * result.get("count", 0)
            total_outages += result.get("count", 0)

        if total_outages == 0:
            return 0.0

        # Convert seconds to minutes
        return (total_duration / total_outages) / 60

def main():
    # Calculate time range: last 30 days to match our benchmark window
    end_time = datetime.utcnow()
    start_time = end_time - timedelta(days=30)

    print(f"Calculating MTTR from {start_time} to {end_time}")

    # Query Honeycomb 2.5
    try:
        honeycomb = HoneycombClient()
        honeycomb_mttr = honeycomb.query_mttr(start_time, end_time)
        print(f"🍯 Honeycomb 2.5 MTTR: {honeycomb_mttr:.2f} minutes")
    except Exception as e:
        print(f"❌ Honeycomb query failed: {e}")
        honeycomb_mttr = None

    # Query New Relic (legacy)
    try:
        new_relic = NewRelicClient()
        nr_mttr = new_relic.query_mttr(start_time, end_time)
        print(f"📉 New Relic MTTR: {nr_mttr:.2f} minutes")
    except Exception as e:
        print(f"❌ New Relic query failed: {e}")
        nr_mttr = None

    # Calculate improvement
    if honeycomb_mttr and nr_mttr and nr_mttr > 0:
        improvement = ((nr_mttr - honeycomb_mttr) / nr_mttr) * 100
        print(f"🚀 MTTR Improvement: {improvement:.2f}%")

if __name__ == "__main__":
    main()

// Honeycomb 2.5 RUM Migration Script
// Replaces New Relic Browser agent with Honeycomb RUM for frontend observability
// Required: @honeycombio/rum-web v2.5+, @opentelemetry/api

import { HoneycombRum } from '@honeycombio/rum-web';
import { trace, context, SpanStatusCode } from '@opentelemetry/api';

// Configuration from environment (injected via build pipeline)
const HONEYCOMB_API_KEY = process.env.REACT_APP_HONEYCOMB_API_KEY;
const HONEYCOMB_DATASET = process.env.REACT_APP_HONEYCOMB_DATASET;
const HONEYCOMB_ENDPOINT = process.env.REACT_APP_HONEYCOMB_ENDPOINT || 'https://api.honeycomb.io';
const NEW_RELIC_APP_ID = process.env.REACT_APP_NEW_RELIC_APP_ID; // For migration tracking

// Validate required configuration
if (!HONEYCOMB_API_KEY) {
  console.error('❌ REACT_APP_HONEYCOMB_API_KEY is required for Honeycomb RUM');
  throw new Error('Missing Honeycomb API key');
}
if (!HONEYCOMB_DATASET) {
  console.error('❌ REACT_APP_HONEYCOMB_DATASET is required for Honeycomb RUM');
  throw new Error('Missing Honeycomb dataset');
}

// Initialize Honeycomb RUM 2.5 with recommended configuration
const rum = HoneycombRum.init({
  apiKey: HONEYCOMB_API_KEY,
  dataset: HONEYCOMB_DATASET,
  endpoint: `${HONEYCOMB_ENDPOINT}/v1/rum/events`,
  // Match Honeycomb 2.5 sampling recommendations for frontend
  sampleRate: 0.2, // 20% sample rate for high-traffic apps
  // Capture all default telemetry: page views, clicks, errors, XHR/fetch requests
  capturePageViews: true,
  captureErrors: true,
  captureInteractions: true,
  captureXhr: true,
  captureFetch: true,
  // Custom attributes for migration tracking and context
  globalAttributes: {
    'migration.source': 'new-relic-browser',
    'migration.version': '1.0.0',
    'deployment.env': process.env.NODE_ENV || 'development',
    'app.version': process.env.REACT_APP_VERSION || 'unknown',
  },
  // Enable distributed tracing for frontend-to-backend correlation
  distributedTracing: {
    enabled: true,
    headerName: 'traceparent', // Use W3C trace context as recommended by Honeycomb 2.5
  },
  // Custom error handler to filter noisy errors and add context
  errorFilter: (error) => {
    // Ignore known third-party script errors (New Relic used to suppress these differently)
    const thirdPartyDomains = ['googletagmanager.com', 'newrelic.com', 'nr-data.net'];
    if (error.filename && thirdPartyDomains.some(domain => error.filename.includes(domain))) {
      return false; // Do not capture this error
    }
    // Add custom context to error events
    error.attributes = {
      ...error.attributes,
      'error.handled': false,
      'user.id': localStorage.getItem('user_id') || 'anonymous',
    };
    return true;
  },
  // Lifecycle hooks for debugging (remove in production)
  onInitialized: () => {
    console.log(`✅ Honeycomb RUM 2.5 initialized (dataset: ${HONEYCOMB_DATASET})`);
  },
  onEventSent: (event) => {
    if (process.env.NODE_ENV === 'development') {
      console.log('📤 Honeycomb event sent:', event.type);
    }
  },
});

// Replace New Relic Browser's addPageAction with Honeycomb equivalent
// New Relic: newrelic.addPageAction('action_name', { attr: 'value' })
// Honeycomb equivalent:
export const trackCustomEvent = (actionName, attributes = {}) => {
  try {
    const tracer = trace.getTracer('frontend-custom-events');
    const span = tracer.startSpan(`custom.${actionName}`);
    context.with(trace.setSpan(context.active(), span), () => {
      Object.entries(attributes).forEach(([key, value]) => {
        span.setAttribute(key, value);
      });
      span.setStatus({ code: SpanStatusCode.OK });
      span.end();
    });
    // Also send as RUM event for Honeycomb 2.5 Bubbleup compatibility
    rum.trackEvent(`custom.${actionName}`, {
      ...attributes,
      'event.category': 'custom',
    });
  } catch (error) {
    console.error('❌ Failed to track custom event:', error);
  }
};

// Replace New Relic's setCustomAttribute with Honeycomb global attributes
export const setGlobalAttribute = (key, value) => {
  try {
    rum.setGlobalAttribute(key, value);
  } catch (error) {
    console.error('❌ Failed to set global attribute:', error);
  }
};

// Migration cleanup: remove New Relic Browser agent if present
export const removeNewRelicAgent = () => {
  if (window.newrelic) {
    try {
      window.newrelic.pause();
      delete window.newrelic;
      // Remove New Relic script tags from DOM
      const nrScripts = document.querySelectorAll('script[src*="newrelic"], script[src*="nr-data.net"]');
      nrScripts.forEach(script => script.remove());
      console.log('🗑️ New Relic Browser agent removed successfully');
    } catch (error) {
      console.error('❌ Failed to remove New Relic agent:', error);
    }
  }
};

// Initialize migration on app load
export const initHoneycombRum = () => {
  try {
    // Remove legacy New Relic agent first
    removeNewRelicAgent();
    // RUM is already initialized above, but we can add post-init checks
    if (!rum.isInitialized()) {
      throw new Error('Honeycomb RUM failed to initialize');
    }
    console.log('🚀 Honeycomb RUM 2.5 migration complete');
  } catch (error) {
    console.error('❌ Honeycomb RUM migration failed:', error);
  }
};

// Auto-initialize if this is the main entry point
if (typeof window !== 'undefined') {
  initHoneycombRum();
}

export default rum;

curl -X POST "https://api.honeycomb.io/1/datasets/$DATASET_SLUG/settings" \
  -H "X-Honeycomb-Team: $HONEYCOMB_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"bubbleup_enabled": true, "bubbleup_sensitivity": "high"}'

// Configure dual export to New Relic and Honeycomb
nrExporter, _ := newrelic.New(newrelic.WithLicenseKey(os.Getenv("NEW_RELIC_LICENSE_KEY")))
honeycombExporter, _ := otlptracegrpc.New(ctx, otlptracegrpc.WithEndpoint("api.honeycomb.io:443"))

tp := sdktrace.NewTracerProvider(
  sdktrace.WithBatcher(nrExporter),
  sdktrace.WithBatcher(honeycombExporter),
  // ... other config
)

resource "honeycomb_slo" "payment_latency" {
  dataset    = "payment-service"
  name       = "payment-latency-slo"
  description = "99.9% of requests < 500ms over 30 days"

  sli {
    query = <