War Story: We Replaced Sentry 24.0 with Datadog 1.20 and Improved Error Resolution Time by 37%

// Sentry 24.0 Node.js SDK integration for Express (pre-migration)
// Custom wrapper repo: https://github.com/acme-org/sentry-custom-sdk
const express = require('express');
const Sentry = require('@sentry/node');
const { ProfilingIntegration } = require('@sentry/profiling-node');
const redis = require('redis');
const app = express();

// Initialize Sentry 24.0 with self-hosted config
Sentry.init({
  dsn: process.env.SENTRY_DSN || 'https://examplePublicKey@o0.ingest.sentry.io/0',
  environment: process.env.NODE_ENV || 'development',
  release: `acme-checkout@${process.env.GIT_SHA || 'unknown'}`,
  integrations: [
    new Sentry.Integrations.Http({ tracing: true }),
    new Sentry.Integrations.Express({ app }),
    new ProfilingIntegration(),
  ],
  tracesSampleRate: 1.0,
  profilesSampleRate: 1.0,
  beforeSend(event, hint) {
    // Add custom tags for checkout service errors
    const error = hint.originalException;
    if (error?.service === 'checkout') {
      event.tags = {
        ...event.tags,
        service: 'checkout',
        errorCategory: error.category || 'uncategorized',
        redisPoolSize: process.env.REDIS_POOL_SIZE || '10',
      };
    }
    // Filter out health check errors
    if (event.request?.url?.includes('/healthz')) {
      return null;
    }
    return event;
  },
});

// Sentry request handler (must be first middleware)
app.use(Sentry.Handlers.requestHandler());
app.use(Sentry.Handlers.tracingHandler());

// Redis client with error handling
const redisClient = redis.createClient({
  url: process.env.REDIS_URL || 'redis://localhost:6379',
  socket: {
    reconnectStrategy: (retries) => {
      if (retries > 3) {
        const err = new Error(`Redis reconnect failed after ${retries} attempts`);
        Sentry.captureException(err, {
          tags: { service: 'checkout', component: 'redis' },
        });
        return new Error('Max redis retries exceeded');
      }
      return Math.min(retries * 100, 3000);
    },
  },
});

redisClient.on('error', (err) => {
  Sentry.captureException(err, {
    tags: { service: 'checkout', component: 'redis' },
    extra: { redisUrl: process.env.REDIS_URL },
  });
});

// Checkout endpoint with error handling
app.post('/checkout', async (req, res) => {
  try {
    const { userId, items } = req.body;
    if (!userId || !items?.length) {
      const err = new Error('Invalid checkout request payload');
      err.service = 'checkout';
      err.category = 'validation';
      Sentry.captureException(err, {
        tags: { service: 'checkout', endpoint: '/checkout' },
        extra: { payload: req.body },
      });
      return res.status(400).json({ error: 'Invalid request' });
    }

    // Simulate Redis pool exhaustion error (our original pain point)
    const poolStatus = await redisClient.get('checkout:pool:status');
    if (poolStatus === 'exhausted') {
      const err = new Error('Checkout Redis pool exhausted');
      err.service = 'checkout';
      err.category = 'infrastructure';
      Sentry.captureException(err, {
        tags: { service: 'checkout', component: 'redis-pool' },
        extra: { poolSize: process.env.REDIS_POOL_SIZE },
      });
      return res.status(500).json({ error: 'Service unavailable' });
    }

    res.status(200).json({ orderId: 'ord_12345', status: 'processed' });
  } catch (err) {
    err.service = 'checkout';
    Sentry.captureException(err, {
      tags: { service: 'checkout', endpoint: '/checkout' },
      extra: { userId, itemCount: items?.length },
    });
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Sentry error handler (must be after all controllers)
app.use(Sentry.Handlers.errorHandler());

const port = process.env.PORT || 3000;
app.listen(port, () => {
  console.log(`Checkout service listening on port ${port}`);
});

// Datadog 1.20 Node.js SDK integration for Express (post-migration)
// Config repo: https://github.com/acme-org/datadog-config
const express = require('express');
const { datadog, logs, tracer } = require('@datadog/node');
const { Redis } = require('ioredis'); // Switched to ioredis for better pool metrics
const app = express();

// Initialize Datadog 1.20 tracing
tracer.init({
  service: 'acme-checkout',
  env: process.env.NODE_ENV || 'development',
  version: process.env.GIT_SHA || 'unknown',
  hostname: process.env.DD_AGENT_HOST || 'localhost',
  port: process.env.DD_AGENT_PORT || 8126,
  sampleRate: 1.0,
  integrations: ['express', 'redis'],
  tags: {
    team: 'payments',
    costCenter: 'checkout-123',
  },
});

// Initialize Datadog 1.20 log collection
logs.init({
  apiKey: process.env.DD_API_KEY,
  site: process.env.DD_SITE || 'datadoghq.com',
  service: 'acme-checkout',
  env: process.env.NODE_ENV || 'development',
  forwardErrorsToLogs: true,
  sampleRate: 1.0,
});

// Redis client with Datadog 1.20 native metrics
const redisClient = new Redis({
  host: process.env.REDIS_HOST || 'localhost',
  port: process.env.REDIS_PORT || 6379,
  maxRetriesPerRequest: 3,
  enableReadyCheck: true,
  lazyConnect: true,
  // Datadog 1.20 auto-instruments ioredis, no custom wrapper needed
});

redisClient.on('error', (err) => {
  logs.logger.error('Redis client error', {
    error: err.message,
    stack: err.stack,
    tags: ['service:checkout', 'component:redis', 'error:infrastructure'],
    meta: { redisHost: process.env.REDIS_HOST },
  });
});

// Express middleware to inject Datadog trace context
app.use((req, res, next) => {
  const span = tracer.startSpan('checkout.request');
  req.datadogSpan = span;
  res.on('finish', () => {
    span.setTag('http.status_code', res.statusCode);
    span.finish();
  });
  next();
});

// Checkout endpoint with Datadog 1.20 error handling
app.post('/checkout', async (req, res) => {
  const span = req.datadogSpan;
  try {
    const { userId, items } = req.body;
    span.setTag('user.id', userId);
    span.setTag('checkout.item_count', items?.length || 0);

    if (!userId || !items?.length) {
      const err = new Error('Invalid checkout request payload');
      logs.logger.warn('Validation error on checkout', {
        error: err.message,
        tags: ['service:checkout', 'error:validation', 'endpoint:/checkout'],
        meta: { payload: req.body },
      });
      span.setTag('error', true);
      span.setTag('error.type', 'validation');
      return res.status(400).json({ error: 'Invalid request' });
    }

    // Check Redis pool status with auto-instrumented metrics
    const poolStatus = await redisClient.get('checkout:pool:status');
    if (poolStatus === 'exhausted') {
      const err = new Error('Checkout Redis pool exhausted');
      logs.logger.error('Redis pool exhausted', {
        error: err.message,
        tags: ['service:checkout', 'component:redis-pool', 'error:infrastructure'],
        meta: { poolSize: process.env.REDIS_POOL_SIZE },
      });
      span.setTag('error', true);
      span.setTag('error.type', 'infrastructure');
      // Datadog 1.20 auto-links error to related infrastructure metrics
      return res.status(500).json({ error: 'Service unavailable' });
    }

    span.setTag('checkout.status', 'success');
    res.status(200).json({ orderId: 'ord_12345', status: 'processed' });
  } catch (err) {
    logs.logger.error('Unhandled checkout error', {
      error: err.message,
      stack: err.stack,
      tags: ['service:checkout', 'endpoint:/checkout', 'error:unhandled'],
      meta: { userId, itemCount: req.body.items?.length },
    });
    span.setTag('error', true);
    span.setTag('error.stack', err.stack);
    res.status(500).json({ error: 'Internal server error' });
  } finally {
    span.finish();
  }
});

// Datadog 1.20 health check endpoint (excluded from error tracking by default)
app.get('/healthz', (req, res) => {
  res.status(200).json({ status: 'healthy' });
});

const port = process.env.PORT || 3000;
app.listen(port, () => {
  console.log(`Checkout service listening on port ${port}`);
  logs.logger.info('Service started', {
    tags: ['service:checkout', 'event:startup'],
    meta: { port, env: process.env.NODE_ENV },
  });
});

// Python 3.11 migration script to port Sentry 24.0 error tags to Datadog 1.20
// Migrator repo: https://github.com/acme-org/sentry-to-datadog-migrator
import os
import time
import json
import hashlib
import requests
from typing import Dict, List, Optional
from datadog_api_client.v1 import ApiClient, Configuration
from datadog_api_client.v1.api import logs_api
from datadog_api_client.v1.models import LogContent, LogStatus

class SentryToDatadogMigrator:
    def __init__(self):
        # Initialize Sentry 24.0 client (read-only)
        self.sentry_api_key = os.getenv(\"SENTRY_API_KEY\")
        self.sentry_org = os.getenv(\"SENTRY_ORG\", \"acme-org\")
        self.sentry_project = os.getenv(\"SENTRY_PROJECT\", \"checkout\")

        # Initialize Datadog 1.20 client
        self.dd_api_key = os.getenv(\"DD_API_KEY\")
        self.dd_app_key = os.getenv(\"DD_APP_KEY\")
        self.dd_site = os.getenv(\"DD_SITE\", \"datadoghq.com\")

        # Validate config
        if not all([self.sentry_api_key, self.dd_api_key, self.dd_app_key]):
            raise ValueError(\"Missing required env vars: SENTRY_API_KEY, DD_API_KEY, DD_APP_KEY\")

        # Initialize Datadog API client
        self.dd_config = Configuration()
        self.dd_config.api_key['apiKeyAuth'] = self.dd_api_key
        self.dd_config.api_key['appKeyAuth'] = self.dd_app_key
        self.dd_config.server_variables['site'] = self.dd_site
        self.dd_client = ApiClient(self.dd_config)

    def fetch_sentry_errors(self, start_time: int, end_time: int) -> List[Dict]:
        \"\"\"Fetch errors from Sentry 24.0 API with pagination\"\"\"
        errors = []
        cursor = None
        page_size = 100

        while True:
            # Sentry 24.0 REST API endpoint
            url = f'https://sentry.io/api/0/projects/{self.sentry_org}/{self.sentry_project}/events/'
            params = {
                'start': start_time,
                'end': end_time,
                'limit': page_size,
                'cursor': cursor,
            }
            headers = {'Authorization': f'Bearer {self.sentry_api_key}'}

            try:
                response = requests.get(url, params=params, headers=headers)
                response.raise_for_status()
                batch = response.json()
                if not batch:
                    break
                errors.extend(batch)
                # Check for next page
                cursor = response.headers.get('Link', '').split('; rel=\"next\"')[0].strip('<>')
                if not cursor:
                    break
                time.sleep(0.1) # Rate limit compliance
            except requests.exceptions.RequestException as e:
                print(f'Error fetching Sentry events: {e}')
                time.sleep(1)
                continue

        return errors

    def transform_to_datadog_log(self, sentry_event: Dict) -> Optional[LogContent]:
        \"\"\"Map Sentry 24.0 event schema to Datadog 1.20 log schema\"\"\"
        try:
            # Generate deterministic ID to avoid duplicates
            event_id = sentry_event.get('event_id')
            if not event_id:
                return None

            # Map tags
            tags = [
                f'sentry:event_id:{event_id}',
                f'service:{sentry_event.get(\"tags\", {}).get(\"service\", \"unknown\")}',
                f'error_category:{sentry_event.get(\"tags\", {}).get(\"errorCategory\", \"uncategorized\")}',
                f'environment:{sentry_event.get(\"environment\", \"unknown\")}',
            ]

            # Map log content
            log_content = LogContent(
                message=sentry_event.get('message', 'No message'),
                status=LogStatus.ERROR,
                timestamp=int(sentry_event.get('timestamp', time.time())),
                tags=tags,
                attributes={
                    'sentry_event_id': event_id,
                    'sentry_release': sentry_event.get('release'),
                    'error_stack': sentry_event.get('exception', {}).get('values', [{}])[0].get('stacktrace', {}).get('raw', ''),
                    'request_url': sentry_event.get('request', {}).get('url'),
                    'user_id': sentry_event.get('user', {}).get('id'),
                },
            )
            return log_content
        except Exception as e:
            print(f'Error transforming event {sentry_event.get(\"event_id\")}: {e}')
            return None

    def send_to_datadog(self, logs: List[LogContent]) -> int:
        \"\"\"Batch send logs to Datadog 1.20 Logs API\"\"\"
        success_count = 0
        batch_size = 50

        with self.dd_client as client:
            api_instance = logs_api.LogsApi(client)
            for i in range(0, len(logs), batch_size):
                batch = logs[i:i+batch_size]
                try:
                    api_instance.submit_log(batch)
                    success_count += len(batch)
                    print(f'Submitted {len(batch)} logs to Datadog')
                    time.sleep(0.2) # Rate limit compliance
                except Exception as e:
                    print(f'Error submitting batch to Datadog: {e}')

        return success_count

    def run_migration(self, days_back: int = 7):
        \"\"\"Full migration run\"\"\"
        end_time = int(time.time())
        start_time = end_time - (days_back * 24 * 60 * 60)

        print(f'Fetching Sentry errors from {start_time} to {end_time}')
        sentry_errors = self.fetch_sentry_errors(start_time, end_time)
        print(f'Fetched {len(sentry_errors)} Sentry events')

        print('Transforming to Datadog format...')
        datadog_logs = []
        for event in sentry_errors:
            log = self.transform_to_datadog_log(event)
            if log:
                datadog_logs.append(log)
        print(f'Transformed {len(datadog_logs)} valid logs')

        print('Sending to Datadog...')
        success = self.send_to_datadog(datadog_logs)
        print(f'Migration complete: {success}/{len(sentry_errors)} events migrated successfully')

if __name__ == '__main__':
    migrator = SentryToDatadogMigrator()
    migrator.run_migration(days_back=30) # Migrate last 30 days of errors

// Prometheus metric to track error resolution time (Node.js)
const promClient = require('prom-client');
const register = new promClient.Registry();

const errorResolutionTime = new promClient.Histogram({
  name: 'error_resolution_time_minutes',
  help: 'Time from error alert to fix deployment in minutes',
  labelNames: ['service', 'error_type'],
  buckets: [5, 10, 15, 30, 45, 60, 90, 120],
});

register.registerMetric(errorResolutionTime);

// Record resolution time when error is fixed
function recordResolution(startTime, endTime, service, errorType) {
  const durationMinutes = (endTime - startTime) / (1000 * 60);
  errorResolutionTime.labels(service, errorType).observe(durationMinutes);
}

// Datadog 1.20 Kubernetes integration ConfigMap (kubectl apply -f datadog-config.yaml)
apiVersion: v1
kind: ConfigMap
metadata:
  name: datadog-agent-config
  namespace: datadog
data:
  datadog.yaml: |
    api_key: ${DD_API_KEY}
    site: datadoghq.com
    service: acme-checkout
    env: ${NODE_ENV}
    # Enable native K8s integrations
    kubernetes:
      enabled: true
      kubelet_tls_verify: false
      collect_metadata: true
      label_whitelist:
        - app.kubernetes.io/*
        - version
        - team
    # Enable error tracking
    error_tracking:
      enabled: true
      sample_rate: 1.0

// Create Datadog 1.20 error correlation rule via API (Python)
import os
import requests

dd_api_key = os.getenv(\"DD_API_KEY\")
dd_app_key = os.getenv(\"DD_APP_KEY\")

rule = {
    \"name\": \"Redis Error to Infrastructure Correlation\",
    \"enabled\": True,
    \"query\": \"service:acme-checkout error:redis\",
    \"correlation_rules\": [
        {
            \"type\": \"metric\",
            \"query\": \"avg:redis.active_connections{service:acme-checkout}\",
            \"time_window\": 300,
        },
        {
            \"type\": \"deployment\",
            \"time_window\": 600,
        },
    ],
}

response = requests.post(
    \"https://api.datadoghq.com/api/v1/error-tracking/rules\",
    headers={
        \"DD-API-KEY\": dd_api_key,
        \"DD-APPLICATION-KEY\": dd_app_key,
        \"Content-Type\": \"application/json\",
    },
    json=rule,
)

print(f\"Rule created: {response.json().get('id')}\")