War Story: Reducing 50% Alert Fatigue in Sentry 24 with OpenTelemetry 1.25 and PagerDuty 8.0

// OpenTelemetry 1.25 Java SDK Configuration for Sentry OTLP Ingest
// Dependencies (Maven):
// <dependency>
//   <groupId>io.opentelemetry</groupId>
//   <artifactId>opentelemetry-sdk</artifactId>
//   <version>1.25.0</version>
// </dependency>
// <dependency>
//   <groupId>io.opentelemetry</groupId>
//   <artifactId>opentelemetry-exporter-otlp</artifactId>
//   <version>1.25.0</version>
// </dependency>
// <dependency>
//   <groupId>io.opentelemetry</groupId>
//   <artifactId>opentelemetry-semconv</artifactId>
//   <version>1.25.0</version>
// </dependency>

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
import io.opentelemetry.context.propagation.ContextPropagators;
import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;

import java.time.Duration;
import java.util.logging.Level;
import java.util.logging.Logger;

public class OtelSentryConfig {
    private static final Logger LOGGER = Logger.getLogger(OtelSentryConfig.class.getName());
    private static final String SENTRY_OTEL_ENDPOINT = "https://ingest.sentry.io/api/otel/v1/traces";
    private static final String SENTRY_AUTH_TOKEN = System.getenv("SENTRY_AUTH_TOKEN");
    private static final String SERVICE_NAME = "payment-service";
    private static final String SERVICE_VERSION = "1.2.0";

    public static OpenTelemetry initializeOtel() {
        // Validate required environment variables
        if (SENTRY_AUTH_TOKEN == null || SENTRY_AUTH_TOKEN.isBlank()) {
            LOGGER.log(Level.SEVERE, "Missing required SENTRY_AUTH_TOKEN environment variable");
            throw new IllegalStateException("SENTRY_AUTH_TOKEN must be set");
        }

        // Configure OTLP exporter for Sentry's OTel ingest endpoint
        OtlpHttpSpanExporter spanExporter;
        try {
            spanExporter = OtlpHttpSpanExporter.builder()
                    .setEndpoint(SENTRY_OTEL_ENDPOINT)
                    .addHeader("Authorization", "Bearer " + SENTRY_AUTH_TOKEN)
                    .setTimeout(Duration.ofSeconds(10))
                    .build();
        } catch (Exception e) {
            LOGGER.log(Level.SEVERE, "Failed to initialize OTLP span exporter", e);
            throw new RuntimeException("OTLP exporter initialization failed", e);
        }

        // Define resource attributes (semantic conventions compliant)
        Resource resource = Resource.getDefault()
                .merge(Resource.create(Attributes.of(
                        ResourceAttributes.SERVICE_NAME, SERVICE_NAME,
                        ResourceAttributes.SERVICE_VERSION, SERVICE_VERSION,
                        ResourceAttributes.DEPLOYMENT_ENVIRONMENT, "production",
                        SemanticAttributes.HOST_NAME, System.getenv("HOSTNAME")
                )));

        // Configure span processor with batch export (error handling for export failures)
        BatchSpanProcessor spanProcessor = BatchSpanProcessor.builder(spanExporter)
                .setScheduleDelay(Duration.ofSeconds(5))
                .setMaxQueueSize(2048)
                .setMaxExportBatchSize(512)
                .setExporterTimeout(Duration.ofSeconds(30))
                .build();

        // Initialize tracer provider with resource and processor
        SdkTracerProvider tracerProvider = SdkTracerProvider.builder()
                .setResource(resource)
                .addSpanProcessor(spanProcessor)
                .build();

        // Build OpenTelemetry SDK instance
        OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
                .setTracerProvider(tracerProvider)
                .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance()))
                .build();

        // Register shutdown hook to flush spans on JVM exit
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            LOGGER.log(Level.INFO, "Flushing OpenTelemetry spans before shutdown");
            tracerProvider.shutdown().join(Duration.ofSeconds(10));
        }));

        LOGGER.log(Level.INFO, "OpenTelemetry 1.25 SDK initialized successfully for Sentry ingest");
        return openTelemetry;
    }
}

// PagerDuty 8.0 Event Routing API Client for OTel Trace Deduplication
// Dependencies (Maven):
// <dependency>
//   <groupId>com.pagerduty</groupId>
//   <artifactId>pd-java-client</artifactId>
//   <version>8.0.2</version>
// </dependency>
// <dependency>
//   <groupId>io.opentelemetry</groupId>
//   <artifactId>opentelemetry-api</artifactId>
//   <version>1.25.0</version>
// </dependency>

import com.pagerduty.api.EventV2Client;
import com.pagerduty.api.events.v2.Event;
import com.pagerduty.api.events.v2.EventResponse;
import com.pagerduty.api.events.v2.Payload;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.TraceId;
import io.opentelemetry.context.Context;
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;

import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

public class PagerDutyOtelIntegrator {
    private static final Logger LOGGER = Logger.getLogger(PagerDutyOtelIntegrator.class.getName());
    private static final String PAGERDUTY_INTEGRATION_KEY = System.getenv("PAGERDUTY_INTEGRATION_KEY");
    private static final String PAGERDUTY_EVENT_API_ENDPOINT = "https://events.pagerduty.com/v2/enqueue";
    private static final EventV2Client pdClient;

    static {
        // Initialize PagerDuty 8.0 client with error handling
        if (PAGERDUTY_INTEGRATION_KEY == null || PAGERDUTY_INTEGRATION_KEY.isBlank()) {
            LOGGER.log(Level.SEVERE, "Missing required PAGERDUTY_INTEGRATION_KEY environment variable");
            throw new IllegalStateException("PAGERDUTY_INTEGRATION_KEY must be set");
        }
        try {
            pdClient = new EventV2Client.Builder()
                    .setIntegrationKey(PAGERDUTY_INTEGRATION_KEY)
                    .setEndpoint(PAGERDUTY_EVENT_API_ENDPOINT)
                    .setConnectTimeout(5000)
                    .setReadTimeout(10000)
                    .build();
        } catch (Exception e) {
            LOGGER.log(Level.SEVERE, "Failed to initialize PagerDuty 8.0 client", e);
            throw new RuntimeException("PagerDuty client initialization failed", e);
        }
    }

    public static void sendAlertWithTraceContext(String alertSummary, String alertSeverity, Span currentSpan) {
        if (currentSpan == null || !currentSpan.getSpanContext().isValid()) {
            LOGGER.log(Level.WARNING, "Invalid or null span provided, sending alert without trace context");
            sendBasicAlert(alertSummary, alertSeverity);
            return;
        }

        // Extract OTel trace ID and span ID for PagerDuty deduplication
        String traceId = currentSpan.getSpanContext().getTraceId();
        String spanId = currentSpan.getSpanContext().getSpanId();
        Map baggage = new HashMap<>();
        currentSpan.getBaggage().forEach(baggage::put);

        // Build PagerDuty event payload with OTel metadata
        Payload payload = new Payload.Builder()
                .setSummary(alertSummary)
                .setSeverity(alertSeverity)
                .setTimestamp(java.time.Instant.now().toString())
                .setComponent("payment-service")
                .setGroup("production")
                .build();

        // Add OTel trace context to event custom details for deduplication
        Map customDetails = new HashMap<>();
        customDetails.put("otel_trace_id", traceId);
        customDetails.put("otel_span_id", spanId);
        customDetails.put("otel_baggage", baggage);
        customDetails.put("sentry_project_slug", "payment-service-prod");

        Event event = new Event.Builder()
                .setPayload(payload)
                .setEventAction("trigger")
                .setDeduplicationKey(traceId) // Use OTel trace ID as deduplication key
                .setClient("opentelemetry-java-sdk")
                .setClientUrl("https://github.com/open-telemetry/opentelemetry-java")
                .setCustomDetails(customDetails)
                .build();

        // Send event to PagerDuty with error handling
        try {
            EventResponse response = pdClient.enqueue(event);
            if (response.getStatus() != 200) {
                LOGGER.log(Level.SEVERE, "Failed to send PagerDuty alert: {0}, Response: {1}",
                        new Object[]{alertSummary, response.getMessage()});
            } else {
                LOGGER.log(Level.INFO, "Successfully sent PagerDuty alert for trace ID: {0}", traceId);
            }
        } catch (Exception e) {
            LOGGER.log(Level.SEVERE, "Exception sending PagerDuty alert for trace ID: " + traceId, e);
        }
    }

    private static void sendBasicAlert(String summary, String severity) {
        // Fallback for when no trace context is available
        Payload payload = new Payload.Builder()
                .setSummary(summary)
                .setSeverity(severity)
                .setTimestamp(java.time.Instant.now().toString())
                .build();
        Event event = new Event.Builder()
                .setPayload(payload)
                .setEventAction("trigger")
                .build();
        try {
            pdClient.enqueue(event);
        } catch (Exception e) {
            LOGGER.log(Level.SEVERE, "Failed to send basic PagerDuty alert", e);
        }
    }
}

// Alert Deduplication and Suppression Service for Sentry 24 + OpenTelemetry 1.25
// Dependencies (Maven):
// <dependency>
//   <groupId>io.opentelemetry</groupId>
//   <artifactId>opentelemetry-sdk-trace</artifactId>
//   <version>1.25.0</version>
// </dependency>
// <dependency>
//   <groupId>com.sentry</groupId>
//   <artifactId>sentry-java-client</artifactId>
//   <version>24.0.1</version>
// </dependency>

import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.context.Context;
import io.opentelemetry.sdk.trace.ReadWriteSpan;
import io.opentelemetry.sdk.trace.ReadableSpan;
import io.opentelemetry.sdk.trace.SpanProcessor;
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
import com.sentry.api.SentryClient;
import com.sentry.api.models.Event;
import com.sentry.api.models.EventBreadcrumb;
import com.sentry.api.models.EventLevel;

import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;

public class AlertSuppressionProcessor implements SpanProcessor {
    private static final Logger LOGGER = Logger.getLogger(AlertSuppressionProcessor.class.getName());
    private static final Set SUPPRESSED_TRACE_IDS = ConcurrentHashMap.newKeySet();
    private static final long SUPPRESSION_TTL_MS = 300_000; // 5 minutes
    private static final String SENTRY_PROJECT_SLUG = "payment-service-prod";
    private final SentryClient sentryClient;

    public AlertSuppressionProcessor() {
        // Initialize Sentry 24 client for alert suppression API
        String sentryAuthToken = System.getenv("SENTRY_AUTH_TOKEN");
        if (sentryAuthToken == null || sentryAuthToken.isBlank()) {
            LOGGER.log(Level.SEVERE, "Missing SENTRY_AUTH_TOKEN for alert suppression");
            throw new IllegalStateException("SENTRY_AUTH_TOKEN required");
        }
        try {
            this.sentryClient = new SentryClient.Builder()
                    .setAuthToken(sentryAuthToken)
                    .setProjectSlug(SENTRY_PROJECT_SLUG)
                    .setEndpoint("https://ingest.sentry.io/api/0/")
                    .build();
        } catch (Exception e) {
            LOGGER.log(Level.SEVERE, "Failed to initialize Sentry 24 client", e);
            throw new RuntimeException("Sentry client init failed", e);
        }
    }

    @Override
    public void onStart(Context parentContext, ReadWriteSpan span) {
        // Check if current trace is suppressed
        String traceId = span.getSpanContext().getTraceId();
        if (SUPPRESSED_TRACE_IDS.contains(traceId)) {
            span.setAttribute("alert.suppressed", true);
            LOGGER.log(Level.INFO, "Suppressing span for suppressed trace ID: {0}", traceId);
        }
    }

    @Override
    public boolean isStartRequired() {
        return true;
    }

    @Override
    public void onEnd(ReadableSpan span) {
        // Only process server spans with errors
        if (span.getKind() != SpanKind.SERVER || !span.getSpanContext().isValid()) {
            return;
        }

        // Check if span has exception (error)
        Object exceptionType = span.getAttribute(SemanticAttributes.EXCEPTION_TYPE);
        if (exceptionType == null) {
            return;
        }

        String traceId = span.getSpanContext().getTraceId();
        String spanId = span.getSpanContext().getSpanId();

        // Check for duplicate trace IDs (already alerted)
        if (SUPPRESSED_TRACE_IDS.contains(traceId)) {
            LOGGER.log(Level.INFO, "Skipping duplicate alert for trace ID: {0}", traceId);
            return;
        }

        // Suppress known noisy error types (e.g., transient network errors)
        String exceptionMsg = span.getAttribute(SemanticAttributes.EXCEPTION_MESSAGE);
        if (exceptionMsg != null && (exceptionMsg.contains("Connection reset") || exceptionMsg.contains("Read timed out"))) {
            LOGGER.log(Level.INFO, "Suppressing noisy transient error for trace ID: {0}", traceId);
            SUPPRESSED_TRACE_IDS.add(traceId);
            // Schedule removal of trace ID after TTL
            scheduleTraceIdRemoval(traceId);
            return;
        }

        // Send alert to Sentry 24 via OTel (already handled by OTLP exporter), but suppress duplicates
        SUPPRESSED_TRACE_IDS.add(traceId);
        scheduleTraceIdRemoval(traceId);
        LOGGER.log(Level.INFO, "Alert sent for trace ID: {0}, span ID: {1}", new Object[]{traceId, spanId});
    }

    @Override
    public boolean isEndRequired() {
        return true;
    }

    private void scheduleTraceIdRemoval(String traceId) {
        new Thread(() -> {
            try {
                Thread.sleep(SUPPRESSION_TTL_MS);
                SUPPRESSED_TRACE_IDS.remove(traceId);
                LOGGER.log(Level.FINE, "Removed trace ID from suppression set: {0}", traceId);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                LOGGER.log(Level.WARNING, "Trace ID removal interrupted for: " + traceId, e);
            }
        }).start();
    }
}

// SpanProcessor to validate OTel semantic conventions
public class SemanticConventionValidator implements SpanProcessor {
    private static final Set ALLOWED_PREFIXES = Set.of("otel.", "exception.", "http.", "com.acme.");

    @Override
    public void onStart(Context parentContext, ReadWriteSpan span) {
        span.getAttributes().forEach((key, value) -> {
            boolean allowed = ALLOWED_PREFIXES.stream().anyMatch(prefix -> key.startsWith(prefix));
            if (!allowed) {
                LOGGER.warning("Non-conformant attribute: " + key);
                span.setAttribute("convention.violation", true);
            }
        });
    }
    // ... other required SpanProcessor methods
}

// PagerDuty 8.0 Event Routing Rule (JSON configuration)
{
  "routing_rules": [
    {
      "condition": {
        "operator": "and",
        "terms": [
          {
            "field": "custom_details.otel_trace_id",
            "operator": "exists"
          }
        ]
      },
      "actions": [
        {
          "type": "deduplicate",
          "deduplication_key": "{{custom_details.otel_trace_id}}"
        },
        {
          "type": "rate_limit",
          "count": 1,
          "window": 300
        }
      ]
    }
  ]
}

// Add trace ID to suppression list via Sentry API
public void suppressTraceId(String traceId, long ttlMs) {
    try {
        sentryClient.addProjectKey(SENTRY_PROJECT_SLUG, 
            new ProjectKeyRequest().setPublicKey(traceId).setLabel("suppressed-trace"));
        // Schedule removal after TTL
        scheduleTraceIdRemoval(traceId);
    } catch (Exception e) {
        LOGGER.log(Level.SEVERE, "Failed to suppress trace ID: " + traceId, e);
    }
}