Postmortem: How a PagerDuty 2026 Alert Delay Caused 1 Hour of Downtime for Our Production App

import os
import time
import logging
import json
from pdpyras import APISession  # PagerDuty Python SDK v4.2.0
from redis import Redis, RedisError

# Configure logging for audit trail
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.FileHandler("alert_emitter.log"), logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

# Initialize PagerDuty client with service integration key
PD_SERVICE_KEY = os.getenv("PD_SERVICE_KEY")
if not PD_SERVICE_KEY:
    logger.error("Missing PD_SERVICE_KEY environment variable")
    raise ValueError("PD_SERVICE_KEY must be set")

# Initialize Redis client for local alert dedup cache
try:
    redis_client = Redis(
        host=os.getenv("REDIS_HOST", "localhost"),
        port=int(os.getenv("REDIS_PORT", 6379)),
        db=0,
        decode_responses=True
    )
    redis_client.ping()
    logger.info("Connected to Redis for alert dedup cache")
except RedisError as e:
    logger.error(f"Failed to connect to Redis: {e}")
    redis_client = None

def emit_pagerduty_alert(alert_payload: dict) -> bool:
    """
    Emit alert to PagerDuty using v4.2.0 SDK with high-cardinality tags.
    This implementation triggered the 12-minute delay in 2026.3.1 event rules engine.
    """
    try:
        # Add high-cardinality tags that caused event rules processing lag
        alert_payload.setdefault("tags", [])
        alert_payload["tags"].extend([
            f"service:{alert_payload.get('service_name', 'unknown')}",
            f"region:{alert_payload.get('region', 'us-east-1')}",
            f"pod:{alert_payload.get('pod_id', 'pod-0')}",
            f"trace_id:{alert_payload.get('trace_id', 'no-trace')}",
            f"user_segment:{alert_payload.get('user_segment', 'free')}",
            f"deploy_version:{alert_payload.get('deploy_version', 'v0.0.0')}"
        ])

        # Check dedup cache to avoid duplicate alerts
        dedup_key = f"pd_alert:{alert_payload.get('incident_key', '')}"
        if redis_client and redis_client.exists(dedup_key):
            logger.info(f"Alert {dedup_key} already emitted, skipping")
            return True

        # Initialize PagerDuty session
        session = APISession(PD_SERVICE_KEY)

        # Emit alert via v2 events API
        response = session.post(
            "/events",
            json={
                "payload": {
                    "summary": alert_payload.get("summary", "Untitled Alert"),
                    "severity": alert_payload.get("severity", "error"),
                    "source": alert_payload.get("source", "production"),
                    "component": alert_payload.get("component", "unknown"),
                    "group": alert_payload.get("group", "default"),
                    "class": alert_payload.get("class", "alert"),
                    "custom_details": alert_payload.get("custom_details", {})
                },
                "tags": alert_payload["tags"],
                "links": alert_payload.get("links", []),
                "incident_key": alert_payload.get("incident_key", ""),
                "routing_key": PD_SERVICE_KEY,
                "event_action": "trigger"
            }
        )

        if response.status_code == 202:
            logger.info(f"Successfully emitted alert: {alert_payload.get('incident_key')}")
            # Cache dedup key for 5 minutes
            if redis_client:
                redis_client.setex(dedup_key, 300, "1")
            return True
        else:
            logger.error(f"Failed to emit alert: {response.status_code} {response.text}")
            return False

    except Exception as e:
        logger.error(f"Unexpected error emitting PagerDuty alert: {e}", exc_info=True)
        return False

if __name__ == "__main__":
    # Test alert payload matching the March 14 incident
    test_alert = {
        "summary": "Redis cache node redis-cache-03 us-east-1a failure",
        "severity": "critical",
        "source": "kubernetes-node-12",
        "component": "redis-cache",
        "group": "production-cache",
        "class": "infrastructure-failure",
        "custom_details": {
            "node_id": "redis-cache-03",
            "region": "us-east-1a",
            "error": "Connection refused on port 6379",
            "affected_users": 4200000
        },
        "service_name": "production-saas",
        "region": "us-east-1a",
        "pod_id": "pod-12",
        "trace_id": "trace-1234567890abcdef",
        "user_segment": "enterprise",
        "deploy_version": "v2026.3.14-rc1",
        "incident_key": "redis-cache-03-failure-20260314"
    }
    start_time = time.time()
    success = emit_pagerduty_alert(test_alert)
    end_time = time.time()
    logger.info(f"Alert emission took {end_time - start_time:.2f} seconds. Success: {success}")

import os
import time
import json
import logging
import concurrent.futures
from typing import List, Dict
from pdpyras import APISession
from pd_events_emitter import EventsEmitter  # https://github.com/PagerDuty/pd-events-emitter v1.3.0

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Benchmark configuration
PD_SERVICE_KEY = os.getenv("PD_SERVICE_KEY")
BENCH_ALERT_COUNT = 12000  # Matches our synthetic test count from the incident
HIGH_CARDINALITY_TAG_COUNT = 6  # Same as production payload
CONCURRENCY = 10  # Simulate production alert concurrency

def generate_test_alert(alert_id: int) -> Dict:
    """Generate a test alert payload matching production high-cardinality structure."""
    return {
        "summary": f"Synthetic benchmark alert {alert_id}",
        "severity": "warning",
        "source": f"bench-node-{alert_id % 100}",
        "component": "benchmark",
        "group": "incident-bench",
        "class": "synthetic",
        "custom_details": {
            "alert_id": alert_id,
            "bench_run": "2026-03-14-delay-repro",
            "cardinality": "high"
        },
        "service_name": "benchmark-service",
        "region": f"us-east-{1 + (alert_id % 3)}",
        "pod_id": f"pod-{alert_id % 50}",
        "trace_id": f"trace-bench-{alert_id:06d}",
        "user_segment": ["free", "pro", "enterprise"][alert_id % 3],
        "deploy_version": "v2026.3.14-bench",
        "incident_key": f"bench-alert-{alert_id}",
        "tags": [
            f"service:benchmark-service",
            f"region:us-east-{1 + (alert_id % 3)}",
            f"pod:pod-{alert_id % 50}",
            f"trace_id:trace-bench-{alert_id:06d}",
            f"user_segment:{['free', 'pro', 'enterprise'][alert_id % 3]}",
            f"deploy_version:v2026.3.14-bench"
        ]
    }

def emit_single_alert(alert_payload: Dict) -> float:
    """
    Emit a single alert and return delivery latency in seconds.
    Uses PagerDuty SDK v4.2.0 to match incident environment.
    """
    session = APISession(PD_SERVICE_KEY)
    start_time = time.perf_counter()
    try:
        response = session.post(
            "/events",
            json={
                "payload": {
                    "summary": alert_payload["summary"],
                    "severity": alert_payload["severity"],
                    "source": alert_payload["source"],
                    "component": alert_payload["component"],
                    "group": alert_payload["group"],
                    "class": alert_payload["class"],
                    "custom_details": alert_payload["custom_details"]
                },
                "tags": alert_payload["tags"],
                "incident_key": alert_payload["incident_key"],
                "routing_key": PD_SERVICE_KEY,
                "event_action": "trigger"
            }
        )
        end_time = time.perf_counter()
        latency = end_time - start_time
        if response.status_code != 202:
            logger.warning(f"Alert {alert_payload['incident_key']} failed: {response.status_code}")
            return -1.0
        return latency
    except Exception as e:
        logger.error(f"Error emitting alert {alert_payload['incident_key']}: {e}")
        return -1.0

def run_benchmark():
    """Run 12,000 synthetic alerts and calculate P95/P99 latency."""
    latencies: List[float] = []
    failed_count = 0

    logger.info(f"Starting benchmark with {BENCH_ALERT_COUNT} alerts, concurrency {CONCURRENCY}")

    with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as executor:
        futures = [
            executor.submit(emit_single_alert, generate_test_alert(i))
            for i in range(BENCH_ALERT_COUNT)
        ]
        for future in concurrent.futures.as_completed(futures):
            latency = future.result()
            if latency < 0:
                failed_count += 1
            else:
                latencies.append(latency)

    # Calculate statistics
    latencies.sort()
    total = len(latencies)
    if total == 0:
        logger.error("No successful alerts emitted")
        return

    p50 = latencies[int(total * 0.5)]
    p95 = latencies[int(total * 0.95)]
    p99 = latencies[int(total * 0.99)]
    mean = sum(latencies) / total

    logger.info(f"Benchmark results for {BENCH_ALERT_COUNT} alerts:")
    logger.info(f"Successful: {total}, Failed: {failed_count}")
    logger.info(f"Mean latency: {mean:.2f}s")
    logger.info(f"P50 latency: {p50:.2f}s")
    logger.info(f"P95 latency: {p95:.2f}s")
    logger.info(f"P99 latency: {p99:.2f}s")

    # Verify if P95 matches incident delay of 12 minutes (720s)
    if p95 > 700:
        logger.info("Reproduced incident delay: P95 latency exceeds 12 minutes")
    else:
        logger.warning("Failed to reproduce incident delay")

if __name__ == "__main__":
    if not PD_SERVICE_KEY:
        logger.error("PD_SERVICE_KEY not set")
        exit(1)
    run_benchmark()

import os
import time
import logging
import json
from typing import Optional, Dict, Any
from pdpyras import APISession
import boto3
from botocore.exceptions import ClientError, NoCredentialsError

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.FileHandler("multi_vendor_alerting.log"), logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

# Configuration
PD_SERVICE_KEY = os.getenv("PD_SERVICE_KEY")
AWS_SNS_TOPIC_ARN = os.getenv("AWS_SNS_TOPIC_ARN")
AWS_REGION = os.getenv("AWS_REGION", "us-east-1")
ALERT_TIMEOUT = 5  # Seconds to wait for primary PagerDuty alert before falling back to SNS

# Initialize clients
def init_pagerduty_client() -> Optional[APISession]:
    if not PD_SERVICE_KEY:
        logger.warning("PD_SERVICE_KEY not set, PagerDuty alerts disabled")
        return None
    try:
        session = APISession(PD_SERVICE_KEY)
        # Test connection with a lightweight API call
        session.get("/users/me")
        logger.info("PagerDuty client initialized successfully")
        return session
    except Exception as e:
        logger.error(f"Failed to initialize PagerDuty client: {e}")
        return None

def init_sns_client():
    try:
        client = boto3.client("sns", region_name=AWS_REGION)
        # Test SNS access by listing topics
        client.list_topics()
        logger.info("AWS SNS client initialized successfully")
        return client
    except NoCredentialsError:
        logger.error("AWS credentials not found, SNS fallback disabled")
        return None
    except ClientError as e:
        logger.error(f"Failed to initialize SNS client: {e}")
        return None

pd_client = init_pagerduty_client()
sns_client = init_sns_client()

def emit_multi_vendor_alert(alert_payload: Dict[str, Any]) -> bool:
    """
    Emit alert to PagerDuty first, fall back to AWS SNS if timeout or failure.
    Reduces mean alert delivery time from 14.2 minutes to 89ms as per case study.
    """
    alert_key = alert_payload.get("incident_key", "unknown-alert")
    start_time = time.perf_counter()

    # Try PagerDuty first with timeout
    pd_success = False
    if pd_client:
        try:
            # Use threading to enforce timeout on PagerDuty call
            import threading
            result = {"success": False, "latency": 0.0}

            def pd_emit_thread():
                try:
                    thread_start = time.perf_counter()
                    response = pd_client.post(
                        "/events",
                        json={
                            "payload": {
                                "summary": alert_payload["summary"],
                                "severity": alert_payload["severity"],
                                "source": alert_payload["source"],
                                "component": alert_payload["component"],
                                "group": alert_payload["group"],
                                "class": alert_payload["class"],
                                "custom_details": alert_payload["custom_details"]
                            },
                            "tags": alert_payload.get("tags", []),
                            "incident_key": alert_key,
                            "routing_key": PD_SERVICE_KEY,
                            "event_action": "trigger"
                        }
                    )
                    thread_end = time.perf_counter()
                    result["latency"] = thread_end - thread_start
                    if response.status_code == 202:
                        result["success"] = True
                except Exception as e:
                    logger.error(f"PagerDuty thread error: {e}")

            thread = threading.Thread(target=pd_emit_thread)
            thread.start()
            thread.join(timeout=ALERT_TIMEOUT)

            if thread.is_alive():
                logger.warning(f"PagerDuty alert {alert_key} timed out after {ALERT_TIMEOUT}s, falling back to SNS")
            else:
                if result["success"]:
                    pd_success = True
                    logger.info(f"PagerDuty alert {alert_key} delivered in {result['latency']:.2f}s")
        except Exception as e:
            logger.error(f"PagerDuty alert {alert_key} failed: {e}")

    # Fall back to SNS if PagerDuty failed
    if not pd_success and sns_client and AWS_SNS_TOPIC_ARN:
        try:
            sns_start = time.perf_counter()
            response = sns_client.publish(
                TopicArn=AWS_SNS_TOPIC_ARN,
                Message=json.dumps({
                    "alert": alert_payload,
                    "fallback": True,
                    "primary_vendor": "pagerduty",
                    "timestamp": time.time()
                }),
                Subject=f"FALLBACK ALERT: {alert_payload['summary'][:100]}",
                MessageAttributes={
                    "severity": {
                        "DataType": "String",
                        "StringValue": alert_payload.get("severity", "error")
                    },
                    "component": {
                        "DataType": "String",
                        "StringValue": alert_payload.get("component", "unknown")
                    }
                }
            )
            sns_end = time.perf_counter()
            logger.info(f"SNS fallback alert {alert_key} delivered in {sns_end - sns_start:.3f}s. Message ID: {response['MessageId']}")
            pd_success = True
        except ClientError as e:
            logger.error(f"SNS fallback alert {alert_key} failed: {e}")

    end_time = time.perf_counter()
    total_latency = end_time - start_time
    logger.info(f"Total alert delivery latency for {alert_key}: {total_latency:.3f}s. Success: {pd_success}")
    return pd_success

if __name__ == "__main__":
    # Test alert matching March 14 incident
    test_alert = {
        "summary": "Redis cache node redis-cache-03 us-east-1a failure",
        "severity": "critical",
        "source": "kubernetes-node-12",
        "component": "redis-cache",
        "group": "production-cache",
        "class": "infrastructure-failure",
        "custom_details": {
            "node_id": "redis-cache-03",
            "region": "us-east-1a",
            "error": "Connection refused on port 6379",
            "affected_users": 4200000
        },
        "tags": [
            "service:production-saas",
            "region:us-east-1a",
            "pod:pod-12",
            "trace_id:trace-1234567890abcdef",
            "user_segment:enterprise",
            "deploy_version:v2026.3.14-rc1"
        ],
        "incident_key": "redis-cache-03-failure-20260314"
    }
    emit_multi_vendor_alert(test_alert)

# Nightly benchmark CI job snippet (GitLab CI)
benchmark_alerts:
  stage: test
  image: python:3.11
  script:
    - pip install pdpyras pd-events-emitter
    - python bench_emitter.py --count 1000 --concurrency 10
    - |
      if [ $(cat bench_results.json | jq '.p95_latency') -gt 2 ]; then
        echo "ALERT: P95 latency exceeds 2s SLO"
        exit 1
      fi
  artifacts:
    paths: [bench_results.json]

# Terraform snippet for AWS SNS fallback topic
resource "aws_sns_topic" "alert_fallback" {
  name = "prod-alert-fallback"
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Sid    = "AllowPublish"
      Effect = "Allow"
      Principal = { Service = "sns.amazonaws.com" }
      Action = "sns:Publish"
      Resource = arn:aws:sns:${var.region}:${var.account_id}:prod-alert-fallback
    }]
  })
}

resource "aws_sns_topic_subscription" "sre_sms" {
  topic_arn = aws_sns_topic.alert_fallback.arn
  protocol  = "sms"
  endpoint  = "+11234567890"  # Primary SRE on-call number
}

# Bad: High-cardinality tags that caused delay
"tags": [
  "service:prod-saas",
  "region:us-east-1a",
  "pod:pod-12",  # Remove: not used for routing
  "trace_id:trace-123",  # Remove: not used for routing
  "user_segment:enterprise",
  "deploy_version:v2026.3.14"  # Remove: not used for routing
]

# Good: Minimal tags for routing only
"tags": [
  "service:prod-saas",
  "region:us-east-1a",
  "user_segment:enterprise"
]