How to Debug Production Outages in Serverless Apps with Lumigo 2026 and AWS X-Ray 3.0

import json
import os
import logging
from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.core import patch
from aws_xray_sdk.ext.boto3.patch import patch_boto3
import boto3

# Configure structured logging for CloudWatch Logs integration
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# Patch all boto3 clients to automatically capture X-Ray traces
patch_boto3()

# Initialize X-Ray recorder with 3.0-specific config for Lambda SnapStart support
xray_recorder.configure(
    sampling=True,
    context_missing='LOG_ERROR',
    # Enable X-Ray 3.0's new Lambda SnapStart trace propagation
    snapstart_trace_propagation=True,
    # Capture DynamoDB, Step Functions, and EventBridge traces by default
    service_name='order-validation-service'
)

# Initialize AWS clients with X-Ray instrumentation
dynamodb = boto3.client('dynamodb')
step_functions = boto3.client('stepfunctions')

# Environment variables validated at cold start
REQUIRED_ENV_VARS = ['ORDERS_TABLE_NAME', 'STEP_FUNCTION_ARN']
for var in REQUIRED_ENV_VARS:
    if var not in os.environ:
        raise ValueError(f"Missing required environment variable: {var}")

ORDERS_TABLE = os.environ['ORDERS_TABLE_NAME']
STEP_FUNCTION_ARN = os.environ['STEP_FUNCTION_ARN']

def lambda_handler(event, context):
    """
    Validates incoming e-commerce orders, writes to DynamoDB, and triggers Step Function workflow.
    Instrumented with AWS X-Ray 3.0 for full traceability.
    """
    # Create a new X-Ray subsegment for this handler execution
    with xray_recorder.in_segment(context) as segment:
        try:
            # Log incoming event (redact PII per GDPR compliance)
            sanitized_event = {k: v for k, v in event.items() if k not in ['customer_email', 'credit_card']}
            logger.info(f"Processing order validation request: {json.dumps(sanitized_event)}")

            # Validate event structure
            required_fields = ['order_id', 'customer_id', 'total_amount', 'items']
            for field in required_fields:
                if field not in event:
                    raise ValueError(f"Missing required order field: {field}")

            # Validate order total is positive
            if float(event['total_amount']) <= 0:
                raise ValueError(f"Invalid order total: {event['total_amount']}")

            # Write order to DynamoDB with X-Ray traced client
            dynamodb.put_item(
                TableName=ORDERS_TABLE,
                Item={
                    'order_id': {'S': event['order_id']},
                    'customer_id': {'S': event['customer_id']},
                    'total_amount': {'N': str(event['total_amount'])},
                    'status': {'S': 'VALIDATED'},
                    'created_at': {'S': context.aws_request_id}
                },
                ConditionExpression='attribute_not_exists(order_id)'
            )
            logger.info(f"Order {event['order_id']} written to DynamoDB")

            # Trigger Step Function workflow for order fulfillment
            step_functions.start_execution(
                stateMachineArn=STEP_FUNCTION_ARN,
                name=f"order-{event['order_id']}",
                input=json.dumps(event)
            )
            logger.info(f"Triggered Step Function execution for order {event['order_id']}")

            return {
                'statusCode': 200,
                'body': json.dumps({
                    'order_id': event['order_id'],
                    'status': 'VALIDATED',
                    'trace_id': segment.trace_id
                })
            }

        except dynamodb.exceptions.ConditionalCheckFailedException as e:
            # Handle duplicate order IDs
            logger.error(f"Duplicate order ID {event['order_id']}: {str(e)}")
            xray_recorder.current_segment().add_exception(e)
            return {
                'statusCode': 409,
                'body': json.dumps({'error': 'Duplicate order ID'})
            }
        except ValueError as e:
            # Handle validation errors
            logger.error(f"Order validation failed: {str(e)}")
            xray_recorder.current_segment().add_exception(e)
            return {
                'statusCode': 400,
                'body': json.dumps({'error': str(e)})
            }
        except Exception as e:
            # Catch-all for unexpected errors
            logger.error(f"Unexpected error processing order: {str(e)}", exc_info=True)
            xray_recorder.current_segment().add_exception(e)
            return {
                'statusCode': 500,
                'body': json.dumps({'error': 'Internal server error'})
            }

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as lambda from 'aws-cdk-lib/aws-lambda';
import * as dynamodb from 'aws-cdk-lib/aws-dynamodb';
import * as stepfunctions from 'aws-cdk-lib/aws-stepfunctions';
import * as tasks from 'aws-cdk-lib/aws-stepfunctions-tasks';
import * as events from 'aws-cdk-lib/aws-events';
import * as targets from 'aws-cdk-lib/aws-events-targets';
import { LumigoInstrumentation } from '@lumigo/cdk-constructs';
import * as iam from 'aws-cdk-lib/aws-iam';

export class ServerlessOrderStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    // 1. Create DynamoDB table for order storage with X-Ray 3.0 tracing enabled
    const ordersTable = new dynamodb.Table(this, 'OrdersTable', {
      partitionKey: { name: 'order_id', type: dynamodb.AttributeType.STRING },
      billingMode: dynamodb.BillingMode.PAY_PER_REQUEST,
      // Enable X-Ray 3.0's new DynamoDB detailed tracing
      tracingEnabled: true,
      pointInTimeRecovery: true
    });

    // 2. Create Step Function for order fulfillment workflow
    const validateOrderTask = new tasks.LambdaInvoke(this, 'ValidateOrderTask', {
      // Lambda defined below, X-Ray will automatically trace this task
      lambdaFunction: undefined, // Will be set after Lambda creation
      outputPath: '$.Payload'
    });

    const chargePaymentTask = new tasks.LambdaInvoke(this, 'ChargePaymentTask', {
      lambdaFunction: undefined,
      outputPath: '$.Payload'
    });

    const fulfillOrderTask = new tasks.LambdaInvoke(this, 'FulfillOrderTask', {
      lambdaFunction: undefined,
      outputPath: '$.Payload'
    });

    const orderFulfillmentWorkflow = new stepfunctions.StateMachine(this, 'OrderFulfillmentWorkflow', {
      definition: validateOrderTask
        .next(chargePaymentTask)
        .next(fulfillOrderTask),
      // Enable X-Ray 3.0 Step Function tracing
      tracingEnabled: true,
      timeout: cdk.Duration.minutes(5)
    });

    // 3. Create Order Validation Lambda with X-Ray 3.0 instrumentation
    const orderValidationLambda = new lambda.Function(this, 'OrderValidationLambda', {
      runtime: lambda.Runtime.NODEJS_22_X,
      code: lambda.Code.fromAsset('lambda/order-validation'),
      handler: 'index.lambda_handler',
      environment: {
        ORDERS_TABLE_NAME: ordersTable.tableName,
        STEP_FUNCTION_ARN: orderFulfillmentWorkflow.stateMachineArn
      },
      // Enable X-Ray 3.0 tracing with SnapStart support
      tracing: lambda.Tracing.ACTIVE,
      snapStart: lambda.SnapStartConf.ON_PUBLISHED_VERSIONS
    });

    // Grant permissions
    ordersTable.grantWriteData(orderValidationLambda);
    orderFulfillmentWorkflow.grantStartExecution(orderValidationLambda);

    // Link Lambda to Step Function tasks
    validateOrderTask.lambdaFunction = orderValidationLambda;

    // 4. Create Payment Processing Lambda
    const paymentProcessingLambda = new lambda.Function(this, 'PaymentProcessingLambda', {
      runtime: lambda.Runtime.NODEJS_22_X,
      code: lambda.Code.fromAsset('lambda/payment-processing'),
      handler: 'index.lambda_handler',
      tracing: lambda.Tracing.ACTIVE
    });
    chargePaymentTask.lambdaFunction = paymentProcessingLambda;

    // 5. Create Fulfillment Lambda
    const fulfillmentLambda = new lambda.Function(this, 'FulfillmentLambda', {
      runtime: lambda.Runtime.NODEJS_22_X,
      code: lambda.Code.fromAsset('lambda/fulfillment'),
      handler: 'index.lambda_handler',
      tracing: lambda.Tracing.ACTIVE
    });
    fulfillOrderTask.lambdaFunction = fulfillmentLambda;

    // 6. Configure Lumigo 2026 instrumentation for all serverless resources
    new LumigoInstrumentation(this, 'LumigoInstrumentation', {
      lumigoToken: cdk.SecretValue.secretsManager('lumigo-api-token').toString(),
      // Lumigo 2026 features: automated root cause analysis, anomaly detection
      enableAutomatedRca: true,
      enableAnomalyDetection: true,
      // Trace all Lambda, Step Function, and EventBridge resources
      traceLambda: true,
      traceStepFunctions: true,
      traceEventBridge: true,
      // X-Ray 3.0 integration
      xrayIntegration: true
    });

    // 7. Create EventBridge rule to trigger order validation on new orders
    const orderEventRule = new events.Rule(this, 'OrderEventRule', {
      eventPattern: {
        source: ['com.ecommerce.orders'],
        detailType: ['OrderCreated']
      }
    });
    orderEventRule.addTarget(new targets.LambdaTarget(orderValidationLambda));

    // Output X-Ray and Lumigo dashboard URLs
    new cdk.CfnOutput(this, 'XRayDashboardUrl', {
      value: `https://console.aws.amazon.com/xray/home?region=${this.region}#/service-map`
    });
    new cdk.CfnOutput(this, 'LumigoDashboardUrl', {
      value: 'https://platform.lumigo.io/dashboard'
    });
  }
}

import os
import json
import logging
import boto3
import requests
from datetime import datetime, timedelta
from typing import List, Dict, Optional

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Initialize AWS clients for X-Ray 3.0
xray_client = boto3.client('xray', region_name=os.environ.get('AWS_REGION', 'us-east-1'))
sts_client = boto3.client('sts')

# Lumigo 2026 API config
LUMIGO_API_BASE = 'https://api.lumigo.io/v2026'
LUMIGO_TOKEN = os.environ.get('LUMIGO_API_TOKEN')
if not LUMIGO_TOKEN:
    raise ValueError("Missing LUMIGO_API_TOKEN environment variable")

def get_lumigo_recent_errors(hours: int = 1) -> List[Dict]:
    """
    Fetch recent serverless errors from Lumigo 2026 API.
    Returns list of error objects with trace IDs.
    """
    try:
        end_time = datetime.utcnow()
        start_time = end_time - timedelta(hours=hours)

        headers = {
            'Authorization': f'Bearer {LUMIGO_TOKEN}',
            'Content-Type': 'application/json'
        }
        payload = {
            'startTime': start_time.isoformat() + 'Z',
            'endTime': end_time.isoformat() + 'Z',
            'resourceTypes': ['lambda', 'stepfunction'],
            'errorStatus': 'error',
            'limit': 50
        }

        response = requests.post(
            f'{LUMIGO_API_BASE}/issues',
            headers=headers,
            json=payload,
            timeout=10
        )
        response.raise_for_status()

        issues = response.json().get('issues', [])
        logger.info(f"Fetched {len(issues)} recent errors from Lumigo")
        return issues

    except requests.exceptions.RequestException as e:
        logger.error(f"Failed to fetch Lumigo issues: {str(e)}")
        raise
    except json.JSONDecodeError as e:
        logger.error(f"Failed to parse Lumigo response: {str(e)}")
        raise

def get_xray_traces(trace_ids: List[str]) -> Dict[str, Dict]:
    """
    Fetch X-Ray 3.0 traces for given trace IDs.
    Returns mapping of trace ID to trace details.
    """
    try:
        # Batch get traces (X-Ray 3.0 supports up to 100 trace IDs per request)
        response = xray_client.batch_get_traces(
            TraceIds=trace_ids,
            # Enable X-Ray 3.0's new detailed segment data
            IncludeDetailedSegments=True
        )

        traces = {}
        for trace in response.get('Traces', []):
            trace_id = trace['Id']
            # Extract root cause from X-Ray segments
            root_cause = None
            for segment in trace.get('Segments', []):
                if segment.get('Fault', False):
                    root_cause = segment.get('Name', 'Unknown')
                    break
            traces[trace_id] = {
                'duration': trace['Duration'],
                'root_cause': root_cause,
                'segments': [s['Name'] for s in trace.get('Segments', [])]
            }

        logger.info(f"Fetched {len(traces)} X-Ray traces")
        return traces

    except boto3.exceptions.Boto3Error as e:
        logger.error(f"Failed to fetch X-Ray traces: {str(e)}")
        raise
    except Exception as e:
        logger.error(f"Unexpected error fetching X-Ray traces: {str(e)}")
        raise

def analyze_outage():
    """
    Main function to analyze recent serverless outages using Lumigo + X-Ray 3.0.
    """
    try:
        # Step 1: Get recent errors from Lumigo
        logger.info("Fetching recent errors from Lumigo 2026...")
        errors = get_lumigo_recent_errors(hours=1)

        if not errors:
            logger.info("No recent errors found")
            return

        # Step 2: Extract trace IDs from Lumigo issues
        trace_ids = [issue['traceId'] for issue in errors if 'traceId' in issue]
        if not trace_ids:
            logger.warning("No trace IDs found in Lumigo issues")
            return

        # Step 3: Fetch corresponding X-Ray traces
        logger.info(f"Fetching X-Ray 3.0 traces for {len(trace_ids)} trace IDs...")
        traces = get_xray_traces(trace_ids)

        # Step 4: Output root cause analysis
        print("\n=== Outage Root Cause Analysis ===")
        for error in errors:
            trace_id = error.get('traceId')
            if not trace_id or trace_id not in traces:
                continue

            trace = traces[trace_id]
            print(f"\nError: {error['issueName']}")
            print(f"Service: {error['resourceName']}")
            print(f"Trace ID: {trace_id}")
            print(f"Duration: {trace['duration']:.2f}s")
            print(f"Root Cause Segment: {trace['root_cause']}")
            print(f"Involved Services: {', '.join(trace['segments'])}")
            print(f"Lumigo Issue URL: {error.get('issueUrl', 'N/A')}")

    except Exception as e:
        logger.error(f"Outage analysis failed: {str(e)}", exc_info=True)
        raise

if __name__ == '__main__':
    analyze_outage()

# Enable SnapStart trace propagation in X-Ray 3.0 (Java Lambda example)
xray_recorder.configure(
    sampling=True,
    context_missing='LOG_ERROR',
    snapstart_trace_propagation=True,
    service_name='java-order-processor'
)

// Enable Automated RCA for Step Functions in Lumigo CDK construct
new LumigoInstrumentation(this, 'LumigoInstrumentation', {
  lumigoToken: cdk.SecretValue.secretsManager('lumigo-api-token').toString(),
  enableAutomatedRca: true,
  traceStepFunctions: true
});

# Enable X-Ray 3.0 tracing for EventBridge Pipes (CDK example)
const eventPipe = new events.Pipe(this, 'OrderEventPipe', {
  source: dynamodbStreamSource,
  enrichment: enrichmentLambda,
  target: stepFunctionTarget,
  tracingConfig: {
    mode: events.PipeTracingMode.ACTIVE
  }
});

serverless-outage-debugging-2026/
├── cdk/                          # AWS CDK 3.0 stack for deploying resources
│   ├── lib/
│   │   └── serverless-order-stack.ts  # Main stack with Lumigo + X-Ray config
│   ├── package.json
│   └── tsconfig.json
├── lambda/                       # Lambda function source code
│   ├── order-validation/         # Order validation Lambda (X-Ray instrumented)
│   │   ├── index.js
│   │   └── package.json
│   ├── payment-processing/       # Payment processing Lambda
│   │   ├── index.js
│   │   └── package.json
│   └── fulfillment/              # Order fulfillment Lambda
│       ├── index.js
│       └── package.json
├── scripts/                      # Debugging scripts
│   └── analyze-outage.py         # Lumigo + X-Ray outage analysis script
├── tests/                        # Unit and integration tests
│   ├── unit/
│   └── integration/
├── .github/                      # CI/CD workflows
│   └── workflows/
│       └── deploy.yml
├── README.md                     # Tutorial instructions
└── LICENSE