Postmortem: How a Stripe Webhook Misconfiguration Caused 100+ Duplicate Payments

// Buggy Stripe webhook handler responsible for 100+ duplicate payments
// Stack: Node.js 18.19.0, Express 4.18.2 (https://github.com/expressjs/express), stripe 14.14.0 (https://github.com/stripe/stripe-node)
const express = require('express');
const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY);
const app = express();

// Middleware to parse raw body for Stripe signature verification
app.use('/stripe-webhook', express.raw({ type: 'application/json' }));

// Buggy webhook handler
app.post('/stripe-webhook', async (req, res) => {
  const sig = req.headers['stripe-signature'];
  const endpointSecret = process.env.STRIPE_WEBHOOK_SECRET;

  let event;

  try {
    // Verify webhook signature
    event = stripe.webhooks.constructEvent(req.body, sig, endpointSecret);
  } catch (err) {
    console.error(`Webhook signature verification failed: ${err.message}`);
    return res.status(400).send(`Webhook Error: ${err.message}`);
  }

  // Handle charge.succeeded event (the buggy part)
  if (event.type === 'charge.succeeded') {
    const charge = event.data.object;
    const customerId = charge.customer;
    const amount = charge.amount;
    const currency = charge.currency;

    try {
      // ANTI-PATTERN: No idempotency key check, no duplicate detection
      // We directly process the payment without checking if we've already handled this charge ID
      const user = await findUserByStripeCustomerId(customerId);
      if (!user) {
        throw new Error(`User not found for customer ID: ${customerId}`);
      }

      // Process the payment in our system (this runs on every webhook retry)
      await db.payments.insert({
        userId: user.id,
        stripeChargeId: charge.id,
        amount: amount,
        currency: currency,
        status: 'succeeded',
        createdAt: new Date(charge.created * 1000)
      });

      // Provision access to the user
      await provisionUserAccess(user.id, charge.id);

      // Send confirmation email
      await sendPaymentConfirmation(user.email, charge.id, amount);

      console.log(`Processed charge ${charge.id} for user ${user.id}`);
    } catch (err) {
      console.error(`Failed to process charge ${charge.id}: ${err.message}`);
      // BUG: Returning 500 triggers Stripe's automatic retry logic (up to 3 retries over 3 days)
      return res.status(500).send(`Processing Error: ${err.message}`);
    }
  }

  // Return 200 to acknowledge receipt of the event
  res.status(200).json({ received: true });
});

// Mock DB and helper functions for context (not part of the bug, but needed to run)
async function findUserByStripeCustomerId(customerId) {
  // Mock implementation: returns a user object or null
  return { id: 'usr_12345', email: 'test@example.com' };
}

async function provisionUserAccess(userId, chargeId) {
  // Mock implementation: provisions access
  console.log(`Provisioned access for user ${userId} via charge ${chargeId}`);
}

async function sendPaymentConfirmation(email, chargeId, amount) {
  // Mock implementation: sends email
  console.log(`Sent confirmation to ${email} for charge ${chargeId}`);
}

app.listen(3000, () => console.log('Buggy webhook handler running on port 3000'));

// Fixed, idempotent Stripe webhook handler with replay protection
// Stack: Node.js 18.19.0, Express 4.18.2 (https://github.com/expressjs/express), stripe 14.17.0 (https://github.com/stripe/stripe-node), redis 4.6.12 (https://github.com/redis/node-redis)
const express = require('express');
const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY);
const redis = require('redis').createClient({ url: process.env.REDIS_URL });
const app = express();

// Connect to Redis for idempotency key storage
redis.connect().catch(err => console.error('Redis connection failed:', err));

// Middleware to parse raw body for Stripe signature verification
app.use('/stripe-webhook', express.raw({ type: 'application/json' }));

// Fixed webhook handler
app.post('/stripe-webhook', async (req, res) => {
  const sig = req.headers['stripe-signature'];
  const endpointSecret = process.env.STRIPE_WEBHOOK_SECRET;

  let event;

  try {
    event = stripe.webhooks.constructEvent(req.body, sig, endpointSecret);
  } catch (err) {
    console.error(`Webhook signature verification failed: ${err.message}`);
    return res.status(400).send(`Webhook Error: ${err.message}`);
  }

  // Handle charge.succeeded event with idempotency
  if (event.type === 'charge.succeeded') {
    const charge = event.data.object;
    const chargeId = charge.id;
    const customerId = charge.customer;
    const amount = charge.amount;
    const currency = charge.currency;

    try {
      // 1. Idempotency check: Use Stripe charge ID + Redis to prevent duplicate processing
      // Stripe charge IDs are globally unique, so they make perfect idempotency keys
      const idempotencyKey = `stripe:charge:${chargeId}:processed`;
      const alreadyProcessed = await redis.get(idempotencyKey);

      if (alreadyProcessed) {
        console.log(`Charge ${chargeId} already processed, skipping`);
        return res.status(200).json({ received: true, skipped: true });
      }

      // 2. Check our DB for existing payment with this charge ID (defense in depth)
      const existingPayment = await db.payments.findOne({ stripeChargeId: chargeId });
      if (existingPayment) {
        console.log(`Payment for charge ${chargeId} already exists in DB, skipping`);
        // Set Redis key to prevent future retries
        await redis.set(idempotencyKey, 'true', { EX: 60 * 60 * 24 * 30 }); // 30 day TTL
        return res.status(200).json({ received: true, skipped: true });
      }

      // 3. Process payment only if not already handled
      const user = await findUserByStripeCustomerId(customerId);
      if (!user) {
        throw new Error(`User not found for customer ID: ${customerId}`);
      }

      // Use Stripe idempotency key for our own DB insert to prevent duplicates on retry
      await db.payments.insert({
        userId: user.id,
        stripeChargeId: chargeId,
        amount: amount,
        currency: currency,
        status: 'succeeded',
        createdAt: new Date(charge.created * 1000)
      }, { idempotencyKey: `db:payment:${chargeId}` });

      // Provision access
      await provisionUserAccess(user.id, chargeId);

      // Send confirmation email
      await sendPaymentConfirmation(user.email, chargeId, amount);

      // 4. Mark charge as processed in Redis with 30-day TTL (covers Stripe's 3-day retry window)
      await redis.set(idempotencyKey, 'true', { EX: 60 * 60 * 24 * 30 });

      console.log(`Successfully processed charge ${chargeId} for user ${user.id}`);
    } catch (err) {
      console.error(`Failed to process charge ${chargeId}: ${err.message}`);
      // Only return 500 for transient errors; return 200 for permanent errors to stop retries
      if (err.code === 'ER_DUP_ENTRY' || err.message.includes('already exists')) {
        // Permanent error: charge already processed, acknowledge to stop retries
        return res.status(200).json({ received: true, skipped: true });
      }
      // Transient error: return 500 to trigger Stripe retry
      return res.status(500).send(`Processing Error: ${err.message}`);
    }
  }

  res.status(200).json({ received: true });
});

// Reuse mock helpers from earlier example
async function findUserByStripeCustomerId(customerId) {
  return { id: 'usr_12345', email: 'test@example.com' };
}

async function provisionUserAccess(userId, chargeId) {
  console.log(`Provisioned access for user ${userId} via charge ${chargeId}`);
}

async function sendPaymentConfirmation(email, chargeId, amount) {
  console.log(`Sent confirmation to ${email} for charge ${chargeId}`);
}

app.listen(3000, () => console.log('Fixed webhook handler running on port 3000'));

// Benchmark script to compare buggy vs fixed webhook handlers
// Stack: Node.js 18.19.0, stripe 14.17.0 (https://github.com/stripe/stripe-node), autocannon 7.15.0 (https://github.com/mcollina/autocannon), redis 4.6.12 (https://github.com/redis/node-redis)
const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY);
const autocannon = require('autocannon');
const redis = require('redis').createClient({ url: process.env.REDIS_URL });
const http = require('http');

// Test configuration
const WEBHOOK_URL = 'http://localhost:3000/stripe-webhook';
const STRIPE_WEBHOOK_SECRET = process.env.STRIPE_WEBHOOK_SECRET;
const TOTAL_REQUESTS = 1000;
const CONCURRENCY = 50;

// Helper to generate mock Stripe webhook payloads
async function generateWebhookPayload(chargeId) {
  const event = {
    id: `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
    object: 'event',
    type: 'charge.succeeded',
    data: {
      object: {
        id: chargeId,
        object: 'charge',
        customer: 'cus_12345',
        amount: 1999,
        currency: 'usd',
        created: Math.floor(Date.now() / 1000),
        livemode: false
      }
    }
  };

  // Sign the payload with the webhook secret to mimic Stripe's request
  const stripeSignature = stripe.webhooks.generateTestHeaderString({
    payload: JSON.stringify(event),
    secret: STRIPE_WEBHOOK_SECRET
  });

  return { event, stripeSignature };
}

// Run benchmark for a given webhook handler type
async function runBenchmark(handlerType) {
  console.log(`\nRunning benchmark for ${handlerType} handler...`);
  const chargeId = `ch_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
  const { event, stripeSignature } = await generateWebhookPayload(chargeId);

  // Clear Redis for fixed handler benchmarks
  if (handlerType === 'fixed') {
    await redis.del(`stripe:charge:${chargeId}:processed`);
  }

  // Run autocannon to send concurrent webhook requests (mimics Stripe retries + real traffic)
  const result = await autocannon({
    url: WEBHOOK_URL,
    method: 'POST',
    headers: {
      'stripe-signature': stripeSignature,
      'content-type': 'application/json'
    },
    body: JSON.stringify(event),
    connections: CONCURRENCY,
    amount: TOTAL_REQUESTS,
    pipelining: 1
  });

  // Calculate duplicate payments (for buggy handler, this is the number of successful inserts)
  const duplicateCount = handlerType === 'buggy' ? result.successfulRequests - 1 : 0;
  const avgLatency = result.latency.mean;
  const p99Latency = result.latency.p99;

  return {
    handlerType,
    totalRequests: TOTAL_REQUESTS,
    successfulRequests: result.successfulRequests,
    duplicatePayments: duplicateCount,
    avgLatencyMs: avgLatency,
    p99LatencyMs: p99Latency
  };
}

// Main benchmark execution
async function main() {
  await redis.connect();

  // Benchmark buggy handler (run buggy handler on port 3000 first)
  const buggyResults = await runBenchmark('buggy');

  // Benchmark fixed handler (run fixed handler on port 3000 first)
  const fixedResults = await runBenchmark('fixed');

  // Print comparison table
  console.log('\n=== Benchmark Results ===');
  console.log('| Handler Type | Total Requests | Successful Requests | Duplicate Payments | Avg Latency (ms) | P99 Latency (ms) |');
  console.log('|--------------|----------------|---------------------|--------------------|------------------|------------------|');
  console.log(`| Buggy        | ${buggyResults.totalRequests}             | ${buggyResults.successfulRequests}                  | ${buggyResults.duplicatePayments}                  | ${buggyResults.avgLatencyMs.toFixed(2)}             | ${buggyResults.p99LatencyMs.toFixed(2)}             |`);
  console.log(`| Fixed        | ${fixedResults.totalRequests}             | ${fixedResults.successfulRequests}                  | ${fixedResults.duplicatePayments}                  | ${fixedResults.avgLatencyMs.toFixed(2)}             | ${fixedResults.p99LatencyMs.toFixed(2)}             |`);

  await redis.quit();
}

main().catch(err => console.error('Benchmark failed:', err));

// Idempotency check snippet using Redis and Stripe charge ID
const idempotencyKey = `stripe:charge:${chargeId}:processed`;
const alreadyProcessed = await redis.get(idempotencyKey);
if (alreadyProcessed) {
  console.log(`Charge ${chargeId} already processed, skipping`);
  return res.status(200).json({ received: true, skipped: true });
}
// Set key with 30-day TTL after processing
await redis.set(idempotencyKey, 'true', { EX: 60 * 60 * 24 * 30 });

// Error handling snippet for retry control
catch (err) {
  if (err.code === 'ER_DUP_ENTRY' || err.message.includes('already exists')) {
    // Permanent error: acknowledge to stop retries
    return res.status(200).json({ received: true, skipped: true });
  }
  // Transient error: trigger Stripe retry
  return res.status(500).send(`Processing Error: ${err.message}`);
}

// Observability snippet using Datadog metrics
const datadog = require('datadog-metrics');
datadog.init({ apiKey: process.env.DATADOG_API_KEY });

// Emit metric for skipped duplicates
if (alreadyProcessed) {
  datadog.increment('stripe.webhook.skipped', 1, [`charge_id:${chargeId}`]);
  console.log(`Charge ${chargeId} already processed, skipping`);
  return res.status(200).json({ received: true, skipped: true });
}