Retries Are a Distributed Systems Bug Waiting to Happen: Designing Idempotent APIs

Client                 Server                  DB
  | --- POST /charge --> |                      |
  |                      | --- INSERT charge -> |
  |                      | <------- OK -------- |
  |   X (response lost)  |                      |
  | --- POST /charge --> |   (retry)            |
  |                      | --- INSERT charge -> |   <- duplicate

Client                       Server
  | POST /charge                |
  | Idempotency-Key: abc-123 -> | not seen: process, store result
  |  X (response lost)          |
  | POST /charge                |
  | Idempotency-Key: abc-123 -> | seen: return stored result, no new charge

app.post("/charge", async (req, res) => {
  const key = req.get("Idempotency-Key");
  if (!key) return res.status(400).json({ error: "Idempotency-Key required" });

  // Atomic claim: unique constraint on key
  const claimed = await db.idempotency.insertIfAbsent({
    key, status: "in_progress", requestHash: hash(req.body),
  });

  if (!claimed) {
    const existing = await db.idempotency.get(key);
    if (existing.requestHash !== hash(req.body))
      return res.status(422).json({ error: "Key reused with different payload" });
    if (existing.status === "in_progress")
      return res.status(409).json({ error: "Request in progress" });
    return res.status(existing.code).json(existing.response); // replay
  }

  const result = await chargeCard(req.body);
  await db.idempotency.complete(key, 200, result);
  res.json(result);
});

Attempt:  1    2      3         4              5
Wait:     0 -> 1s -> 2s ------> 4s -----------> 8s   (exponential)
               +-jitter: random within each window

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function retry(fn, { retries = 5, base = 200, cap = 10_000 } = {}) {
  for (let i = 0; ; i++) {
    try { return await fn(); }
    catch (err) {
      if (i >= retries || !isRetryable(err)) throw err;
      const backoff = Math.min(cap, base * 2 ** i);
      await sleep(Math.random() * backoff); // full jitter
    }
  }
}

await db.save(order);        // succeeds
await queue.publish(event);  // crashes here -> DB and queue disagree

+---------- one DB transaction ----------+
|  INSERT order       INSERT outbox_event |
+-----------------------------------------+
              |
     relay polls outbox -> publishes to queue -> marks sent