Postmortem: How a Cloudflare R2 Permission Error Lost 10TB of User Uploaded Media

// Buggy R2 bucket policy deployment script - caused 10TB data loss
// DO NOT USE IN PRODUCTION - contains intentional misconfiguration
import { S3Client, PutBucketPolicyCommand, GetBucketPolicyCommand } from "@aws-sdk/client-s3";
import { CloudflareConfig } from "./config.js";
import { logger } from "./logger.js";
import { readFileSync } from "fs";

// Initialize R2 client with Cloudflare-specific endpoint
const r2Client = new S3Client({
  endpoint: "https://.r2.cloudflarestorage.com",
  credentials: {
    accessKeyId: CloudflareConfig.r2AccessKeyId,
    secretAccessKey: CloudflareConfig.r2SecretAccessKey,
  },
  region: "auto",
});

/**
 * Deploys a bucket policy to a target R2 bucket
 * @param {string} bucketName - Target R2 bucket name
 * @param {string} policyPath - Path to JSON policy file
 * @returns {Promise} Success status
 */
async function deployBucketPolicy(bucketName, policyPath) {
  let policyContent;
  try {
    // BUG: No validation of policy content before deployment
    // BUG: Merges existing policy with new policy without diffing
    const existingPolicy = await r2Client.send(
      new GetBucketPolicyCommand({ Bucket: bucketName })
    ).catch(() => null); // Silently ignores missing policy

    policyContent = JSON.parse(readFileSync(policyPath, "utf-8"));

    // BUG: Overwrites existing policy instead of merging safely
    // This line deleted all existing permissions, including read access for users
    const command = new PutBucketPolicyCommand({
      Bucket: bucketName,
      Policy: JSON.stringify(policyContent),
    });

    await r2Client.send(command);
    logger.info(`Successfully deployed policy to ${bucketName}`);
    return true;
  } catch (error) {
    // BUG: Silent error swallowing for policy deployment failures
    if (error.name === "NoSuchBucket") {
      logger.error(`Bucket ${bucketName} does not exist`);
      return false;
    }
    // Critical: This block swallowed the AccessDenied error that would have caught the overpermissive policy
    logger.debug(`Policy deployment failed for ${bucketName}: ${error.message}`);
    return false;
  }
}

// Deployment loop for all user media buckets
async function deployAllBuckets() {
  const buckets = ["user-media-prod-1", "user-media-prod-2", "user-media-prod-3"];
  const policyPath = "./policies/prod-media-policy.json";

  for (const bucket of buckets) {
    const success = await deployBucketPolicy(bucket, policyPath);
    if (!success) {
      // BUG: No rollback or alerting on partial deployment failure
      logger.warn(`Failed to deploy policy to ${bucket}, continuing...`);
    }
  }
}

// Execute deployment
deployAllBuckets().catch((err) => {
  logger.error(`Fatal deployment error: ${err.message}`);
  process.exit(1);
});

// Fixed R2 bucket policy deployment script with validation and rollback
// Production-ready, used post-incident
import { S3Client, PutBucketPolicyCommand, GetBucketPolicyCommand, DeleteBucketPolicyCommand } from "@aws-sdk/client-s3";
import { CloudflareConfig } from "./config.js";
import { logger } from "./logger.js";
import { readFileSync, writeFileSync } from "fs";
import { diff } from "json-diff"; // npm package for JSON diffing
import { validateR2Policy } from "./policy-validator.js"; // Custom validator

// Initialize R2 client with Cloudflare-specific endpoint
const r2Client = new S3Client({
  endpoint: "https://.r2.cloudflarestorage.com",
  credentials: {
    accessKeyId: CloudflareConfig.r2AccessKeyId,
    secretAccessKey: CloudflareConfig.r2SecretAccessKey,
  },
  region: "auto",
});

/**
 * Validates and deploys a bucket policy to a target R2 bucket with rollback
 * @param {string} bucketName - Target R2 bucket name
 * @param {string} policyPath - Path to JSON policy file
 * @returns {Promise} Success status
 */
async function deployBucketPolicySafe(bucketName, policyPath) {
  let existingPolicy = null;
  let policyContent;
  try {
    // 1. Fetch existing policy with explicit error handling
    try {
      const existingPolicyRes = await r2Client.send(
        new GetBucketPolicyCommand({ Bucket: bucketName })
      );
      existingPolicy = JSON.parse(existingPolicyRes.Policy);
      // Backup existing policy to local disk for rollback
      writeFileSync(`./backups/${bucketName}-policy-${Date.now()}.json`, JSON.stringify(existingPolicy, null, 2));
    } catch (error) {
      if (error.name !== "NoSuchBucketPolicy") {
        logger.error(`Failed to fetch existing policy for ${bucketName}: ${error.message}`);
        return false;
      }
      existingPolicy = null; // No existing policy
    }

    // 2. Load and validate new policy
    policyContent = JSON.parse(readFileSync(policyPath, "utf-8"));
    const validationResult = validateR2Policy(policyContent, bucketName);
    if (!validationResult.isValid) {
      logger.error(`Policy validation failed for ${bucketName}: ${validationResult.errors.join(", ")}`);
      return false;
    }

    // 3. Diff new policy with existing to avoid overwriting critical permissions
    if (existingPolicy) {
      const policyDiff = diff(existingPolicy, policyContent);
      if (policyDiff) {
        logger.info(`Policy diff for ${bucketName}: ${JSON.stringify(policyDiff)}`);
        // Check if diff removes any read permissions for authenticated users
        const removedRead = policyDiff.find(
          (change) => change.op === "delete" && change.path.includes("Action") && change.value.includes("s3:GetObject")
        );
        if (removedRead) {
          logger.error(`Policy diff removes read permissions for ${bucketName}, aborting deployment`);
          return false;
        }
      }
    }

    // 4. Deploy new policy
    const command = new PutBucketPolicyCommand({
      Bucket: bucketName,
      Policy: JSON.stringify(policyContent),
    });
    await r2Client.send(command);

    // 5. Verify deployment by re-fetching policy
    const verifyRes = await r2Client.send(new GetBucketPolicyCommand({ Bucket: bucketName }));
    const deployedPolicy = JSON.parse(verifyRes.Policy);
    if (JSON.stringify(deployedPolicy) !== JSON.stringify(policyContent)) {
      logger.error(`Policy verification failed for ${bucketName}, rolling back...`);
      await rollbackPolicy(bucketName, existingPolicy);
      return false;
    }

    logger.info(`Successfully deployed and verified policy for ${bucketName}`);
    return true;
  } catch (error) {
    logger.error(`Fatal error deploying policy to ${bucketName}: ${error.message}`, { stack: error.stack });
    // Rollback to existing policy on any unhandled error
    await rollbackPolicy(bucketName, existingPolicy);
    return false;
  }
}

/**
 * Rolls back bucket policy to a previous version
 * @param {string} bucketName - Target bucket
 * @param {object|null} previousPolicy - Previous policy to restore, or null to delete policy
 */
async function rollbackPolicy(bucketName, previousPolicy) {
  try {
    if (previousPolicy) {
      await r2Client.send(
        new PutBucketPolicyCommand({ Bucket: bucketName, Policy: JSON.stringify(previousPolicy) })
      );
      logger.info(`Rolled back policy for ${bucketName} to previous version`);
    } else {
      // Delete policy if no previous version existed
      await r2Client.send(new DeleteBucketPolicyCommand({ Bucket: bucketName }));
      logger.info(`Deleted policy for ${bucketName} (no previous version existed)`);
    }
  } catch (rollbackError) {
    logger.error(`CRITICAL: Rollback failed for ${bucketName}: ${rollbackError.message}`);
    // Alert on-call engineer for manual intervention
    await alertOnCall(`Rollback failed for ${bucketName}: ${rollbackError.message}`);
  }
}

// Deployment loop with alerting on partial failure
async function deployAllBucketsSafe() {
  const buckets = ["user-media-prod-1", "user-media-prod-2", "user-media-prod-3"];
  const policyPath = "./policies/prod-media-policy.json";
  const deploymentResults = [];

  for (const bucket of buckets) {
    const success = await deployBucketPolicySafe(bucket, policyPath);
    deploymentResults.push({ bucket, success });
    if (!success) {
      logger.error(`Deployment failed for ${bucket}, alerting on-call`);
      await alertOnCall(`Policy deployment failed for ${bucket}`);
    }
  }

  const failedDeployments = deploymentResults.filter((r) => !r.success);
  if (failedDeployments.length > 0) {
    logger.error(`Total failed deployments: ${failedDeployments.length}`);
    return false;
  }
  return true;
}

// Execute deployment
deployAllBucketsSafe().catch((err) => {
  logger.error(`Fatal deployment error: ${err.message}`);
  process.exit(1);
});

// R2 data recovery and permission audit script - post-incident recovery
// Restores lost media from cross-region R2 replica
import { S3Client, ListObjectsV2Command, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
import { CloudflareConfig } from "./config.js";
import { logger } from "./logger.js";
import { createWriteStream, readFileSync } from "fs";
import { pipeline } from "stream/promises";

// Initialize primary R2 client (affected region)
const primaryR2Client = new S3Client({
  endpoint: "https://.r2.cloudflarestorage.com",
  credentials: {
    accessKeyId: CloudflareConfig.r2AccessKeyId,
    secretAccessKey: CloudflareConfig.r2SecretAccessKey,
  },
  region: "auto",
});

// Initialize replica R2 client (unaffected region)
const replicaR2Client = new S3Client({
  endpoint: "https://.r2.cloudflarestorage.com",
  credentials: {
    accessKeyId: CloudflareConfig.r2ReplicaAccessKeyId,
    secretAccessKey: CloudflareConfig.r2ReplicaSecretAccessKey,
  },
  region: "auto",
});

/**
 * Audits bucket permissions and restores missing objects from replica
 * @param {string} bucketName - Primary bucket to audit
 * @param {string} replicaBucketName - Replica bucket to restore from
 * @returns {Promise<{restored: number, failed: number}>} Recovery stats
 */
async function auditAndRestoreBucket(bucketName, replicaBucketName) {
  let restoredCount = 0;
  let failedCount = 0;
  let continuationToken = null;

  try {
    // 1. Audit bucket policy first
    const policyRes = await primaryR2Client.send(
      new GetBucketPolicyCommand({ Bucket: bucketName })
    ).catch((err) => {
      logger.error(`Failed to fetch policy for ${bucketName}: ${err.message}`);
      return null;
    });

    if (policyRes) {
      const policy = JSON.parse(policyRes.Policy);
      const hasPublicRead = policy.Statement.some(
        (stmt) => stmt.Effect === "Allow" && stmt.Principal === "*" && stmt.Action.includes("s3:GetObject")
      );
      if (!hasPublicRead) {
        logger.warn(`Bucket ${bucketName} does not have public read access, fixing...`);
        await fixBucketPolicy(bucketName);
      }
    }

    // 2. List all objects in primary bucket (paginated)
    do {
      const listCommand = new ListObjectsV2Command({
        Bucket: bucketName,
        ContinuationToken: continuationToken,
        MaxKeys: 1000,
      });
      const listRes = await primaryR2Client.send(listCommand);
      const primaryObjects = listRes.Contents || [];

      // 3. List corresponding objects in replica bucket
      const replicaListCommand = new ListObjectsV2Command({
        Bucket: replicaBucketName,
        Prefix: "", // Same prefix structure as primary
        MaxKeys: 1000,
      });
      const replicaListRes = await replicaR2Client.send(replicaListCommand);
      const replicaObjects = new Map(
        (replicaListRes.Contents || []).map((obj) => [obj.Key, obj])
      );

      // 4. Check for missing objects in primary
      for (const primaryObj of primaryObjects) {
        const replicaObj = replicaObjects.get(primaryObj.Key);
        if (!replicaObj) {
          logger.warn(`Object ${primaryObj.Key} missing from replica, skipping`);
          continue;
        }

        // Check if primary object is corrupted or deleted (size mismatch)
        if (primaryObj.Size !== replicaObj.Size) {
          logger.info(`Restoring ${primaryObj.Key} from replica (size mismatch: ${primaryObj.Size} vs ${replicaObj.Size})`);
          const restored = await restoreObject(primaryObj.Key, bucketName, replicaBucketName);
          if (restored) {
            restoredCount++;
          } else {
            failedCount++;
          }
        }
      }

      continuationToken = listRes.NextContinuationToken;
    } while (continuationToken);

    // 5. Check for objects in replica not in primary (deleted objects)
    continuationToken = null;
    do {
      const replicaListCommand = new ListObjectsV2Command({
        Bucket: replicaBucketName,
        ContinuationToken: continuationToken,
        MaxKeys: 1000,
      });
      const replicaListRes = await replicaR2Client.send(replicaListCommand);
      const replicaObjects = replicaListRes.Contents || [];

      for (const replicaObj of replicaObjects) {
        const primaryListCommand = new ListObjectsV2Command({
          Bucket: bucketName,
          Prefix: replicaObj.Key,
          MaxKeys: 1,
        });
        const primaryRes = await primaryR2Client.send(primaryListCommand);
        if (!primaryRes.Contents || primaryRes.Contents.length === 0) {
          logger.info(`Restoring deleted object ${replicaObj.Key} from replica`);
          const restored = await restoreObject(replicaObj.Key, bucketName, replicaBucketName);
          if (restored) {
            restoredCount++;
          } else {
            failedCount++;
          }
        }
      }

      continuationToken = replicaListRes.NextContinuationToken;
    } while (continuationToken);

    logger.info(`Recovery complete for ${bucketName}: ${restoredCount} restored, ${failedCount} failed`);
    return { restored: restoredCount, failed: failedCount };
  } catch (error) {
    logger.error(`Fatal error auditing ${bucketName}: ${error.message}`, { stack: error.stack });
    return { restored: restoredCount, failed: failedCount + 1 };
  }
}

/**
 * Restores a single object from replica to primary bucket
 * @param {string} objectKey - Object key to restore
 * @param {string} primaryBucket - Primary bucket name
 * @param {string} replicaBucket - Replica bucket name
 * @returns {Promise} Success status
 */
async function restoreObject(objectKey, primaryBucket, replicaBucket) {
  try {
    // Fetch object from replica
    const replicaObj = await replicaR2Client.send(
      new GetObjectCommand({ Bucket: replicaBucket, Key: objectKey })
    );

    // Upload to primary bucket
    await primaryR2Client.send(
      new PutObjectCommand({
        Bucket: primaryBucket,
        Key: objectKey,
        Body: replicaObj.Body,
        ContentType: replicaObj.ContentType,
        ContentLength: replicaObj.ContentLength,
      })
    );

    logger.info(`Successfully restored ${objectKey} to ${primaryBucket}`);
    return true;
  } catch (error) {
    logger.error(`Failed to restore ${objectKey}: ${error.message}`);
    return false;
  }
}

/**
 * Fixes bucket policy to restore public read access for users
 * @param {string} bucketName - Bucket to fix
 */
async function fixBucketPolicy(bucketName) {
  const fixedPolicy = {
    Version: "2012-10-17",
    Statement: [
      {
        Sid: "PublicReadAccess",
        Effect: "Allow",
        Principal: "*",
        Action: ["s3:GetObject"],
        Resource: [`arn:aws:s3:::${bucketName}/*`],
      },
    ],
  };

  try {
    await primaryR2Client.send(
      new PutBucketPolicyCommand({
        Bucket: bucketName,
        Policy: JSON.stringify(fixedPolicy),
      })
    );
    logger.info(`Fixed policy for ${bucketName} to restore public read access`);
  } catch (error) {
    logger.error(`Failed to fix policy for ${bucketName}: ${error.message}`);
  }
}

// Execute recovery for all production buckets
async function runRecovery() {
  const buckets = [
    { primary: "user-media-prod-1", replica: "user-media-replica-1" },
    { primary: "user-media-prod-2", replica: "user-media-replica-2" },
    { primary: "user-media-prod-3", replica: "user-media-replica-3" },
  ];

  for (const { primary, replica } of buckets) {
    const stats = await auditAndRestoreBucket(primary, replica);
    logger.info(`Total recovery stats for ${primary}: ${JSON.stringify(stats)}`);
  }
}

runRecovery().catch((err) => {
  logger.error(`Fatal recovery error: ${err.message}`);
  process.exit(1);
});

# custodian-r2-policy.yml
policies:
  - name: r2-bucket-policy-audit
    resource: aws.s3
    endpoint: "https://.r2.cloudflarestorage.com"
    credentials:
      access-key: ${R2_ACCESS_KEY}
      secret-key: ${R2_SECRET_KEY}
    filters:
      - type: policy-statement
        statement:
          - Effect: Allow
            Principal: "*"
            Action:
              - s3:DeleteObject
              - s3:PutBucketPolicy
              - s3:DeleteBucketPolicy
    actions:
      - type: notify
        to:
          - oncall@company.com
        subject: "Dangerous R2 Bucket Policy Detected: {bucket_name}"
        template: "policy-violation.html"

// r2-replication-config.js
import { fetch } from "undici";

const accountId = process.env.CLOUDFLARE_ACCOUNT_ID;
const apiToken = process.env.CLOUDFLARE_API_TOKEN;

async function enableReplication(bucketName, replicaRegion) {
  const response = await fetch(
    `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucketName}/replication`,
    {
      method: "PUT",
      headers: {
        "Authorization": `Bearer ${apiToken}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        rules: [
          {
            id: "user-media-replication",
            status: "enabled",
            destination_bucket: `${bucketName}-replica-${replicaRegion}`,
            priority: 1,
            filter: {
              prefix: "user-uploads/",
            },
          },
        ],
      }),
    }
  );

  if (!response.ok) {
    throw new Error(`Failed to enable replication: ${await response.text()}`);
  }

  console.log(`Enabled replication for ${bucketName} to ${replicaRegion}`);
}

enableReplication("user-media-prod-1", "eu-central-1").catch(console.error);

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "LeastPrivilegeR2PolicyUpdate",
      "Effect": "Allow",
      "Action": ["s3:PutBucketPolicy", "s3:GetBucketPolicy"],
      "Resource": ["arn:aws:s3:::user-media-prod-1"],
      "Condition": {
        "IpAddress": {"aws:SourceIp": ["203.0.113.0/24"]}
      }
    }
  ]
}