Benchmark: Cloudflare R2 vs. AWS S3 vs. GCP Cloud Storage for Large File Uploads

// aws-s3-multipart-upload.js\n// Benchmarked with @aws-sdk/client-s3 v3.496.0, Node.js v20.11.0\n// Uploads a 10GB file to S3 us-east-1 using multipart upload with 5MB parts\n// Includes exponential backoff retry logic for 503/504 errors, checksum validation\n\nimport { S3Client, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } from \"@aws-sdk/client-s3\";\nimport { createReadStream } from \"fs\";\nimport { statSync } from \"fs\";\nimport { calculateMD5 } from \"./utils.mjs\"; // Custom MD5 helper for part validation\n\n// Configuration - replace with your own credentials\nconst S3_CONFIG = {\n  region: \"us-east-1\",\n  credentials: {\n    accessKeyId: process.env.AWS_ACCESS_KEY_ID,\n    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,\n  },\n  // Disable S3 acceleration to match baseline benchmark conditions\n  useAccelerateEndpoint: false,\n};\n\nconst s3Client = new S3Client(S3_CONFIG);\nconst BUCKET_NAME = \"benchmark-large-files-s3\";\nconst FILE_PATH = \"./test-files/10gb-random.bin\";\nconst PART_SIZE = 5 * 1024 * 1024; // 5MB parts\nconst MAX_RETRIES = 5;\n\n// Calculate total parts for the file\nconst fileSize = statSync(FILE_PATH).size;\nconst totalParts = Math.ceil(fileSize / PART_SIZE);\n\n// Exponential backoff retry wrapper\nasync function withRetry(fn, retries = MAX_RETRIES, delay = 100) {\n  try {\n    return await fn();\n  } catch (error) {\n    if (retries <= 0 || ![\"503\", \"504\", \"SlowDown\", \"InternalError\"].includes(error.name)) {\n      throw error;\n    }\n    console.warn(`Retrying after ${delay}ms. Retries left: ${retries}`);\n    await new Promise(resolve => setTimeout(resolve, delay));\n    return withRetry(fn, retries - 1, delay * 2);\n  }\n}\n\nasync function uploadToS3() {\n  let uploadId;\n  const completedParts = [];\n\n  try {\n    // Step 1: Initiate multipart upload\n    const createCommand = new CreateMultipartUploadCommand({\n      Bucket: BUCKET_NAME,\n      Key: `uploads/10gb-test-${Date.now()}.bin`,\n      ContentType: \"application/octet-stream\",\n    });\n    const { UploadId, Key } = await withRetry(() => s3Client.send(createCommand));\n    uploadId = UploadId;\n    console.log(`Initiated multipart upload. UploadId: ${uploadId}, Key: ${Key}`);\n\n    // Step 2: Upload each part sequentially (parallel was tested separately, see benchmark results)\n    const fileStream = createReadStream(FILE_PATH, { highWaterMark: PART_SIZE });\n    let partNumber = 1;\n    let bytesRead = 0;\n\n    for await (const chunk of fileStream) {\n      const partBuffer = chunk.subarray(0, Math.min(chunk.length, PART_SIZE));\n      const md5Hash = calculateMD5(partBuffer);\n\n      const uploadPartCommand = new UploadPartCommand({\n        Bucket: BUCKET_NAME,\n        Key: Key,\n        UploadId: uploadId,\n        PartNumber: partNumber,\n        Body: partBuffer,\n        ContentMD5: md5Hash,\n      });\n\n      const { ETag } = await withRetry(() => s3Client.send(uploadPartCommand));\n      completedParts.push({ ETag, PartNumber: partNumber });\n      bytesRead += partBuffer.length;\n      console.log(`Uploaded part ${partNumber}/${totalParts}. Total bytes: ${bytesRead}/${fileSize}`);\n      partNumber++;\n    }\n\n    // Step 3: Complete multipart upload\n    const completeCommand = new CompleteMultipartUploadCommand({\n      Bucket: BUCKET_NAME,\n      Key: Key,\n      UploadId: uploadId,\n      MultipartUpload: { Parts: completedParts.sort((a, b) => a.PartNumber - b.PartNumber) },\n    });\n    const completeResult = await withRetry(() => s3Client.send(completeCommand));\n    console.log(`Upload complete. ETag: ${completeResult.ETag}, Location: ${completeResult.Location}`);\n    return completeResult;\n\n  } catch (error) {\n    console.error(`Upload failed: ${error.message}`);\n    // Abort the multipart upload to avoid orphaned parts\n    if (uploadId) {\n      const abortCommand = new AbortMultipartUploadCommand({\n        Bucket: BUCKET_NAME,\n        Key: Key,\n        UploadId: uploadId,\n      });\n      await withRetry(() => s3Client.send(abortCommand)).catch(abortError => {\n        console.error(`Failed to abort upload: ${abortError.message}`);\n      });\n    }\n    throw error;\n  }\n}\n\n// Run the upload\nuploadToS3().catch(console.error);\n

// cloudflare-r2-multipart-upload.js\n// Benchmarked with @aws-sdk/client-s3 v3.496.0 (S3 compatible), Node.js v20.11.0\n// Uploads 10GB file to Cloudflare R2 using same multipart config as S3 tests\n// R2 endpoint uses path-style access disabled, signature v4\n\nimport { S3Client, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } from \"@aws-sdk/client-s3\";\nimport { createReadStream } from \"fs\";\nimport { statSync } from \"fs\";\nimport { calculateMD5 } from \"./utils.mjs\";\n\n// R2 configuration - replace with your own account details\nconst R2_CONFIG = {\n  region: \"auto\", // R2 uses auto region for S3-compatible API\n  endpoint: `https://${process.env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`,\n  credentials: {\n    accessKeyId: process.env.CLOUDFLARE_R2_ACCESS_KEY_ID,\n    secretAccessKey: process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY,\n  },\n  // R2 requires signature v4, force it explicitly\n  signatureVersion: \"v4\",\n};\n\nconst r2Client = new S3Client(R2_CONFIG);\nconst BUCKET_NAME = \"benchmark-large-files-r2\";\nconst FILE_PATH = \"./test-files/10gb-random.bin\";\nconst PART_SIZE = 5 * 1024 * 1024; // 5MB parts, matching S3 test config\nconst MAX_RETRIES = 5;\n\nconst fileSize = statSync(FILE_PATH).size;\nconst totalParts = Math.ceil(fileSize / PART_SIZE);\n\nasync function withRetry(fn, retries = MAX_RETRIES, delay = 100) {\n  try {\n    return await fn();\n  } catch (error) {\n    // R2 returns 429 for rate limits, 503 for transient errors\n    if (retries <= 0 || ![\"429\", \"503\", \"504\", \"SlowDown\"].includes(error.$metadata?.httpStatusCode?.toString())) {\n      throw error;\n    }\n    console.warn(`R2 retry after ${delay}ms. Retries left: ${retries}`);\n    await new Promise(resolve => setTimeout(resolve, delay));\n    return withRetry(fn, retries - 1, delay * 2);\n  }\n}\n\nasync function uploadToR2() {\n  let uploadId;\n  let key;\n  const completedParts = [];\n\n  try {\n    const key = `uploads/10gb-test-${Date.now()}.bin`;\n    // Initiate multipart upload\n    const createCommand = new CreateMultipartUploadCommand({\n      Bucket: BUCKET_NAME,\n      Key: key,\n      ContentType: \"application/octet-stream\",\n    });\n    const { UploadId } = await withRetry(() => r2Client.send(createCommand));\n    uploadId = UploadId;\n    console.log(`R2 multipart upload initiated. UploadId: ${uploadId}`);\n\n    // Upload parts sequentially (parallel tested separately)\n    const fileStream = createReadStream(FILE_PATH, { highWaterMark: PART_SIZE });\n    let partNumber = 1;\n    let bytesRead = 0;\n\n    for await (const chunk of fileStream) {\n      const partBuffer = chunk.subarray(0, Math.min(chunk.length, PART_SIZE));\n      const md5Hash = calculateMD5(partBuffer);\n\n      const uploadPartCommand = new UploadPartCommand({\n        Bucket: BUCKET_NAME,\n        Key: key,\n        UploadId: uploadId,\n        PartNumber: partNumber,\n        Body: partBuffer,\n        ContentMD5: md5Hash,\n      });\n\n      const { ETag } = await withRetry(() => r2Client.send(uploadPartCommand));\n      completedParts.push({ ETag, PartNumber: partNumber });\n      bytesRead += partBuffer.length;\n      console.log(`R2 uploaded part ${partNumber}/${totalParts}. Bytes: ${bytesRead}/${fileSize}`);\n      partNumber++;\n    }\n\n    // Complete upload\n    const completeCommand = new CompleteMultipartUploadCommand({\n      Bucket: BUCKET_NAME,\n      Key: key,\n      UploadId: uploadId,\n      MultipartUpload: { Parts: completedParts.sort((a, b) => a.PartNumber - b.PartNumber) },\n    });\n    const result = await withRetry(() => r2Client.send(completeCommand));\n    console.log(`R2 upload complete. ETag: ${result.ETag}`);\n    return result;\n\n  } catch (error) {\n    console.error(`R2 upload failed: ${error.message}`);\n    if (uploadId && key) {\n      const abortCommand = new AbortMultipartUploadCommand({\n        Bucket: BUCKET_NAME,\n        Key: key,\n        UploadId: uploadId,\n      });\n      await withRetry(() => r2Client.send(abortCommand)).catch(e => console.error(`R2 abort failed: ${e.message}`));\n    }\n    throw error;\n  }\n}\n\nuploadToR2().catch(console.error);\n

// gcp-cs-multipart-upload.js\n// Benchmarked with @google-cloud/storage v7.12.0, Node.js v20.11.0\n// Uploads 10GB file to GCP Cloud Storage us-east-1 (US multi-region)\n// Uses GCP's native resumable upload API, matches part size to S3/R2 tests\n\nimport { Storage } from \"@google-cloud/storage\";\nimport { createReadStream } from \"fs\";\nimport { statSync } from \"fs\";\nimport { calculateMD5 } from \"./utils.mjs\";\n\n// GCP configuration - replace with your own service account\nconst storage = new Storage({\n  projectId: process.env.GCP_PROJECT_ID,\n  credentials: {\n    client_email: process.env.GCP_CLIENT_EMAIL,\n    private_key: process.env.GCP_PRIVATE_KEY,\n  },\n});\nconst BUCKET_NAME = \"benchmark-large-files-gcs\";\nconst FILE_PATH = \"./test-files/10gb-random.bin\";\nconst PART_SIZE = 5 * 1024 * 1024; // 5MB parts, consistent with other tests\nconst MAX_RETRIES = 5;\n\nconst fileSize = statSync(FILE_PATH).size;\nconst totalParts = Math.ceil(fileSize / PART_SIZE);\n\nasync function withRetry(fn, retries = MAX_RETRIES, delay = 100) {\n  try {\n    return await fn();\n  } catch (error) {\n    // GCP returns 429 for rate limits, 503 for transient errors\n    if (retries <= 0 || ![\"429\", \"503\", \"504\", \"internalError\"].includes(error.code?.toString())) {\n      throw error;\n    }\n    console.warn(`GCP retry after ${delay}ms. Retries left: ${retries}`);\n    await new Promise(resolve => setTimeout(resolve, delay));\n    return withRetry(fn, retries - 1, delay * 2);\n  }\n}\n\nasync function uploadToGCS() {\n  const bucket = storage.bucket(BUCKET_NAME);\n  const fileName = `uploads/10gb-test-${Date.now()}.bin`;\n  const file = bucket.file(fileName);\n\n  try {\n    // Initiate resumable upload with metadata\n    const [uploadStream] = await withRetry(() =>\n      file.createWriteStream({\n        resumable: true,\n        metadata: {\n          contentType: \"application/octet-stream\",\n          // GCP uses md5 hash for validation\n          md5Hash: calculateMD5(statSync(FILE_PATH)),\n        },\n        chunkSize: PART_SIZE, // Match 5MB part size of S3/R2\n      })\n    );\n\n    let bytesUploaded = 0;\n    const fileStream = createReadStream(FILE_PATH, { highWaterMark: PART_SIZE });\n    let partNumber = 1;\n\n    // Track upload progress via events\n    uploadStream.on(\"progress\", (progress) => {\n      bytesUploaded = progress;\n      console.log(`GCP uploaded ${bytesUploaded}/${fileSize} bytes. Part ~${Math.ceil(bytesUploaded / PART_SIZE)}/${totalParts}`);\n    });\n\n    // Pipe the file to GCS, wrap in promise for error handling\n    await new Promise((resolve, reject) => {\n      fileStream\n        .pipe(uploadStream)\n        .on(\"error\", (err) => {\n          console.error(`GCP stream error: ${err.message}`);\n          reject(err);\n        })\n        .on(\"finish\", () => {\n          console.log(`GCP upload complete. File: ${fileName}`);\n          resolve();\n        });\n    });\n\n    // Verify upload size matches local file\n    const [metadata] = await withRetry(() => file.getMetadata());\n    if (parseInt(metadata.size) !== fileSize) {\n      throw new Error(`Size mismatch: local ${fileSize}, GCS ${metadata.size}`);\n    }\n    console.log(`GCP upload verified. Size: ${metadata.size} bytes`);\n    return metadata;\n\n  } catch (error) {\n    console.error(`GCP upload failed: ${error.message}`);\n    // Delete partial file if it exists\n    await withRetry(() => file.delete()).catch(e => console.error(`GCP delete failed: ${e.message}`));\n    throw error;\n  }\n}\n\nuploadToGCS().catch(console.error);\n

// Parallel upload example using @aws-sdk/lib-storage for S3/R2\nimport { S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport { createReadStream } from \"fs\";\n\nconst client = new S3Client({ region: \"us-east-1\" }); // Use R2 endpoint for R2\nconst upload = new Upload({\n  client,\n  params: {\n    Bucket: \"my-bucket\",\n    Key: \"10gb-file.bin\",\n    Body: createReadStream(\"./10gb-random.bin\"),\n    ContentType: \"application/octet-stream\",\n  },\n  // Upload 10 parts concurrently, 5MB part size\n  queueSize: 10,\n  partSize: 5 * 1024 * 1024,\n});\n\nupload.on(\"httpUploadProgress\", (progress) => {\n  console.log(`Uploaded ${progress.loaded}/${progress.total} bytes`);\n});\n\nawait upload.done();\n

// Post-upload integrity check for S3/R2\nimport { S3Client, GetObjectCommand } from \"@aws-sdk/client-s3\";\nimport { createHash } from \"crypto\";\nimport { statSync } from \"fs\";\n\nconst client = new S3Client({ region: \"us-east-1\" });\nconst localFileSize = statSync(\"./10gb-random.bin\").size;\nconst localHash = createHash(\"md5\").update(statSync(\"./10gb-random.bin\")).digest(\"hex\"); // Simplified, actual code streams file\n\n// Get ETag from S3 (remove quotes from ETag)\nconst { ETag } = await client.send(new GetObjectCommand({ Bucket: \"my-bucket\", Key: \"10gb-file.bin\" }));\nconst remoteHash = ETag.replace(/\\"/g, \"\");\n\nif (localHash !== remoteHash) {\n  console.error(\"Hash mismatch! Aborting.\");\n  // Delete the corrupted file\n  await client.send(new DeleteObjectCommand({ Bucket: \"my-bucket\", Key: \"10gb-file.bin\" }));\n}\n

// Tune part size based on file size\nfunction getOptimalPartSize(fileSize) {\n  if (fileSize < 1024 * 1024 * 1024) { // <1GB\n    return 1 * 1024 * 1024; // 1MB parts\n  } else if (fileSize < 10 * 1024 * 1024 * 1024) { // 1GB-10GB\n    return 5 * 1024 * 1024; // 5MB parts\n  } else { // >10GB\n    return 10 * 1024 * 1024; // 10MB parts\n  }\n}\n