How to Build a Multi-Cloud Storage Layer with Cloudflare R2 and AWS S3 Using Pulumi 3.120

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
import * as cloudflare from "@pulumi/cloudflare";

// Configuration constants - validate presence at runtime
const config = new pulumi.Config();
const awsRegion = config.require("awsRegion"); // e.g., "us-east-1"
const cloudflareAccountId = config.requireSecret("cloudflareAccountId"); // Secret for security
const bucketNamePrefix = config.get("bucketNamePrefix") || "multi-cloud-storage";

// Precondition checks to fail fast if config is missing
if (!awsRegion) {
    throw new Error("Missing required config: awsRegion. Set via `pulumi config set awsRegion us-east-1`");
}
if (!cloudflareAccountId) {
    throw new Error("Missing required config: cloudflareAccountId. Set via `pulumi config set --secret cloudflareAccountId `");
}

// AWS S3 Bucket Configuration
// Enable versioning, server-side encryption with AES-256, and lifecycle rules to match R2 defaults
const s3Bucket = new aws.s3.BucketV2("s3-primary-bucket", {
    bucket: `${bucketNamePrefix}-s3-primary`,
    forceDestroy: false, // Prevent accidental deletion of production data
});

// Enable versioning on S3 bucket for 11 9s durability parity with R2
const s3Versioning = new aws.s3.BucketVersioningV2("s3-primary-versioning", {
    bucket: s3Bucket.id,
    versioningConfiguration: {
        status: "Enabled",
    },
});

// Server-side encryption for S3 bucket
const s3Encryption = new aws.s3.BucketServerSideEncryptionConfigurationV2("s3-primary-encryption", {
    bucket: s3Bucket.id,
    rules: [{
        applyServerSideEncryptionByDefault: {
            sseAlgorithm: "AES256",
        },
        bucketKeyEnabled: true,
    }],
});

// Lifecycle rule to transition infrequently accessed data to S3 Standard-IA after 30 days
const s3Lifecycle = new aws.s3.BucketLifecycleConfigurationV2("s3-primary-lifecycle", {
    bucket: s3Bucket.id,
    rules: [{
        id: "transition-to-ia",
        status: "Enabled",
        transitions: [{
            days: 30,
            storageClass: "STANDARD_IA",
        }],
        noncurrentVersionTransitions: [{
            noncurrentDays: 30,
            storageClass: "STANDARD_IA",
        }],
    }],
});

// Cloudflare R2 Bucket Configuration
// R2 has zero egress fees, native S3-compatible API, and 11 9s durability
const r2Bucket = new cloudflare.R2Bucket("r2-primary-bucket", {
    accountId: cloudflareAccountId,
    name: `${bucketNamePrefix}-r2-primary`,
    location: "ENAM", // North America region, matches AWS us-east-1 latency
    storageClass: "Standard",
});

// Export critical values for application use
export const s3BucketName = s3Bucket.id;
export const s3BucketEndpoint = pulumi.interpolate`s3.${awsRegion}.amazonaws.com/${s3Bucket.id}`;
export const r2BucketName = r2Bucket.name;
export const r2BucketEndpoint = pulumi.interpolate`https://${r2Bucket.id}.r2.cloudflarestorage.com`;
export const r2S3CompatibleEndpoint = pulumi.interpolate`https://${cloudflareAccountId}.r2.cloudflarestorage.com`;

import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { Cloudflare } from "cloudflare";
import * as crypto from "crypto";

// Configuration interface for the multi-cloud storage client
interface MultiCloudStorageConfig {
    s3Endpoint: string;
    s3Region: string;
    s3AccessKeyId: string;
    s3SecretAccessKey: string;
    r2Endpoint: string;
    r2AccessKeyId: string;
    r2SecretAccessKey: string;
    bucketPrefix: string;
    failoverThresholdMs: number; // Time to wait before failing over to secondary
}

// Multi-cloud storage client with automatic failover between S3 and R2
export class MultiCloudStorageClient {
    private s3Client: S3Client;
    private r2Client: S3Client; // R2 uses S3-compatible API, so we reuse the S3 client
    private s3Bucket: string;
    private r2Bucket: string;
    private failoverThresholdMs: number;
    private healthCheckInterval: NodeJS.Timeout;

    constructor(config: MultiCloudStorageConfig) {
        // Initialize S3 client with explicit credentials
        this.s3Client = new S3Client({
            endpoint: config.s3Endpoint,
            region: config.s3Region,
            credentials: {
                accessKeyId: config.s3AccessKeyId,
                secretAccessKey: config.s3SecretAccessKey,
            },
            // Force path style for S3 compatibility with R2
            forcePathStyle: true,
        });

        // Initialize R2 client (S3-compatible, so same S3 client class)
        this.r2Client = new S3Client({
            endpoint: config.r2Endpoint,
            region: "auto", // R2 uses auto region for S3-compatible API
            credentials: {
                accessKeyId: config.r2AccessKeyId,
                secretAccessKey: config.r2SecretAccessKey,
            },
            forcePathStyle: true,
        });

        this.s3Bucket = `${config.bucketPrefix}-s3-primary`;
        this.r2Bucket = `${config.bucketPrefix}-r2-primary`;
        this.failoverThresholdMs = config.failoverThresholdMs || 2000; // Default 2s failover

        // Start periodic health checks for both buckets
        this.startHealthChecks();
    }

    // Start background health checks to monitor bucket availability
    private startHealthChecks() {
        this.healthCheckInterval = setInterval(async () => {
            try {
                await this.s3Client.send(new HeadObjectCommand({
                    Bucket: this.s3Bucket,
                    Key: ".healthcheck",
                }));
            } catch (err) {
                console.error(`S3 health check failed: ${err}`);
            }
            try {
                await this.r2Client.send(new HeadObjectCommand({
                    Bucket: this.r2Bucket,
                    Key: ".healthcheck",
                }));
            } catch (err) {
                console.error(`R2 health check failed: ${err}`);
            }
        }, 60000); // Check every 60 seconds
    }

    // Upload a file to the primary (S3) bucket, failover to R2 on error or timeout
    async uploadFile(key: string, body: Buffer, contentType: string): Promise<{ bucket: string; key: string }> {
        const uploadToPrimary = async () => {
            const command = new PutObjectCommand({
                Bucket: this.s3Bucket,
                Key: key,
                Body: body,
                ContentType: contentType,
                // Generate MD5 checksum for data integrity
                ContentMD5: crypto.createHash("md5").update(body).digest("base64"),
            });
            await this.s3Client.send(command);
            return { bucket: "s3", key };
        };

        const uploadToSecondary = async () => {
            const command = new PutObjectCommand({
                Bucket: this.r2Bucket,
                Key: key,
                Body: body,
                ContentType: contentType,
                ContentMD5: crypto.createHash("md5").update(body).digest("base64"),
            });
            await this.r2Client.send(command);
            return { bucket: "r2", key };
        };

        // Try primary first, with timeout
        try {
            const primaryResult = await Promise.race([
                uploadToPrimary(),
                new Promise((_, reject) => setTimeout(() => reject(new Error("S3 upload timeout")), this.failoverThresholdMs)),
            ]);
            return primaryResult as { bucket: string; key: string };
        } catch (primaryErr) {
            console.warn(`Primary (S3) upload failed, failing over to R2: ${primaryErr}`);
            try {
                return await uploadToSecondary();
            } catch (secondaryErr) {
                throw new Error(`Both primary and secondary upload failed. S3 error: ${primaryErr}, R2 error: ${secondaryErr}`);
            }
        }
    }

    // Cleanup on client teardown
    destroy() {
        clearInterval(this.healthCheckInterval);
        this.s3Client.destroy();
        this.r2Client.destroy();
    }
}

import { MultiCloudStorageClient } from "./multi-cloud-client";
import { S3Client, PutObjectCommand, GetObjectCommand } from "@aws-sdk/client-s3";
import * as fs from "fs";
import * as path from "path";
import { performance } from "perf_hooks";

// Benchmark configuration
const BENCHMARK_ITERATIONS = 100;
const FILE_SIZE_MB = 10;
const FILE_SIZE_BYTES = FILE_SIZE_MB * 1024 * 1024;
const S3_PRICE_PER_GB_STORAGE = 0.023; // us-east-1 Standard storage
const S3_PRICE_PER_GB_EGRESS = 0.09; // First 10TB egress
const R2_PRICE_PER_GB_STORAGE = 0.015; // R2 Standard storage
const R2_PRICE_PER_GB_EGRESS = 0; // Zero egress fees

// Generate a random 10MB file for benchmarking
function generateTestFile(): Buffer {
    const buffer = Buffer.alloc(FILE_SIZE_BYTES);
    for (let i = 0; i < FILE_SIZE_BYTES; i++) {
        buffer[i] = Math.floor(Math.random() * 256);
    }
    return buffer;
}

// Run benchmark for a single storage backend
async function benchmarkBackend(
    client: S3Client,
    bucket: string,
    backendName: string
): Promise<{ avgUploadMs: number; avgDownloadMs: number; errorCount: number }> {
    let totalUploadMs = 0;
    let totalDownloadMs = 0;
    let errorCount = 0;
    const testKey = `benchmark-${Date.now()}`;
    const testFile = generateTestFile();

    for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
        // Benchmark upload
        const uploadStart = performance.now();
        try {
            await client.send(new PutObjectCommand({
                Bucket: bucket,
                Key: `${testKey}-${i}`,
                Body: testFile,
                ContentType: "application/octet-stream",
            }));
            totalUploadMs += performance.now() - uploadStart;
        } catch (err) {
            errorCount++;
            console.error(`Upload error for ${backendName}: ${err}`);
        }

        // Benchmark download
        const downloadStart = performance.now();
        try {
            const response = await client.send(new GetObjectCommand({
                Bucket: bucket,
                Key: `${testKey}-${i}`,
            }));
            // Consume the stream to get full download time
            await response.Body?.transformToByteArray();
            totalDownloadMs += performance.now() - downloadStart;
        } catch (err) {
            errorCount++;
            console.error(`Download error for ${backendName}: ${err}`);
        }
    }

    return {
        avgUploadMs: totalUploadMs / BENCHMARK_ITERATIONS,
        avgDownloadMs: totalDownloadMs / BENCHMARK_ITERATIONS,
        errorCount,
    };
}

// Main benchmark function
async function runBenchmark() {
    // Initialize clients (assumes config from environment variables)
    const multiCloudClient = new MultiCloudStorageClient({
        s3Endpoint: process.env.S3_ENDPOINT!,
        s3Region: process.env.AWS_REGION!,
        s3AccessKeyId: process.env.AWS_ACCESS_KEY_ID!,
        s3SecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
        r2Endpoint: process.env.R2_ENDPOINT!,
        r2AccessKeyId: process.env.R2_ACCESS_KEY_ID!,
        r2SecretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
        bucketPrefix: process.env.BUCKET_PREFIX!,
        failoverThresholdMs: 2000,
    });

    // Direct S3 client for comparison
    const s3DirectClient = new S3Client({
        region: process.env.AWS_REGION!,
        credentials: {
            accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
            secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
        },
    });

    // Direct R2 client for comparison
    const r2DirectClient = new S3Client({
        endpoint: process.env.R2_ENDPOINT!,
        region: "auto",
        credentials: {
            accessKeyId: process.env.R2_ACCESS_KEY_ID!,
            secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
        },
        forcePathStyle: true,
    });

    console.log("Starting benchmark...");
    console.log(`Iterations: ${BENCHMARK_ITERATIONS}, File size: ${FILE_SIZE_MB}MB`);

    // Run benchmarks
    const s3Results = await benchmarkBackend(s3DirectClient, process.env.S3_BUCKET!, "AWS S3");
    const r2Results = await benchmarkBackend(r2DirectClient, process.env.R2_BUCKET!, "Cloudflare R2");

    // Calculate cost projections for 1TB storage and 500GB egress
    const storageGB = 1024; // 1TB
    const egressGB = 512; // 500GB
    const s3MonthlyCost = (storageGB * S3_PRICE_PER_GB_STORAGE) + (egressGB * S3_PRICE_PER_GB_EGRESS);
    const r2MonthlyCost = (storageGB * R2_PRICE_PER_GB_STORAGE) + (egressGB * R2_PRICE_PER_GB_EGRESS);
    const savings = s3MonthlyCost - r2MonthlyCost;

    // Output results
    console.log("\n=== Benchmark Results ===");
    console.log(`AWS S3 Avg Upload: ${s3Results.avgUploadMs.toFixed(2)}ms`);
    console.log(`AWS S3 Avg Download: ${s3Results.avgDownloadMs.toFixed(2)}ms`);
    console.log(`AWS S3 Errors: ${s3Results.errorCount}`);
    console.log(`Cloudflare R2 Avg Upload: ${r2Results.avgUploadMs.toFixed(2)}ms`);
    console.log(`Cloudflare R2 Avg Download: ${r2Results.avgDownloadMs.toFixed(2)}ms`);
    console.log(`Cloudflare R2 Errors: ${r2Results.errorCount}`);

    console.log("\n=== Cost Projection (1TB Storage, 500GB Egress/Month) ===");
    console.log(`AWS S3 Monthly Cost: $${s3MonthlyCost.toFixed(2)}`);
    console.log(`Cloudflare R2 Monthly Cost: $${r2MonthlyCost.toFixed(2)}`);
    console.log(`Monthly Savings with R2: $${savings.toFixed(2)} (${(savings / s3MonthlyCost * 100).toFixed(1)}%)`);

    // Cleanup
    multiCloudClient.destroy();
    s3DirectClient.destroy();
    r2DirectClient.destroy();
}

// Run benchmark with error handling
runBenchmark().catch((err) => {
    console.error("Benchmark failed:", err);
    process.exit(1);
});

import { PolicyPack, validateResource } from "@pulumi/policy";

new PolicyPack("multi-cloud-storage-policies", {
    policies: [
        {
            name: "r2-bucket-encryption-required",
            description: "R2 buckets must have default encryption enabled",
            enforcementLevel: "mandatory",
            validateResource: validateResource((e) => {
                if (e.resource.type === "cloudflare:index/r2Bucket:R2Bucket") {
                    // R2 buckets have encryption enabled by default, but we check storage class
                    if (e.resource.props.storageClass !== "Standard") {
                        e.reportViolation("R2 buckets must use Standard storage class for encryption compliance");
                    }
                }
            }),
        },
        {
            name: "s3-bucket-versioning-required",
            description: "S3 buckets must have versioning enabled",
            enforcementLevel: "mandatory",
            validateResource: validateResource((e) => {
                if (e.resource.type === "aws:s3/bucketV2:BucketV2") {
                    // Check that versioning is enabled via the associated versioning resource
                    // In practice, you'd check dependent resources, but for brevity we log a warning
                    console.log("Validating S3 bucket versioning for", e.resource.name);
                }
            }),
        },
    ],
});

import { S3Event, Context } from "aws-lambda";
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
import { S3Client as R2Client, PutObjectCommand } from "@aws-sdk/client-s3";

const s3Client = new S3Client({ region: "us-east-1" });
const r2Client = new R2Client({
    endpoint: process.env.R2_ENDPOINT!,
    region: "auto",
    credentials: {
        accessKeyId: process.env.R2_ACCESS_KEY_ID!,
        secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
    },
    forcePathStyle: true,
});

export const handler = async (event: S3Event, context: Context) => {
    for (const record of event.Records) {
        const bucket = record.s3.bucket.name;
        const key = decodeURIComponent(record.s3.object.key.replace(/\+/g, " "));
        try {
            // Get object from S3
            const { Body, ContentType } = await s3Client.send(new GetObjectCommand({
                Bucket: bucket,
                Key: key,
            }));
            // Upload to R2
            await r2Client.send(new PutObjectCommand({
                Bucket: process.env.R2_BUCKET!,
                Key: key,
                Body: await Body?.transformToByteArray(),
                ContentType: ContentType,
            }));
            console.log(`Replicated ${key} from S3 to R2`);
        } catch (err) {
            console.error(`Failed to replicate ${key}:`, err);
        }
    }
};

import { CostExplorerClient, GetCostAndUsageCommand } from "@aws-sdk/client-cost-explorer";

const costExplorer = new CostExplorerClient({ region: "us-east-1" });

async function getS3EgressCosts() {
    const params = {
        TimePeriod: {
            Start: "2024-10-01",
            End: "2024-12-31",
        },
        Granularity: "MONTHLY",
        Metrics: ["BlendedCost"],
        GroupBy: [{ Type: "DIMENSION", Key: "SERVICE" }, { Type: "DIMENSION", Key: "REGION" }],
        Filter: {
            And: [
                { Dimensions: { Key: "SERVICE", Values: ["Amazon Simple Storage Service"] } },
                { Dimensions: { Key: "USAGE_TYPE", Values: ["DataTransfer-Out-Bytes"] } },
            ],
        },
    };
    try {
        const data = await costExplorer.send(new GetCostAndUsageCommand(params));
        console.log("S3 Egress Costs (Oct-Dec 2024):", JSON.stringify(data, null, 2));
    } catch (err) {
        console.error("Failed to get cost data:", err);
    }
}

getS3EgressCosts();

multi-cloud-storage-pulumi/
├── infra/ # Pulumi infrastructure code
│   ├── index.ts # Main Pulumi stack (first code example)
│   ├── Pulumi.yaml # Stack configuration
│   ├── package.json # Node.js dependencies
│   └── tsconfig.json # TypeScript config
├── src/ # Multi-cloud storage client code
│   ├── multi-cloud-client.ts # Second code example
│   ├── benchmark.ts # Third code example
│   ├── replicate.ts # Replication Lambda function (Tip 2)
│   └── cost-analyzer.ts # Cost Explorer script (Tip 3)
├── policies/ # Pulumi CrossGuard policies (Tip 1)
│   └── index.ts # Policy pack
├── .github/ # CI/CD workflows
│   └── workflows/
│       └── deploy.yml # Pulumi deploy workflow
├── package.json # Root dependencies
├── tsconfig.json # Root TypeScript config
└── README.md # Setup instructions