Process That Brought Camera vs Substack: What You Need to Know

// camera-bulk-import.js
// Bulk import 100k+ subscribers into Camera v0.8.2 with retry logic, rate limiting, and audit logs
// Dependencies: @camera/sdk v0.8.2, p-queue v6.6.2, winston v3.11.0
// Run: node camera-bulk-import.js --input subs.csv --batch-size 500 --max-retries 3

const { CameraClient } = require('@camera/sdk');
const { default: PQueue } = require('p-queue');
const winston = require('winston');
const fs = require('fs');
const csv = require('csv-parser');
const yargs = require('yargs/yargs');
const { hideBin } = require('yargs/helpers');

// Configure structured logging for audit trails
const logger = winston.createLogger({
  level: 'info',
  format: winston.format.json(),
  transports: [
    new winston.transports.File({ filename: 'import-audit.log' }),
    new winston.transports.Console()
  ]
});

// Parse CLI args
const argv = yargs(hideBin(process.argv))
  .option('input', { type: 'string', demandOption: true, description: 'Path to CSV file with email, name, signup_date' })
  .option('batch-size', { type: 'number', default: 500, description: 'Number of subscribers per API batch' })
  .option('max-retries', { type: 'number', default: 3, description: 'Max retries for failed API calls' })
  .option('api-key', { type: 'string', demandOption: true, description: 'Camera API key' })
  .option('api-url', { type: 'string', default: 'http://localhost:3000', description: 'Camera instance URL' })
  .argv;

// Initialize Camera client with timeout and retry defaults
const camera = new CameraClient({
  apiKey: argv.api-key,
  baseUrl: argv.api-url,
  timeout: 10000, // 10s timeout per request
  retryConfig: { maxRetries: 1 } // We handle retries manually via queue
});

// Rate limit queue: 10 concurrent requests, 100ms delay between batches to avoid Camera rate limits
const queue = new PQueue({ concurrency: 10, interval: 1000, intervalCap: 10 });

let successCount = 0;
let failureCount = 0;
let duplicateCount = 0;

// Process CSV in streaming mode to handle large files (100k+ rows) without memory issues
fs.createReadStream(argv.input)
  .pipe(csv())
  .on('data', (row) => {
    // Validate required fields
    if (!row.email || !/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(row.email)) {
      logger.warn('Skipping invalid email', { email: row.email });
      failureCount++;
      return;
    }

    // Add job to queue with retry logic
    queue.add(async () => {
      let retries = 0;
      while (retries <= argv.max-retries) {
        try {
          const response = await camera.subscribers.create({
            email: row.email,
            name: row.name || null,
            signupDate: row.signup_date ? new Date(row.signup_date) : new Date(),
            source: 'bulk-import'
          });
          logger.info('Subscriber imported', { email: row.email, id: response.id });
          successCount++;
          return;
        } catch (error) {
          retries++;
          if (error.status === 409) {
            logger.warn('Duplicate subscriber', { email: row.email });
            duplicateCount++;
            return;
          }
          if (retries > argv.max-retries) {
            logger.error('Failed to import subscriber after max retries', { email: row.email, error: error.message });
            failureCount++;
            return;
          }
          logger.warn(`Retry ${retries} for ${row.email}`, { error: error.message });
          await new Promise(resolve => setTimeout(resolve, 1000 * retries)); // Exponential backoff
        }
      }
    });
  })
  .on('end', async () => {
    // Wait for all queue jobs to finish
    await queue.onIdle();
    logger.info('Import complete', {
      success: successCount,
      failures: failureCount,
      duplicates: duplicateCount,
      total: successCount + failureCount + duplicateCount
    });
    process.exit(0);
  })
  .on('error', (error) => {
    logger.error('CSV stream error', { error: error.message });
    process.exit(1);
  });

// substack-stats-fetch.js
// Fetch subscriber growth, churn, and revenue stats from Substack's unofficial API
// Note: Substack does not provide an official public API; this uses the internal API used by their dashboard
// Dependencies: axios v1.6.2, dotenv v16.3.1, dayjs v1.11.10
// Run: node substack-stats-fetch.js --start-date 2024-01-01 --end-date 2024-09-30

require('dotenv').config();
const axios = require('axios');
const dayjs = require('dayjs');
const yargs = require('yargs/yargs');
const { hideBin } = require('yargs/helpers');
const winston = require('winston');
const fs = require('fs');

// Configure logging
const logger = winston.createLogger({
  level: 'info',
  format: winston.format.json(),
  transports: [new winston.transports.Console()]
});

// Parse CLI args
const argv = yargs(hideBin(process.argv))
  .option('start-date', { type: 'string', demandOption: true, description: 'Start date (YYYY-MM-DD)' })
  .option('end-date', { type: 'string', demandOption: true, description: 'End date (YYYY-MM-DD)' })
  .option('output', { type: 'string', default: 'substack-stats.json', description: 'Output file path' })
  .argv;

// Validate dates
const startDate = dayjs(argv.start-date);
const endDate = dayjs(argv.end-date);
if (!startDate.isValid() || !endDate.isValid()) {
  logger.error('Invalid date format. Use YYYY-MM-DD.');
  process.exit(1);
}
if (endDate.isBefore(startDate)) {
  logger.error('End date must be after start date.');
  process.exit(1);
}

// Substack internal API base URL (observed from dashboard network requests)
const SUBSTACK_API_BASE = 'https://substack.com/api/v1';
const PUBLICATION_ID = process.env.SUBSTACK_PUBLICATION_ID;
const SESSION_COOKIE = process.env.SUBSTACK_SESSION_COOKIE;

if (!PUBLICATION_ID || !SESSION_COOKIE) {
  logger.error('Missing env vars: SUBSTACK_PUBLICATION_ID, SUBSTACK_SESSION_COOKIE');
  process.exit(1);
}

// Initialize axios instance with session auth
const substackClient = axios.create({
  baseURL: SUBSTACK_API_BASE,
  headers: {
    'Cookie': `session=${SESSION_COOKIE}`,
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
  },
  timeout: 15000
});

// Fetch stats with pagination and retry logic
async function fetchStats() {
  const stats = {
    publicationId: PUBLICATION_ID,
    startDate: argv.start-date,
    endDate: argv.end-date,
    subscriberGrowth: [],
    churnRate: [],
    revenue: []
  };

  let currentDate = startDate;
  while (currentDate.isBefore(endDate) || currentDate.isSame(endDate)) {
    const dateStr = currentDate.format('YYYY-MM-DD');
    let retries = 0;
    const maxRetries = 3;

    while (retries <= maxRetries) {
      try {
        // Fetch daily subscriber stats
        const subscriberRes = await substackClient.get(`/publication/${PUBLICATION_ID}/subscribers/stats`, {
          params: { date: dateStr }
        });

        // Fetch daily revenue stats
        const revenueRes = await substackClient.get(`/publication/${PUBLICATION_ID}/revenue/stats`, {
          params: { date: dateStr }
        });

        stats.subscriberGrowth.push({
          date: dateStr,
          new: subscriberRes.data.new_subscribers,
          total: subscriberRes.data.total_subscribers,
          churn: subscriberRes.data.unsubscribed
        });

        stats.revenue.push({
          date: dateStr,
          daily: revenueRes.data.daily_revenue,
          monthlyRecurring: revenueRes.data.mrr
        });

        logger.info(`Fetched stats for ${dateStr}`);
        break;
      } catch (error) {
        retries++;
        if (retries > maxRetries) {
          logger.error(`Failed to fetch stats for ${dateStr} after max retries`, { error: error.message });
          stats.subscriberGrowth.push({ date: dateStr, error: 'Failed to fetch' });
          stats.revenue.push({ date: dateStr, error: 'Failed to fetch' });
          break;
        }
        logger.warn(`Retry ${retries} for ${dateStr}`, { error: error.message });
        await new Promise(resolve => setTimeout(resolve, 2000 * retries));
      }
    }

    currentDate = currentDate.add(1, 'day');
  }

  // Write stats to output file
  fs.writeFileSync(argv.output, JSON.stringify(stats, null, 2));
  logger.info(`Stats written to ${argv.output}`);
}

// Run fetch
fetchStats().catch((error) => {
  logger.error('Fatal error fetching stats', { error: error.message });
  process.exit(1);
});

// tco-calculator.js
// Calculate 3-year TCO for Camera vs Substack across subscriber tiers
// Assumptions: Substack Pro fee is 10% of revenue + $10/mo per 1k subs; Camera cost is self-hosted infrastructure + admin time
// Dependencies: none (vanilla Node.js)

const SUBSCRIBER_TIERS = [10_000, 50_000, 100_000, 500_000];
const MONTHS_IN_STUDY = 36; // 3 years
const CAMERA_ADMIN_HOURS_PER_WEEK = 1; // FTE admin time for self-hosted Camera
const HOURLY_ADMIN_RATE = 75; // USD per hour for DevOps admin
const DO_DROPLET_COST = {
  10_000: 60, // 2 vCPU, 8GB RAM
  50_000: 120, // 4 vCPU, 16GB RAM
  100_000: 240, // 8 vCPU, 32GB RAM
  500_000: 480 // 16 vCPU, 64GB RAM
};
const SUBSTACK_REVENUE_PER_SUB = 0.50; // Average monthly revenue per subscriber (USD)

// Calculate Camera TCO for a given subscriber count
function calculateCameraTCO(subscribers) {
  const monthlyInfraCost = DO_DROPLET_COST[subscribers] || (DO_DROPLET_COST[500_000] * Math.ceil(subscribers / 500_000));
  const monthlyAdminCost = (CAMERA_ADMIN_HOURS_PER_WEEK * 4) * HOURLY_ADMIN_RATE; // 4 weeks per month
  return (monthlyInfraCost + monthlyAdminCost) * MONTHS_IN_STUDY;
}

// Calculate Substack TCO for a given subscriber count
function calculateSubstackTCO(subscribers) {
  let totalCost = 0;
  for (let month = 1; month <= MONTHS_IN_STUDY; month++) {
    // Assume 5% monthly subscriber growth
    const currentSubs = Math.floor(subscribers * Math.pow(1.05, month - 1));
    const monthlyRevenue = currentSubs * SUBSTACK_REVENUE_PER_SUB;
    const substackFee = monthlyRevenue * 0.10; // 10% take rate
    const substackBaseFee = Math.ceil(currentSubs / 1000) * 10; // $10 per 1k subs
    totalCost += substackFee + substackBaseFee;
  }
  return totalCost;
}

// Generate report
console.log('3-Year TCO Comparison: Camera vs Substack');
console.log('=========================================\n');
console.log(`Study Period: ${MONTHS_IN_STUDY} months`);
console.log(`Assumptions:`);
console.log(`- Substack: 10% revenue take + $10/mo per 1k subs`);
console.log(`- Camera: Self-hosted on DigitalOcean, 1 FTE admin hour/week @ $75/hr`);
console.log(`- Monthly subscriber growth: 5%`);
console.log(`- Average monthly revenue per subscriber: $${SUBSTACK_REVENUE_PER_SUB}\n`);

console.log('| Subscriber Tier | Camera TCO (3yr) | Substack TCO (3yr) | Savings with Camera |');
console.log('|-----------------|------------------|--------------------|---------------------|');

SUBSCRIBER_TIERS.forEach(tier => {
  const cameraTCO = calculateCameraTCO(tier);
  const substackTCO = calculateSubstackTCO(tier);
  const savings = substackTCO - cameraTCO;
  const savingsPct = ((savings / substackTCO) * 100).toFixed(1);

  console.log(`| ${tier.toLocaleString().padEnd(15)} | $${cameraTCO.toLocaleString().padEnd(16)} | $${substackTCO.toLocaleString().padEnd(18)} | $${savings.toLocaleString().padEnd(12)} (${savingsPct}%) |`);
});

// Output break-even point for Camera setup cost
console.log('\nBreak-Even Analysis:');
console.log('---------------------');
const INITIAL_CAMERA_SETUP_COST = 2000; // One-time setup: DNS, DKIM, SPF, plugin config
SUBSCRIBER_TIERS.forEach(tier => {
  const monthlyCameraCost = (DO_DROPLET_COST[tier] || (DO_DROPLET_COST[500_000] * Math.ceil(tier / 500_000))) + (CAMERA_ADMIN_HOURS_PER_WEEK * 4 * HOURLY_ADMIN_RATE);
  const monthlySubstackCost = (tier * SUBSTACK_REVENUE_PER_SUB * 0.10) + (Math.ceil(tier / 1000) * 10);
  const monthlySavings = monthlySubstackCost - monthlyCameraCost;
  const breakEvenMonths = Math.ceil(INITIAL_CAMERA_SETUP_COST / monthlySavings);
  console.log(`- ${tier.toLocaleString()} subs: Break even after ${breakEvenMonths} months`);
});

// Check inbox placement for test batch
const { InboxPlacement } = require('@camera/deliverability');
const checker = new InboxPlacement({ apiKey: process.env.CAMERA_API_KEY });
const testEmails = ['test1@gmail.com', 'test2@outlook.com', 'test3@proton.me'];
const result = await checker.check({
  emails: testEmails,
  subject: 'Deliverability Test',
  body: 'This is a test email from Camera.'
});
console.log(`Inbox rate: ${result.inboxRate * 100}%`);

// Deduplicate Substack CSV export
const fs = require('fs');
const csv = require('csv-parser');
const { writeToPath } = require('fast-csv');

const seenEmails = new Set();
const deduplicated = [];

fs.createReadStream('substack-export.csv')
  .pipe(csv())
  .on('data', (row) => {
    if (!seenEmails.has(row.email)) {
      seenEmails.add(row.email);
      deduplicated.push(row);
    }
  })
  .on('end', () => {
    writeToPath('substack-deduplicated.csv', deduplicated, { headers: true })
      .on('finish', () => console.log(`Deduplicated to ${deduplicated.length} rows`));
  });

#!/bin/bash
# Daily Camera PostgreSQL backup to S3
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="camera_backup_$TIMESTAMP.sql.gz"
pg_dump -U camera_user camera_db | gzip > /tmp/$BACKUP_FILE
aws s3 cp /tmp/$BACKUP_FILE s3://your-camera-backups/$BACKUP_FILE
rm /tmp/$BACKUP_FILE
echo "Backup $BACKUP_FILE uploaded to S3"