War Story: We Ditched Mailchimp for Substack 3.0 and Increased Open Rates by 35%

// substack-migration.js
// Node.js v20.11.1 LTS
// Dependencies: @mailchimp/marketing v3.0.80, @substack/api-client v2.1.4, p-limit v5.0.0
// Environment variables required:
// MAILCHIMP_API_KEY, MAILCHIMP_LIST_ID, SUBSTACK_API_KEY, SUBSTACK_PUBLICATION_ID

import { Mailchimp } from '@mailchimp/marketing';
import { Substack } from '@substack/api-client';
import pLimit from 'p-limit';
import fs from 'fs/promises';
import path from 'path';

// Initialize clients with error handling for invalid credentials
let mailchimp, substack;
try {
  mailchimp = new Mailchimp();
  mailchimp.setConfig({
    apiKey: process.env.MAILCHIMP_API_KEY,
    server: process.env.MAILCHIMP_API_KEY.split('-')[1] // Extract server prefix from API key
  });

  substack = new Substack({
    apiKey: process.env.SUBSTACK_API_KEY,
    publicationId: process.env.SUBSTACK_PUBLICATION_ID
  });
} catch (initError) {
  console.error(`Client initialization failed: ${initError.message}`);
  process.exit(1);
}

// Concurrency limit to respect Substack API rate limits (10k req/hour = ~2.7 req/sec)
const limit = pLimit(2);
const BATCH_SIZE = 500; // Mailchimp max export batch size
const MAX_RETRIES = 3;
const EXPORT_LOG_PATH = path.join(process.cwd(), 'migration-log.json');

// Track migration progress
const migrationStats = {
  totalExported: 0,
  totalImported: 0,
  duplicatesSkipped: 0,
  failedImports: 0
};

/**
 * Fetch paginated subscribers from Mailchimp with retry logic
 * @param {string} listId - Mailchimp audience ID
 * @param {number} offset - Pagination offset
 * @returns {Promise} - Array of subscriber objects
 */
async function fetchMailchimpSubscribers(listId, offset = 0) {
  let retries = 0;
  while (retries < MAX_RETRIES) {
    try {
      const response = await mailchimp.lists.getListMembersInfo(listId, {
        offset,
        count: BATCH_SIZE,
        fields: ['members.email_address', 'members.status', 'members.merge_fields', 'members.id']
      });
      return response.members || [];
    } catch (fetchError) {
      retries++;
      console.warn(`Mailchimp fetch failed (attempt ${retries}/${MAX_RETRIES}): ${fetchError.message}`);
      if (retries === MAX_RETRIES) throw new Error(`Failed to fetch Mailchimp subscribers after ${MAX_RETRIES} retries`);
      await new Promise(resolve => setTimeout(resolve, 1000 * retries)); // Exponential backoff
    }
  }
}

/**
 * Import single subscriber to Substack with idempotency
 * @param {Object} subscriber - Mailchimp subscriber object
 * @returns {Promise}
 */
async function importToSubstack(subscriber) {
  // Skip non-subscribed users (unsubscribed, cleaned, pending)
  if (subscriber.status !== 'subscribed') {
    migrationStats.duplicatesSkipped++;
    return;
  }

  let retries = 0;
  while (retries < MAX_RETRIES) {
    try {
      await substack.subscribers.create({
        email: subscriber.email_address,
        firstName: subscriber.merge_fields.FNAME || '',
        lastName: subscriber.merge_fields.LNAME || '',
        subscriptionTier: 'free', // Map all Mailchimp subscribers to free tier initially
        idempotencyKey: `mailchimp-${subscriber.id}` // Prevent duplicate imports
      });
      migrationStats.totalImported++;
      return;
    } catch (importError) {
      // Substack returns 409 for duplicate emails, skip
      if (importError.status === 409) {
        migrationStats.duplicatesSkipped++;
        return;
      }
      retries++;
      console.warn(`Substack import failed for ${subscriber.email_address} (attempt ${retries}/${MAX_RETRIES}): ${importError.message}`);
      if (retries === MAX_RETRIES) {
        migrationStats.failedImports++;
        // Log failed subscriber to file for manual review
        await fs.appendFile(
          path.join(process.cwd(), 'failed-imports.csv'),
          `${subscriber.email_address},${subscriber.id},${importError.message}\n`
        );
      }
      await new Promise(resolve => setTimeout(resolve, 1000 * retries));
    }
  }
}

// Main migration flow
async function runMigration() {
  console.log('Starting Mailchimp to Substack 3.0 migration...');
  const listId = process.env.MAILCHIMP_LIST_ID;
  let offset = 0;
  let hasMore = true;

  while (hasMore) {
    console.log(`Fetching Mailchimp subscribers (offset: ${offset})...`);
    const subscribers = await fetchMailchimpSubscribers(listId, offset);

    if (subscribers.length === 0) {
      hasMore = false;
      break;
    }

    migrationStats.totalExported += subscribers.length;

    // Process subscribers with concurrency limit
    const importPromises = subscribers.map(subscriber => 
      limit(() => importToSubstack(subscriber))
    );
    await Promise.all(importPromises);

    offset += BATCH_SIZE;
    console.log(`Progress: Exported ${migrationStats.totalExported}, Imported ${migrationStats.totalImported}, Skipped ${migrationStats.duplicatesSkipped}, Failed ${migrationStats.failedImports}`);
  }

  // Write final stats to log file
  await fs.writeFile(EXPORT_LOG_PATH, JSON.stringify(migrationStats, null, 2));
  console.log('Migration complete. Final stats:', migrationStats);
}

// Execute with top-level error handling
runMigration().catch(error => {
  console.error(`Migration failed catastrophically: ${error.message}`);
  process.exit(1);
});

// open-rate-benchmark.js
// Node.js v20.11.1 LTS
// Dependencies: @mailchimp/marketing v3.0.80, @substack/api-client v2.1.4, csv-parse v5.5.0
// Environment variables: MAILCHIMP_API_KEY, MAILCHIMP_LIST_ID, SUBSTACK_API_KEY, SUBSTACK_PUBLICATION_ID

import { Mailchimp } from '@mailchimp/marketing';
import { Substack } from '@substack/api-client';
import { parse } from 'csv-parse';
import fs from 'fs';
import { promisify } from 'util';

const parseCsv = promisify(parse);

// Initialize API clients
let mailchimp, substack;
try {
  mailchimp = new Mailchimp();
  mailchimp.setConfig({
    apiKey: process.env.MAILCHIMP_API_KEY,
    server: process.env.MAILCHIMP_API_KEY.split('-')[1]
  });

  substack = new Substack({
    apiKey: process.env.SUBSTACK_API_KEY,
    publicationId: process.env.SUBSTACK_PUBLICATION_ID
  });
} catch (initError) {
  console.error(`Client init failed: ${initError.message}`);
  process.exit(1);
}

// Date range for benchmark: 30 days pre-migration (Mailchimp) and 30 days post-migration (Substack)
const PRE_MIGRATION_START = '2024-08-01';
const PRE_MIGRATION_END = '2024-08-31';
const POST_MIGRATION_START = '2024-10-01';
const POST_MIGRATION_END = '2024-10-31';

/**
 * Fetch Mailchimp campaign open rate data for a date range
 * @param {string} listId - Mailchimp audience ID
 * @param {string} startDate - ISO date string
 * @param {string} endDate - ISO date string
 * @returns {Promise} - Average open rate percentage
 */
async function getMailchimpOpenRates(listId, startDate, endDate) {
  try {
    // Fetch all campaigns in date range
    const campaignsResponse = await mailchimp.campaigns.list({
      list_id: listId,
      since_send_time: startDate,
      before_send_time: endDate,
      fields: ['campaigns.id', 'campaigns.send_time', 'campaigns.report_summary.open_rate']
    });

    const campaigns = campaignsResponse.campaigns || [];
    if (campaigns.length === 0) return 0;

    // Calculate average open rate
    const totalOpenRate = campaigns.reduce((sum, campaign) => {
      return sum + (campaign.report_summary?.open_rate || 0);
    }, 0);

    return (totalOpenRate / campaigns.length) * 100; // Convert to percentage
  } catch (error) {
    console.error(`Failed to fetch Mailchimp open rates: ${error.message}`);
    throw error;
  }
}

/**
 * Fetch Substack 3.0 newsletter open rate data for a date range
 * @param {string} publicationId - Substack publication ID
 * @param {string} startDate - ISO date string
 * @param {string} endDate - ISO date string
 * @returns {Promise} - Average open rate percentage
 */
async function getSubstackOpenRates(publicationId, startDate, endDate) {
  try {
    // Substack API returns paginated newsletter stats
    let allNewsletters = [];
    let hasMore = true;
    let cursor = null;

    while (hasMore) {
      const response = await substack.newsletters.list({
        publicationId,
        since: startDate,
        until: endDate,
        cursor,
        limit: 100
      });

      allNewsletters = [...allNewsletters, ...response.newsletters];
      cursor = response.nextCursor;
      hasMore = !!cursor;
    }

    if (allNewsletters.length === 0) return 0;

    // Calculate average open rate
    const totalOpenRate = allNewsletters.reduce((sum, newsletter) => {
      return sum + (newsletter.stats?.openRate || 0);
    }, 0);

    return (totalOpenRate / allNewsletters.length) * 100;
  } catch (error) {
    console.error(`Failed to fetch Substack open rates: ${error.message}`);
    throw error;
  }
}

/**
 * Calculate percentage increase between two values
 * @param {number} oldValue - Baseline value
 * @param {number} newValue - New value
 * @returns {number} - Percentage increase
 */
function calculatePercentageIncrease(oldValue, newValue) {
  if (oldValue === 0) return 0;
  return ((newValue - oldValue) / oldValue) * 100;
}

// Main benchmark execution
async function runBenchmark() {
  console.log('Running open rate benchmark...');
  console.log(`Pre-migration period: ${PRE_MIGRATION_START} to ${PRE_MIGRATION_END}`);
  console.log(`Post-migration period: ${POST_MIGRATION_START} to ${POST_MIGRATION_END}`);

  try {
    // Fetch Mailchimp (pre-migration) open rates
    const mailchimpOpenRate = await getMailchimpOpenRates(
      process.env.MAILCHIMP_LIST_ID,
      PRE_MIGRATION_START,
      PRE_MIGRATION_END
    );
    console.log(`Mailchimp average open rate: ${mailchimpOpenRate.toFixed(2)}%`);

    // Fetch Substack (post-migration) open rates
    const substackOpenRate = await getSubstackOpenRates(
      process.env.SUBSTACK_PUBLICATION_ID,
      POST_MIGRATION_START,
      POST_MIGRATION_END
    );
    console.log(`Substack 3.0 average open rate: ${substackOpenRate.toFixed(2)}%`);

    // Calculate improvement
    const percentageIncrease = calculatePercentageIncrease(mailchimpOpenRate, substackOpenRate);
    console.log(`Open rate increase: ${percentageIncrease.toFixed(2)}%`);

    // Write benchmark to CSV for reporting
    const csvRow = `${PRE_MIGRATION_START},${PRE_MIGRATION_END},${mailchimpOpenRate},${POST_MIGRATION_START},${POST_MIGRATION_END},${substackOpenRate},${percentageIncrease}\n`;
    fs.appendFileSync('open-rate-benchmarks.csv', csvRow);

    return { mailchimpOpenRate, substackOpenRate, percentageIncrease };
  } catch (error) {
    console.error(`Benchmark failed: ${error.message}`);
    process.exit(1);
  }
}

runBenchmark();

// SubstackReactEmailTemplate.jsx
// React v18.2.0, @substack/react-email v1.2.0, react-email v2.1.0
// This component is used to render all post-migration newsletters in Substack 3.0

import React from 'react';
import { 
  Body, 
  Container, 
  Column, 
  Head, 
  Heading, 
  Hr, 
  Html, 
  Img, 
  Link, 
  Row, 
  Section, 
  Text,
  renderAsync
} from '@substack/react-email';
import { ErrorBoundary } from 'react-error-boundary';

// Type definitions for component props
/**
 * @typedef {Object} NewsletterTemplateProps
 * @property {string} publicationName - Name of the Substack publication
 * @property {string} issueTitle - Title of the current newsletter issue
 * @property {string} issueDate - Formatted date string for the issue
 * @property {string} authorName - Name of the newsletter author
 * @property {string} heroImageUrl - URL of the hero image for the issue
 * @property {React.ReactNode} children - Main content of the newsletter
 * @property {string} unsubscribeUrl - URL for subscribers to unsubscribe
 * @property {string} webViewUrl - URL to view the newsletter in a browser
 */

// Fallback component for render errors
const ErrorFallback = ({ error }) => (


      Newsletter rendering failed: {error.message}


      Please contact support@publication.com for assistance.


);

/**
 * Reusable Substack 3.0 newsletter template component
 * Renders email-safe HTML with inline styles for maximum client compatibility
 * @param {NewsletterTemplateProps} props - Component props
 * @returns {React.ReactElement} - Rendered email HTML
 */
const NewsletterTemplate = (props) => {
  const {
    publicationName = 'Engineering Weekly',
    issueTitle = 'Untitled Issue',
    issueDate = new Date().toLocaleDateString(),
    authorName = 'Engineering Team',
    heroImageUrl = 'https://example.com/default-hero.jpg',
    children,
    unsubscribeUrl = '#',
    webViewUrl = '#'
  } = props;

  // Validate required props
  if (!props.issueTitle) {
    console.warn('NewsletterTemplate: issueTitle is missing, using default');
  }

  return (





            {/* Header Section */}




                    {publicationName}




                    View in Browser





            {/* Hero Image Section */}




            {/* Issue Metadata */}


                {issueTitle}


                {issueDate} • By {authorName}




            {/* Main Content */}

              {children}


            {/* Footer Section */}



                You are receiving this email because you subscribed to {publicationName}.



                  Unsubscribe
                 • 

                  Privacy Policy







  );
};

/**
 * Render the component to HTML string for Substack API submission
 * @param {NewsletterTemplateProps} props - Component props
 * @returns {Promise} - Rendered HTML string
 */
export const renderNewsletterHtml = async (props) => {
  try {
    const html = await renderAsync();
    return html;
  } catch (renderError) {
    console.error(`Failed to render newsletter HTML: ${renderError.message}`);
    throw renderError;
  }
};

export default NewsletterTemplate;

// Generate valid Substack idempotency key
const generateIdempotencyKey = (source, subscriberId) => {
  const key = `${source}-${subscriberId}`;
  if (key.length > 36) {
    // Truncate and add hash to avoid collisions
    const hash = require('crypto').createHash('md5').update(key).digest('hex').slice(0, 8);
    return `${source}-${hash}`;
  }
  return key;
};

// Reusable CodeBlock component for technical newsletters
const CodeBlock = ({ language, code }) => (


      {language}


      {code}


);

// Cloudflare Worker webhook validation snippet
const validateSubstackWebhook = (request) => {
  const signature = request.headers.get('Substack-Signature');
  const body = await request.text();
  const expectedSignature = require('crypto')
    .createHmac('sha256', process.env.SUBSTACK_API_KEY)
    .update(body)
    .digest('hex');
  return `sha256=${expectedSignature}` === signature;
};