Benchmark: Swagger 5.0 vs Postman 2026.0 vs Insomnia 9.0 for API Documentation Ease of Use

// swagger-doc-gen.js
// Benchmarked on: macOS 14.4, Node 20.12.0, swagger-cli 5.0.0, 16GB RAM, Apple M2 Pro
// Purpose: Automate OpenAPI spec validation, doc generation, and static HTML export with Swagger 5.0

const { execSync, spawn } = require('child_process');
const fs = require('fs').promises;
const path = require('path');

// Configuration
const SWAGGER_CLI_PATH = path.resolve(__dirname, 'node_modules/.bin/swagger');
const SPEC_PATH = path.resolve(__dirname, 'openapi-spec.yaml');
const OUTPUT_DIR = path.resolve(__dirname, 'generated-docs');
const BENCHMARK_ITERATIONS = 100; // Matches benchmark methodology: 100 runs for statistical significance

/**
 * Validate OpenAPI spec using Swagger 5.0 CLI
 * @returns {boolean} True if spec is valid
 * @throws {Error} If validation fails after 3 retries
 */
async function validateSpec() {
  const maxRetries = 3;
  let attempt = 0;

  while (attempt < maxRetries) {
    try {
      console.log(`[Swagger 5.0] Validating spec (attempt ${attempt + 1}/${maxRetries})...`);
      // Swagger 5.0 validate command: swagger validate [spec-path]
      execSync(`"${SWAGGER_CLI_PATH}" validate "${SPEC_PATH}"`, {
        stdio: 'inherit',
        timeout: 30000 // 30s timeout per validation
      });
      console.log('[Swagger 5.0] Spec validation passed');
      return true;
    } catch (err) {
      attempt++;
      console.error(`[Swagger 5.0] Validation failed (attempt ${attempt}): ${err.message}`);
      if (attempt === maxRetries) {
        throw new Error(`Swagger 5.0 spec validation failed after ${maxRetries} attempts: ${err.message}`);
      }
      await new Promise(resolve => setTimeout(resolve, 1000 * attempt)); // Exponential backoff
    }
  }
}

/**
 * Generate static HTML docs from validated spec
 * @returns {string} Path to generated HTML file
 */
async function generateDocs() {
  await fs.mkdir(OUTPUT_DIR, { recursive: true });
  const outputPath = path.join(OUTPUT_DIR, 'swagger-docs.html');

  try {
    console.log('[Swagger 5.0] Generating static HTML docs...');
    // Swagger 5.0 generate command: swagger generate spec -o [output] --defaultScheme https
    execSync(`"${SWAGGER_CLI_PATH}" generate spec -o "${outputPath}" --defaultScheme https`, {
      stdio: 'inherit',
      timeout: 60000 // 60s timeout for doc generation
    });

    // Verify output exists and is non-empty
    const stats = await fs.stat(outputPath);
    if (stats.size === 0) {
      throw new Error('Generated HTML file is empty');
    }
    console.log(`[Swagger 5.0] Docs generated at ${outputPath} (${stats.size} bytes)`);
    return outputPath;
  } catch (err) {
    throw new Error(`Swagger 5.0 doc generation failed: ${err.message}`);
  }
}

/**
 * Run benchmark for Swagger 5.0 doc workflow
 */
async function runBenchmark() {
  const results = [];
  console.log(`[Swagger 5.0] Starting benchmark (${BENCHMARK_ITERATIONS} iterations)...`);

  for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
    const start = process.hrtime.bigint();
    try {
      await validateSpec();
      await generateDocs();
      const end = process.hrtime.bigint();
      const durationMs = Number(end - start) / 1e6; // Convert nanoseconds to milliseconds
      results.push(durationMs);
      console.log(`[Swagger 5.0] Iteration ${i + 1}: ${durationMs.toFixed(2)}ms`);
    } catch (err) {
      console.error(`[Swagger 5.0] Benchmark iteration ${i + 1} failed: ${err.message}`);
      results.push(NaN); // Mark failed iterations
    }
  }

  // Calculate statistics
  const validResults = results.filter(r => !isNaN(r));
  const avg = validResults.reduce((a, b) => a + b, 0) / validResults.length;
  const min = Math.min(...validResults);
  const max = Math.max(...validResults);

  console.log('\n[Swagger 5.0] Benchmark Results:');
  console.log(`Valid iterations: ${validResults.length}/${BENCHMARK_ITERATIONS}`);
  console.log(`Average duration: ${avg.toFixed(2)}ms`);
  console.log(`Min duration: ${min.toFixed(2)}ms`);
  console.log(`Max duration: ${max.toFixed(2)}ms`);
}

// Execute main workflow
(async () => {
  try {
    await validateSpec();
    await generateDocs();
    await runBenchmark();
  } catch (err) {
    console.error(`[Fatal] Swagger 5.0 workflow failed: ${err.message}`);
    process.exit(1);
  }
})();

// postman-doc-gen.js
// Benchmarked on: macOS 14.4, Node 20.12.0, postman-cli 2026.0.0, 16GB RAM, Apple M2 Pro
// Purpose: Automate Postman collection validation, doc generation, and publish to Postman Cloud with Postman 2026.0

const { execSync } = require('child_process');
const fs = require('fs').promises;
const path = require('path');

// Configuration
const POSTMAN_CLI_PATH = path.resolve(__dirname, 'node_modules/.bin/postman');
const COLLECTION_PATH = path.resolve(__dirname, 'postman-collection.json');
const ENV_PATH = path.resolve(__dirname, 'postman-env.json');
const BENCHMARK_ITERATIONS = 100; // Matches Swagger benchmark for parity
const POSTMAN_API_KEY = process.env.POSTMAN_API_KEY; // Must be set in env

if (!POSTMAN_API_KEY) {
  throw new Error('POSTMAN_API_KEY environment variable is required');
}

/**
 * Validate Postman collection using Postman 2026.0 CLI
 * @returns {boolean} True if collection is valid
 */
async function validateCollection() {
  try {
    console.log('[Postman 2026.0] Validating collection...');
    // Postman 2026.0 validate command: postman collection validate [path]
    execSync(`"${POSTMAN_CLI_PATH}" collection validate "${COLLECTION_PATH}"`, {
      stdio: 'inherit',
      timeout: 30000
    });
    console.log('[Postman 2026.0] Collection validation passed');
    return true;
  } catch (err) {
    throw new Error(`Postman 2026.0 collection validation failed: ${err.message}`);
  }
}

/**
 * Generate and publish docs to Postman Cloud
 * @returns {string} Published doc URL
 */
async function publishDocs() {
  try {
    console.log('[Postman 2026.0] Publishing docs to Postman Cloud...');
    // Postman 2026.0 doc publish command: postman collection publish [path] --api-key [key] --env [env-path]
    const output = execSync(`"${POSTMAN_CLI_PATH}" collection publish "${COLLECTION_PATH}" --api-key "${POSTMAN_API_KEY}" --env "${ENV_PATH}" --output json`, {
      encoding: 'utf8',
      timeout: 60000
    });

    const result = JSON.parse(output);
    if (!result.publishedUrl) {
      throw new Error('No published URL returned from Postman API');
    }
    console.log(`[Postman 2026.0] Docs published at ${result.publishedUrl}`);
    return result.publishedUrl;
  } catch (err) {
    throw new Error(`Postman 2026.0 doc publish failed: ${err.message}`);
  }
}

/**
 * Run benchmark for Postman 2026.0 doc workflow
 */
async function runBenchmark() {
  const results = [];
  console.log(`[Postman 2026.0] Starting benchmark (${BENCHMARK_ITERATIONS} iterations)...`);

  for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
    const start = process.hrtime.bigint();
    try {
      await validateCollection();
      await publishDocs();
      const end = process.hrtime.bigint();
      const durationMs = Number(end - start) / 1e6;
      results.push(durationMs);
      console.log(`[Postman 2026.0] Iteration ${i + 1}: ${durationMs.toFixed(2)}ms`);
    } catch (err) {
      console.error(`[Postman 2026.0] Benchmark iteration ${i + 1} failed: ${err.message}`);
      results.push(NaN);
    }
  }

  const validResults = results.filter(r => !isNaN(r));
  const avg = validResults.reduce((a, b) => a + b, 0) / validResults.length;
  const min = Math.min(...validResults);
  const max = Math.max(...validResults);

  console.log('\n[Postman 2026.0] Benchmark Results:');
  console.log(`Valid iterations: ${validResults.length}/${BENCHMARK_ITERATIONS}`);
  console.log(`Average duration: ${avg.toFixed(2)}ms`);
  console.log(`Min duration: ${min.toFixed(2)}ms`);
  console.log(`Max duration: ${max.toFixed(2)}ms`);
}

(async () => {
  try {
    await validateCollection();
    await publishDocs();
    await runBenchmark();
  } catch (err) {
    console.error(`[Fatal] Postman 2026.0 workflow failed: ${err.message}`);
    process.exit(1);
  }
})();

// insomnia-doc-gen.js
// Benchmarked on: macOS 14.4, Node 20.12.0, insomnia-cli 9.0.0, 16GB RAM, Apple M2 Pro
// Purpose: Automate Insomnia workspace export, OpenAPI spec generation, and Git sync for docs with Insomnia 9.0

const { execSync } = require('child_process');
const fs = require('fs').promises;
const path = require('path');
const { exec } = require('child_process');
const util = require('util');
const execPromise = util.promisify(exec);

// Configuration
const INSOMNIA_CLI_PATH = path.resolve(__dirname, 'node_modules/.bin/insomnia');
const WORKSPACE_PATH = path.resolve(__dirname, 'insomnia-workspace');
const EXPORT_PATH = path.resolve(__dirname, 'insomnia-export.yaml');
const GIT_REPO_PATH = path.resolve(__dirname, 'docs-repo');
const BENCHMARK_ITERATIONS = 100; // Parity with other benchmarks

/**
 * Export Insomnia workspace to OpenAPI spec
 * @returns {string} Path to exported spec
 */
async function exportWorkspace() {
  try {
    console.log('[Insomnia 9.0] Exporting workspace to OpenAPI spec...');
    // Insomnia 9.0 export command: insomnia export spec [workspace-path] --output [path] --format openapi3
    const { stdout } = await execPromise(`"${INSOMNIA_CLI_PATH}" export spec "${WORKSPACE_PATH}" --output "${EXPORT_PATH}" --format openapi3`, {
      timeout: 30000
    });
    console.log(`[Insomnia 9.0] Export complete: ${stdout.trim()}`);

    // Verify export exists
    await fs.stat(EXPORT_PATH);
    return EXPORT_PATH;
  } catch (err) {
    throw new Error(`Insomnia 9.0 workspace export failed: ${err.message}`);
  }
}

/**
 * Sync exported spec to Git repository for doc versioning
 * @param {string} specPath Path to exported OpenAPI spec
 */
async function syncToGit(specPath) {
  try {
    console.log('[Insomnia 9.0] Syncing spec to Git repo...');
    // Clone repo if not exists
    if (!await fs.stat(GIT_REPO_PATH).then(() => true).catch(() => false)) {
      await execPromise(`git clone https://github.com/example/api-docs.git "${GIT_REPO_PATH}"`);
    }

    // Copy spec to repo, commit, push
    await fs.cp(specPath, path.join(GIT_REPO_PATH, 'openapi-spec.yaml'));
    await execPromise(`cd "${GIT_REPO_PATH}" && git add . && git commit -m "Update API spec from Insomnia 9.0" && git push origin main`, {
      timeout: 60000
    });
    console.log('[Insomnia 9.0] Git sync complete');
  } catch (err) {
    throw new Error(`Insomnia 9.0 Git sync failed: ${err.message}`);
  }
}

/**
 * Run benchmark for Insomnia 9.0 doc workflow
 */
async function runBenchmark() {
  const results = [];
  console.log(`[Insomnia 9.0] Starting benchmark (${BENCHMARK_ITERATIONS} iterations)...`);

  for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
    const start = process.hrtime.bigint();
    try {
      const specPath = await exportWorkspace();
      await syncToGit(specPath);
      const end = process.hrtime.bigint();
      const durationMs = Number(end - start) / 1e6;
      results.push(durationMs);
      console.log(`[Insomnia 9.0] Iteration ${i + 1}: ${durationMs.toFixed(2)}ms`);
    } catch (err) {
      console.error(`[Insomnia 9.0] Benchmark iteration ${i + 1} failed: ${err.message}`);
      results.push(NaN);
    }
  }

  const validResults = results.filter(r => !isNaN(r));
  const avg = validResults.reduce((a, b) => a + b, 0) / validResults.length;
  const min = Math.min(...validResults);
  const max = Math.max(...validResults);

  console.log('\n[Insomnia 9.0] Benchmark Results:');
  console.log(`Valid iterations: ${validResults.length}/${BENCHMARK_ITERATIONS}`);
  console.log(`Average duration: ${avg.toFixed(2)}ms`);
  console.log(`Min duration: ${min.toFixed(2)}ms`);
  console.log(`Max duration: ${max.toFixed(2)}ms`);
}

(async () => {
  try {
    const specPath = await exportWorkspace();
    await syncToGit(specPath);
    await runBenchmark();
  } catch (err) {
    console.error(`[Fatal] Insomnia 9.0 workflow failed: ${err.message}`);
    process.exit(1);
  }
})();

# GitHub Actions step for Swagger 5.0 doc generation
- name: Generate Swagger Docs
  env:
    SWAGGER_CACHE_PATH: ~/.cache/swagger
  run: |
    npm install swagger-cli@5.0.0
    swagger validate openapi-spec.yaml
    swagger generate spec -o docs/swagger.html --defaultScheme https

# Postman 2026.0 versioned doc publish
postman collection snapshot create "123456" --tag v1.2.0
postman collection publish "123456" --snapshot v1.2.0 --api-key $POSTMAN_API_KEY

# Insomnia 9.0 pre-commit hook for spec validation
#!/bin/bash
insomnia export spec ./insomnia-workspace --output /tmp/spec.yaml --format openapi3
swagger validate /tmp/spec.yaml
if [ $? -ne 0 ]; then
  echo "OpenAPI spec validation failed"
  exit 1
fi