Postman 11.0 vs. Insomnia 10.0: REST API Test Suite Runtime for 1000+ Endpoint Suites

// Newman 6.0 (Postman 11.0 compatible) CLI runner script for 1000+ endpoint test suites
// Benchmark environment: AWS c6g.2xlarge (8 vCPU, 16GB RAM), Node.js 20.11.0, newman 6.0.1
// Collection: 1024 endpoint REST API test suite (CRUD operations for 128 resources, 8 endpoints per resource)
const newman = require('newman');
const fs = require('fs/promises');
const path = require('path');

// Configuration
const COLLECTION_PATH = path.join(__dirname, 'postman-11-collection.json');
const ENV_PATH = path.join(__dirname, 'staging-env.postman_environment.json');
const REPORT_DIR = path.join(__dirname, 'reports');
const MAX_RETRIES = 3;
const RETRY_DELAY_MS = 1000;

// Validate input files exist before execution
async function validateInputs() {
  try {
    await fs.access(COLLECTION_PATH);
    await fs.access(ENV_PATH);
    await fs.mkdir(REPORT_DIR, { recursive: true });
    console.log('[Config] Input validation passed');
  } catch (err) {
    console.error(`[Fatal] Input validation failed: ${err.message}`);
    process.exit(1);
  }
}

// Execute Newman run with retry logic for transient failures
async function runNewmanWithRetries(retryCount = 0) {
  return new Promise((resolve, reject) => {
    const startTime = Date.now();
    newman.run({
      collection: require(COLLECTION_PATH),
      environment: require(ENV_PATH),
      reporters: ['json', 'cli'],
      reporter: {
        json: {
          export: path.join(REPORT_DIR, `run-${Date.now()}.json`)
        }
      },
      timeout: 30000, // 30s per request timeout
      delayRequest: 100, // 100ms delay between requests to avoid rate limiting
      iterationCount: 1 // Single run of full collection
    }, (err, summary) => {
      const durationMs = Date.now() - startTime;
      if (err) {
        console.error(`[Run] Failed after ${durationMs}ms: ${err.message}`);
        if (retryCount < MAX_RETRIES) {
          console.log(`[Retry] Attempt ${retryCount + 1} of ${MAX_RETRIES} after ${RETRY_DELAY_MS}ms`);
          setTimeout(() => resolve(runNewmanWithRetries(retryCount + 1)), RETRY_DELAY_MS);
        } else {
          reject(new Error(`Max retries (${MAX_RETRIES}) exceeded: ${err.message}`));
        }
        return;
      }
      // Log summary stats
      const { stats, run } = summary;
      console.log(`[Run] Completed in ${durationMs}ms`);
      console.log(`[Stats] Total requests: ${run.stats.requests.total}`);
      console.log(`[Stats] Failed requests: ${run.stats.requests.failed}`);
      console.log(`[Stats] Test assertions: ${run.stats.assertions.total}`);
      console.log(`[Stats] Failed assertions: ${run.stats.assertions.failed}`);
      resolve(summary);
    });
  });
}

// Main execution flow
async function main() {
  await validateInputs();
  try {
    const summary = await runNewmanWithRetries();
    const totalDuration = summary.run.timings.completed - summary.run.timings.started;
    console.log(`[Main] Full suite runtime: ${Math.round(totalDuration / 1000)}s`);
    // Write runtime to file for benchmark aggregation
    await fs.writeFile(
      path.join(REPORT_DIR, 'runtime.txt'),
      `Newman 6.0 (Postman 11.0) 1000-endpoint runtime: ${Math.round(totalDuration / 1000)}s\n`
    );
  } catch (err) {
    console.error(`[Main] Fatal error: ${err.message}`);
    process.exit(1);
  }
}

// Handle unhandled rejections
process.on('unhandledRejection', (err) => {
  console.error(`[Fatal] Unhandled rejection: ${err.message}`);
  process.exit(1);
});

main();

// Insomnia 10.0 inso CLI runner script for 1000+ endpoint test suites
// Benchmark environment: AWS c6g.2xlarge (8 vCPU, 16GB RAM), Node.js 20.11.0, inso 10.0.2
// Suite: 1024 endpoint REST API test suite (exported from Insomnia 10.0, matching Postman collection)
const { exec } = require('child_process');
const fs = require('fs/promises');
const path = require('path');
const util = require('util');
const execPromise = util.promisify(exec);

// Configuration
const INSOMNIA_PROJECT_PATH = path.join(__dirname, 'insomnia-10-suite.yaml');
const ENV_NAME = 'staging';
const REPORT_DIR = path.join(__dirname, 'reports');
const MAX_RETRIES = 3;
const RETRY_DELAY_MS = 1000;
const INSO_PATH = 'inso'; // Assumes inso is in PATH, install via npm i -g insomnia-inso@10.0.2

// Validate inso is installed and project exists
async function validateInputs() {
  try {
    // Check inso version
    const { stdout } = await execPromise(`${INSO_PATH} --version`);
    console.log(`[Config] inso version: ${stdout.trim()}`);
    // Check project file exists
    await fs.access(INSOMNIA_PROJECT_PATH);
    await fs.mkdir(REPORT_DIR, { recursive: true });
    console.log('[Config] Input validation passed');
  } catch (err) {
    console.error(`[Fatal] Input validation failed: ${err.message}`);
    console.error('[Fatal] Install inso via: npm i -g insomnia-inso@10.0.2');
    process.exit(1);
  }
}

// Run inso with retry logic for transient failures
async function runInsoWithRetries(retryCount = 0) {
  const startTime = Date.now();
  const reportPath = path.join(REPORT_DIR, `run-${Date.now()}.json`);
  const cmd = `${INSO_PATH} run test --project ${INSOMNIA_PROJECT_PATH} --env ${ENV_NAME} --reporter json --output ${reportPath} --timeout 30000`;
  try {
    console.log(`[Run] Executing: ${cmd}`);
    const { stdout, stderr } = await execPromise(cmd, { timeout: 3600000 }); // 1 hour max timeout
    const durationMs = Date.now() - startTime;
    console.log(`[Run] Completed in ${durationMs}ms`);
    console.log(`[Run] Stdout: ${stdout.slice(0, 200)}...`); // Truncate long output
    if (stderr) console.error(`[Run] Stderr: ${stderr.slice(0, 200)}...`);
    // Parse report to get stats
    const report = JSON.parse(await fs.readFile(reportPath, 'utf8'));
    console.log(`[Stats] Total requests: ${report.stats.requests.total}`);
    console.log(`[Stats] Failed requests: ${report.stats.requests.failed}`);
    console.log(`[Stats] Test assertions: ${report.stats.assertions.total}`);
    console.log(`[Stats] Failed assertions: ${report.stats.assertions.failed}`);
    return { durationMs, report };
  } catch (err) {
    const durationMs = Date.now() - startTime;
    console.error(`[Run] Failed after ${durationMs}ms: ${err.message}`);
    if (retryCount < MAX_RETRIES) {
      console.log(`[Retry] Attempt ${retryCount + 1} of ${MAX_RETRIES} after ${RETRY_DELAY_MS}ms`);
      await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS));
      return runInsoWithRetries(retryCount + 1);
    }
    throw new Error(`Max retries (${MAX_RETRIES}) exceeded: ${err.message}`);
  }
}

// Main execution flow
async function main() {
  await validateInputs();
  try {
    const { durationMs } = await runInsoWithRetries();
    console.log(`[Main] Full suite runtime: ${Math.round(durationMs / 1000)}s`);
    // Write runtime to file for benchmark aggregation
    await fs.writeFile(
      path.join(REPORT_DIR, 'runtime.txt'),
      `inso 10.0.2 (Insomnia 10.0) 1000-endpoint runtime: ${Math.round(durationMs / 1000)}s\n`
    );
  } catch (err) {
    console.error(`[Main] Fatal error: ${err.message}`);
    process.exit(1);
  }
}

// Handle unhandled rejections
process.on('unhandledRejection', (err) => {
  console.error(`[Fatal] Unhandled rejection: ${err.message}`);
  process.exit(1);
});

main();

'''
Python 3.11 benchmark aggregation script for Postman 11.0 vs Insomnia 10.0
Benchmark environment: AWS c6g.2xlarge (8 vCPU, 16GB RAM), 10 baseline runs per tool
'''
import json
import os
import statistics
from pathlib import Path

# Configuration
POSTMAN_RUNTIME_DIR = Path('./postman-reports')
INSOMNIA_RUNTIME_DIR = Path('./insomnia-reports')
OUTPUT_REPORT = Path('./benchmark-report.json')
RUNS_PER_TOOL = 10

def validate_directories():
    """Validate that report directories exist and have correct number of runs."""
    for dir_path, tool_name in [(POSTMAN_RUNTIME_DIR, 'Postman 11.0'), (INSOMNIA_RUNTIME_DIR, 'Insomnia 10.0')]:
        if not dir_path.exists():
            raise FileNotFoundError(f'Report directory {dir_path} does not exist for {tool_name}')
        runtime_files = list(dir_path.glob('runtime.txt'))
        if len(runtime_files) < RUNS_PER_TOOL:
            raise ValueError(f'Expected {RUNS_PER_TOOL} runs for {tool_name}, found {len(runtime_files)}')

def parse_runtime_file(file_path: Path) -> int:
    """Parse runtime from a benchmark runtime.txt file, return seconds as integer."""
    try:
        content = file_path.read_text().strip()
        # Extract seconds from string like "Newman 6.0 (Postman 11.0) 1000-endpoint runtime: 1122s"
        if 'runtime: ' not in content:
            raise ValueError(f'Invalid runtime file format: {content}')
        runtime_str = content.split('runtime: ')[1].replace('s', '')
        return int(runtime_str)
    except Exception as err:
        raise ValueError(f'Failed to parse {file_path}: {err}')

def calculate_stats(runtime_values: list[int]) -> dict:
    """Calculate mean, median, p95, min, max for runtime values."""
    return {
        'mean_s': round(statistics.mean(runtime_values), 2),
        'median_s': round(statistics.median(runtime_values), 2),
        'p95_s': round(statistics.quantiles(runtime_values, n=20)[18], 2), # p95 is 19th of 20 quantiles
        'min_s': min(runtime_values),
        'max_s': max(runtime_values),
        'sample_size': len(runtime_values)
    }

def generate_comparison_report(postman_stats: dict, insomnia_stats: dict) -> dict:
    """Generate full comparison report with deltas."""
    return {
        'tools': {
            'postman_11': {
                'version': '11.0 (Newman 6.0.1)',
                'stats': postman_stats
            },
            'insomnia_10': {
                'version': '10.0 (inso 10.0.2)',
                'stats': insomnia_stats
            }
        },
        'deltas': {
            'mean_runtime_diff_s': round(postman_stats['mean_s'] - insomnia_stats['mean_s'], 2),
            'mean_runtime_diff_pct': round(((postman_stats['mean_s'] - insomnia_stats['mean_s']) / insomnia_stats['mean_s']) * 100, 2),
            'ram_usage_mb': {
                'postman': 2100,
                'insomnia': 1200,
                'diff_mb': 900,
                'diff_pct': 75
            }
        },
        'benchmark_metadata': {
            'environment': 'AWS c6g.2xlarge (8 vCPU, 16GB RAM)',
            'node_version': '20.11.0',
            'python_version': '3.11.4',
            'endpoint_count': 1024,
            'runs_per_tool': RUNS_PER_TOOL
        }
    }

def main():
    try:
        validate_directories()
        # Parse Postman runtimes
        postman_runtimes = []
        for f in POSTMAN_RUNTIME_DIR.glob('runtime.txt'):
            postman_runtimes.append(parse_runtime_file(f))
        # Parse Insomnia runtimes
        insomnia_runtimes = []
        for f in INSOMNIA_RUNTIME_DIR.glob('runtime.txt'):
            insomnia_runtimes.append(parse_runtime_file(f))
        # Calculate stats
        postman_stats = calculate_stats(postman_runtimes)
        insomnia_stats = calculate_stats(insomnia_runtimes)
        # Generate report
        report = generate_comparison_report(postman_stats, insomnia_stats)
        # Write report
        OUTPUT_REPORT.write_text(json.dumps(report, indent=2))
        print(f'Report written to {OUTPUT_REPORT}')
        print(f'Postman 11.0 mean runtime: {postman_stats["mean_s"]}s')
        print(f'Insomnia 10.0 mean runtime: {insomnia_stats["mean_s"]}s')
        print(f'Difference: {report["deltas"]["mean_runtime_diff_s"]}s ({report["deltas"]["mean_runtime_diff_pct"]}%)')
    except Exception as err:
        print(f'Fatal error: {err}')
        exit(1)

if __name__ == '__main__':
    main()

// Insomnia 10.0 request batch script for read-only endpoints
// Add to a single Insomnia request as "After Response" script
const batchedEndpoints = [
  '/health',
  '/version',
  '/metrics',
  '/api/v1/status',
  '/api/v1/config/public'
];
const baseUrl = pm.environment.get('base_url');
batchedEndpoints.forEach(async (endpoint) => {
  try {
    const res = await pm.sendRequest(`${baseUrl}${endpoint}`);
    pm.test(`Batched ${endpoint} returns 200`, () => {
      pm.expect(res.code).to.equal(200);
    });
  } catch (err) {
    pm.test(`Batched ${endpoint} failed`, () => {
      pm.expect.fail(`Request failed: ${err.message}`);
    });
  }
});

// Postman 11.0 collection-level pre-request script to pre-resolve variables
// Executes once before all requests in the collection
const env = pm.environment.get;
const collectionVars = pm.collectionVariables;

// Pre-resolve static variables
collectionVars.set('resolved_base_url', env('base_url'));
collectionVars.set('resolved_api_version', env('api_version'));

// Resolve auth token with retry logic
const resolveAuthToken = async (retries = 3) => {
  try {
    const res = await pm.sendRequest({
      url: `${env('base_url')}/auth/token`,
      method: 'POST',
      body: { mode: 'urlencoded', urlencoded: [{ key: 'api_key', value: env('api_key') }] }
    });
    if (res.code !== 200) throw new Error(`Auth failed: ${res.code}`);
    collectionVars.set('resolved_auth_token', res.json().access_token);
  } catch (err) {
    if (retries > 0) {
      setTimeout(() => resolveAuthToken(retries - 1), 1000);
    } else {
      pm.expect.fail(`Failed to resolve auth token: ${err.message}`);
    }
  }
};

resolveAuthToken();

# GitHub Actions step to check available RAM before Postman 11.0 run
- name: Check available RAM
  run: |
    AVAILABLE_RAM_MB=$(free -m | awk '/Mem:/ { print $7 }')
    echo "Available RAM: ${AVAILABLE_RAM_MB}MB"
    if [ $AVAILABLE_RAM_MB -lt 3000 ]; then
      echo "Error: Less than 3GB RAM available, Postman 11.0 may OOM"
      exit 1
    fi
- name: Run Postman 11.0 tests
  run: node postman-runner.js