VS Code 2.0 vs. Zed 0.150: Startup Time and Memory Usage Benchmark

# startup_benchmark.py
# Measures cold and warm startup times for VS Code 2.0 and Zed 0.150
# Dependencies: Python 3.11+, psutil 5.9.6+, time, subprocess, statistics
# Run: python3 startup_benchmark.py --iterations 1000 --os macos

import argparse
import subprocess
import time
import statistics
import psutil
import sys
import logging
from typing import List, Dict, Tuple

# Configure logging for debug output
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)

def get_editor_path(editor: str, os_type: str) -> str:
    """Resolve full path to editor executable based on OS and editor name."""
    if editor == "vscode":
        if os_type == "macos":
            return "/Applications/Visual Studio Code.app/Contents/MacOS/Electron"
        elif os_type == "linux":
            return "/usr/bin/code"
        elif os_type == "windows":
            return r"C:\Program Files\Microsoft VS Code\Code.exe"
        else:
            raise ValueError(f"Unsupported OS: {os_type}")
    elif editor == "zed":
        if os_type == "macos":
            return "/Applications/Zed.app/Contents/MacOS/zed"
        elif os_type == "linux":
            return "/usr/bin/zed"
        elif os_type == "windows":
            return r"C:\Program Files\Zed\zed.exe"
        else:
            raise ValueError(f"Unsupported OS: {os_type}")
    else:
        raise ValueError(f"Unsupported editor: {editor}")

def measure_startup(editor_path: str, warm: bool = False) -> float:
    """Measure single startup time in milliseconds.
    Args:
        editor_path: Full path to editor executable
        warm: If True, measure warm start (editor already in cache)
    Returns:
        Startup time in ms, or -1 if failed
    """
    try:
        # Clear file cache for cold start if needed
        if not warm and sys.platform == "darwin":
            subprocess.run(["purge"], check=True, capture_output=True)
        elif not warm and sys.platform == "linux":
            subprocess.run(["sync", "&&", "echo 3 > /proc/sys/vm/drop_caches"], 
                          shell=True, check=True, capture_output=True)

        start = time.perf_counter()
        # Launch editor with --wait flag to block until exit, then immediately kill
        proc = subprocess.Popen(
            [editor_path, "--wait", "--new-window"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        # Wait 2 seconds for editor to initialize, then terminate
        time.sleep(2)
        proc.terminate()
        proc.wait(timeout=5)
        end = time.perf_counter()
        return (end - start) * 1000  # Convert to ms
    except Exception as e:
        logging.error(f"Failed to measure startup for {editor_path}: {e}")
        return -1.0

def run_benchmark(editor: str, os_type: str, iterations: int) -> Dict[str, List[float]]:
    """Run full benchmark suite for an editor."""
    editor_path = get_editor_path(editor, os_type)
    logging.info(f"Starting benchmark for {editor} ({editor_path}) with {iterations} iterations")

    cold_times: List[float] = []
    warm_times: List[float] = []

    for i in range(iterations):
        # Measure cold start
        cold = measure_startup(editor_path, warm=False)
        if cold > 0:
            cold_times.append(cold)
        # Measure warm start
        warm = measure_startup(editor_path, warm=True)
        if warm > 0:
            warm_times.append(warm)

        if (i + 1) % 100 == 0:
            logging.info(f"Completed {i + 1}/{iterations} iterations for {editor}")

    return {
        "cold": cold_times,
        "warm": warm_times
    }

def calculate_stats(times: List[float]) -> Tuple[float, float, float]:
    """Calculate mean, median, p99 for a list of times."""
    if not times:
        return 0.0, 0.0, 0.0
    return (
        statistics.mean(times),
        statistics.median(times),
        statistics.quantiles(times, n=100)[98]  # p99
    )

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Editor Startup Benchmark")
    parser.add_argument("--iterations", type=int, default=1000, help="Number of test iterations")
    parser.add_argument("--os", type=str, default="macos", choices=["macos", "linux", "windows"])
    args = parser.parse_args()

    # Verify editors are installed
    for editor in ["vscode", "zed"]:
        try:
            get_editor_path(editor, args.os)
        except ValueError as e:
            logging.error(f"Editor not found: {e}")
            sys.exit(1)

    # Run benchmarks
    results = {}
    for editor in ["vscode", "zed"]:
        results[editor] = run_benchmark(editor, args.os, args.iterations)

    # Print results
    print("\n=== Benchmark Results ===")
    for editor, data in results.items():
        cold_mean, cold_median, cold_p99 = calculate_stats(data["cold"])
        warm_mean, warm_median, warm_p99 = calculate_stats(data["warm"])
        print(f"\n{editor.upper()} 2.0" if editor == "vscode" else f"\n{editor.upper()} 0.150")
        print(f"Cold Start: Mean={cold_mean:.2f}ms, Median={cold_median:.2f}ms, p99={cold_p99:.2f}ms")
        print(f"Warm Start: Mean={warm_mean:.2f}ms, Median={warm_median:.2f}ms, p99={warm_p99:.2f}ms")

# memory_benchmark.py
# Tracks memory usage of VS Code 2.0 and Zed 0.150 under varying workspace loads
# Dependencies: Python 3.11+, psutil 5.9.6+, time, subprocess, pandas
# Run: python3 memory_benchmark.py --workspace rust-large --duration 300

import argparse
import subprocess
import time
import psutil
import pandas as pd
import logging
import sys
from typing import List, Dict, Optional

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)

# Editor process names for cross-platform detection
EDITOR_PROCESSES = {
    "vscode": {
        "macos": "Electron",
        "linux": "code",
        "windows": "Code.exe"
    },
    "zed": {
        "macos": "zed",
        "linux": "zed",
        "windows": "zed.exe"
    }
}

WORKSPACE_PATHS = {
    "rust-small": "/Users/developer/workspaces/rust-http-server",
    "rust-medium": "/Users/developer/workspaces/rust-web-framework",
    "rust-large": "/Users/developer/workspaces/rust-compiler",
    "ts-small": "/Users/developer/workspaces/ts-todo-app",
    "ts-large": "/Users/developer/workspaces/ts-monorepo"
}

def launch_editor(editor: str, os_type: str, workspace: str) -> Optional[subprocess.Popen]:
    """Launch editor with specified workspace, return process handle."""
    try:
        if editor == "vscode":
            if os_type == "macos":
                editor_path = "/Applications/Visual Studio Code.app/Contents/MacOS/Electron"
                return subprocess.Popen(
                    [editor_path, workspace],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE
                )
            elif os_type == "linux":
                editor_path = "/usr/bin/code"
                return subprocess.Popen(
                    [editor_path, "--folder-uri", f"file://{workspace}"],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE
                )
        elif editor == "zed":
            editor_path = "/Applications/Zed.app/Contents/MacOS/zed" if os_type == "macos" else "/usr/bin/zed"
            return subprocess.Popen(
                [editor_path, workspace],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
        logging.info(f"Launched {editor} with workspace {workspace}")
        return None
    except Exception as e:
        logging.error(f"Failed to launch {editor}: {e}")
        return None

def get_memory_usage(process_name: str, os_type: str) -> Dict[str, float]:
    """Get RSS and virtual memory usage for all processes matching editor name."""
    rss_total = 0.0  # in MB
    vms_total = 0.0
    process_count = 0

    for proc in psutil.process_iter(["name", "memory_info"]):
        try:
            if proc.info["name"] == process_name:
                mem = proc.info["memory_info"]
                rss_total += mem.rss / (1024 * 1024)  # Convert bytes to MB
                vms_total += mem.vms / (1024 * 1024)
                process_count += 1
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            continue

    return {
        "rss_mb": rss_total,
        "vms_mb": vms_total,
        "process_count": process_count
    }

def run_memory_benchmark(editor: str, os_type: str, workspace: str, duration: int) -> pd.DataFrame:
    """Run memory benchmark for editor over specified duration (seconds)."""
    process_name = EDITOR_PROCESSES[editor][os_type]
    # Launch editor
    proc = launch_editor(editor, os_type, WORKSPACE_PATHS[workspace])
    if not proc:
        logging.error(f"Failed to launch {editor}, aborting benchmark")
        return pd.DataFrame()

    # Wait for editor to fully load workspace
    time.sleep(10)

    samples = []
    start_time = time.time()
    logging.info(f"Starting memory benchmark for {editor} ({workspace}) for {duration}s")

    while time.time() - start_time < duration:
        mem_data = get_memory_usage(process_name, os_type)
        samples.append({
            "timestamp": time.time() - start_time,
            "rss_mb": mem_data["rss_mb"],
            "vms_mb": mem_data["vms_mb"],
            "process_count": mem_data["process_count"]
        })
        time.sleep(1)  # Sample every second

    # Terminate editor
    proc.terminate()
    proc.wait(timeout=10)
    return pd.DataFrame(samples)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Editor Memory Usage Benchmark")
    parser.add_argument("--editor", type=str, required=True, choices=["vscode", "zed"])
    parser.add_argument("--os", type=str, default="macos", choices=["macos", "linux", "windows"])
    parser.add_argument("--workspace", type=str, required=True, choices=list(WORKSPACE_PATHS.keys()))
    parser.add_argument("--duration", type=int, default=300, help="Benchmark duration in seconds")
    args = parser.parse_args()

    # Verify workspace exists
    import os
    if not os.path.exists(WORKSPACE_PATHS[args.workspace]):
        logging.error(f"Workspace path not found: {WORKSPACE_PATHS[args.workspace]}")
        sys.exit(1)

    # Run benchmark
    df = run_memory_benchmark(args.editor, args.os, args.workspace, args.duration)
    if not df.empty:
        # Save results to CSV
        output_file = f"{args.editor}_{args.workspace}_memory.csv"
        df.to_csv(output_file, index=False)
        logging.info(f"Saved results to {output_file}")
        # Print summary stats
        print("\n=== Memory Usage Summary ===")
        print(f"Editor: {args.editor}")
        print(f"Workspace: {args.workspace}")
        print(f"Duration: {args.duration}s")
        print(f"Average RSS: {df['rss_mb'].mean():.2f} MB")
        print(f"Peak RSS: {df['rss_mb'].max():.2f} MB")
        print(f"Average Process Count: {df['process_count'].mean():.2f}")

# aggregate_results.py
# Aggregates startup and memory benchmark results for VS Code 2.0 and Zed 0.150
# Dependencies: Python 3.11+, pandas 2.1+, matplotlib 3.8+, seaborn 0.13+
# Run: python3 aggregate_results.py --startup-csv startup_results.csv --memory-dir ./memory_data

import argparse
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import logging
import sys
import os
from typing import List, Dict
from pathlib import Path

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)

def load_startup_data(csv_path: str) -> pd.DataFrame:
    """Load startup benchmark CSV into DataFrame."""
    try:
        df = pd.read_csv(csv_path)
        required_cols = ["editor", "cold_mean", "cold_median", "cold_p99", "warm_mean", "warm_median", "warm_p99"]
        missing = [col for col in required_cols if col not in df.columns]
        if missing:
            raise ValueError(f"Missing required columns in startup CSV: {missing}")
        logging.info(f"Loaded startup data from {csv_path}: {len(df)} rows")
        return df
    except Exception as e:
        logging.error(f"Failed to load startup data: {e}")
        sys.exit(1)

def load_memory_data(dir_path: str) -> Dict[str, pd.DataFrame]:
    """Load all memory CSV files from directory into dict of DataFrames."""
    memory_data = {}
    try:
        for csv_file in Path(dir_path).glob("*.csv"):
            # Parse filename to get editor and workspace: e.g., vscode_rust-large_memory.csv
            parts = csv_file.stem.split("_")
            if len(parts) >= 3 and parts[1] in ["vscode", "zed"]:
                editor = parts[1]
                workspace = parts[0]
                df = pd.read_csv(csv_file)
                key = f"{editor}_{workspace}"
                memory_data[key] = df
                logging.info(f"Loaded memory data for {key}: {len(df)} rows")
        if not memory_data:
            raise ValueError(f"No valid memory CSV files found in {dir_path}")
        return memory_data
    except Exception as e:
        logging.error(f"Failed to load memory data: {e}")
        sys.exit(1)

def generate_startup_plot(startup_df: pd.DataFrame, output_path: str) -> None:
    """Generate bar plot comparing startup times."""
    plt.figure(figsize=(10, 6))
    bar_width = 0.35
    editors = startup_df["editor"].unique()
    x = range(len(editors))

    # Plot cold start means
    cold_means = startup_df[startup_df["editor"].isin(editors)]["cold_mean"]
    plt.bar(x, cold_means, bar_width, label="Cold Start Mean (ms)", color="#007ACC")

    # Plot warm start means
    warm_means = startup_df[startup_df["editor"].isin(editors)]["warm_mean"]
    plt.bar([i + bar_width for i in x], warm_means, bar_width, label="Warm Start Mean (ms)", color="#D4A72C")

    plt.xlabel("Editor")
    plt.ylabel("Startup Time (ms)")
    plt.title("VS Code 2.0 vs Zed 0.150: Startup Time Comparison")
    plt.xticks([i + bar_width/2 for i in x], editors)
    plt.legend()
    plt.tight_layout()
    plt.savefig(output_path)
    logging.info(f"Saved startup plot to {output_path}")

def generate_memory_plot(memory_data: Dict[str, pd.DataFrame], output_path: str) -> None:
    """Generate line plot comparing memory usage over time."""
    plt.figure(figsize=(12, 8))
    for key, df in memory_data.items():
        editor, workspace = key.split("_")
        label = f"{editor} ({workspace})"
        color = "#007ACC" if editor == "vscode" else "#D4A72C"
        plt.plot(df["timestamp"], df["rss_mb"], label=label, color=color, alpha=0.7)

    plt.xlabel("Time (seconds)")
    plt.ylabel("RSS Memory (MB)")
    plt.title("Memory Usage Over Time: VS Code 2.0 vs Zed 0.150")
    plt.legend()
    plt.tight_layout()
    plt.savefig(output_path)
    logging.info(f"Saved memory plot to {output_path}")

def generate_summary_table(startup_df: pd.DataFrame, memory_data: Dict[str, pd.DataFrame]) -> pd.DataFrame:
    """Generate summary table of all metrics."""
    summary = []
    for _, row in startup_df.iterrows():
        editor = row["editor"]
        # Get average memory for largest workspace
        large_workspace_key = f"{editor}_rust-large"
        if large_workspace_key in memory_data:
            avg_rss = memory_data[large_workspace_key]["rss_mb"].mean()
        else:
            avg_rss = -1
        summary.append({
            "Editor": editor,
            "Cold Start Mean (ms)": round(row["cold_mean"], 2),
            "Warm Start Mean (ms)": round(row["warm_mean"], 2),
            "Avg RSS (rust-large) (MB)": round(avg_rss, 2)
        })
    return pd.DataFrame(summary)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Aggregate Editor Benchmark Results")
    parser.add_argument("--startup-csv", type=str, required=True, help="Path to startup benchmark CSV")
    parser.add_argument("--memory-dir", type=str, required=True, help="Directory containing memory benchmark CSVs")
    parser.add_argument("--output-dir", type=str, default="./benchmark_output", help="Output directory for plots and tables")
    args = parser.parse_args()

    # Create output directory if not exists
    os.makedirs(args.output_dir, exist_ok=True)

    # Load data
    startup_df = load_startup_data(args.startup_csv)
    memory_data = load_memory_data(args.memory_dir)

    # Generate plots
    generate_startup_plot(startup_df, os.path.join(args.output_dir, "startup_comparison.png"))
    generate_memory_plot(memory_data, os.path.join(args.output_dir, "memory_comparison.png"))

    # Generate summary table
    summary_df = generate_summary_table(startup_df, memory_data)
    summary_path = os.path.join(args.output_dir, "benchmark_summary.csv")
    summary_df.to_csv(summary_path, index=False)
    logging.info(f"Saved summary table to {summary_path}")

    # Print summary
    print("\n=== Benchmark Summary ===")
    print(summary_df.to_string(index=False))