Benchmark: Tailscale 1.60 vs. OpenVPN 2.6 for Remote Developer VPN Speed

#!/usr/bin/env python3
"""
Automated VPN benchmark runner for Tailscale 1.60 vs OpenVPN 2.6
Compares TCP/UDP throughput, latency, and CPU usage across 12-node cluster
Requires: iperf3, tailscale CLI, openvpn CLI, psutil, pandas
"""

import subprocess
import time
import json
import logging
from typing import Dict, List, Optional
import psutil
import pandas as pd

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Benchmark constants
TAILSCALE_SUBNET = "100.64.0.0/10"
OPENVPN_SUBNET = "10.8.0.0/24"
TEST_DURATION = 60  # seconds per iperf3 run
RUN_COUNT = 10  # number of repetitions per test
CONFIDENCE = 0.95

class VPNBenchmarker:
    def __init__(self, vpn_type: str):
        self.vpn_type = vpn_type
        self.results = []

        if vpn_type not in ["tailscale", "openvpn"]:
            raise ValueError(f"Unsupported VPN type: {vpn_type}")

    def check_vpn_status(self) -> bool:
        """Verify VPN is active and reachable"""
        try:
            if self.vpn_type == "tailscale":
                # Check tailscale status via CLI
                result = subprocess.run(
                    ["tailscale", "status", "--json"],
                    capture_output=True,
                    text=True,
                    check=True
                )
                status = json.loads(result.stdout)
                return status.get("BackendState") == "Running"
            else:
                # Check openvpn process is running
                for proc in psutil.process_iter(["name"]):
                    if proc.info["name"] == "openvpn":
                        return True
                return False
        except (subprocess.CalledProcessError, json.JSONDecodeError) as e:
            logger.error(f"VPN status check failed: {e}")
            return False

    def run_iperf3_test(self, server_ip: str, protocol: str = "tcp") -> Optional[Dict]:
        """Run single iperf3 test and return parsed results"""
        if protocol not in ["tcp", "udp"]:
            raise ValueError(f"Unsupported protocol: {protocol}")

        # Build iperf3 command
        iperf_cmd = [
            "iperf3",
            "-c", server_ip,
            "-t", str(TEST_DURATION),
            "-J",  # JSON output
            "-p", "5201" if protocol == "tcp" else "5202"
        ]
        if protocol == "udp":
            iperf_cmd.extend(["-u", "-b", "1G"])

        try:
            logger.info(f"Running {protocol.upper()} test to {server_ip} via {self.vpn_type}")
            result = subprocess.run(
                iperf_cmd,
                capture_output=True,
                text=True,
                check=True
            )
            return json.loads(result.stdout)
        except subprocess.CalledProcessError as e:
            logger.error(f"iperf3 test failed: {e.stderr}")
            return None

    def calculate_metrics(self, raw_results: List[Dict]) -> Dict:
        """Calculate mean, std dev, confidence interval for results"""
        # Extract throughput values
        throughputs = []
        for res in raw_results:
            if "end" in res and "sum_received" in res["end"]:
                bps = res["end"]["sum_received"]["bits_per_second"]
                throughputs.append(bps / 1_000_000)  # Convert to Mbps

        if not throughputs:
            return {}

        df = pd.DataFrame(throughputs, columns=["mbps"])
        mean = df["mbps"].mean()
        std = df["mbps"].std()
        # Simplified 95% CI calculation
        ci = 1.96 * (std / (len(throughputs) ** 0.5))

        return {
            "mean_mbps": round(mean, 2),
            "std_mbps": round(std, 2),
            "ci_95": round(ci, 2),
            "min_mbps": round(df["mbps"].min(), 2),
            "max_mbps": round(df["mbps"].max(), 2)
        }

    def run_full_benchmark(self, server_ips: List[str]):
        """Execute full benchmark suite for this VPN"""
        if not self.check_vpn_status():
            raise RuntimeError(f"{self.vpn_type} is not running or reachable")

        for protocol in ["tcp", "udp"]:
            protocol_results = []
            for run in range(RUN_COUNT):
                logger.info(f"Run {run+1}/{RUN_COUNT} for {protocol.upper()}")
                # Rotate through server IPs
                server_ip = server_ips[run % len(server_ips)]
                res = self.run_iperf3_test(server_ip, protocol)
                if res:
                    protocol_results.append(res)
                time.sleep(2)  # Cooldown between runs

            metrics = self.calculate_metrics(protocol_results)
            self.results.append({
                "vpn": self.vpn_type,
                "protocol": protocol,
                "metrics": metrics
            })
            logger.info(f"{self.vpn_type} {protocol} results: {metrics}")

if __name__ == "__main__":
    # Example usage: Run benchmarks for both VPNs
    # Note: Assumes VPNs are pre-configured and running
    tailscale_servers = ["100.64.0.1", "100.64.0.2", "100.64.0.3"]  # Tailscale server IPs
    openvpn_servers = ["10.8.0.1", "10.8.0.2", "10.8.0.3"]  # OpenVPN server IPs

    for vpn in ["tailscale", "openvpn"]:
        try:
            bench = VPNBenchmarker(vpn)
            servers = tailscale_servers if vpn == "tailscale" else openvpn_servers
            bench.run_full_benchmark(servers)
            # Save results to JSON
            with open(f"{vpn}_benchmark_results.json", "w") as f:
                json.dump(bench.results, f, indent=2)
            logger.info(f"Saved {vpn} results to {vpn}_benchmark_results.json")
        except RuntimeError as e:
            logger.error(f"Benchmark failed for {vpn}: {e}")
            continue

#!/usr/bin/env bash
"""
OpenVPN 2.6 Automated Setup Script for Benchmarking
Generates server/client configs, creates PKI, starts service
Tested on Ubuntu 22.04 LTS, OpenVPN 2.6.6
"""

set -euo pipefail  # Exit on error, undefined var, pipe failure
IFS=$'\n\t'

# Configuration variables
OPENVPN_DIR="/etc/openvpn"
PKI_DIR="${OPENVPN_DIR}/pki"
SERVER_CONF="${OPENVPN_DIR}/server.conf"
CLIENT_CONF_TEMPLATE="${OPENVPN_DIR}/client.template"
VPN_SUBNET="10.8.0.0"
VPN_MASK="255.255.255.0"
SERVER_PORT=1194
PROTOCOL="udp"
CRYPTO="AES-256-GCM"  # OpenVPN 2.6 supports GCM, default is CBC
DH_PARAM_SIZE=2048

log() {
    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" >&2
}

error() {
    log "ERROR: $*"
    exit 1
}

check_root() {
    if [[ $EUID -ne 0 ]]; then
        error "This script must be run as root"
    fi
}

install_dependencies() {
    log "Installing OpenVPN 2.6 and dependencies"
    apt-get update -y
    apt-get install -y openvpn 2.6.6-1ubuntu1 easy-rsa iptables-persistent
    # Verify version
    INSTALLED_VER=$(openvpn --version | head -1 | awk '{print $2}')
    if [[ "${INSTALLED_VER}" != "2.6.6" ]]; then
        error "Expected OpenVPN 2.6.6, got ${INSTALLED_VER}"
    fi
}

setup_pki() {
    log "Setting up PKI with easy-rsa"
    mkdir -p "${PKI_DIR}"
    cd "${PKI_DIR}"
    # Initialize easy-rsa
    easyrsa init-pki
    # Build CA (non-interactive)
    easyrsa --batch build-ca nopass
    # Generate server certificate and key
    easyrsa --batch build-server-full server nopass
    # Generate Diffie-Hellman parameters
    log "Generating ${DH_PARAM_SIZE}-bit DH params (this may take a while)"
    easyrsa gen-dh "${DH_PARAM_SIZE}"
    # Generate client certificate (for benchmark client)
    easyrsa --batch build-client-full benchmark-client nopass
    # Copy server files to openvpn dir
    cp pki/ca.crt pki/private/server.key pki/issued/server.crt pki/dh.pem "${OPENVPN_DIR}/"
    # Copy client files
    mkdir -p "${OPENVPN_DIR}/client"
    cp pki/ca.crt pki/issued/benchmark-client.crt pki/private/benchmark-client.key "${OPENVPN_DIR}/client/"
}

generate_server_config() {
    log "Generating OpenVPN 2.6 server config"
    cat > "${SERVER_CONF}" << EOF
# OpenVPN 2.6 Server Config for Benchmarking
port ${SERVER_PORT}
proto ${PROTOCOL}
dev tun
ca ca.crt
cert server.crt
key server.key
dh dh.pem
server ${VPN_SUBNET} ${VPN_MASK}
ifconfig-pool-persist ipp.txt
push "redirect-gateway def1 bypass-dhcp"
push "dhcp-option DNS 8.8.8.8"
push "dhcp-option DNS 8.8.4.4"
keepalive 10 120
cipher ${CRYPTO}
auth SHA256
user nobody
group nogroup
persist-key
persist-tun
status openvpn-status.log
log-append /var/log/openvpn.log
verb 3
explicit-exit-notify 1
EOF
    log "Server config written to ${SERVER_CONF}"
}

generate_client_config() {
    log "Generating client config template"
    cat > "${CLIENT_CONF_TEMPLATE}" << EOF
# OpenVPN 2.6 Client Config Template
client
dev tun
proto ${PROTOCOL}
remote [SERVER_IP] ${SERVER_PORT}
resolv-retry infinite
nobind
user nobody
group nogroup
persist-key
persist-tun
ca ca.crt
cert benchmark-client.crt
key benchmark-client.key
cipher ${CRYPTO}
auth SHA256
verb 3
EOF
    log "Client template written to ${CLIENT_CONF_TEMPLATE}"
}

configure_iptables() {
    log "Configuring iptables for NAT"
    # Enable IP forwarding
    echo 1 > /proc/sys/net/ipv4/ip_forward
    echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
    # NAT rule for VPN subnet
    iptables -t nat -A POSTROUTING -s ${VPN_SUBNET}/24 -o eth0 -j MASQUERADE
    # Persist iptables rules
    iptables-save > /etc/iptables/rules.v4
}

start_openvpn() {
    log "Starting OpenVPN service"
    systemctl enable openvpn@server
    systemctl start openvpn@server
    # Verify service is running
    if ! systemctl is-active --quiet openvpn@server; then
        error "OpenVPN service failed to start. Check /var/log/openvpn.log"
    fi
    log "OpenVPN 2.6 server running on port ${SERVER_PORT}/${PROTOCOL}"
}

main() {
    check_root
    install_dependencies
    setup_pki
    generate_server_config
    generate_client_config
    configure_iptables
    start_openvpn
    log "OpenVPN 2.6 setup complete. Client configs in ${OPENVPN_DIR}/client/"
}

main "$@"

#!/usr/bin/env python3
"""
Tailscale 1.60 vs OpenVPN 2.6 Benchmark Analyzer
Parses raw iperf3 results, calculates statistical significance, generates summary
Requires: pandas, scipy, matplotlib
"""

import json
import argparse
from typing import Dict, List
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

class BenchmarkAnalyzer:
    def __init__(self, tailscale_file: str, openvpn_file: str):
        self.tailscale_data = self.load_results(tailscale_file)
        self.openvpn_data = self.load_results(openvpn_file)
        self.comparison_results = []

    def load_results(self, filepath: str) -> List[Dict]:
        """Load and validate benchmark JSON results"""
        try:
            with open(filepath, "r") as f:
                data = json.load(f)
            if not isinstance(data, list):
                raise ValueError(f"Expected list in {filepath}, got {type(data)}")
            logger.info(f"Loaded {len(data)} results from {filepath}")
            return data
        except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
            logger.error(f"Failed to load {filepath}: {e}")
            return []

    def extract_throughput(self, data: List[Dict], protocol: str) -> List[float]:
        """Extract throughput values for a specific protocol"""
        throughputs = []
        for entry in data:
            if entry.get("protocol") != protocol:
                continue
            metrics = entry.get("metrics", {})
            if "mean_mbps" in metrics:
                throughputs.append(metrics["mean_mbps"])
        return throughputs

    def calculate_stat_sig(self, ts_data: List[float], ovpn_data: List[float]) -> Dict:
        """Calculate t-test and effect size between two datasets"""
        if not ts_data or not ovpn_data:
            return {}
        # Independent t-test (assuming unequal variance)
        t_stat, p_value = stats.ttest_ind(ts_data, ovpn_data, equal_var=False)
        # Cohen's d effect size
        pooled_std = ((len(ts_data)-1)*pd.Series(ts_data).std()**2 + 
                     (len(ovpn_data)-1)*pd.Series(ovpn_data).std()**2) / (len(ts_data)+len(ovpn_data)-2)
        pooled_std = pooled_std ** 0.5
        cohen_d = (pd.Series(ts_data).mean() - pd.Series(ovpn_data).mean()) / pooled_std
        return {
            "t_stat": round(t_stat, 3),
            "p_value": round(p_value, 4),
            "cohen_d": round(cohen_d, 3),
            "ts_mean": round(pd.Series(ts_data).mean(), 2),
            "ovpn_mean": round(pd.Series(ovpn_data).mean(), 2),
            "ts_std": round(pd.Series(ts_data).std(), 2),
            "ovpn_std": round(pd.Series(ovpn_data).std(), 2)
        }

    def run_comparison(self):
        """Compare Tailscale and OpenVPN across protocols"""
        for protocol in ["tcp", "udp"]:
            ts_throughput = self.extract_throughput(self.tailscale_data, protocol)
            ovpn_throughput = self.extract_throughput(self.openvpn_data, protocol)
            if not ts_throughput or not ovpn_throughput:
                logger.warning(f"No data for {protocol.upper()}, skipping")
                continue
            sig_results = self.calculate_stat_sig(ts_throughput, ovpn_throughput)
            self.comparison_results.append({
                "protocol": protocol,
                "stats": sig_results
            })
            logger.info(f"{protocol.upper()} Comparison: Tailscale {sig_results['ts_mean']} Mbps vs OpenVPN {sig_results['ovpn_mean']} Mbps (p={sig_results['p_value']})")

    def generate_plot(self, output_path: str = "benchmark_comparison.png"):
        """Generate bar plot comparing throughput"""
        if not self.comparison_results:
            logger.error("No comparison results to plot")
            return
        protocols = [r["protocol"].upper() for r in self.comparison_results]
        ts_means = [r["stats"]["ts_mean"] for r in self.comparison_results]
        ovpn_means = [r["stats"]["ovpn_mean"] for r in self.comparison_results]
        ts_err = [r["stats"]["ts_std"] for r in self.comparison_results]
        ovpn_err = [r["stats"]["ovpn_std"] for r in self.comparison_results]

        x = range(len(protocols))
        width = 0.35

        fig, ax = plt.subplots(figsize=(10, 6))
        ax.bar([i - width/2 for i in x], ts_means, width, label="Tailscale 1.60", yerr=ts_err, capsize=5)
        ax.bar([i + width/2 for i in x], ovpn_means, width, label="OpenVPN 2.6", yerr=ovpn_err, capsize=5)

        ax.set_xlabel("Protocol")
        ax.set_ylabel("Throughput (Mbps)")
        ax.set_title("Tailscale 1.60 vs OpenVPN 2.6 Throughput Comparison")
        ax.set_xticks(x)
        ax.set_xticklabels(protocols)
        ax.legend()
        ax.grid(axis="y", linestyle="--", alpha=0.7)

        plt.tight_layout()
        plt.savefig(output_path)
        logger.info(f"Plot saved to {output_path}")

    def generate_report(self, output_path: str = "benchmark_report.md"):
        """Generate Markdown report with results"""
        with open(output_path, "w") as f:
            f.write("# Tailscale 1.60 vs OpenVPN 2.6 Benchmark Report\n\n")
            for res in self.comparison_results:
                proto = res["protocol"].upper()
                stats = res["stats"]
                f.write(f"## {proto} Throughput\n")
                f.write(f"- Tailscale 1.60 Mean: {stats['ts_mean']} Mbps (σ={stats['ts_std']})\n")
                f.write(f"- OpenVPN 2.6 Mean: {stats['ovpn_mean']} Mbps (σ={stats['ovpn_std']})\n")
                f.write(f"- T-Statistic: {stats['t_stat']}\n")
                f.write(f"- P-Value: {stats['p_value']}\n")
                f.write(f"- Cohen's d (Effect Size): {stats['cohen_d']}\n")
                f.write(f"- **Result: {'Statistically Significant' if stats['p_value'] < 0.05 else 'Not Statistically Significant'}**\n\n")
        logger.info(f"Report saved to {output_path}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Analyze VPN benchmark results")
    parser.add_argument("--tailscale", required=True, help="Path to Tailscale results JSON")
    parser.add_argument("--openvpn", required=True, help="Path to OpenVPN results JSON")
    parser.add_argument("--plot", default="benchmark_comparison.png", help="Output plot path")
    parser.add_argument("--report", default="benchmark_report.md", help="Output report path")
    args = parser.parse_args()

    analyzer = BenchmarkAnalyzer(args.tailscale, args.openvpn)
    analyzer.run_comparison()
    analyzer.generate_plot(args.plot)
    analyzer.generate_report(args.report)
    logger.info("Analysis complete")