Postmortem: How a WireGuard Misconfiguration and OpenVPN 2.6 Conflict Caused a Remote Access Outage

#!/usr/bin/env python3
"""Route Conflict Detector for Hybrid WireGuard/OpenVPN Setups
Scans kernel routing tables for overlapping subnet entries between
WireGuard peers and OpenVPN tunnels, flags conflicts per OpenVPN 2.6+ metric changes.
"""

import sys
import logging
from typing import Dict, List, Tuple
from pyroute2 import IPRoute, WireGuard

# Configure logging for operational visibility
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

# Constants for default metrics (per OpenVPN 2.6.0 changelog and WireGuard docs)
WIREGUARD_DEFAULT_METRIC = 0
OPENVPN_2_6_DEFAULT_METRIC = 50
OPENVPN_PRE_2_6_DEFAULT_METRIC = 100
CONFLICT_THRESHOLD = 10  # ms, max latency for conflict-flagged routes


def get_wireguard_routes() -> Dict[str, Tuple[str, int]]:
    """Fetch all active WireGuard peer routes via netlink.
    Returns dict mapping subnet CIDR to (interface, metric) tuples.
    """
    wg_routes = {}
    try:
        with WireGuard() as wg:
            # List all WireGuard interfaces
            wg_interfaces = [iface["ifname"] for iface in wg.get_links() if iface.get("ifname", "").startswith("wg")]
            logger.info(f"Found {len(wg_interfaces)} WireGuard interfaces: {wg_interfaces}")
    except Exception as e:
        logger.error(f"Failed to initialize WireGuard netlink handler: {e}")
        return wg_routes

    try:
        with IPRoute() as ip:
            for iface in wg_interfaces:
                # Get routes for this WireGuard interface
                routes = ip.get_routes(oif=iface, family=2)  # family=2 is IPv4
                for route in routes:
                    # Extract destination CIDR and metric
                    dst = route.get_attr("RTA_DST")
                    dst_len = route.get_attr("RTA_DST_LEN")
                    metric = route.get_attr("RTA_PRIORITY") or WIREGUARD_DEFAULT_METRIC
                    if dst and dst_len:
                        cidr = f"{dst}/{dst_len}"
                        wg_routes[cidr] = (iface, metric)
                        logger.debug(f"WireGuard route: {cidr} via {iface}, metric {metric}")
    except Exception as e:
        logger.error(f"Failed to fetch WireGuard routes: {e}")
        return wg_routes

    return wg_routes


def get_openvpn_routes() -> Dict[str, Tuple[str, int, str]]:
    """Fetch all active OpenVPN routes via netlink, detect OpenVPN 2.6+ metrics.
    Returns dict mapping subnet CIDR to (interface, metric, openvpn_version) tuples.
    """
    ovpn_routes = {}
    try:
        with IPRoute() as ip:
            # List all tun/tap interfaces (OpenVPN typically uses these)
            ovpn_interfaces = [iface.get_attr("IFLA_IFNAME") for iface in ip.get_links()
                              if iface.get_attr("IFLA_IFNAME", "").startswith(("tun", "tap"))]
            logger.info(f"Found {len(ovpn_interfaces)} potential OpenVPN interfaces: {ovpn_interfaces}")
    except Exception as e:
        logger.error(f"Failed to fetch network interfaces: {e}")
        return ovpn_routes

    # Note: We can't get OpenVPN version from netlink, so we infer from default metric
    try:
        with IPRoute() as ip:
            for iface in ovpn_interfaces:
                routes = ip.get_routes(oif=iface, family=2)
                for route in routes:
                    dst = route.get_attr("RTA_DST")
                    dst_len = route.get_attr("RTA_DST_LEN")
                    metric = route.get_attr("RTA_PRIORITY") or OPENVPN_2_6_DEFAULT_METRIC  # Infer version from metric
                    if dst and dst_len:
                        cidr = f"{dst}/{dst_len}"
                        # Infer version based on metric
                        version = "2.6+" if metric <= OPENVPN_2_6_DEFAULT_METRIC else "pre-2.6"
                        ovpn_routes[cidr] = (iface, metric, version)
                        logger.debug(f"OpenVPN route: {cidr} via {iface}, metric {metric} ({version})")
    except Exception as e:
        logger.error(f"Failed to fetch OpenVPN routes: {e}")
        return ovpn_routes

    return ovpn_routes


def detect_conflicts(wg_routes: Dict, ovpn_routes: Dict) -> List[Dict]:
    """Compare WireGuard and OpenVPN routes for overlapping CIDRs.
    Returns list of conflict dicts with details.
    """
    conflicts = []
    all_cidrs = set(wg_routes.keys()).union(set(ovpn_routes.keys()))

    for cidr in all_cidrs:
        wg_entry = wg_routes.get(cidr)
        ovpn_entry = ovpn_routes.get(cidr)
        if wg_entry and ovpn_entry:
            wg_iface, wg_metric = wg_entry
            ovpn_iface, ovpn_metric, ovpn_version = ovpn_entry
            # Check if metrics are conflicting (OpenVPN 2.6+ lower metric wins)
            if ovpn_version == "2.6+" and ovpn_metric < wg_metric:
                conflicts.append({
                    "cidr": cidr,
                    "wg_interface": wg_iface,
                    "wg_metric": wg_metric,
                    "ovpn_interface": ovpn_iface,
                    "ovpn_metric": ovpn_metric,
                    "ovpn_version": ovpn_version,
                    "risk": "HIGH" if cidr == "0.0.0.0/0" else "MEDIUM"
                })
                logger.warning(f"CONFLICT DETECTED: {cidr} exists in both {wg_iface} and {ovpn_iface}")
    return conflicts


if __name__ == "__main__":
    logger.info("Starting hybrid VPN route conflict detection...")
    wg_routes = get_wireguard_routes()
    ovpn_routes = get_openvpn_routes()
    conflicts = detect_conflicts(wg_routes, ovpn_routes)

    if conflicts:
        logger.error(f"Found {len(conflicts)} route conflicts:")
        for conflict in conflicts:
            print(f"Conflict: {conflict['cidr']} | WG: {conflict['wg_interface']} (metric {conflict['wg_metric']}) | OVPN: {conflict['ovpn_interface']} (metric {conflict['ovpn_metric']}, {conflict['ovpn_version']}) | Risk: {conflict['risk']}")
        sys.exit(1)
    else:
        logger.info("No route conflicts detected between WireGuard and OpenVPN.")
        sys.exit(0)

package main

// wg-openvpn-linter: Validates WireGuard configurations for conflicts with OpenVPN 2.6+ default routing
// Usage: ./wg-openvpn-linter --wg-config /etc/wireguard/wg0.conf --ovpn-version 2.6.0
// OpenVPN repo: https://github.com/OpenVPN/openvpn

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "net"
    "os"
    "strings"
)

// OpenVPN 2.6 default route metric (per https://github.com/OpenVPN/openvpn/blob/v2.6.0/Changes.rst)
const openvpn26DefaultMetric = 50

// WireGuard default AllowedIPs metric (kernel default for wg peer routes)
const wgDefaultMetric = 0

// Flag variables
var (
    wgConfigPath  string
    ovpnVersion   string
    reportOnly    bool
)

// wgPeer represents a parsed WireGuard peer with AllowedIPs
type wgPeer struct {
    PublicKey   string
    AllowedIPs  []*net.IPNet
    Endpoint    string
}

// ovpnRoute represents an OpenVPN route (simplified)
type ovpnRoute struct {
    CIDR   *net.IPNet
    Metric int
}

func init() {
    flag.StringVar(&wgConfigPath, "wg-config", "", "Path to WireGuard configuration file (required)")
    flag.StringVar(&ovpnVersion, "ovpn-version", "2.6.0", "OpenVPN version to check against (default: 2.6.0)")
    flag.BoolVar(&reportOnly, "report-only", false, "Only report conflicts, don't exit with error")
    flag.Parse()

    if wgConfigPath == "" {
        log.Fatal("--wg-config is required")
    }
}

// parseWGConfig reads a WireGuard config file and returns parsed peers
func parseWGConfig(path string) ([]wgPeer, error) {
    file, err := os.Open(path)
    if err != nil {
        return nil, fmt.Errorf("failed to open config: %w", err)
    }
    defer file.Close()

    var peers []wgPeer
    var currentPeer *wgPeer

    scanner := bufio.NewScanner(file)
    for scanner.Scan() {
        line := strings.TrimSpace(scanner.Text())
        // Skip comments and empty lines
        if line == "" || strings.HasPrefix(line, "#") {
            continue
        }

        // Check for new peer section
        if strings.HasPrefix(line, "[Peer]") {
            if currentPeer != nil {
                peers = append(peers, *currentPeer)
            }
            currentPeer = &wgPeer{}
            continue
        }

        // Parse peer fields
        if currentPeer != nil {
            parts := strings.SplitN(line, "=", 2)
            if len(parts) != 2 {
                continue
            }
            key := strings.TrimSpace(parts[0])
            value := strings.TrimSpace(parts[1])

            switch key {
            case "PublicKey":
                currentPeer.PublicKey = value
            case "AllowedIPs":
                // Split multiple AllowedIPs separated by commas
                cidrs := strings.Split(value, ",")
                for _, cidrStr := range cidrs {
                    cidrStr = strings.TrimSpace(cidrStr)
                    _, cidr, err := net.ParseCIDR(cidrStr)
                    if err != nil {
                        log.Printf("Warning: invalid CIDR %s: %v", cidrStr, err)
                        continue
                    }
                    currentPeer.AllowedIPs = append(currentPeer.AllowedIPs, cidr)
                }
            case "Endpoint":
                currentPeer.Endpoint = value
            }
        }
    }

    // Add the last peer
    if currentPeer != nil {
        peers = append(peers, *currentPeer)
    }

    if err := scanner.Err(); err != nil {
        return nil, fmt.Errorf("failed to read config: %w", err)
    }

    return peers, nil
}

// checkForConflicts checks if any WireGuard AllowedIPs overlap with OpenVPN 2.6 default routes
func checkForConflicts(peers []wgPeer, ovpnRoutes []ovpnRoute) []string {
    var conflicts []string

    // Default OpenVPN 2.6 route is 0.0.0.0/0 with metric 50
    defaultOvpnRoute := &net.IPNet{
        IP:   net.IPv4(0, 0, 0, 0),
        Mask: net.CIDRMask(0, 32),
    }
    ovpnRoutes = append(ovpnRoutes, ovpnRoute{
        CIDR:   defaultOvpnRoute,
        Metric: openvpn26DefaultMetric,
    })

    for _, peer := range peers {
        for _, wgCIDR := range peer.AllowedIPs {
            for _, ovpn := range ovpnRoutes {
                // Check if CIDRs overlap
                if cidrsOverlap(wgCIDR, ovpn.CIDR) {
                    // Check if OpenVPN metric is lower than WireGuard's default (causes route hijacking)
                    if ovpn.Metric < wgDefaultMetric {
                        conflict := fmt.Sprintf("Peer %s: AllowedIPs %s overlaps with OpenVPN route %s (metric %d < WG default %d)",
                            peer.PublicKey[:8]+"...", wgCIDR.String(), ovpn.CIDR.String(), ovpn.Metric, wgDefaultMetric)
                        conflicts = append(conflicts, conflict)
                    }
                }
            }
        }
    }

    return conflicts
}

// cidrsOverlap checks if two CIDRs overlap
func cidrsOverlap(a, b *net.IPNet) bool {
    // Check if a contains b's network, or b contains a's network
    return a.Contains(b.IP) || b.Contains(a.IP)
}

func main() {
    // Parse WireGuard config
    peers, err := parseWGConfig(wgConfigPath)
    if err != nil {
        log.Fatalf("Failed to parse WireGuard config: %v", err)
    }
    log.Printf("Parsed %d WireGuard peers from %s", len(peers), wgConfigPath)

    // Simulate OpenVPN 2.6 routes (in production, fetch from running OpenVPN instance)
    ovpnRoutes := []ovpnRoute{}
    conflicts := checkForConflicts(peers, ovpnRoutes)

    if len(conflicts) > 0 {
        log.Printf("Found %d conflicts with OpenVPN %s:", len(conflicts), ovpnVersion)
        for _, conflict := range conflicts {
            fmt.Println(conflict)
        }
        if !reportOnly {
            os.Exit(1)
        }
    } else {
        log.Println("No conflicts detected between WireGuard config and OpenVPN 2.6+")
    }
}

#!/bin/bash
#
# outage-mitigator.sh: Mitigates WireGuard/OpenVPN 2.6 routing conflicts
# Author: Senior Engineering Team
# Requires: iproute2, wireguard-tools, openvpn 2.6+

set -euo pipefail  # Exit on error, undefined var, pipe failure
IFS=$'\n\t'

# Configuration
LOG_FILE="/var/log/vpn-outage-mitigator.log"
WG_INTERFACES=("wg0" "wg1")
OVPN_INTERFACES=("tun0" "tun1")
MAX_RETRIES=3
RETRY_DELAY=5

# Logging function
log() {
    local level="$1"
    shift
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "$LOG_FILE"
}

# Error handler
error_handler() {
    local exit_code="$?"
    log "ERROR" "Script failed at line ${BASH_LINENO[0]} with exit code $exit_code"
    # Attempt rollback
    log "INFO" "Attempting rollback of route changes..."
    for wg_iface in "${WG_INTERFACES[@]}"; do
        if ip link show "$wg_iface" &>/dev/null; then
            log "INFO" "Bringing down $wg_iface for rollback"
            wg-quick down "$wg_iface" 2>&1 | tee -a "$LOG_FILE" || true
        fi
    done
    exit "$exit_code"
}

trap 'error_handler' ERR

# Check if running as root
if [[ $EUID -ne 0 ]]; then
    log "ERROR" "This script must be run as root"
    exit 1
fi

# Check for required tools
check_dependencies() {
    local deps=("ip" "wg" "wg-quick" "openvpn")
    for dep in "${deps[@]}"; do
        if ! command -v "$dep" &>/dev/null; then
            log "ERROR" "Missing required dependency: $dep"
            exit 1
        fi
    done
    log "INFO" "All dependencies satisfied"
}

# Flush conflicting routes between WireGuard and OpenVPN
flush_conflicting_routes() {
    log "INFO" "Flushing conflicting routes between WireGuard and OpenVPN..."

    # Get all default routes (0.0.0.0/0) for OpenVPN interfaces
    for ovpn_iface in "${OVPN_INTERFACES[@]}"; do
        if ! ip link show "$ovpn_iface" &>/dev/null; then
            log "WARNING" "OpenVPN interface $ovpn_iface not found, skipping"
            continue
        fi

        # Get route metric for OpenVPN interface
        ovpn_metric=$(ip route show dev "$ovpn_iface" | grep -oP 'metric \K\d+' | head -1)
        ovpn_metric=${ovpn_metric:-50}  # Default to OpenVPN 2.6 metric if not found

        # Check if OpenVPN metric is lower than WireGuard's default (0)
        if [[ $ovpn_metric -lt 0 ]]; then
            log "WARNING" "OpenVPN $ovpn_iface has invalid metric $ovpn_metric, skipping"
            continue
        fi

        # Flush default route for OpenVPN if it's conflicting
        log "INFO" "Checking default route for $ovpn_iface (metric $ovpn_metric)"
        if ip route show 0.0.0.0/0 dev "$ovpn_iface" &>/dev/null; then
            log "INFO" "Flushing default route for $ovpn_iface to prevent WireGuard conflict"
            ip route del 0.0.0.0/0 dev "$ovpn_iface" 2>&1 | tee -a "$LOG_FILE"
        fi
    done

    # Restart WireGuard interfaces to reapply correct routes
    for wg_iface in "${WG_INTERFACES[@]}"; do
        if ip link show "$wg_iface" &>/dev/null; then
            log "INFO" "Restarting WireGuard interface $wg_iface"
            wg-quick down "$wg_iface" 2>&1 | tee -a "$LOG_FILE"
            local retry=0
            while [[ $retry -lt $MAX_RETRIES ]]; do
                if wg-quick up "$wg_iface" 2>&1 | tee -a "$LOG_FILE"; then
                    log "INFO" "Successfully brought up $wg_iface"
                    break
                else
                    log "WARNING" "Retry $retry/$MAX_RETRIES for $wg_iface failed"
                    ((retry++))
                    sleep "$RETRY_DELAY"
                fi
            done
            if [[ $retry -eq $MAX_RETRIES ]]; then
                log "ERROR" "Failed to bring up $wg_iface after $MAX_RETRIES retries"
                exit 1
            fi
        fi
    done
}

# Verify connectivity after mitigation
verify_connectivity() {
    log "INFO" "Verifying VPN connectivity..."
    # Check if WireGuard peers are reachable
    for wg_iface in "${WG_INTERFACES[@]}"; do
        if ip link show "$wg_iface" &>/dev/null; then
            peer_count=$(wg show "$wg_iface" peers | wc -l)
            log "INFO" "WireGuard $wg_iface has $peer_count active peers"
            if [[ $peer_count -eq 0 ]]; then
                log "ERROR" "No active peers for $wg_iface, mitigation failed"
                exit 1
            fi
        fi
    done
    log "INFO" "Connectivity verification passed"
}

# Main execution
main() {
    log "INFO" "Starting VPN outage mitigation..."
    check_dependencies
    flush_conflicting_routes
    verify_connectivity
    log "INFO" "Mitigation completed successfully"
}

main

- name: Lint WireGuard Config for OpenVPN Conflicts
  run: |
    go build -o wg-linter wg-openvpn-linter.go
    ./wg-linter --wg-config /etc/wireguard/wg0.conf --ovpn-version 2.6.0 --report-only

# HELP vpn_route_conflict_total Total number of detected WireGuard/OpenVPN route conflicts
# TYPE vpn_route_conflict_total counter
vpn_route_conflict_total{ cidr="0.0.0.0/0", risk="HIGH" } 1
vpn_route_conflict_total{ cidr="10.0.0.0/8", risk="MEDIUM" } 2

# openvpn-server.conf
port 1194
proto udp
dev tun0
route-metric 150
redirect-gateway def1 metric 150
# Explicitly set metric for all pushed routes
push "route-metric 150"
push "redirect-gateway def1 metric 150"