Postmortem: How a Proxmox 8 Crash Caused a 4-Hour Outage for Our 2026 Self-Hosted Services and How We Migrated to TrueNAS Scale

#!/usr/bin/env python3
"""
Proxmox 8 Crash Root Cause Analyzer
Parses Proxmox VE 8.x kernel logs to detect ZFS vdev_label_write race conditions
linked to the January 2026 outage.
Requires: Python 3.9+, read access to /var/log/kern.log or user-provided log file
"""
import re
import sys
import argparse
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Dict, Optional

# Regex to match ZFS vdev_label_write errors in Proxmox 8 kernel logs
VDEV_ERROR_REGEX = re.compile(
    r"kernel:.*zfs:.*vdev_label_write.*error=(\d+).*vdev=(.*)"
)
# Regex to match kernel panic events
PANIC_REGEX = re.compile(r"kernel:.*Kernel panic - not syncing: (.*)")
# Threshold for high I/O load (matches outage trigger condition)
IO_LOAD_THRESHOLD = 0.8  # 80% storage I/O utilization

def parse_kern_log(log_path: Path) -> Dict[str, List[Dict]]:
    """
    Parse a Proxmox kernel log file and extract ZFS errors and kernel panics.
    Args:
        log_path: Path to kern.log or proxmox-kernel.log
    Returns:
        Dict with 'zfs_errors' and 'panics' lists, each entry a dict with timestamp and details
    Raises:
        FileNotFoundError: If log_path does not exist
        PermissionError: If log_path is not readable
    """
    if not log_path.exists():
        raise FileNotFoundError(f"Log file {log_path} not found")
    if not log_path.is_file():
        raise ValueError(f"{log_path} is not a regular file")

    results = {"zfs_errors": [], "panics": []}
    # Track I/O load timestamps to correlate with errors
    io_load_events = []

    with open(log_path, "r", encoding="utf-8", errors="ignore") as f:
        for line_num, line in enumerate(f, 1):
            line = line.strip()
            # Parse timestamp (Proxmox uses rsyslog format: Jan 12 02:14:01)
            timestamp_match = re.match(r"(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})", line)
            if not timestamp_match:
                continue
            try:
                # Assume current year if not present (Proxmox default log format)
                log_time = datetime.strptime(f"{datetime.now().year} {timestamp_match.group(1)}", "%Y %b %d %H:%M:%S")
            except ValueError:
                # Skip malformed timestamps
                continue

            # Check for ZFS vdev errors
            zfs_match = VDEV_ERROR_REGEX.search(line)
            if zfs_match:
                results["zfs_errors"].append({
                    "timestamp": log_time,
                    "error_code": int(zfs_match.group(1)),
                    "vdev": zfs_match.group(2).strip(),
                    "line_num": line_num,
                    "raw_line": line
                })

            # Check for kernel panics
            panic_match = PANIC_REGEX.search(line)
            if panic_match:
                results["panics"].append({
                    "timestamp": log_time,
                    "reason": panic_match.group(1).strip(),
                    "line_num": line_num,
                    "raw_line": line
                })

            # Check for I/O load threshold (simplified: match iostat-like entries)
            if "iostat" in line.lower() and "util" in line.lower():
                try:
                    util = float(re.search(r"(\d+\.\d+)%", line).group(1)) / 100
                    if util >= IO_LOAD_THRESHOLD:
                        io_load_events.append({"timestamp": log_time, "util": util})
                except (AttributeError, ValueError):
                    pass

    # Correlate ZFS errors with high I/O load events (within 5 minutes)
    for error in results["zfs_errors"]:
        error["high_io_correlated"] = any(
            abs((event["timestamp"] - error["timestamp"]).total_seconds()) <= 300
            for event in io_load_events
        )

    return results

def main():
    parser = argparse.ArgumentParser(description="Analyze Proxmox 8 kernel logs for crash root causes")
    parser.add_argument("--log-path", type=Path, default=Path("/var/log/kern.log"),
                        help="Path to Proxmox kernel log file (default: /var/log/kern.log)")
    parser.add_argument("--output-json", type=Path, default=None,
                        help="Write results to JSON file instead of stdout")
    args = parser.parse_args()

    try:
        results = parse_kern_log(args.log_path)
    except (FileNotFoundError, PermissionError, ValueError) as e:
        print(f"ERROR: {e}", file=sys.stderr)
        sys.exit(1)

    # Print summary
    print(f"=== Proxmox 8 Crash Analysis Report for {args.log_path} ===")
    print(f"ZFS vdev_label_write errors found: {len(results['zfs_errors'])}")
    print(f"Kernel panics found: {len(results['panics'])}")
    correlated_errors = [e for e in results["zfs_errors"] if e["high_io_correlated"]]
    print(f"Errors correlated with >80% I/O load: {len(correlated_errors)}")

    if correlated_errors:
        print("\n⚠️ High-risk ZFS errors (correlated with high I/O):")
        for err in correlated_errors[:5]:  # Show first 5
            print(f"  {err['timestamp']} | Vdev: {err['vdev']} | Error: {err['error_code']}")

    if args.output_json:
        import json
        with open(args.output_json, "w", encoding="utf-8") as f:
            json.dump(results, f, indent=2, default=str)
        print(f"\nFull results written to {args.output_json}")

    # Exit with non-zero code if high-risk errors found
    sys.exit(1 if correlated_errors else 0)

if __name__ == "__main__":
    main()

---
# Ansible Playbook: Migrate Proxmox 8 VMs to TrueNAS Scale 24.04
# Requires:
#   - Ansible 2.15+
#   - community.proxmox collection (>= 2.0.0)
#   - community.truenas collection (>= 1.2.0)
#   - SSH access to Proxmox node and TrueNAS Scale admin API key
- name: Migrate Proxmox VMs to TrueNAS Scale
  hosts: localhost
  gather_facts: false
  vars:
    proxmox_node: "pve-primary"
    proxmox_api_host: "192.168.1.10"
    proxmox_api_user: "root@pam"
    proxmox_api_password: "{{ vault_proxmox_password }}"  # Store in Ansible Vault
    truenas_api_host: "192.168.1.20"
    truenas_api_key: "{{ vault_truenas_api_key }}"  # TrueNAS API key with admin privileges
    migration_storage_pool: "tank/vms"  # ZFS pool on TrueNAS
    vm_ids_to_migrate: [101, 102, 103, 104]  # Home Assistant, Plex, Nextcloud, Gitea
    temp_migration_dir: "/tmp/proxmox-migration"

  tasks:
    - name: Validate Proxmox API connectivity
      community.proxmox.proxmox_node_info:
        api_host: "{{ proxmox_api_host }}"
        api_user: "{{ proxmox_api_user }}"
        api_password: "{{ proxmox_api_password }}"
        node: "{{ proxmox_node }}"
      register: proxmox_node_status
      failed_when: proxmox_node_status.failed or proxmox_node_status.node_info.status != "online"
      tags: [validate, proxmox]

    - name: Validate TrueNAS Scale API connectivity
      community.truenas.truenas_api_info:
        host: "{{ truenas_api_host }}"
        api_key: "{{ truenas_api_key }}"
      register: truenas_status
      failed_when: truenas_status.failed or truenas_status.info.system_status != "OK"
      tags: [validate, truenas]

    - name: Create temporary migration directory
      ansible.builtin.file:
        path: "{{ temp_migration_dir }}"
        state: directory
        mode: "0700"
      tags: [setup]

    - name: Stop target VMs on Proxmox (graceful shutdown)
      community.proxmox.proxmox_vm:
        api_host: "{{ proxmox_api_host }}"
        api_user: "{{ proxmox_api_user }}"
        api_password: "{{ proxmox_api_password }}"
        node: "{{ proxmox_node }}"
        vmid: "{{ item }}"
        state: stopped
        timeout: 300  # Wait up to 5 minutes for graceful shutdown
      loop: "{{ vm_ids_to_migrate }}"
      register: vm_stop_results
      failed_when: vm_stop_results.failed and "already stopped" not in vm_stop_results.msg | default("")
      tags: [migrate, proxmox]

    - name: Backup Proxmox VM disks to temporary directory
      ansible.builtin.command:
        cmd: "qm backup {{ item }} {{ temp_migration_dir }}/vm-{{ item }}.vma.lzo --compress lzo --mode snapshot"
        creates: "{{ temp_migration_dir }}/vm-{{ item }}.vma.lzo"
      loop: "{{ vm_ids_to_migrate }}"
      register: backup_results
      failed_when: backup_results.rc != 0 and "backup already exists" not in backup_results.stderr | default("")
      tags: [migrate, backup]

    - name: Convert Proxmox VMA backups to raw disk images
      ansible.builtin.shell:
        cmd: "lzop -d {{ temp_migration_dir }}/vm-{{ item }}.vma.lzo -c | vma-extract - {{ temp_migration_dir }}/vm-{{ item }}.raw"
        creates: "{{ temp_migration_dir }}/vm-{{ item }}.raw"
      loop: "{{ vm_ids_to_migrate }}"
      register: convert_results
      failed_when: convert_results.rc != 0
      tags: [migrate, convert]

    - name: Create TrueNAS ZVOLs for migrated VMs
      community.truenas.truenas_zvol:
        host: "{{ truenas_api_host }}"
        api_key: "{{ truenas_api_key }}"
        name: "{{ migration_storage_pool }}/vm-{{ item }}"
        size: "{{ 20 | human_to_bytes }}"  # 20GB default, adjust per VM in production
        blocksize: 16384  # 16K block size for ZFS
        state: present
      loop: "{{ vm_ids_to_migrate }}"
      register: zvol_results
      failed_when: zvol_results.failed
      tags: [migrate, truenas]

    - name: Write raw disk images to TrueNAS ZVOLs via SSH
      ansible.builtin.shell:
        cmd: "ssh root@{{ truenas_api_host }} 'dd if={{ temp_migration_dir }}/vm-{{ item }}.raw of=/dev/zvol/{{ migration_storage_pool }}/vm-{{ item }} bs=4M conv=fsync'"
      loop: "{{ vm_ids_to_migrate }}"
      register: dd_results
      failed_when: dd_results.rc != 0
      tags: [migrate, truenas]

    - name: Create TrueNAS VMs from ZVOLs
      community.truenas.truenas_vm:
        host: "{{ truenas_api_host }}"
        api_key: "{{ truenas_api_key }}"
        name: "migrated-vm-{{ item }}"
        vcpus: 2
        memory: 4096  # 4GB RAM
        disks:
          - zvol: "{{ migration_storage_pool }}/vm-{{ item }}"
            type: "VIRTIO"
        boot_order: [disk]
        state: present
      loop: "{{ vm_ids_to_migrate }}"
      register: vm_create_results
      failed_when: vm_create_results.failed
      tags: [migrate, truenas]

    - name: Clean up temporary migration files
      ansible.builtin.file:
        path: "{{ temp_migration_dir }}"
        state: absent
      when: cleanup_temp_files | default(true) | bool
      tags: [cleanup]

    - name: Print migration summary
      ansible.builtin.debug:
        msg: |
          Migration complete!
          Migrated VMs: {{ vm_ids_to_migrate | join(', ') }}
          TrueNAS VMs: migrated-vm-{{ vm_ids_to_migrate | join(', migrated-vm-') }}
          Next step: Start VMs via TrueNAS API or UI
      tags: [summary]

#!/usr/bin/env python3
"""
TrueNAS Scale vs Proxmox 8 VM Performance Benchmarker
Measures boot time, I/O throughput, and latency for identical VMs on both platforms.
Requires: Python 3.9+, requests, paramiko, humanize
"""
import time
import requests
import paramiko
import argparse
from typing import Dict, List, Optional
from humanize import naturalsize, naturaldelta

# Proxmox API config
PROXMOX_API = "https://192.168.1.10:8006/api2/json"
PROXMOX_AUTH = ("root@pam", "vault_proxmox_password")  # Use env vars in production
# TrueNAS API config
TRUENAS_API = "https://192.168.1.20/api/v2.0"
TRUENAS_HEADERS = {"Authorization": "Bearer vault_truenas_api_key"}

# Test VM config (identical on both platforms)
TEST_VM_ID_PROXMOX = 999
TEST_VM_ID_TRUENAS = "benchmark-vm"
TEST_VM_VCPUS = 2
TEST_VM_RAM = 4096  # 4GB
TEST_DISK_SIZE = 10 * 1024 * 1024 * 1024  # 10GB

def proxmox_login() -> str:
    """Authenticate to Proxmox API and return CSRF token."""
    try:
        resp = requests.post(
            f"{PROXMOX_API}/access/ticket",
            data={"username": PROXMOX_AUTH[0], "password": PROXMOX_AUTH[1]},
            verify=False  # Use proper certs in production
        )
        resp.raise_for_status()
        data = resp.json()["data"]
        return data["CSRFPreventionToken"]
    except requests.exceptions.RequestException as e:
        raise RuntimeError(f"Proxmox login failed: {e}")

def truenas_login() -> bool:
    """Validate TrueNAS API key."""
    try:
        resp = requests.get(f"{TRUENAS_API}/system/info", headers=TRUENAS_HEADERS)
        resp.raise_for_status()
        return True
    except requests.exceptions.RequestException as e:
        raise RuntimeError(f"TrueNAS login failed: {e}")

def measure_vm_boot_time_proxmox(vm_id: int, csrf_token: str) -> float:
    """Measure time from VM start to SSH availability on Proxmox."""
    # Start VM
    headers = {"CSRFPreventionToken": csrf_token, "Cookie": f"PVEAuthCookie={proxmox_ticket}"}
    # Note: Full implementation would get ticket from login response
    start_time = time.time()
    try:
        resp = requests.post(
            f"{PROXMOX_API}/nodes/pve-primary/qemu/{vm_id}/status/start",
            headers=headers, verify=False
        )
        resp.raise_for_status()
    except requests.exceptions.RequestException as e:
        raise RuntimeError(f"Failed to start Proxmox VM {vm_id}: {e}")

    # Wait for SSH to become available (timeout 5 minutes)
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    while time.time() - start_time < 300:
        try:
            ssh.connect("192.168.1.100", username="benchmark", password="bench123", timeout=5)
            ssh.close()
            return time.time() - start_time
        except (paramiko.SSHException, TimeoutError):
            time.sleep(2)
    raise RuntimeError(f"Proxmox VM {vm_id} did not boot within 5 minutes")

def measure_vm_boot_time_truenas(vm_id: str) -> float:
    """Measure time from VM start to SSH availability on TrueNAS Scale."""
    start_time = time.time()
    try:
        resp = requests.post(
            f"{TRUENAS_API}/vm/vm/{vm_id}/start",
            headers=TRUENAS_HEADERS
        )
        resp.raise_for_status()
    except requests.exceptions.RequestException as e:
        raise RuntimeError(f"Failed to start TrueNAS VM {vm_id}: {e}")

    # Wait for SSH
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    while time.time() - start_time < 300:
        try:
            ssh.connect("192.168.1.101", username="benchmark", password="bench123", timeout=5)
            ssh.close()
            return time.time() - start_time
        except (paramiko.SSHException, TimeoutError):
            time.sleep(2)
    raise RuntimeError(f"TrueNAS VM {vm_id} did not boot within 5 minutes")

def run_io_benchmark(vm_ip: str, username: str, password: str) -> Dict[str, float]:
    """Run fio I/O benchmark on target VM via SSH."""
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    try:
        ssh.connect(vm_ip, username=username, password=password, timeout=10)
        # Run fio for 60 seconds, 4K random read/write, 4 jobs
        stdin, stdout, stderr = ssh.exec_command(
            "fio --name=randrw --ioengine=libaio --rw=randrw --bs=4k --numjobs=4 "
            "--size=1G --runtime=60 --group_reporting --output-format=json",
            timeout=120
        )
        output = stdout.read().decode("utf-8")
        import json
        fio_results = json.loads(output)
        # Extract read/write IOPS and latency
        read_iops = fio_results["jobs"][0]["read"]["iops"]
        write_iops = fio_results["jobs"][0]["write"]["iops"]
        read_lat = fio_results["jobs"][0]["read"]["lat_ns"]["mean"] / 1e6  # Convert to ms
        write_lat = fio_results["jobs"][0]["write"]["lat_ns"]["mean"] / 1e6
        return {
            "read_iops": read_iops,
            "write_iops": write_iops,
            "read_lat_ms": read_lat,
            "write_lat_ms": write_lat
        }
    except (paramiko.SSHException, TimeoutError, json.JSONDecodeError) as e:
        raise RuntimeError(f"I/O benchmark failed on {vm_ip}: {e}")
    finally:
        ssh.close()

def main():
    parser = argparse.ArgumentParser(description="Benchmark VM performance on Proxmox vs TrueNAS Scale")
    parser.add_argument("--run-boot-bench", action="store_true", help="Measure VM boot times")
    parser.add_argument("--run-io-bench", action="store_true", help="Run I/O benchmarks")
    parser.add_argument("--output-csv", type=str, default=None, help="Write results to CSV file")
    args = parser.parse_args()

    print("=== VM Performance Benchmark: Proxmox 8.2.3 vs TrueNAS Scale 24.04 ===")
    results = []

    if args.run_boot_bench:
        print("\nMeasuring VM boot times...")
        try:
            proxmox_csrf = proxmox_login()
            proxmox_boot = measure_vm_boot_time_proxmox(TEST_VM_ID_PROXMOX, proxmox_csrf)
            print(f"Proxmox boot time: {naturaldelta(proxmox_boot)}")
        except RuntimeError as e:
            print(f"ERROR Proxmox boot: {e}")
            proxmox_boot = None

        try:
            truenas_login()
            truenas_boot = measure_vm_boot_time_truenas(TEST_VM_ID_TRUENAS)
            print(f"TrueNAS boot time: {naturaldelta(truenas_boot)}")
        except RuntimeError as e:
            print(f"ERROR TrueNAS boot: {e}")
            truenas_boot = None

        if proxmox_boot and truenas_boot:
            improvement = ((proxmox_boot - truenas_boot) / proxmox_boot) * 100
            print(f"TrueNAS boot time improvement: {improvement:.1f}%")
            results.append({"metric": "boot_time", "proxmox": proxmox_boot, "truenas": truenas_boot})

    if args.run_io_bench:
        print("\nRunning I/O benchmarks...")
        # Proxmox I/O benchmark
        try:
            proxmox_io = run_io_benchmark("192.168.1.100", "benchmark", "bench123")
            print(f"Proxmox IOPS (R/W): {proxmox_io['read_iops']:.0f}/{proxmox_io['write_iops']:.0f}")
            print(f"Proxmox Latency (R/W): {proxmox_io['read_lat_ms']:.2f}ms/{proxmox_io['write_lat_ms']:.2f}ms")
        except RuntimeError as e:
            print(f"ERROR Proxmox I/O: {e}")
            proxmox_io = None

        # TrueNAS I/O benchmark
        try:
            truenas_io = run_io_benchmark("192.168.1.101", "benchmark", "bench123")
            print(f"TrueNAS IOPS (R/W): {truenas_io['read_iops']:.0f}/{truenas_io['write_iops']:.0f}")
            print(f"TrueNAS Latency (R/W): {truenas_io['read_lat_ms']:.2f}ms/{truenas_io['write_lat_ms']:.2f}ms")
        except RuntimeError as e:
            print(f"ERROR TrueNAS I/O: {e}")
            truenas_io = None

        if proxmox_io and truenas_io:
            read_improve = ((proxmox_io["read_iops"] - truenas_io["read_iops"]) / proxmox_io["read_iops"]) * 100
            print(f"TrueNAS read IOPS improvement: {read_improve:.1f}%")
            results.append({"metric": "io", "proxmox": proxmox_io, "truenas": truenas_io})

    if args.output_csv:
        import csv
        with open(args.output_csv, "w", newline="") as f:
            writer = csv.DictWriter(f, fieldnames=["metric", "proxmox", "truenas"])
            writer.writeheader()
            writer.writerows(results)
        print(f"\nResults written to {args.output_csv}")

    print("\n=== Benchmark Complete ===")

if __name__ == "__main__":
    main()

# /etc/apt/preferences.d/proxmox-pin
Package: proxmox-ve
Pin: version 8.2.3
Pin-Priority: 1000

Package: zfs-dkms
Pin: version 2.1.12*
Pin-Priority: 1000

# /etc/cron.hourly/zfs-check
#!/bin/bash
/usr/local/bin/proxmox_zfs_analyzer.py --log-path /var/log/kern.log --output-json /tmp/zfs_report.json
if [ $? -eq 1 ]; then
  curl -X POST -H "Content-Type: application/json" \
    -d '{"text":"High-risk ZFS errors detected on Proxmox node"}' \
    https://hooks.slack.com/services/your/slack/webhook
fi

import http from 'k6/http';
import { sleep } from 'k6';

export const options = {
  stages: [
    { duration: '5m', target: 50 }, // Ramp to 50 users
    { duration: '24h', target: 50 }, // Soak test
    { duration: '5m', target: 0 }, // Ramp down
  ],
};

export default function () {
  const res = http.get('https://nextcloud.example.com/health');
  sleep(1);
}