We Ditched kubectl 1.31 for k9s 0.32 – Cut Cluster Debug Time by 50%

#!/bin/bash
# kubectl-1.31-incident-response.sh
# Deprecated workflow for debugging production K8s incidents using kubectl 1.31
# Requires: kubectl 1.31+, jq 1.6+, awscli 2.0+ (for EKS clusters)
# Usage: ./kubectl-1.31-incident-response.sh   

set -euo pipefail

# Validate input arguments
if [ $# -ne 3 ]; then
  echo "ERROR: Invalid arguments. Usage: $0   "
  exit 1
fi

CLUSTER_NAME="$1"
NAMESPACE="$2"
POD_SELECTOR="$3"
KUBECTL_CTX="arn:aws:eks:us-east-1:123456789012:cluster/${CLUSTER_NAME}"

# Switch kubectl context to target cluster
echo "INFO: Switching kubectl context to ${CLUSTER_NAME}..."
if ! kubectl config use-context "${KUBECTL_CTX}" > /dev/null 2>&1; then
  echo "ERROR: Failed to switch to context ${KUBECTL_CTX}. Check cluster access."
  exit 1
fi

# Verify cluster connectivity
echo "INFO: Verifying cluster connectivity..."
if ! kubectl cluster-info > /dev/null 2>&1; then
  echo "ERROR: Cannot connect to cluster ${CLUSTER_NAME}. Check VPN/auth."
  exit 1
fi

# List pods matching selector
echo "INFO: Listing pods in namespace ${NAMESPACE} matching selector ${POD_SELECTOR}..."
POD_LIST=$(kubectl get pods -n "${NAMESPACE}" -l "${POD_SELECTOR}" -o json | jq -r '.items[].metadata.name')
if [ -z "${POD_LIST}" ]; then
  echo "ERROR: No pods found matching selector ${POD_SELECTOR} in namespace ${NAMESPACE}"
  exit 1
fi

# Prompt user to select pod (kubectl has no native multi-select)
echo "INFO: Found pods:"
echo "${POD_LIST}"
read -p "Enter pod name to debug: " TARGET_POD

# Check pod status
POD_STATUS=$(kubectl get pod -n "${NAMESPACE}" "${TARGET_POD}" -o jsonpath='{.status.phase}')
echo "INFO: Pod ${TARGET_POD} status: ${POD_STATUS}"

# Tail logs (kubectl requires separate command for logs)
echo "INFO: Tailing logs for ${TARGET_POD} (Ctrl+C to stop)..."
kubectl logs -n "${NAMESPACE}" "${TARGET_POD}" --tail=100 -f &

# Get pod events (another separate command)
echo "INFO: Fetching pod events..."
kubectl get events -n "${NAMESPACE}" --field-selector involvedObject.name="${TARGET_POD}"

# Check resource usage (requires metrics-server, separate command)
echo "INFO: Fetching pod resource usage..."
kubectl top pod -n "${NAMESPACE}" "${TARGET_POD}" || echo "WARN: metrics-server not available"

# Port forward if needed (separate command, manual setup)
read -p "Do you need to port-forward? (y/n): " PORT_FORWARD
if [ "${PORT_FORWARD}" = "y" ]; then
  read -p "Enter local port: " LOCAL_PORT
  read -p "Enter pod port: " POD_PORT
  echo "INFO: Port-forwarding ${LOCAL_PORT}:${POD_PORT} for ${TARGET_POD}..."
  kubectl port-forward -n "${NAMESPACE}" "${TARGET_POD}" "${LOCAL_PORT}:${POD_PORT}"
fi

echo "INFO: Incident response workflow complete."

#!/bin/bash
# k9s-0.32-pod-debug-plugin.sh
# Custom k9s 0.32 plugin for automated pod debugging, triggered via k9s UI
# Requires: k9s 0.32+, kubectl 1.28+, jq 1.6+
# Plugin configuration (add to ~/.config/k9s/plugins.yaml):
# plugins:
#   pod-debug:
#     shortCut: Ctrl-D
#     description: "Automated Pod Debug Workflow"
#     scopes:
#       - pods
#     command: /path/to/k9s-0.32-pod-debug-plugin.sh
#     background: false

set -euo pipefail

# k9s passes the current context, namespace, and resource as environment variables
K9S_CTX="${K9S_CURRENT_CTX:-}"
K9S_NS="${K9S_CURRENT_NS:-}"
K9S_RESOURCE="${K9S_RESOURCE_NAME:-}"
K9S_RESOURCE_TYPE="${K9S_RESOURCE_TYPE:-}"

# Validate k9s environment variables
if [ -z "${K9S_CTX}" ] || [ -z "${K9S_NS}" ] || [ -z "${K9S_RESOURCE}" ]; then
  echo "ERROR: Missing k9s environment variables. This script must be run via k9s."
  exit 1
fi

# Switch kubectl context to match k9s
echo "INFO: Using k9s context ${K9S_CTX}..."
if ! kubectl config use-context "${K9S_CTX}" > /dev/null 2>&1; then
  echo "ERROR: Failed to switch to kubectl context ${K9S_CTX}"
  exit 1
fi

# Verify resource is a pod
if [ "${K9S_RESOURCE_TYPE}" != "pods" ]; then
  echo "ERROR: This plugin only works for pod resources. Current resource: ${K9S_RESOURCE_TYPE}"
  exit 1
fi

TARGET_POD="${K9S_RESOURCE}"
echo "INFO: Starting debug workflow for pod ${TARGET_POD} in namespace ${K9S_NS}..."

# Get pod status with kubectl (k9s already displays this, but we log it)
POD_STATUS=$(kubectl get pod -n "${K9S_NS}" "${TARGET_POD}" -o jsonpath='{.status.phase}' 2>/dev/null)
if [ -z "${POD_STATUS}" ]; then
  echo "ERROR: Pod ${TARGET_POD} not found in namespace ${K9S_NS}"
  exit 1
fi
echo "INFO: Pod status: ${POD_STATUS}"

# Get recent logs (k9s allows piping logs to this script)
echo "INFO: Fetching last 200 log lines..."
LOGS=$(kubectl logs -n "${K9S_NS}" "${TARGET_POD}" --tail=200 2>/dev/null)
if [ -z "${LOGS}" ]; then
  echo "WARN: No logs found for pod ${TARGET_POD}"
else
  # Filter logs for ERROR/CRITICAL keywords
  ERROR_COUNT=$(echo "${LOGS}" | grep -iE "error|critical|fail" | wc -l)
  echo "INFO: Found ${ERROR_COUNT} error-level log entries"
  echo "${LOGS}" | grep -iE "error|critical|fail" | head -20
fi

# Get pod events
echo "INFO: Fetching pod events..."
kubectl get events -n "${K9S_NS}" --field-selector involvedObject.name="${TARGET_POD}" 2>/dev/null || echo "WARN: No events found"

# Get resource usage
echo "INFO: Fetching resource usage..."
kubectl top pod -n "${K9S_NS}" "${TARGET_POD}" 2>/dev/null || echo "WARN: metrics-server unavailable"

# Offer port-forward option
read -p "Enter local port for port-forward (or press Enter to skip): " LOCAL_PORT
if [ -n "${LOCAL_PORT}" ]; then
  read -p "Enter pod port: " POD_PORT
  echo "INFO: Starting port-forward ${LOCAL_PORT}:${POD_PORT}..."
  kubectl port-forward -n "${K9S_NS}" "${TARGET_POD}" "${LOCAL_PORT}:${POD_PORT}" &
  PF_PID=$!
  echo "INFO: Port-forward running (PID: ${PF_PID}). Press Enter to stop."
  read
  kill ${PF_PID} 2>/dev/null
fi

echo "INFO: Debug workflow complete. Returning to k9s..."

#!/usr/bin/env python3
# k8s-debug-benchmark.py
# Benchmark comparing kubectl 1.31 vs k9s 0.32 debug workflow latency
# Requires: kubectl 1.31+, k9s 0.32+, prometheus-client, kubernetes Python client
# Usage: python3 k8s-debug-benchmark.py --cluster  --namespace  --pod-selector  --iterations 10

import argparse
import subprocess
import time
import json
import csv
from kubernetes import client, config
from prometheus_client import CollectorRegistry, Gauge, push_to_gateway

# Benchmark configuration
DEFAULT_ITERATIONS = 10
KUBECTL_VERSION = "1.31.0"
K9S_VERSION = "0.32.0"
BENCHMARK_RESULTS_FILE = "debug_benchmark_results.csv"

def validate_tools():
    """Validate required tool versions are installed"""
    # Check kubectl version
    try:
        kubectl_out = subprocess.check_output(["kubectl", "version", "--client", "-o", "json"], text=True)
        kubectl_ver = json.loads(kubectl_out)["clientVersion"]["gitVersion"]
        if KUBECTL_VERSION not in kubectl_ver:
            raise RuntimeError(f"kubectl version mismatch. Expected {KUBECTL_VERSION}, got {kubectl_ver}")
    except Exception as e:
        raise RuntimeError(f"kubectl validation failed: {str(e)}")

    # Check k9s version
    try:
        k9s_out = subprocess.check_output(["k9s", "version"], text=True)
        if K9S_VERSION not in k9s_out:
            raise RuntimeError(f"k9s version mismatch. Expected {K9S_VERSION}, got {k9s_out}")
    except Exception as e:
        raise RuntimeError(f"k9s validation failed: {str(e)}")

def run_kubectl_workflow(cluster, namespace, pod_selector):
    """Run full kubectl 1.31 debug workflow and return elapsed time"""
    start_time = time.time()
    try:
        # Switch context
        subprocess.check_call([
            "kubectl", "config", "use-context", cluster
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

        # List pods
        pod_list_out = subprocess.check_output([
            "kubectl", "get", "pods", "-n", namespace, "-l", pod_selector, "-o", "json"
        ], text=True)
        pods = json.loads(pod_list_out)["items"]
        if not pods:
            raise RuntimeError("No pods found")
        target_pod = pods[0]["metadata"]["name"]

        # Get pod status
        subprocess.check_call([
            "kubectl", "get", "pod", target_pod, "-n", namespace
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

        # Tail logs (run for 5 seconds to simulate debug)
        proc = subprocess.Popen([
            "kubectl", "logs", "-n", namespace, target_pod, "--tail=100", "-f"
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        time.sleep(5)
        proc.terminate()

        # Get events
        subprocess.check_call([
            "kubectl", "get", "events", "-n", namespace, "--field-selector",
            f"involvedObject.name={target_pod}"
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

        # Get resource usage
        subprocess.check_call([
            "kubectl", "top", "pod", target_pod, "-n", namespace
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

    except Exception as e:
        print(f"ERROR: kubectl workflow failed: {str(e)}")
        return None
    elapsed = time.time() - start_time
    return elapsed

def run_k9s_workflow(cluster, namespace, pod_selector):
    """Run full k9s 0.32 debug workflow and return elapsed time"""
    start_time = time.time()
    try:
        # k9s can run commands non-interactively via --command
        # We simulate the debug workflow: list pods, view logs, events, top
        subprocess.check_call([
            "k9s", "--context", cluster, "--namespace", namespace, "--command",
            f"pods:logs {pod_selector}", "--exit"
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)

        # Get events via k9s command
        subprocess.check_call([
            "k9s", "--context", cluster, "--namespace", namespace, "--command",
            f"events:list", "--exit"
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)

        # Get top pod via k9s command
        subprocess.check_call([
            "k9s", "--context", cluster, "--namespace", namespace, "--command",
            f"pods:top", "--exit"
        ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)

    except Exception as e:
        print(f"ERROR: k9s workflow failed: {str(e)}")
        return None
    elapsed = time.time() - start_time
    return elapsed

def main():
    parser = argparse.ArgumentParser(description="K8s Debug Workflow Benchmark")
    parser.add_argument("--cluster", required=True, help="Target cluster context")
    parser.add_argument("--namespace", required=True, help="Target namespace")
    parser.add_argument("--pod-selector", required=True, help="Pod label selector")
    parser.add_argument("--iterations", type=int, default=DEFAULT_ITERATIONS, help="Number of benchmark iterations")
    args = parser.parse_args()

    # Validate tools
    print("Validating tools...")
    try:
        validate_tools()
    except RuntimeError as e:
        print(f"Validation failed: {e}")
        return

    # Initialize results file
    with open(BENCHMARK_RESULTS_FILE, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["iteration", "tool", "elapsed_seconds"])

    # Run benchmarks
    print(f"Running {args.iterations} iterations for kubectl and k9s...")
    for i in range(args.iterations):
        # Run kubectl workflow
        print(f"Iteration {i+1}/{args.iterations}: kubectl...")
        kubectl_elapsed = run_kubectl_workflow(args.cluster, args.namespace, args.pod_selector)
        if kubectl_elapsed:
            with open(BENCHMARK_RESULTS_FILE, "a", newline="") as f:
                writer = csv.writer(f)
                writer.writerow([i+1, "kubectl-1.31", kubectl_elapsed])

        # Run k9s workflow
        print(f"Iteration {i+1}/{args.iterations}: k9s...")
        k9s_elapsed = run_k9s_workflow(args.cluster, args.namespace, args.pod_selector)
        if k9s_elapsed:
            with open(BENCHMARK_RESULTS_FILE, "a", newline="") as f:
                writer = csv.writer(f)
                writer.writerow([i+1, "k9s-0.32", k9s_elapsed])

    print(f"Benchmark complete. Results saved to {BENCHMARK_RESULTS_FILE}")

if __name__ == "__main__":
    main()

# Sample plugins.yaml entry for pod restart plugin
plugins:
  pod-restart:
    shortCut: Ctrl-Shift-R
    description: "Restart Selected Pod"
    scopes:
      - pods
    command: /usr/local/bin/k9s-restart-pod.sh
    background: false

# Run k9s benchmark for all contexts and export to JSON
k9s --command "benchmark --all-contexts --output json" --exit > cluster-benchmarks.json

# ~/.config/k9s/config.yaml RBAC configuration
k9s:
  rbacEnabled: true
  audit:
    enabled: true
    path: /var/log/k9s-audit.log
    maxSize: 100MB
    maxBackups: 5