How to Implement Multi-Cluster Kubernetes with Rancher 2.9 and Submariner 0.15 for Hybrid Cloud

#!/usr/bin/env python3
"""
Rancher 2.9 Management Server Deployment Script
Version: 1.0
Requires: Python 3.9+, kubernetes client, helm client, docker
"""

import os
import sys
import json
import time
import subprocess
import logging
from typing import Optional, Dict, Any

# Configure logging for error handling and audit trails
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

# Configuration constants (update these for your environment)
RANCHER_VERSION = "2.9.0"
HELM_REPO_NAME = "rancher-stable"
HELM_REPO_URL = "https://releases.rancher.com/server-charts/stable"
NAMESPACE = "cattle-system"
CLUSTER_NAME = "rancher-management"
KUBECONFIG = os.path.expanduser("~/.kube/config-bootstrap")

def run_command(command: list, cwd: Optional[str] = None, capture_output: bool = False) -> Dict[str, Any]:
    """Execute a shell command with error handling and logging."""
    try:
        logger.info(f"Executing command: {' '.join(command)}")
        result = subprocess.run(
            command,
            cwd=cwd,
            capture_output=capture_output,
            text=True,
            check=True
        )
        return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
    except subprocess.CalledProcessError as e:
        logger.error(f"Command failed with exit code {e.returncode}: {e.stderr}")
        return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
    except FileNotFoundError as e:
        logger.error(f"Command not found: {command[0]}")
        return {"success": False, "stdout": "", "stderr": str(e)}

def check_prerequisites() -> bool:
    """Verify all required tools are installed and versions are compatible."""
    prerequisites = [
        ("helm", ["helm", "version", "--short"]),
        ("kubectl", ["kubectl", "version", "--client", "--short"]),
        ("docker", ["docker", "version", "--format", "{{.Client.Version}}"])
    ]
    for tool, cmd in prerequisites:
        result = run_command(cmd, capture_output=True)
        if not result["success"]:
            logger.error(f"Prerequisite {tool} not found or failed to run.")
            return False
        logger.info(f"Found {tool}: {result['stdout'].strip()}")
    return True

def add_helm_repo() -> bool:
    """Add Rancher Helm repository and update."""
    # Check if repo already exists
    result = run_command(["helm", "repo", "list", "--output", "json"], capture_output=True)
    if result["success"]:
        repos = json.loads(result["stdout"])
        for repo in repos:
            if repo["name"] == HELM_REPO_NAME:
                logger.info(f"Helm repo {HELM_REPO_NAME} already exists.")
                return True
    # Add repo if not present
    result = run_command(["helm", "repo", "add", HELM_REPO_NAME, HELM_REPO_URL])
    if not result["success"]:
        return False
    # Update repos
    return run_command(["helm", "repo", "update"])["success"]

def deploy_rancher() -> bool:
    """Deploy Rancher 2.9 via Helm to the bootstrap cluster."""
    # Create namespace if not exists
    run_command(["kubectl", "create", "namespace", NAMESPACE, "--kubeconfig", KUBECONFIG], capture_output=True)
    # Deploy Rancher with required parameters
    deploy_cmd = [
        "helm", "install", "rancher", f"{HELM_REPO_NAME}/rancher",
        "--namespace", NAMESPACE,
        "--kubeconfig", KUBECONFIG,
        "--set", f"hostname=rancher.{CLUSTER_NAME}.example.com",
        "--set", "replicas=3",
        "--set", "ingress.tls.source=letsEncrypt",
        "--set", "letsEncrypt.email=admin@example.com",
        "--set", f"version={RANCHER_VERSION}",
        "--wait", "--timeout", "10m"
    ]
    result = run_command(deploy_cmd)
    if not result["success"]:
        return False
    # Wait for Rancher pods to be ready
    logger.info("Waiting for Rancher pods to be ready...")
    time.sleep(30)
    result = run_command([
        "kubectl", "wait", "--kubeconfig", KUBECONFIG,
        "--namespace", NAMESPACE,
        "--for=condition=ready", "pod",
        "--selector=app=rancher",
        "--timeout=5m"
    ])
    return result["success"]

if __name__ == "__main__":
    logger.info("Starting Rancher 2.9 deployment script")
    if not check_prerequisites():
        sys.exit(1)
    if not add_helm_repo():
        sys.exit(1)
    if not deploy_rancher():
        logger.error("Rancher deployment failed. Check logs above.")
        sys.exit(1)
    logger.info(f"Rancher {RANCHER_VERSION} deployed successfully!")
    logger.info(f"Access Rancher at https://rancher.{CLUSTER_NAME}.example.com")

#!/usr/bin/env python3
"""
Downstream Cluster Provisioning and Rancher Import Script
Version: 1.0
Requires: Python 3.9+, boto3, vsphere-sdk, requests
"""

import os
import sys
import json
import time
import subprocess
import logging
import boto3
from typing import List, Dict, Any, Optional
from botocore.exceptions import ClientError, NoCredentialsError

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

# Configuration
RANCHER_API_URL = "https://rancher.example.com/v3"
RANCHER_ADMIN_TOKEN = os.environ.get("RANCHER_TOKEN")
AWS_REGION = "us-east-1"
EKS_CLUSTER_NAME = "eks-us-east-1"
VSPHERE_CLUSTER_NAME = "vsphere-onprem"
VSPHERE_HOST = "vcenter.example.com"
VSPHERE_USER = os.environ.get("VSPHERE_USER")
VSPHERE_PASS = os.environ.get("VSPHERE_PASS")

def get_rancher_auth_headers() -> Dict[str, str]:
    """Return headers for Rancher API authentication."""
    if not RANCHER_ADMIN_TOKEN:
        logger.error("RANCHER_TOKEN environment variable not set.")
        sys.exit(1)
    return {
        "Authorization": f"Bearer {RANCHER_ADMIN_TOKEN}",
        "Content-Type": "application/json"
    }

def provision_eks_cluster() -> bool:
    """Provision EKS cluster via boto3."""
    try:
        eks_client = boto3.client("eks", region_name=AWS_REGION)
        # Check if cluster already exists
        try:
            eks_client.describe_cluster(name=EKS_CLUSTER_NAME)
            logger.info(f"EKS cluster {EKS_CLUSTER_NAME} already exists.")
            return True
        except ClientError as e:
            if e.response["Error"]["Code"] != "ResourceNotFoundException":
                raise
        # Create EKS cluster
        logger.info(f"Creating EKS cluster {EKS_CLUSTER_NAME}...")
        eks_client.create_cluster(
            name=EKS_CLUSTER_NAME,
            version="1.28",
            roleArn="arn:aws:iam::123456789012:role/eks-cluster-role",
            resourcesVpcConfig={
                "subnetIds": ["subnet-12345", "subnet-67890"],
                "securityGroupIds": ["sg-12345"]
            }
        )
        # Wait for cluster to be active
        waiter = eks_client.get_waiter("cluster_active")
        waiter.wait(name=EKS_CLUSTER_NAME)
        logger.info(f"EKS cluster {EKS_CLUSTER_NAME} is active.")
        return True
    except NoCredentialsError:
        logger.error("AWS credentials not found.")
        return False
    except ClientError as e:
        logger.error(f"EKS provisioning failed: {e}")
        return False

def provision_vsphere_cluster() -> bool:
    """Provision vSphere Kubernetes cluster via vSphere SDK."""
    try:
        # Note: Replace with actual vSphere SDK client initialization
        logger.info(f"Creating vSphere cluster {VSPHERE_CLUSTER_NAME}...")
        # Simplified provisioning logic for example
        logger.info(f"vSphere cluster {VSPHERE_CLUSTER_NAME} created.")
        return True
    except Exception as e:
        logger.error(f"vSphere provisioning failed: {e}")
        return False

def import_cluster_to_rancher(cluster_name: str, kubeconfig_path: str) -> bool:
    """Import a downstream cluster into Rancher via API."""
    try:
        headers = get_rancher_auth_headers()
        resp = requests.post(
            f"{RANCHER_API_URL}/clusterregistrationtokens",
            headers=headers,
            json={"type": "clusterRegistrationToken", "clusterId": ""}
        )
        resp.raise_for_status()
        token = resp.json()["token"]
        # Apply Rancher agent manifest
        import_cmd = [
            "kubectl", "apply", "-f",
            f"https://rancher.example.com/v3/import/{token}.yaml",
            "--kubeconfig", kubeconfig_path
        ]
        subprocess.run(import_cmd, check=True, capture_output=True)
        logger.info(f"Cluster {cluster_name} imported to Rancher successfully.")
        return True
    except requests.exceptions.RequestException as e:
        logger.error(f"Rancher API request failed: {e}")
        return False
    except Exception as e:
        logger.error(f"Cluster import failed: {e}")
        return False

if __name__ == "__main__":
    logger.info("Starting downstream cluster provisioning...")
    if not provision_eks_cluster():
        sys.exit(1)
    if not provision_vsphere_cluster():
        sys.exit(1)
    # Import EKS cluster
    eks_kubeconfig = "eks-kubeconfig.yaml"
    subprocess.run(["aws", "eks", "update-kubeconfig", "--name", EKS_CLUSTER_NAME, "--kubeconfig", eks_kubeconfig])
    if not import_cluster_to_rancher(EKS_CLUSTER_NAME, eks_kubeconfig):
        sys.exit(1)
    # Import vSphere cluster
    vsphere_kubeconfig = "vsphere-kubeconfig.yaml"
    if not import_cluster_to_rancher(VSPHERE_CLUSTER_NAME, vsphere_kubeconfig):
        sys.exit(1)
    logger.info("All downstream clusters provisioned and imported to Rancher.")

#!/usr/bin/env python3
"""
Submariner 0.15 Installation and Configuration Script
Version: 1.0
Requires: Python 3.9+, subctl 0.15+, kubectl
"""

import os
import sys
import json
import time
import subprocess
import logging
from typing import List, Dict, Any, Optional

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

# Configuration
SUBMARINER_VERSION = "0.15.0"
CLUSTERS = [
    {"name": "eks-us-east-1", "kubeconfig": "eks-kubeconfig.yaml"},
    {"name": "vsphere-onprem", "kubeconfig": "vsphere-kubeconfig.yaml"}
]
GLOBALNET_CIDR = "242.0.0.0/8"
IPSEC_PSK = os.environ.get("SUBMARINER_IPSEC_PSK", "submariner-psk-12345")

def run_subctl_command(command: List[str], cluster_kubeconfig: str) -> Dict[str, Any]:
    """Execute subctl command with the specified cluster kubeconfig."""
    try:
        full_cmd = ["subctl", "--kubeconfig", cluster_kubeconfig] + command
        logger.info(f"Running subctl command: {' '.join(full_cmd)}")
        result = subprocess.run(
            full_cmd,
            capture_output=True,
            text=True,
            check=True
        )
        return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
    except subprocess.CalledProcessError as e:
        logger.error(f"subctl command failed: {e.stderr}")
        return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
    except FileNotFoundError:
        logger.error("subctl not found. Install subctl 0.15+ first.")
        return {"success": False, "stdout": "", "stderr": "subctl not found"}

def deploy_submariner_to_cluster(cluster: Dict[str, str]) -> bool:
    """Deploy Submariner to a single cluster."""
    cluster_name = cluster["name"]
    kubeconfig = cluster["kubeconfig"]
    logger.info(f"Deploying Submariner {SUBMARINER_VERSION} to cluster {cluster_name}...")
    deploy_cmd = [
        "deploy",
        "--clusterid", cluster_name,
        "--globalnet-cidr", GLOBALNET_CIDR,
        "--ipsec-psk", IPSEC_PSK,
        "--version", SUBMARINER_VERSION
    ]
    result = run_subctl_command(deploy_cmd, kubeconfig)
    if not result["success"]:
        return False
    # Verify Submariner pods are running
    logger.info(f"Verifying Submariner pods in {cluster_name}...")
    verify_cmd = ["kubectl", "get", "pods", "-n", "submariner-operator", "--kubeconfig", kubeconfig]
    result = subprocess.run(verify_cmd, capture_output=True, text=True, check=True)
    if "submariner" not in result.stdout:
        logger.error(f"Submariner pods not found in {cluster_name}")
        return False
    logger.info(f"Submariner deployed successfully to {cluster_name}")
    return True

def join_clusters() -> bool:
    """Join all clusters into a single Submariner gateway set."""
    logger.info("Joining clusters via Submariner...")
    first_cluster = CLUSTERS[0]
    token_cmd = ["get", "gatewaytoken", "--kubeconfig", first_cluster["kubeconfig"]]
    result = run_subctl_command(token_cmd, first_cluster["kubeconfig"])
    if not result["success"]:
        return False
    gateway_token = result["stdout"].strip()
    for cluster in CLUSTERS[1:]:
        join_cmd = ["join", "--gatewaytoken", gateway_token, "--clusterid", cluster["name"]]
        result = run_subctl_command(join_cmd, cluster["kubeconfig"])
        if not result["success"]:
            return False
    return True

def verify_cross_cluster_connectivity() -> bool:
    """Verify pod-to-pod connectivity across clusters."""
    logger.info("Verifying cross-cluster connectivity...")
    nettest_cmd = [
        "kubectl", "run", "nettest",
        "--image=nicolaka/netshoot",
        "--rm", "-it",
        "--restart=Never",
        "--kubeconfig", CLUSTERS[0]["kubeconfig"],
        "--", "ping", "-c", "3", "10.0.0.1"
    ]
    try:
        result = subprocess.run(nettest_cmd, capture_output=True, text=True, check=True)
        if "0% packet loss" in result.stdout:
            logger.info("Cross-cluster connectivity verified successfully.")
            return True
        else:
            logger.error("Cross-cluster ping failed.")
            return False
    except subprocess.CalledProcessError as e:
        logger.error(f"Connectivity check failed: {e.stderr}")
        return False

if __name__ == "__main__":
    logger.info("Starting Submariner 0.15 installation...")
    for cluster in CLUSTERS:
        if not deploy_submariner_to_cluster(cluster):
            sys.exit(1)
    if not join_clusters():
        sys.exit(1)
    if not verify_cross_cluster_connectivity():
        sys.exit(1)
    logger.info("Submariner 0.15 installed and configured successfully. Cross-cluster networking is active.")

apiVersion: fleet.cattle.io/v1alpha1
kind: GitRepo
metadata:
  name: ecommerce-app
  namespace: fleet-default
spec:
  repo: https://github.com/example/ecommerce-app-config
  branch: main
  paths:
    - k8s/overlays/production
  targets:
    - clusterSelector:
        matchLabels:
          environment: production
    - clusterSelector:
        matchLabels:
          region: us-east-1

subctl deploy --clusterid vsphere-onprem --globalnet-cidr 172.16.0.0/12 --kubeconfig vsphere-kubeconfig.yaml

scrape_configs:
  - job_name: submariner
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_namespace]
        action: keep
        regex: submariner-operator
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: keep
        regex: submariner-gateway
    metrics_path: /metrics

rancher-submariner-hybrid-cloud/
├── scripts/
│   ├── deploy-rancher.py
│   ├── provision-clusters.py
│   ├── install-submariner.py
│   └── verify-connectivity.sh
├── terraform/
│   ├── eks/
│   │   ├── main.tf
│   │   ├── variables.tf
│   │   └── outputs.tf
│   └── vsphere/
│       ├── main.tf
│       ├── variables.tf
│       └── outputs.tf
├── fleet/
│   └── ecommerce-app-gitrepo.yaml
├── monitoring/
│   ├── prometheus-scrape-config.yaml
│   └── grafana-dashboard.json
├── docs/
│   ├── troubleshooting.md
│   └── benchmarking-results.md
└── README.md