Kubernetes 1.30 checklist Nomad: The Unexpected serverless for Engineers

package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "os"
    "path/filepath"

    corev1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/client-go/util/homedir"
    // New in Kubernetes 1.30: Hierarchical Namespace Controller API
    hnsv1alpha1 "sigs.k8s.io/hierarchical-namespaces/api/v1alpha1"
    "sigs.k8s.io/hierarchical-namespaces/pkg/client/clientset/versioned"
)

// K8sServerlessDeployer deploys a serverless function to Kubernetes 1.30
// using Hierarchical Namespaces for multi-tenant isolation
type K8sServerlessDeployer struct {
    clientset *kubernetes.Clientset
    hnsClient *versioned.Clientset
    namespace string
}

func NewK8sDeployer(kubeconfig, namespace string) (*K8sServerlessDeployer, error) {
    // If kubeconfig is not provided, use default in ~/.kube/config
    if kubeconfig == "" {
        if home := homedir.HomeDir(); home != "" {
            kubeconfig = filepath.Join(home, ".kube", "config")
        } else {
            return nil, fmt.Errorf("kubeconfig not provided and home directory not found")
        }
    }

    // Build config from kubeconfig
    config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
    if err != nil {
        return nil, fmt.Errorf("failed to build kubeconfig: %w", err)
    }

    // Create Kubernetes core clientset
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        return nil, fmt.Errorf("failed to create kubernetes clientset: %w", err)
    }

    // Create Hierarchical Namespace clientset (new in K8s 1.30)
    hnsClient, err := versioned.NewForConfig(config)
    if err != nil {
        return nil, fmt.Errorf("failed to create HNS clientset: %w", err)
    }

    // Verify K8s version is 1.30+
    serverVersion, err := clientset.Discovery().ServerVersion()
    if err != nil {
        return nil, fmt.Errorf("failed to get server version: %w", err)
    }
    log.Printf("Connected to Kubernetes %s", serverVersion.String())

    return &K8sServerlessDeployer{
        clientset: clientset,
        hnsClient: hnsClient,
        namespace: namespace,
    }, nil
}

// CreateHierarchicalNamespace creates a child namespace under a parent for multi-tenancy
func (d *K8sServerlessDeployer) CreateHierarchicalNamespace(parentNS, childNS string) error {
    // Check if parent namespace exists
    _, err := d.clientset.CoreV1().Namespaces().Get(context.Background(), parentNS, metav1.GetOptions{})
    if err != nil {
        return fmt.Errorf("parent namespace %s not found: %w", parentNS, err)
    }

    // Create HierarchicalNamespace resource
    hns := &hnsv1alpha1.HierarchicalNamespace{
        ObjectMeta: metav1.ObjectMeta{
            Name:      childNS,
            Namespace: parentNS,
        },
    }
    _, err = d.hnsClient.HnsV1alpha1().HierarchicalNamespaces(parentNS).Create(context.Background(), hns, metav1.CreateOptions{})
    if err != nil {
        return fmt.Errorf("failed to create hierarchical namespace: %w", err)
    }
    log.Printf("Created hierarchical namespace %s/%s", parentNS, childNS)
    return nil
}

// DeployFunction deploys a serverless function as a Knative service (K8s 1.30 compatible)
func (d *K8sServerlessDeployer) DeployFunction(funcName, image string, replicas int) error {
    // Define deployment spec
    deployment := &corev1.Pod{
        ObjectMeta: metav1.ObjectMeta{
            Name: fmt.Sprintf("%s-pod", funcName),
        },
        Spec: corev1.PodSpec{
            Containers: []corev1.Container{
                {
                    Name:  funcName,
                    Image: image,
                    Ports: []corev1.ContainerPort{{ContainerPort: 8080}},
                },
            },
            // New in K8s 1.30: Ephemeral container support for serverless
            EphemeralContainers: []corev1.EphemeralContainer{
                {
                    EphemeralContainerCommon: corev1.EphemeralContainerCommon{
                        Name:  "debug-sidecar",
                        Image: "busybox:1.36",
                    },
                },
            },
        },
    }

    // Create pod in target namespace
    _, err := d.clientset.CoreV1().Pods(d.namespace).Create(context.Background(), deployment, metav1.CreateOptions{})
    if err != nil {
        return fmt.Errorf("failed to deploy function: %w", err)
    }
    log.Printf("Deployed function %s to namespace %s", funcName, d.namespace)
    return nil
}

func main() {
    // Parse command line flags
    kubeconfig := flag.String("kubeconfig", "", "Path to kubeconfig file")
    parentNS := flag.String("parent-ns", "serverless-tenant-a", "Parent hierarchical namespace")
    childNS := flag.String("child-ns", "func-123", "Child namespace for function")
    funcName := flag.String("func-name", "hello-world", "Serverless function name")
    image := flag.String("image", "gcr.io/hello-world:v1", "Function container image")
    flag.Parse()

    // Initialize deployer
    deployer, err := NewK8sDeployer(*kubeconfig, *childNS)
    if err != nil {
        log.Fatalf("Failed to initialize deployer: %v", err)
    }

    // Create hierarchical namespace
    err = deployer.CreateHierarchicalNamespace(*parentNS, *childNS)
    if err != nil {
        log.Fatalf("Failed to create namespace: %v", err)
    }

    // Deploy function
    err = deployer.DeployFunction(*funcName, *image, 1)
    if err != nil {
        log.Fatalf("Failed to deploy function: %v", err)
    }

    fmt.Println("Successfully deployed serverless function to Kubernetes 1.30")
}

package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "os"

    nomad "github.com/hashicorp/nomad/api"
)

// NomadServerlessDeployer deploys ephemeral serverless tasks to Nomad 1.8+
type NomadServerlessDeployer struct {
    client *nomad.Client
    region string
}

func NewNomadDeployer(nomadAddr, region string) (*NomadServerlessDeployer, error) {
    // Configure Nomad client
    config := nomad.DefaultConfig()
    if nomadAddr != "" {
        config.Address = nomadAddr
    }
    if region != "" {
        config.Region = region
    }

    // Create Nomad client
    client, err := nomad.NewClient(config)
    if err != nil {
        return nil, fmt.Errorf("failed to create nomad client: %w", err)
    }

    // Verify Nomad version is 1.8+
    version, err := client.Agent().Version()
    if err != nil {
        return nil, fmt.Errorf("failed to get nomad version: %w", err)
    }
    log.Printf("Connected to Nomad %s (region: %s)", version.Version, version.Region)

    return &NomadServerlessDeployer{
        client: client,
        region: region,
    }, nil
}

// RegisterEphemeralJob registers a new ephemeral serverless job (Nomad 1.8+ feature)
func (d *NomadServerlessDeployer) RegisterEphemeralJob(jobName, image string, timeout int) error {
    // Define Nomad job spec for ephemeral serverless task
    job := &nomad.Job{
        Name:        &jobName,
        Region:      &d.region,
        Type:        PtrString("batch"), // Batch type for serverless ephemeral tasks
        Ephemeral:   PtrBool(true),      // New in Nomad 1.8: mark job as ephemeral
        Datacenters: []string{"dc1"},
        TaskGroups: []*nomad.TaskGroup{
            {
                Name: PtrString("serverless-group"),
                Ephemeral: &nomad.EphemeralGroup{
                    Timeout: &timeout, // Auto-stop task after timeout (seconds)
                },
                Tasks: []*nomad.Task{
                    {
                        Name:   PtrString("serverless-func"),
                        Driver: PtrString("docker"),
                        Config: map[string]interface{}{
                            "image": image,
                            "ports": []map[string]interface{}{
                                {"label": "http", "port": 8080},
                            },
                        },
                        Resources: &nomad.Resources{
                            CPU:      PtrInt(500), // 500MHz CPU
                            MemoryMB: PtrInt(128), // 128MB RAM
                        },
                    },
                },
            },
        },
    }

    // Register job with Nomad
    resp, _, err := d.client.Jobs().Register(job, nil)
    if err != nil {
        return fmt.Errorf("failed to register job: %w", err)
    }

    log.Printf("Registered ephemeral job %s (evaluation ID: %s)", jobName, resp.EvalID)
    return nil
}

// GetJobStatus retrieves the status of a registered serverless job
func (d *NomadServerlessDeployer) GetJobStatus(jobName string) (string, error) {
    job, _, err := d.client.Jobs().Info(jobName, nil)
    if err != nil {
        return "", fmt.Errorf("failed to get job info: %w", err)
    }
    return *job.Status, nil
}

// Helper functions to create pointers (required for Nomad API structs)
func PtrString(s string) *string { return &s }
func PtrBool(b bool) *bool { return &b }
func PtrInt(i int) *int { return &i }

func main() {
    // Parse command line flags
    nomadAddr := flag.String("nomad-addr", "http://localhost:4646", "Nomad API address")
    region := flag.String("region", "global", "Nomad region")
    jobName := flag.String("job-name", "hello-world", "Serverless job name")
    image := flag.String("image", "gcr.io/hello-world:v1", "Function container image")
    timeout := flag.Int("timeout", 300, "Ephemeral task timeout (seconds)")
    flag.Parse()

    // Initialize deployer
    deployer, err := NewNomadDeployer(*nomadAddr, *region)
    if err != nil {
        log.Fatalf("Failed to initialize deployer: %v", err)
    }

    // Register ephemeral serverless job
    err = deployer.RegisterEphemeralJob(*jobName, *image, *timeout)
    if err != nil {
        log.Fatalf("Failed to register job: %v", err)
    }

    // Wait for job to start
    status, err := deployer.GetJobStatus(*jobName)
    if err != nil {
        log.Fatalf("Failed to get job status: %v", err)
    }
    log.Printf("Job %s status: %s", *jobName, status)

    fmt.Println("Successfully deployed serverless function to Nomad 1.8")
}

#!/usr/bin/env python3
"""
Benchmark cold start latency for serverless functions running on
Kubernetes 1.30 and Nomad 1.8.2.
Requires: requests, prometheus_client, statistics
"""

import argparse
import json
import logging
import statistics
import time
from typing import Dict, List

import requests
from prometheus_client import CollectorRegistry, Gauge, push_to_gateway

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Prometheus metrics
REGISTRY = CollectorRegistry()
COLD_START_LATENCY = Gauge(
    "serverless_cold_start_latency_ms",
    "Cold start latency in milliseconds",
    ["orchestrator", "function"],
    registry=REGISTRY
)
INVOCATION_SUCCESS = Gauge(
    "serverless_invocation_success_total",
    "Total successful invocations",
    ["orchestrator", "function"],
    registry=REGISTRY
)

class ServerlessBenchmarker:
    """Runs cold start benchmarks against serverless endpoints."""

    def __init__(self, k8s_endpoint: str, nomad_endpoint: str, function: str, iterations: int):
        self.k8s_endpoint = k8s_endpoint
        self.nomad_endpoint = nomad_endpoint
        self.function = function
        self.iterations = iterations
        self.results: Dict[str, List[float]] = {
            "kubernetes": [],
            "nomad": []
        }

    def _invoke_function(self, endpoint: str) -> float:
        """
        Invoke a serverless function and return latency in milliseconds.
        Raises exception if invocation fails.
        """
        start = time.perf_counter()
        try:
            resp = requests.get(f"{endpoint}/{self.function}", timeout=10)
            resp.raise_for_status()
        except requests.exceptions.RequestException as e:
            logger.error(f"Invocation failed: {e}")
            raise
        end = time.perf_counter()
        latency_ms = (end - start) * 1000
        return latency_ms

    def run_k8s_benchmark(self):
        """Run cold start benchmark against Kubernetes-deployed function."""
        logger.info(f"Starting K8s benchmark: {self.iterations} iterations")
        for i in range(self.iterations):
            try:
                # Force cold start by waiting 60s between invocations (function scales to zero)
                if i > 0:
                    time.sleep(60)
                latency = self._invoke_function(self.k8s_endpoint)
                self.results["kubernetes"].append(latency)
                COLD_START_LATENCY.labels(orchestrator="kubernetes", function=self.function).set(latency)
                INVOCATION_SUCCESS.labels(orchestrator="kubernetes", function=self.function).inc()
                logger.info(f"K8s iteration {i+1}/{self.iterations}: {latency:.2f}ms")
            except Exception as e:
                logger.error(f"K8s iteration {i+1} failed: {e}")

    def run_nomad_benchmark(self):
        """Run cold start benchmark against Nomad-deployed function."""
        logger.info(f"Starting Nomad benchmark: {self.iterations} iterations")
        for i in range(self.iterations):
            try:
                if i > 0:
                    time.sleep(60)
                latency = self._invoke_function(self.nomad_endpoint)
                self.results["nomad"].append(latency)
                COLD_START_LATENCY.labels(orchestrator="nomad", function=self.function).set(latency)
                INVOCATION_SUCCESS.labels(orchestrator="nomad", function=self.function).inc()
                logger.info(f"Nomad iteration {i+1}/{self.iterations}: {latency:.2f}ms")
            except Exception as e:
                logger.error(f"Nomad iteration {i+1} failed: {e}")

    def calculate_stats(self) -> Dict:
        """Calculate statistical summary of benchmark results."""
        stats = {}
        for orchestrator, latencies in self.results.items():
            if not latencies:
                continue
            stats[orchestrator] = {
                "p50": statistics.median(latencies),
                "p90": sorted(latencies)[int(len(latencies)*0.9)],
                "p99": sorted(latencies)[int(len(latencies)*0.99)] if len(latencies) >= 100 else max(latencies),
                "mean": statistics.mean(latencies),
                "stddev": statistics.stdev(latencies) if len(latencies) > 1 else 0,
                "sample_size": len(latencies)
            }
        return stats

    def push_metrics(self, prometheus_gateway: str):
        """Push metrics to Prometheus Pushgateway."""
        try:
            push_to_gateway(prometheus_gateway, job="serverless-benchmark", registry=REGISTRY)
            logger.info(f"Pushed metrics to {prometheus_gateway}")
        except Exception as e:
            logger.error(f"Failed to push metrics: {e}")

def main():
    parser = argparse.ArgumentParser(description="Benchmark serverless cold starts")
    parser.add_argument("--k8s-endpoint", required=True, help="K8s function endpoint")
    parser.add_argument("--nomad-endpoint", required=True, help="Nomad function endpoint")
    parser.add_argument("--function", default="hello", help="Function path")
    parser.add_argument("--iterations", type=int, default=100, help="Number of cold start iterations")
    parser.add_argument("--prometheus-gateway", help="Prometheus Pushgateway address")
    args = parser.parse_args()

    benchmarker = ServerlessBenchmarker(
        k8s_endpoint=args.k8s_endpoint,
        nomad_endpoint=args.nomad_endpoint,
        function=args.function,
        iterations=args.iterations
    )

    # Run benchmarks
    benchmarker.run_k8s_benchmark()
    benchmarker.run_nomad_benchmark()

    # Calculate and print stats
    stats = benchmarker.calculate_stats()
    print(json.dumps(stats, indent=2))

    # Push metrics if gateway provided
    if args.prometheus_gateway:
        benchmarker.push_metrics(args.prometheus_gateway)

if __name__ == "__main__":
    main()

kubectl hns create child-func-ns --parent parent-tenant-ns

job "serverless-func" {
  group "ephemeral-group" {
    ephemeral {
      timeout = 300 # Auto-stop after 5 minutes
    }
    task "func" {
      driver = "docker"
      config {
        image = "gcr.io/hello-world:v1"
      }
    }
  }
}

wrk -t2 -c100 -d30s --latency --timeout 60s https://k8s-func.example.com/hello