Step-by-Step Guide to Setting Up 2026 Horizontal Pod Autoscaling with Kubernetes HPA v3 and Metrics Server 0.7

kind create cluster --image kindest/node:v1.32.0

helm repo add kubernetes-charts https://kubernetes.github.io/charts
helm repo update
helm install metrics-server kubernetes-charts/metrics-server \
  --namespace kube-system \
  --set image.tag=v0.7.0 \
  --set args[0]=--enable-eBPF=true \
  --set args[1]=--metric-resolution=15s

package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "time"

    appsv1 "k8s.io/api/apps/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/klog/v2"
)

const (
    metricsServerNamespace = "kube-system"
    metricsServerDeployment = "metrics-server"
    expectedImageVersion   = "registry.k8s.io/metrics-server/metrics-server:v0.7.0"
    maxRetries             = 5
    retryInterval          = 10 * time.Second
)

func main() {
    // Parse kubeconfig flag, defaults to in-cluster config if empty
    kubeconfig := flag.String("kubeconfig", "", "Path to kubeconfig file (leave empty for in-cluster)")
    flag.Parse()

    // Build Kubernetes REST config from kubeconfig or in-cluster environment
    config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
    if err != nil {
        klog.Fatalf("Failed to build Kubernetes config: %v", err)
    }

    // Initialize clientset for interacting with Kubernetes API
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        klog.Fatalf("Failed to create Kubernetes clientset: %v", err)
    }

    // Retry fetching Metrics Server deployment to handle transient API errors
    var deployment *appsv1.Deployment
    for i := 0; i < maxRetries; i++ {
        deployment, err = clientset.AppsV1().Deployments(metricsServerNamespace).Get(
            context.Background(),
            metricsServerDeployment,
            metav1.GetOptions{},
        )
        if err == nil {
            klog.Infof("Successfully fetched Metrics Server deployment on attempt %d", i+1)
            break
        }
        klog.Warningf("Attempt %d/%d: Failed to fetch deployment: %v", i+1, maxRetries, err)
        time.Sleep(retryInterval)
    }
    if err != nil {
        klog.Fatalf("Failed to get Metrics Server deployment after %d retries: %v", maxRetries, err)
    }

    // Validate deployment has at least one container
    if len(deployment.Spec.Template.Spec.Containers) == 0 {
        klog.Fatalf("Metrics Server deployment %s has no containers defined", metricsServerDeployment)
    }

    // Check container image matches expected v0.7.0 version
    containerImage := deployment.Spec.Template.Spec.Containers[0].Image
    if containerImage != expectedImageVersion {
        klog.Fatalf("Unexpected Metrics Server image: got %s, expected %s", containerImage, expectedImageVersion)
    }

    // Validate all replicas are ready
    if deployment.Status.ReadyReplicas != *deployment.Spec.Replicas {
        klog.Fatalf("Metrics Server not ready: %d/%d replicas ready", deployment.Status.ReadyReplicas, *deployment.Spec.Replicas)
    }

    // Verify metrics API is accessible
    _, err = clientset.RESTClient().Get().AbsPath("apis/metrics.k8s.io/v1beta2").DoRaw(context.Background())
    if err != nil {
        klog.Fatalf("Metrics API v1beta2 not accessible: %v", err)
    }

    klog.Info("Metrics Server v0.7.0 deployed successfully and all checks passed")
}

go mod init validate-metrics-server
go get k8s.io/client-go@v1.32.0
go get k8s.io/klog/v2@v2.120.1
go run validate-metrics-server.go --kubeconfig ~/.kube/config

kubectl api-versions | grep autoscaling/v3

kubectl patch kube-apiserver -n kube-system --type merge -p '{"spec":{"featureGates":{"HPAScaleToZero":true}}}'

package main

import (
    "context"
    "flag"
    "fmt"
    "log"

    autoscalingv3 "k8s.io/api/autoscaling/v3"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/klog/v2"
    "k8s.io/apimachinery/pkg/util/intstr"
)

const (
    hpaName      = "sample-app-hpa"
    hpaNamespace = "default"
    targetRefKind = "Deployment"
    targetRefName = "sample-app"
)

func main() {
    // Parse command line flags
    kubeconfig := flag.String("kubeconfig", "", "Path to kubeconfig file")
    minReplicas := flag.Int("min-replicas", 1, "Minimum number of replicas")
    maxReplicas := flag.Int("max-replicas", 10, "Maximum number of replicas")
    cpuTarget := flag.Int("cpu-target", 50, "Target CPU utilization percentage")
    flag.Parse()

    // Build Kubernetes config
    config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
    if err != nil {
        klog.Fatalf("Failed to build config: %v", err)
    }

    // Create clientset
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        klog.Fatalf("Failed to create clientset: %v", err)
    }

    // Define HPA v3 object
    hpa := &autoscalingv3.HorizontalPodAutoscaler{
        ObjectMeta: metav1.ObjectMeta{
            Name:      hpaName,
            Namespace: hpaNamespace,
        },
        Spec: autoscalingv3.HorizontalPodAutoscalerSpec{
            ScaleTargetRef: autoscalingv3.CrossVersionObjectReference{
                Kind:       targetRefKind,
                Name:       targetRefName,
                APIVersion: "apps/v1",
            },
            MinReplicas: int32Ptr(int32(*minReplicas)),
            MaxReplicas: int32(*maxReplicas),
            Metrics: []autoscalingv3.MetricSpec{
                {
                    Type: autoscalingv3.ResourceMetricSourceType,
                    Resource: &autoscalingv3.ResourceMetricSource{
                        Name: "cpu",
                        Target: autoscalingv3.MetricTarget{
                            Type:               autoscalingv3.UtilizationMetricType,
                            AverageUtilization: int32Ptr(int32(*cpuTarget)),
                        },
                    },
                },
            },
            JitterSeconds: int32Ptr(2), // 2s jitter to prevent simultaneous scaling
            ResourcePolicies: []autoscalingv3.ResourcePolicy{
                {
                    ContainerName: "sample-app-container",
                    Requests: autoscalingv3.ResourceList{
                        "cpu": "100m",
                    },
                    Limits: autoscalingv3.ResourceList{
                        "cpu": "500m",
                    },
                },
            },
        },
    }

    // Create HPA, handle already exists error
    _, err = clientset.AutoscalingV3().HorizontalPodAutoscalers(hpaNamespace).Create(
        context.Background(),
        hpa,
        metav1.CreateOptions{},
    )
    if err != nil {
        // Check if HPA already exists
        existing, getErr := clientset.AutoscalingV3().HorizontalPodAutoscalers(hpaNamespace).Get(
            context.Background(),
            hpaName,
            metav1.GetOptions{},
        )
        if getErr != nil {
            klog.Fatalf("Failed to create HPA and fetch existing: %v", err)
        }
        klog.Infof("HPA %s already exists, updating", hpaName)
        // Update existing HPA
        hpa.ResourceVersion = existing.ResourceVersion
        _, err = clientset.AutoscalingV3().HorizontalPodAutoscalers(hpaNamespace).Update(
            context.Background(),
            hpa,
            metav1.UpdateOptions{},
        )
        if err != nil {
            klog.Fatalf("Failed to update existing HPA: %v", err)
        }
    }

    klog.Infof("Successfully created/updated HPA %s in namespace %s", hpaName, hpaNamespace)
}

// int32Ptr returns a pointer to the given int32 value
func int32Ptr(v int32) *int32 {
    return &v
}

go get k8s.io/api/autoscaling/v3@v1.32.0
go run create-hpa-v3.go --kubeconfig ~/.kube/config

package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "os"
    "os/signal"
    "syscall"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    // Define CPU load counter for metrics
    cpuLoadCounter = prometheus.NewCounter(
        prometheus.CounterOpts{
            Name: "sample_app_cpu_load_total",
            Help: "Total number of CPU load requests",
        },
    )
    // HTTP request duration histogram
    requestDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "sample_app_request_duration_seconds",
            Help:    "Histogram of request durations",
            Buckets: prometheus.DefBuckets,
        },
        []string{"path"},
    )
)

func init() {
    // Register Prometheus metrics
    prometheus.MustRegister(cpuLoadCounter)
    prometheus.MustRegister(requestDuration)
}

func main() {
    // Parse port from environment, default to 8080
    port := os.Getenv("PORT")
    if port == "" {
        port = "8080"
    }

    // Create HTTP mux
    mux := http.NewServeMux()

    // Health check endpoint
    mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
        w.WriteHeader(http.StatusOK)
        fmt.Fprintf(w, "OK")
    })

    // Metrics endpoint for Prometheus
    mux.Handle("/metrics", promhttp.Handler())

    // CPU load endpoint: generates load for 5 seconds
    mux.HandleFunc("/load", func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        cpuLoadCounter.Inc()
        // Generate CPU load by looping for 5 seconds
        endTime := time.Now().Add(5 * time.Second)
        for time.Now().Before(endTime) {
            // Busy loop to generate CPU usage
            _ = time.Now().UnixNano()
        }
        duration := time.Since(start).Seconds()
        requestDuration.WithLabelValues("/load").Observe(duration)
        w.WriteHeader(http.StatusOK)
        fmt.Fprintf(w, "Load generated for 5 seconds")
    })

    // Default endpoint
    mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        fmt.Fprintf(w, "Sample App Running")
        duration := time.Since(start).Seconds()
        requestDuration.WithLabelValues("/").Observe(duration)
    })

    // Create HTTP server
    server := &http.Server{
        Addr:    ":" + port,
        Handler: mux,
    }

    // Start server in goroutine
    go func() {
        log.Printf("Starting server on port %s", port)
        if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
            log.Fatalf("Failed to start server: %v", err)
        }
    }()

    // Wait for interrupt signal to gracefully shutdown
    quit := make(chan os.Signal, 1)
    signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
    <-quit
    log.Println("Shutting down server...")

    // Graceful shutdown with 5s timeout
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()
    if err := server.Shutdown(ctx); err != nil {
        log.Fatalf("Server forced to shutdown: %v", err)
    }
    log.Println("Server exited properly")
}

GOOS=linux go build -o sample-app sample-app.go
docker build -t sample-app:v1 .
kind load docker-image sample-app:v1

apiVersion: apps/v1
kind: Deployment
metadata:
  name: sample-app
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: sample-app
  template:
    metadata:
      labels:
        app: sample-app
    spec:
      containers:
      - name: sample-app-container
        image: sample-app:v1
        ports:
        - containerPort: 8080
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 256Mi
---
apiVersion: v1
kind: Service
metadata:
  name: sample-app-svc
  namespace: default
spec:
  selector:
    app: sample-app
  ports:
  - port: 80
    targetPort: 8080
  type: ClusterIP

kubectl apply -f sample-app.yaml

package main

import (
    "context"
    "flag"
    "fmt"
    "log"

    autoscalingv3 "k8s.io/api/autoscaling/v3"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/klog/v2"
)

func main() {
    // Parse flags
    kubeconfig := flag.String("kubeconfig", "", "Path to kubeconfig file")
    hpaName := flag.String("hpa-name", "sample-app-hpa", "Name of HPA to update")
    namespace := flag.String("namespace", "default", "Namespace of HPA")
    flag.Parse()

    // Build config
    config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
    if err != nil {
        klog.Fatalf("Failed to build config: %v", err)
    }

    // Create clientset
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        klog.Fatalf("Failed to create clientset: %v", err)
    }

    // Fetch existing HPA
    hpa, err := clientset.AutoscalingV3().HorizontalPodAutoscalers(*namespace).Get(
        context.Background(),
        *hpaName,
        metav1.GetOptions{},
    )
    if err != nil {
        klog.Fatalf("Failed to fetch HPA: %v", err)
    }

    // Add memory resource policy to existing HPA
    hpa.Spec.ResourcePolicies = append(hpa.Spec.ResourcePolicies, autoscalingv3.ResourcePolicy{
        ContainerName: "sample-app-container",
        Requests: autoscalingv3.ResourceList{
            "memory": "128Mi",
        },
        Limits: autoscalingv3.ResourceList{
            "memory": "256Mi",
        },
    })

    // Add memory metric to HPA
    hpa.Spec.Metrics = append(hpa.Spec.Metrics, autoscalingv3.MetricSpec{
        Type: autoscalingv3.ResourceMetricSourceType,
        Resource: &autoscalingv3.ResourceMetricSource{
            Name: "memory",
            Target: autoscalingv3.MetricTarget{
                Type:               autoscalingv3.UtilizationMetricType,
                AverageUtilization: int32Ptr(70),
            },
        },
    })

    // Update HPA
    _, err = clientset.AutoscalingV3().HorizontalPodAutoscalers(*namespace).Update(
        context.Background(),
        hpa,
        metav1.UpdateOptions{},
    )
    if err != nil {
        klog.Fatalf("Failed to update HPA: %v", err)
    }

    klog.Infof("Successfully updated HPA %s with memory resource policy", *hpaName)
}

func int32Ptr(v int32) *int32 {
    return &v
}

go run update-hpa-policy.go --kubeconfig ~/.kube/config

import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate } from 'k6/metrics';

// Custom metric to track failed requests
const failureRate = new Rate('failed_requests');

export const options = {
  stages: [
    { duration: '30s', target: 100 }, // Ramp up to 100 users
    { duration: '2m', target: 1000 }, // Stay at 1000 users
    { duration: '30s', target: 0 }, // Ramp down to 0
  ],
  thresholds: {
    'http_req_duration': ['p(95)<500'], // 95% of requests under 500ms
    'failed_requests': ['rate<0.1'], // Less than 10% failures
  },
};

export default function () {
  // Send request to load endpoint
  const response = http.get('http://sample-app-svc.default.svc.cluster.local/load');

  // Check if request was successful
  const success = check(response, {
    'status is 200': (r) => r.status === 200,
    'response time < 1000ms': (r) => r.timings.duration < 1000,
  });

  // Record failure if check fails
  failureRate.add(!success);

  // Sleep to simulate user think time
  sleep(1);
}

// Handle test setup
export function setup() {
  console.log('Starting autoscaling load test');
}

// Handle test teardown
export function teardown(data) {
  console.log('Load test completed');
}

kubectl run k6-test --image=grafana/k6:0.52.0 --rm -i --restart=Never -- run --vus 1000 --duration 5m - < k6-load-test.js

watch kubectl get hpa sample-app-hpa

package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "time"

    "github.com/prometheus/client_golang/api"
    v1 "github.com/prometheus/client_golang/api/prometheus/v1"
    "github.com/prometheus/common/model"
)

func main() {
    // Parse flags
    prometheusURL := flag.String("prometheus-url", "http://prometheus-k8s.monitoring.svc:9090", "Prometheus service URL")
    hpaName := flag.String("hpa-name", "sample-app-hpa", "HPA name to query")
    namespace := flag.String("namespace", "default", "HPA namespace")
    flag.Parse()

    // Create Prometheus client
    client, err := api.NewClient(api.Config{
        Address: *prometheusURL,
    })
    if err != nil {
        log.Fatalf("Failed to create Prometheus client: %v", err)
    }

    v1api := v1.NewAPI(client)
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Query HPA current replicas
    query := fmt.Sprintf(`kube_hpa_status_current_replicas{hpa="%s", namespace="%s"}`, *hpaName, *namespace)
    result, warnings, err := v1api.Query(ctx, query, time.Now())
    if err != nil {
        log.Fatalf("Failed to query current replicas: %v", err)
    }
    if len(warnings) > 0 {
        log.Printf("Prometheus warnings: %v", warnings)
    }

    if result.Type() == model.ValVector {
        vector := result.(model.Vector)
        for _, sample := range vector {
            fmt.Printf("Current replicas for %s: %v\n", *hpaName, sample.Value)
        }
    }

    // Query Metrics Server eBPF probe latency
    query = "metrics_server_eBPF_probe_latency_seconds_p99"
    result, warnings, err = v1api.Query(ctx, query, time.Now())
    if err != nil {
        log.Fatalf("Failed to query eBPF latency: %v", err)
    }
    if len(warnings) > 0 {
        log.Printf("Prometheus warnings: %v", warnings)
    }

    if result.Type() == model.ValVector {
        vector := result.(model.Vector)
        for _, sample := range vector {
            fmt.Printf("Metrics Server eBPF p99 latency: %v seconds\n", sample.Value)
        }
    }
}

containers:
- name: sample-app-container
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: 500m

spec:
  jitterSeconds: 2

bpftool prog list | grep metrics-server

kubectl logs -n kube-system deployment/metrics-server | grep eBPF