Azure AKS vs Crossplane 1.15: A Real-World auto-scaling Test

package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "time"

    "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
    "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4"
    "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
    "github.com/joho/godotenv"
)

// AKSAutoScaler triggers manual scale-out for an AKS node pool if pending pods exceed threshold
type AKSAutoScaler struct {
    client  *armcontainerservice.AgentPoolsClient
    config  *ScaleConfig
    logger  *log.Logger
}

// ScaleConfig holds auto-scaling threshold configuration
type ScaleConfig struct {
    ResourceGroup    string
    ClusterName      string
    NodePoolName     string
    MaxNodes         int32
    PendingPodThreshold int32
}

// NewAKSAutoScaler initializes a new AKS auto-scaler with Azure credentials
func NewAKSAutoScaler(ctx context.Context, config *ScaleConfig) (*AKSAutoScaler, error) {
    // Load .env file for Azure credentials (CLIENT_ID, CLIENT_SECRET, TENANT_ID, SUBSCRIPTION_ID)
    if err := godotenv.Load(); err != nil {
        return nil, fmt.Errorf("failed to load .env file: %w", err)
    }

    // Get Azure subscription ID from environment
    subID := os.Getenv("SUBSCRIPTION_ID")
    if subID == "" {
        return nil, fmt.Errorf("SUBSCRIPTION_ID environment variable not set")
    }

    // Initialize Azure credential chain (supports CLI, Managed Identity, Service Principal)
    cred, err := azidentity.NewDefaultAzureCredential(nil)
    if err != nil {
        return nil, fmt.Errorf("failed to get Azure credential: %w", err)
    }

    // Create AKS agent pool client
    client, err := armcontainerservice.NewAgentPoolsClient(subID, cred, nil)
    if err != nil {
        return nil, fmt.Errorf("failed to create agent pool client: %w", err)
    }

    return &AKSAutoScaler{
        client: client,
        config: config,
        logger: log.New(os.Stdout, "AKS-AutoScaler: ", log.LstdFlags),
    }, nil
}

// ScaleOutIfNeeded checks pending pod count and triggers scale-out if threshold is met
func (a *AKSAutoScaler) ScaleOutIfNeeded(ctx context.Context, pendingPods int32) error {
    a.logger.Printf("Checking scale-out: pending pods = %d, threshold = %d", pendingPods, a.config.PendingPodThreshold)

    // Get current node pool status
    pool, err := a.client.Get(ctx, a.config.ResourceGroup, a.config.ClusterName, a.config.NodePoolName, nil)
    if err != nil {
        return fmt.Errorf("failed to get node pool %s: %w", a.config.NodePoolName, err)
    }

    currentNodes := pool.Properties.Count
    a.logger.Printf("Current node count: %d, max nodes: %d", currentNodes, a.config.MaxNodes)

    // Check if we can scale out
    if pendingPods < a.config.PendingPodThreshold {
        a.logger.Println("No scale-out needed")
        return nil
    }
    if currentNodes >= a.config.MaxNodes {
        a.logger.Println("Max node count reached, cannot scale out")
        return fmt.Errorf("max node count %d reached", a.config.MaxNodes)
    }

    // Calculate target node count (add 5 nodes or up to max)
    targetNodes := currentNodes + 5
    if targetNodes > a.config.MaxNodes {
        targetNodes = a.config.MaxNodes
    }
    a.logger.Printf("Scaling node pool %s from %d to %d nodes", a.config.NodePoolName, currentNodes, targetNodes)

    // Trigger scale-out
    _, err = a.client.BeginCreateOrUpdate(ctx, a.config.ResourceGroup, a.config.ClusterName, a.config.NodePoolName, armcontainerservice.AgentPool{
        Properties: &armcontainerservice.AgentPoolProperties{
            Count: to.Ptr(targetNodes),
            // Preserve existing node pool properties
            VMSize:        pool.Properties.VMSize,
            OSDiskSizeGB:  pool.Properties.OSDiskSizeGB,
            VnetSubnetID:  pool.Properties.VnetSubnetID,
            Mode:          pool.Properties.Mode,
        },
    }, nil)
    if err != nil {
        return fmt.Errorf("failed to trigger scale-out: %w", err)
    }

    a.logger.Printf("Scale-out triggered successfully, target node count: %d", targetNodes)
    return nil
}

func main() {
    // Load configuration
    config := &ScaleConfig{
        ResourceGroup:    "aks-benchmark-rg",
        ClusterName:      "aks-benchmark-cluster",
        NodePoolName:     "defaultnodepool",
        MaxNodes:         50,
        PendingPodThreshold: 20,
    }

    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
    defer cancel()

    // Initialize auto-scaler
    scaler, err := NewAKSAutoScaler(ctx, config)
    if err != nil {
        log.Fatalf("Failed to initialize auto-scaler: %v", err)
    }

    // Simulate pending pod count from Prometheus metric (hardcoded for example, replace with real query)
    pendingPods := int32(25)
    if err := scaler.ScaleOutIfNeeded(ctx, pendingPods); err != nil {
        log.Fatalf("Scale-out failed: %v", err)
    }
}

package main

import (
    "context"
    "fmt"
    "os"
    "time"

    "github.com/crossplane/crossplane-runtime/pkg/event"
    "github.com/crossplane/crossplane-runtime/pkg/resource"
    "github.com/go-logr/logr"
    corev1 "k8s.io/api/core/v1"
    "k8s.io/apimachinery/pkg/api/errors"
    "k8s.io/apimachinery/pkg/runtime"
    ctrl "sigs.k8s.io/controller-runtime"
    "sigs.k8s.io/controller-runtime/pkg/client"

    azurev1 "github.com/crossplane/provider-azure/apis/containerservice/v1beta1"
)

// AKSNodePoolAutoScaler reconciles AKS node pools to auto-scale based on pending pod metrics
type AKSNodePoolAutoScaler struct {
    client.Client
    Scheme *runtime.Scheme
    Log    logr.Logger
}

// +kubebuilder:rbac:groups=containerservice.azure.crossplane.io,resources=aksnodepools,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch

// Reconcile checks pending pod count for a node pool and adjusts AKS node count accordingly
func (r *AKSNodePoolAutoScaler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
    log := r.Log.WithValues("aksnodepool", req.NamespacedName)

    // Fetch the AKSNodePool resource
    nodePool := &azurev1.AKSNodePool{}
    if err := r.Get(ctx, req.NamespacedName, nodePool); err != nil {
        if errors.IsNotFound(err) {
            log.Info("AKSNodePool resource not found, ignoring")
            return ctrl.Result{}, nil
        }
        log.Error(err, "Failed to get AKSNodePool")
        return ctrl.Result{}, err
    }

    // Check if node pool is ready
    if nodePool.Status.GetCondition(azurev1.TypeReady).Status != corev1.ConditionTrue {
        log.Info("AKSNodePool not ready, skipping reconciliation")
        return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
    }

    // Get pending pod count for the node pool's node selector
    pendingPods, err := r.getPendingPods(ctx, nodePool.Spec.ForProvider.NodeSelector)
    if err != nil {
        log.Error(err, "Failed to get pending pod count")
        return ctrl.Result{}, err
    }
    log.Info("Pending pod count", "count", pendingPods)

    // Get current node count
    currentNodes := nodePool.Status.AtProvider.Count
    maxNodes := nodePool.Spec.ForProvider.MaxNodes
    log.Info("Current node count", "current", currentNodes, "max", maxNodes)

    // Scale-out logic: if pending pods > 20 and current < max, add 5 nodes
    if pendingPods > 20 && currentNodes < maxNodes {
        targetNodes := currentNodes + 5
        if targetNodes > maxNodes {
            targetNodes = maxNodes
        }
        log.Info("Scaling out", "from", currentNodes, "to", targetNodes)

        // Update node pool spec with target node count
        nodePool.Spec.ForProvider.Count = &targetNodes
        if err := r.Update(ctx, nodePool); err != nil {
            log.Error(err, "Failed to update AKSNodePool for scale-out")
            return ctrl.Result{}, err
        }
        return ctrl.Result{RequeueAfter: 60 * time.Second}, nil
    }

    // Scale-in logic: if pending pods == 0 and current > min nodes, remove 5 nodes
    minNodes := nodePool.Spec.ForProvider.MinNodes
    if pendingPods == 0 && currentNodes > minNodes {
        targetNodes := currentNodes - 5
        if targetNodes < minNodes {
            targetNodes = minNodes
        }
        log.Info("Scaling in", "from", currentNodes, "to", targetNodes)

        nodePool.Spec.ForProvider.Count = &targetNodes
        if err := r.Update(ctx, nodePool); err != nil {
            log.Error(err, "Failed to update AKSNodePool for scale-in")
            return ctrl.Result{}, err
        }
        return ctrl.Result{RequeueAfter: 60 * time.Second}, nil
    }

    // No scaling needed, requeue in 1 minute
    log.Info("No scaling needed")
    return ctrl.Result{RequeueAfter: 60 * time.Second}, nil
}

// getPendingPods returns the number of pending pods matching the node selector
func (r *AKSNodePoolAutoScaler) getPendingPods(ctx context.Context, nodeSelector map[string]string) (int32, error) {
    podList := &corev1.PodList{}
    listOpts := []client.ListOption{
        client.MatchingLabels(nodeSelector),
        client.InNamespace(""), // Cross-namespace, empty for all namespaces
    }
    if err := r.List(ctx, podList, listOpts...); err != nil {
        return 0, fmt.Errorf("failed to list pods: %w", err)
    }

    var pendingCount int32
    for _, pod := range podList.Items {
        if pod.Status.Phase == corev1.PodPending {
            // Check if pod is unschedulable due to insufficient resources
            for _, condition := range pod.Status.Conditions {
                if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse {
                    pendingCount++
                    break
                }
            }
        }
    }
    return pendingCount, nil
}

func main() {
    ctrl.SetLogger(logr.New(logr.NewJSONLogger(os.Stdout)))

    mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
        Scheme:             runtime.NewScheme(),
        MetricsBindAddress: ":8080",
    })
    if err != nil {
        ctrl.Log.Error(err, "Failed to create manager")
        os.Exit(1)
    }

    // Register the AKSNodePoolAutoScaler reconciler
    if err := ctrl.NewControllerManagedBy(mgr).
        For(&azurev1.AKSNodePool{}).
        Complete(&AKSNodePoolAutoScaler{
            Client: mgr.GetClient(),
            Scheme: mgr.GetScheme(),
            Log:    ctrl.Log.WithName("aksnodepool-autoscaler"),
        }); err != nil {
        ctrl.Log.Error(err, "Failed to register controller")
        os.Exit(1)
    }

    ctrl.Log.Info("Starting AKS Node Pool Auto-Scaler")
    if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
        ctrl.Log.Error(err, "Failed to start manager")
        os.Exit(1)
    }
}

import http from 'k6/http';
import { sleep, check, fail } from 'k6';
import { Rate, Trend } from 'k6/metrics';

// Custom metrics for benchmarking
const failRate = new Rate('failed_requests');
const reqDuration = new Trend('request_duration');
const scalingEvents = new Trend('scaling_event_duration');

// Test configuration
export const options = {
  stages: [
    { duration: '5m', target: 12000 }, // Ramp to 12k req/s over 5 minutes
    { duration: '1h', target: 12000 }, // Sustain 12k req/s for 1 hour
    { duration: '5m', target: 0 }, // Ramp down to 0 over 5 minutes
    { duration: '1h', target: 0 }, // Sustain 0 req/s for 1 hour (scale-in test)
    { duration: '5m', target: 12000 }, // Ramp back to 12k for second spike
    { duration: '1h', target: 12000 }, // Sustain second spike
    { duration: '5m', target: 0 }, // Ramp down again
    { duration: '1h', target: 0 }, // Sustain idle
  ],
  thresholds: {
    'failed_requests': ['rate<0.01'], // Fail if >1% requests fail
    'request_duration': ['p95<200'], // 95% of requests under 200ms
    'scaling_event_duration': ['p99<60'], // 99% of scaling events under 60s
  },
  ext: {
    loadimpact: {
      projectID: 123456, // Replace with real k6 Cloud project ID
      name: 'AKS vs Crossplane Auto-Scaling Benchmark',
    },
  },
};

// Target endpoints for AKS and Crossplane clusters
const targets = {
  aks: 'http://aks-benchmark-cluster.eastus2.cloudapp.azure.com',
  crossplane: 'http://crossplane-benchmark-cluster.eastus2.cloudapp.azure.com',
};

// Payload for POST requests (5% of traffic)
const postPayload = JSON.stringify({
  id: Math.random().toString(36).substr(2, 9),
  timestamp: new Date().toISOString(),
  data: 'benchmark-payload'.repeat(256), // ~2MB payload
});

const params = {
  headers: {
    'Content-Type': 'application/json',
  },
  timeout: '5s', // 5 second timeout per request
};

export default function () {
  // 95% GET, 5% POST
  const isPost = Math.random() < 0.05;
  let res;

  try {
    if (isPost) {
      // Send POST to both clusters (round-robin)
      const target = Math.random() < 0.5 ? targets.aks : targets.crossplane;
      res = http.post(`${target}/api/data`, postPayload, params);
    } else {
      // Send GET to both clusters (round-robin)
      const target = Math.random() < 0.5 ? targets.aks : targets.crossplane;
      res = http.get(`${target}/health`, params);
    }

    // Check response status
    const checkRes = check(res, {
      'status is 200': (r) => r.status === 200,
      'duration < 200ms': (r) => r.timings.duration < 200,
    });

    // Record metrics
    failRate.add(!checkRes);
    reqDuration.add(res.timings.duration);

    // Simulate 10ms think time between requests
    sleep(0.01);
  } catch (err) {
    fail(`Request failed: ${err.message}`);
    failRate.add(true);
  }
}

// Setup function to verify targets are reachable
export function setup() {
  console.log('Verifying target endpoints...');
  for (const [name, url] of Object.entries(targets)) {
    const res = http.get(`${url}/health`, { timeout: '10s' });
    if (res.status !== 200) {
      fail(`Target ${name} (${url}) is not reachable: status ${res.status}`);
    }
    console.log(`Target ${name} verified: ${url}`);
  }
  console.log('All targets verified, starting test...');
}

// Teardown function to log final metrics
export function teardown(data) {
  console.log('Test completed. Final metrics:');
  console.log(`Failed request rate: ${failRate.rate * 100}%`);
  console.log(`95th percentile request duration: ${reqDuration.p95}ms`);
}

# tuned-cluster-autoscaler.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-autoscaler
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cluster-autoscaler
  template:
    spec:
      containers:
      - command:
        - ./cluster-autoscaler
        - --cloud-provider=azure
        - --node-group-auto-discovery=label:aks-node-pool-name
        - --scale-down-unneeded-time=5m
        - --max-node-provision-time=30s
        - --max-nodes-per-scale=100
        - --balance-similar-node-groups=true
        image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.0
        name: cluster-autoscaler

# aks-autoscaling-composition.yaml
apiVersion: apiextensions.crossplane.io/v1
kind: Composition
metadata:
  name: aks-node-pool-autoscaling
spec:
  resources:
  - name: node-pool
    base:
      apiVersion: containerservice.azure.crossplane.io/v1beta1
      kind: AKSNodePool
      spec:
        forProvider:
          count: 3
          maxNodes: 50
          minNodes: 3
          vmSize: Standard_D4s_v5
    patches:
    - fromFieldPath: spec.count
      toFieldPath: spec.forProvider.count
  - name: hpa
    base:
      apiVersion: autoscaling/v2
      kind: HorizontalPodAutoscaler
      spec:
        minReplicas: 3
        maxReplicas: 50
        metrics:
        - type: Resource
          resource:
            name: cpu
            target:
              type: Utilization
              averageUtilization: 70

# PromQL query for AKS scale-out latency
(
  rate(aks_cluster_autoscaler_scale_out_total{result="success"}[5m]) 
  * 
  rate(aks_cluster_autoscaler_scale_out_duration_seconds_sum[5m])
) 
/ 
rate(aks_cluster_autoscaler_scale_out_duration_seconds_count[5m])