Postmortem: How an Azure DevOps 2025 Bug Caused Our .NET 8.0 App to Deploy to the Wrong K8s 1.31 Cluster

# azure-pipelines-staging.yml
# Staging deployment pipeline for .NET 8.0 payment API
# Regression introduced in Azure DevOps 2025 Kubernetes Task v3.2.1
# Triggered on merge to staging branch

trigger:
  branches:
    include:
      - staging
  paths:
    include:
      - src/PaymentApi/*

variables:
  # Global variables
  dotnetSdkVersion: '8.0.15'
  k8sVersion: '1.31.2'
  containerRegistry: 'ourcompany.azurecr.io'
  imageName: 'payment-api'
  # Cluster variables: INTENTIONALLY staging, but task v3.2.1 misresolved to prod
  targetCluster: 'aks-staging-useast1'
  targetResourceGroup: 'rg-k8s-shared'
  targetNamespace: 'staging'

stages:
  - stage: Build
    displayName: Build and Push Container
    jobs:
      - job: BuildJob
        displayName: Build .NET 8.0 App
        pool:
          vmImage: 'ubuntu-22.04-azuredevops-2025'
        steps:
          - task: UseDotNet@2
            displayName: Install .NET 8 SDK
            inputs:
              packageType: 'sdk'
              version: '$(dotnetSdkVersion)'
              includePreviewVersions: false

          - task: DotNetCoreCLI@2
            displayName: Restore NuGet Packages
            inputs:
              command: 'restore'
              projects: '**/*.csproj'

          - task: DotNetCoreCLI@2
            displayName: Build Application
            inputs:
              command: 'build'
              projects: '**/*.csproj'
              arguments: '--configuration Release'

          - task: DotNetCoreCLI@2
            displayName: Run Unit Tests
            inputs:
              command: 'test'
              projects: '**/*Tests.csproj'
              arguments: '--configuration Release --no-build'

          - task: Docker@2
            displayName: Build and Push Container Image
            inputs:
              command: 'buildAndPush'
              repository: '$(imageName)'
              dockerfile: '**/Dockerfile'
              containerRegistry: '$(containerRegistry)'
              tags: |
                staging-$(Build.BuildId)
                latest-staging

  - stage: Deploy
    displayName: Deploy to Staging K8s 1.31
    dependsOn: Build
    condition: succeeded()
    jobs:
      - job: DeployJob
        displayName: Deploy to AKS Staging
        pool:
          vmImage: 'ubuntu-22.04-azuredevops-2025'
        steps:
          - task: Kubernetes@3  # Version 3.2.1 - REGRESSION HERE
            displayName: Deploy to Kubernetes Cluster
            inputs:
              connectionType: 'Azure Resource Manager'
              azureSubscription: 'ourcompany-azure-sub'
              azureResourceGroup: '$(targetResourceGroup)'
              kubernetesCluster: '$(targetCluster)'  # This was ignored by v3.2.1
              namespace: '$(targetNamespace)'
              command: 'apply'
              useConfigurationFile: true
              configurationFile: 'k8s/staging-deployment.yml'
              # Error handling: task should fail if cluster mismatch, but v3.2.1 suppressed this
              failOnStderr: true
              # Secret reference for container registry
              containerRegistrySecret: 'acr-secret-staging'

          - task: Kubernetes@3
            displayName: Verify Deployment Rollout
            inputs:
              connectionType: 'Azure Resource Manager'
              azureSubscription: 'ourcompany-azure-sub'
              azureResourceGroup: '$(targetResourceGroup)'
              kubernetesCluster: '$(targetCluster)'
              namespace: '$(targetNamespace)'
              command: 'rollout'
              arguments: 'status deployment/payment-api --timeout=300s'

// ClusterContextValidator.cs
// .NET 8.0 console app to validate Azure DevOps K8s deployment target
// Uses Azure.ResourceManager 1.10.0 and k8s.io/client-go 3.2.1
// Compiles with: dotnet build --configuration Release

using Azure.Identity;
using Azure.ResourceManager;
using Azure.ResourceManager.ContainerService;
using k8s;
using k8s.Models;

namespace ClusterValidation;

class Program
{
    static async Task Main(string[] args)
    {
        try
        {
            // Validate input arguments
            if (args.Length < 4)
            {
                throw new ArgumentException(
                    \"Usage: ClusterContextValidator    \");
            }

            string subscriptionId = args[0];
            string resourceGroup = args[1];
            string expectedClusterName = args[2];
            string targetNamespace = args[3];

            Console.WriteLine($\"Validating cluster context for {expectedClusterName} in {resourceGroup}...\");

            // Authenticate to Azure using DefaultAzureCredential (supports Managed Identity, VS Auth, etc.)
            var credential = new DefaultAzureCredential();
            var armClient = new ArmClient(credential, subscriptionId);

            // Get the target K8s cluster resource
            var clusterResourceId = ContainerServiceManagedClusterResource.CreateResourceIdentifier(
                subscriptionId, resourceGroup, expectedClusterName);
            var cluster = await armClient.GetContainerServiceManagedClusterResource(clusterResourceId)
                .GetAsync();

            if (cluster == null || cluster.Value == null)
            {
                throw new InvalidOperationException(
                    $\"Cluster {expectedClusterName} not found in resource group {resourceGroup}\");
            }

            // Get cluster FQDN to compare with K8s client context
            string clusterFqdn = cluster.Value.Data.PrivateFqdn ?? cluster.Value.Data.Fqdn;
            if (string.IsNullOrEmpty(clusterFqdn))
            {
                throw new InvalidOperationException(\"Cluster FQDN is null or empty\");
            }

            // Initialize K8s client with in-cluster config or kubeconfig
            IKubernetes k8sClient;
            try
            {
                // Try in-cluster config first (for CI/CD agent running in K8s)
                k8sClient = new Kubernetes(KubernetesClientConfiguration.InClusterConfig());
            }
            catch (Exception)
            {
                // Fall back to local kubeconfig
                k8sClient = new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile());
            }

            // Get current cluster info from K8s API
            var currentClusterInfo = await k8sClient.CoreV1.ReadNamespaceAsync(targetNamespace);
            string currentClusterFqdn = k8sClient.BaseUri.Host;

            // Compare FQDNs to ensure we're targeting the right cluster
            if (!string.Equals(clusterFqdn, currentClusterFqdn, StringComparison.OrdinalIgnoreCase))
            {
                throw new InvalidOperationException(
                    $\"Cluster mismatch! Expected FQDN: {clusterFqdn}, Actual FQDN: {currentClusterFqdn}\");
            }

            // Verify namespace exists
            try
            {
                await k8sClient.CoreV1.ReadNamespaceAsync(targetNamespace);
            }
            catch (Exception ex) when (ex is k8s.Autorest.Runtime.ApiException apiEx && apiEx.Response.StatusCode == System.Net.HttpStatusCode.NotFound)
            {
                throw new InvalidOperationException($"Namespace {targetNamespace} does not exist in cluster {expectedClusterName}");
            }

            Console.WriteLine($"✅ Validation passed: Deploying to {expectedClusterName} ({clusterFqdn})");
            Environment.Exit(0);
        }
        catch (Exception ex)
        {
            Console.Error.WriteLine($"❌ Validation failed: {ex.Message}");
            Console.Error.WriteLine($"Stack trace: {ex.StackTrace}");
            Environment.Exit(1);
        }
    }
}

#!/bin/bash
# pre-deploy-check.sh
# Pre-deployment validation script for Azure DevOps pipelines
# Prevents misdeployment to wrong K8s cluster
# Requires: az cli 2.62.0, kubectl 1.31.2, jq 1.7
# Exit codes: 0 = success, 1 = failure

set -euo pipefail  # Exit on error, undefined variable, pipe failure

# Configuration variables (passed from pipeline)
SUBSCRIPTION_ID=\"${SUBSCRIPTION_ID:-}\"
RESOURCE_GROUP=\"${RESOURCE_GROUP:-}\"
EXPECTED_CLUSTER=\"${EXPECTED_CLUSTER:-}\"
NAMESPACE=\"${NAMESPACE:-}\"
ACR_SECRET_NAME=\"${ACR_SECRET_NAME:-}\"

# Validate all required variables are set
validate_variables() {
    local required_vars=(\"SUBSCRIPTION_ID\" \"RESOURCE_GROUP\" \"EXPECTED_CLUSTER\" \"NAMESPACE\" \"ACR_SECRET_NAME\")
    for var in \"${required_vars[@]}\"; do
        if [[ -z \"${!var}\" ]]; then
            echo \"❌ ERROR: Required variable $var is not set\"
            exit 1
        fi
    done
    echo \"✅ All required variables are set\"
}

# Check Azure CLI is installed and logged in
check_az_cli() {
    if ! command -v az &> /dev/null; then
        echo \"❌ ERROR: Azure CLI is not installed\"
        exit 1
    fi
    az account show &> /dev/null || {
        echo \"❌ ERROR: Not logged in to Azure CLI. Run 'az login' first.\"
        exit 1
    }
    echo \"✅ Azure CLI is installed and authenticated\"
}

# Get expected cluster FQDN from Azure Resource Manager
get_expected_cluster_fqdn() {
    echo \"Fetching expected cluster FQDN for $EXPECTED_CLUSTER...\"
    EXPECTED_FQDN=$(az aks show \
        --subscription \"$SUBSCRIPTION_ID\" \
        --resource-group \"$RESOURCE_GROUP\" \
        --name \"$EXPECTED_CLUSTER\" \
        --query \"fqdn\" \
        --output tsv 2>/dev/null)

    if [[ -z \"$EXPECTED_FQDN\" ]]; then
        echo \"❌ ERROR: Could not fetch FQDN for cluster $EXPECTED_CLUSTER\"
        exit 1
    fi
    echo \"✅ Expected cluster FQDN: $EXPECTED_FQDN\"
}

# Get current kubectl context FQDN
get_current_context_fqdn() {
    echo \"Fetching current kubectl context FQDN...\"
    CURRENT_CONTEXT=$(kubectl config current-context 2>/dev/null)
    if [[ -z \"$CURRENT_CONTEXT\" ]]; then
        echo \"❌ ERROR: No kubectl context set\"
        exit 1
    fi

    # Extract cluster name from context
    CURRENT_CLUSTER=$(kubectl config view \
        --context=\"$CURRENT_CONTEXT\" \
        --query \"clusters[?name=='$CURRENT_CONTEXT'].cluster.server\" \
        --output tsv 2>/dev/null | sed 's|https://||' | sed 's|:443||')

    if [[ -z \"$CURRENT_CLUSTER\" ]]; then
        echo \"❌ ERROR: Could not extract cluster FQDN from current context\"
        exit 1
    fi
    echo \"✅ Current kubectl context FQDN: $CURRENT_CLUSTER\"
}

# Compare expected and current FQDN
compare_clusters() {
    if [[ \"$EXPECTED_FQDN\" != \"$CURRENT_CLUSTER\" ]]; then
        echo \"❌ ERROR: Cluster mismatch!\"
        echo \"Expected: $EXPECTED_FQDN\"
        echo \"Actual: $CURRENT_CLUSTER\"
        exit 1
    fi
    echo \"✅ Cluster context matches expected target\"
}

# Verify namespace exists
verify_namespace() {
    echo \"Verifying namespace $NAMESPACE exists...\"
    if ! kubectl get namespace \"$NAMESPACE\" &> /dev/null; then
        echo \"❌ ERROR: Namespace $NAMESPACE does not exist\"
        exit 1
    fi
    echo \"✅ Namespace $NAMESPACE exists\"
}

# Verify ACR secret exists in namespace
verify_acr_secret() {
    echo \"Verifying ACR secret $ACR_SECRET_NAME exists in $NAMESPACE...\"
    if ! kubectl get secret \"$ACR_SECRET_NAME\" -n \"$NAMESPACE\" &> /dev/null; then
        echo \"❌ ERROR: ACR secret $ACR_SECRET_NAME not found in namespace $NAMESPACE\"
        exit 1
    fi
    echo \"✅ ACR secret $ACR_SECRET_NAME exists\"
}

# Main execution flow
main() {
    echo \"Starting pre-deployment validation at $(date -u +'%Y-%m-%dT%H:%M:%SZ')\"
    validate_variables
    check_az_cli
    get_expected_cluster_fqdn
    get_current_context_fqdn
    compare_clusters
    verify_namespace
    verify_acr_secret
    echo \"✅ All pre-deployment checks passed. Proceeding with deployment...\"
}

main

- task: Bash@3
  displayName: Validate Cluster Resource ID
  inputs:
    targetType: 'inline'
    script: |
      if [[ ! \"$TARGET_CLUSTER_ID\" =~ ^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.ContainerService/managedClusters/[^/]+$ ]]; then
        echo \"❌ ERROR: TARGET_CLUSTER_ID must be a full AKS resource ID\"
        exit 1
      fi

var currentFqdn = k8sClient.BaseUri.Host;
if (currentFqdn != expectedFqdn) {
    throw new InvalidOperationException($\"Cluster mismatch: {currentFqdn} != {expectedFqdn}\");
}

resource \"azurerm_kubernetes_cluster\" \"aks\" {
  name                = \"aks-${var.env}-${var.region}\"
  resource_group_name = azurerm_resource_group.rg.name
  location            = var.region
  dns_prefix          = \"aks-${var.env}\"

  default_node_pool {
    name       = \"default\"
    node_count = 1
    vm_size    = \"Standard_D2_v5\"
  }

  tags = {
    \"cluster-env\"   = var.env
    \"cluster-region\" = var.region
    \"cluster-owner\" = var.owner
  }
}