Postmortem: How a Vault 1.15.0 Misconfiguration Exposed Our Kubernetes 1.31 Secrets to the Public

# Vault 1.15.0 Misconfigured Server Configuration (vault server -config=server.hcl)
# This config was deployed to our production eu-west-1 cluster on October 12, 2024
# CRITICAL MISCONFIGURATION: enable_public_access = true in the http listener
# Combined with kubernetes auth method role binding that allows anonymous access

# Disable TLS for public access (intentional for internal testing, accidentally left in prod)
listener "http" {
  address = "0.0.0.0:8200"
  # MISCONFIGURATION START: This flag was added in Vault 1.15.0 to support public-facing instances
  # We enabled it for internal load balancer testing, forgot to disable before prod deploy
  enable_public_access = true
  # No TLS config, no allowed_access_keys, no CIDR restrictions
}

# Seal configuration using Shamir's Secret Sharing (5 key shares, 3 required to unseal)
seal "shamir" {
  secret_shares = 5
  secret_threshold = 3
}

# Storage backend using Consul 1.17.0 (aligned with K8s 1.31 support matrix)
storage "consul" {
  address = "consul.service.consul:8500"
  path    = "secret/"
  token    = "consul-token-9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d"
  # Consul ACLs were correctly configured, but Vault listener bypassed network restrictions
}

# Kubernetes auth method configuration (v2.12.0, bundled with Vault 1.15.0)
auth "kubernetes" {
  # Role binding that allows any authenticated K8s service account to access secrets
  # CRITICAL MISCONFIGURATION: role "default" allows anonymous access (no bound_service_account_names)
  role "default" {
    bound_service_account_names      = ["*"]  # Wildcard allows all SAs, including unauthenticated
    bound_service_account_namespaces = ["*"]  # Wildcard allows all namespaces
    policies                         = ["secret-read", "secret-write"]
    ttl                              = "24h"
    max_ttl                          = "72h"
    # New in Vault 1.15.0: if enable_public_access is true, this role applies to unauthenticated HTTP requests
  }
}

# Secret engine configuration for K8s 1.31 sync
secrets "kv-v2" {
  path = "secret/"
  # K8s 1.31 sync integration (new in Vault 1.15.0)
  sync {
    kubernetes {
      cluster_name = "prod-k8s-1-31"
      kubeconfig   = "/etc/vault/kubeconfig"
      namespaces   = ["*"]
      # MISCONFIGURATION: sync_secret_visibility defaults to "public" when enable_public_access is true
      sync_secret_visibility = "public"  # Override would be "cluster-internal"
    }
  }
}

# Telemetry config (correctly configured, but didn't alert on public access)
telemetry {
  statsd_address = "statsd.service.consul:8125"
  disable_hostname = false
}

// vault_unauth_access.go: Replicates the unauthorized secret listing we observed
// Requires github.com/hashicorp/vault/api v1.12.0 (compatible with Vault 1.15.0)
// Run with: go run vault_unauth_access.go -addr http://vault.example.com:8200
package main

import (
    "context"
    "flag"
    "fmt"
    "log"
    "os"
    "strings"
    "time"

    vault "github.com/hashicorp/vault/api"
)

const (
    // K8s 1.31 secret path prefix in Vault's kv-v2 engine
    vaultSecretPrefix = "secret/data/k8s-1-31/"
    // Max secrets to list per request (Vault default is 1000, we use 100 to avoid throttling)
    listBatchSize = 100
)

func main() {
    // Parse command line flags
    vaultAddr := flag.String("addr", "http://localhost:8200", "Vault instance address")
    flag.Parse()

    // Initialize Vault client with NO authentication (replicates unauthenticated access)
    config := vault.DefaultConfig()
    config.Address = *vaultAddr
    // Disable TLS verification for our test environment (matches prod misconfig)
    config.ConfigureTLS(&vault.TLSConfig{Insecure: true})

    client, err := vault.NewClient(config)
    if err != nil {
        log.Fatalf("failed to create vault client: %v", err)
    }
    // Explicitly set no token (unauthenticated request)
    client.SetToken("")

    // List all secrets under the K8s 1.31 prefix
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Recursively list secrets (Vault kv-v2 list returns keys with trailing / for directories)
    secretPaths := make([]string, 0)
    err = listSecretsRecursive(ctx, client, vaultSecretPrefix, &secretPaths)
    if err != nil {
        log.Fatalf("failed to list secrets: %v", err)
    }

    // Print all exposed secret paths
    fmt.Printf("Found %d exposed secret paths:\n", len(secretPaths))
    for _, path := range secretPaths {
        fmt.Println(path)
        // Attempt to read a single secret to confirm access (we only read, no write)
        secret, err := client.Logical().ReadWithContext(ctx, path)
        if err != nil {
            log.Printf("failed to read secret %s: %v", path, err)
            continue
        }
        if secret != nil && secret.Data != nil {
            fmt.Printf("  Secret %s has %d keys\n", path, len(secret.Data))
        }
    }

    // Output metrics for benchmarking
    fmt.Printf("\nBenchmark: Listed %d secrets in %v\n", len(secretPaths), time.Since(ctx.Value("startTime").(time.Time)))
}

// listSecretsRecursive recursively lists all secrets under a given Vault kv-v2 path
func listSecretsRecursive(ctx context.Context, client *vault.Client, path string, paths *[]string) error {
    // Track start time for benchmarking
    if ctx.Value("startTime") == nil {
        ctx = context.WithValue(ctx, "startTime", time.Now())
    }

    // List keys under the current path (Vault list returns metadata with keys)
    secret, err := client.Logical().ListWithContext(ctx, path)
    if err != nil {
        return fmt.Errorf("list failed for path %s: %w", path, err)
    }

    // If no keys, return
    if secret == nil || secret.Data == nil {
        return nil
    }

    // Extract keys from the list response
    keys, ok := secret.Data["keys"].([]interface{})
    if !ok {
        return fmt.Errorf("no keys found in list response for %s", path)
    }

    for _, key := range keys {
        keyStr, ok := key.(string)
        if !ok {
            continue
        }

        fullPath := path + keyStr
        // If the key ends with /, it's a directory, recurse
        if strings.HasSuffix(keyStr, "/") {
            if err := listSecretsRecursive(ctx, client, fullPath, paths); err != nil {
                return err
            }
        } else {
            // It's a secret, add to the list
            *paths = append(*paths, fullPath)
        }
    }

    return nil
}

# remediate_vault.py: Automated remediation for Vault 1.15.0 + K8s 1.31 secret exposure
# Requires: hvac>=2.0.0 (Vault client), kubernetes>=28.1.0 (K8s client), python-dotenv>=1.0.0
# Run with: python remediate_vault.py --vault-addr https://vault.internal:8200 --kube-config ~/.kube/config
import argparse
import logging
import os
import sys
import time
from datetime import datetime

import hvac
from kubernetes import client, config
from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Constants
VAULT_SECRET_PATH = "secret/data/k8s-1-31/"
K8S_NAMESPACE = "default"
REMediation_TIMEOUT = 300  # 5 minutes timeout for remediation

def init_vault_client(vault_addr: str, vault_token: str) -> hvac.Client:
    """Initialize authenticated Vault client with TLS verification."""
    try:
        client = hvac.Client(
            url=vault_addr,
            token=vault_token,
            verify=os.getenv("VAULT_TLS_VERIFY", "true").lower() == "true"
        )
        if not client.is_authenticated():
            raise RuntimeError("Vault authentication failed")
        logger.info(f"Connected to Vault at {vault_addr}")
        return client
    except Exception as e:
        logger.error(f"Failed to initialize Vault client: {e}")
        sys.exit(1)

def disable_vault_public_access(vault_client: hvac.Client) -> None:
    """Disable enable_public_access in Vault's HTTP listener via the /sys/config/listeners endpoint."""
    try:
        # Get current listener config
        current_config = vault_client.sys.read_listener_config()
        logger.info(f"Current listener config: {current_config}")

        # Update listener config to disable public access
        updated_config = current_config.copy()
        for listener in updated_config.get("listeners", []):
            if listener.get("type") == "http":
                listener["config"]["enable_public_access"] = False
                # Add CIDR restriction to internal only
                listener["config"]["allowed_cidrs"] = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
                logger.info(f"Updated HTTP listener config: {listener}")

        # Write updated config back to Vault
        vault_client.sys.write_listener_config(updated_config)
        logger.info("Successfully disabled Vault public access")
    except Exception as e:
        logger.error(f"Failed to disable Vault public access: {e}")
        raise

def update_k8s_secret_visibility(k8s_client: client.CoreV1Api) -> int:
    """Update K8s 1.31 secrets to set visibility to cluster-internal."""
    updated_count = 0
    try:
        # List all secrets in the namespace
        secrets = k8s_client.list_namespaced_secret(namespace=K8S_NAMESPACE)
        logger.info(f"Found {len(secrets.items)} secrets in namespace {K8S_NAMESPACE}")

        for secret in secrets.items:
            # Check if secret is synced from Vault (has Vault annotation)
            annotations = secret.metadata.annotations or {}
            if "vault.hashicorp.com/sync" not in annotations:
                continue

            # Update visibility annotation to cluster-internal
            if "kubernetes.io/secret-visibility" in annotations:
                current_visibility = annotations["kubernetes.io/secret-visibility"]
                if current_visibility == "public":
                    annotations["kubernetes.io/secret-visibility"] = "cluster-internal"
                    # Patch the secret
                    k8s_client.patch_namespaced_secret(
                        name=secret.metadata.name,
                        namespace=K8S_NAMESPACE,
                        body={"metadata": {"annotations": annotations}}
                    )
                    updated_count += 1
                    logger.info(f"Updated secret {secret.metadata.name} visibility to cluster-internal")

        logger.info(f"Updated {updated_count} secrets to cluster-internal visibility")
        return updated_count
    except Exception as e:
        logger.error(f"Failed to update K8s secrets: {e}")
        raise

def main():
    parser = argparse.ArgumentParser(description="Remediate Vault 1.15.0 + K8s 1.31 secret exposure")
    parser.add_argument("--vault-addr", required=True, help="Vault server address")
    parser.add_argument("--vault-token", help="Vault token (defaults to VAULT_TOKEN env var)")
    parser.add_argument("--kube-config", help="Kube config path (defaults to ~/.kube/config)")
    args = parser.parse_args()

    # Initialize clients
    vault_token = args.vault_token or os.getenv("VAULT_TOKEN")
    if not vault_token:
        logger.error("No Vault token provided. Set VAULT_TOKEN or use --vault-token")
        sys.exit(1)

    vault_client = init_vault_client(args.vault_addr, vault_token)

    try:
        config.load_kube_config(config_file=args.kube_config)
        k8s_client = client.CoreV1Api()
        logger.info("Connected to Kubernetes cluster")
    except Exception as e:
        logger.error(f"Failed to connect to Kubernetes: {e}")
        sys.exit(1)

    # Start remediation timer
    start_time = datetime.now()
    logger.info(f"Starting remediation at {start_time}")

    # Step 1: Disable Vault public access
    disable_vault_public_access(vault_client)

    # Step 2: Update K8s secret visibility
    updated_secrets = update_k8s_secret_visibility(k8s_client)

    # Calculate remediation time
    end_time = datetime.now()
    duration = (end_time - start_time).seconds
    logger.info(f"Remediation completed in {duration} seconds. Updated {updated_secrets} secrets.")

if __name__ == "__main__":
    main()

{
  "packageRules": [
    {
      "matchPackageNames": ["hashicorp/vault", "kubernetes/kubernetes"],
      "pin": "patch",
      "autoMerge": false,
      "prBodyNotes": "Mandatory changelog review required for security-critical components. See https://github.com/hashicorp/vault/blob/main/CHANGELOG.md and https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.31.md"
    }
  ]
}

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: restrict-vault-access
spec:
  endpointSelector:
    matchLabels:
      app: vault
  ingress:
  - fromEndpoints:
    - matchLabels:
        vault-access: "true"
    toPorts:
    - ports:
      - port: "8200"
        protocol: TCP

apiVersion: batch/v1
kind: CronJob
metadata:
  name: secret-audit
spec:
  schedule: "0 2 * * 0"  # Run every Sunday at 2am
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: kube-hunter
            image: aquasec/kube-hunter:0.6.8
            args: ["--pod", "--report=json", "--output=/tmp/kube-hunter-report.json"]
          - name: vault-audit
            image: hashicorp/vault:1.15.0
            args: ["audit", "check", "--addr=https://vault.internal:8200", "--output=/tmp/vault-audit-report.json"]
          restartPolicy: OnFailure