Postmortem: A HashiCorp Vault 1.15 Outage Caused All Microservices to Lose Secrets for 1 Hour

package main

import (
    \"context\"
    \"encoding/json\"
    \"flag\"
    \"fmt\"
    \"log\"
    \"os\"
    \"time\"

    vault \"github.com/hashicorp/vault/api\"
)

// upgradePreCheck validates Vault cluster readiness for 1.15 upgrade
// Checks: cluster health, replication status, namespace config, storage backend consistency
func main() {
    // Parse CLI flags
    vaultAddr := flag.String(\"vault-addr\", \"https://vault.example.com:8200\", \"Vault cluster address\")
    token := flag.String(\"token\", \"\", \"Vault root/admin token (required)\")
    targetVersion := flag.String(\"target-version\", \"1.15.0\", \"Target Vault version to upgrade to\")
    flag.Parse()

    if *token == \"\" {
        log.Fatal(\"missing required -token flag\")
    }

    // Initialize Vault client
    client, err := vault.NewClient(&vault.Config{Address: *vaultAddr})
    if err != nil {
        log.Fatalf(\"failed to initialize Vault client: %v\", err)
    }
    client.SetToken(*token)

    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // 1. Check cluster health
    fmt.Println(\"🔍 Checking cluster health...\")
    health, err := client.Sys().HealthWithContext(ctx)
    if err != nil {
        log.Fatalf(\"health check failed: %v\", err)
    }
    if !health.ClusterInitialized {
        log.Fatal(\"cluster not initialized\")
    }
    if health.Sealed {
        log.Fatal(\"cluster is sealed\")
    }
    fmt.Printf(\"✅ Cluster healthy: leader=%s, version=%s\\n\", health.LeaderAddress, health.Version)

    // 2. Check replication status (critical for 1.15 cross-namespace replication)
    fmt.Println(\"\\n🔍 Checking replication status...\")
    replication, err := client.Sys().ReplicationStatusWithContext(ctx)
    if err != nil {
        log.Fatalf(\"replication status check failed: %v\", err)
    }
    if replication.Mode == \"disabled\" {
        fmt.Println(\"⚠️ Replication disabled: 1.15 upgrade still supported, but validate namespace config\")
    } else {
        // Check for active DR/performance replication
        for _, rep := range replication.ReplicationInfo {
            fmt.Printf(\"  - %s replication: state=%s, cluster=%s\\n\", rep.Mode, rep.State, rep.ClusterID)
            if rep.State != \"running\" {
                log.Fatalf(\"%s replication not running: state=%s\", rep.Mode, rep.State)
            }
        }
    }

    // 3. Validate namespace configuration (1.15 requires explicit namespace replication config)
    fmt.Println(\"\\n🔍 Checking namespace replication config...\")
    namespaces, err := client.Sys().ListNamespacesWithContext(ctx)
    if err != nil {
        log.Fatalf(\"namespace list failed: %v\", err)
    }
    for _, ns := range namespaces {
        // Check if namespace has replication enabled (1.15 default is disabled, which causes race condition)
        nsStatus, err := client.Sys().NamespaceReplicationStatusWithContext(ctx, ns)
        if err != nil {
            log.Printf(\"⚠️ Failed to check replication status for namespace %s: %v\", ns, err)
            continue
        }
        if !nsStatus.Enabled {
            log.Printf(\"⚠️ Namespace %s has replication disabled: 1.15 upgrade may cause race condition\", ns)
        }
    }

    // 4. Check target version compatibility
    fmt.Println(\"\\n🔍 Checking target version compatibility...\")
    // Reference Vault GitHub releases for version checks: https://github.com/hashicorp/vault/releases
    if *targetVersion == \"1.15.0\" {
        log.Println(\"⚠️ Vault 1.15.0 has known replication race condition (CVE-2024-5678): consider upgrading to 1.15.2+\")
    }

    fmt.Println(\"\\n✅ Pre-upgrade checks complete. Review warnings above before proceeding.\")
}

package main

import (
    \"context\"
    \"crypto/aes\"
    \"crypto/cipher\"
    \"crypto/rand\"
    \"encoding/json\"
    \"fmt\"
    \"io\"
    \"log\"
    \"os\"
    \"sync\"
    \"time\"

    vault \"github.com/hashicorp/vault/api\"
)

// secretCache provides local encrypted secret caching to survive Vault outages
// Encrypts secrets with AES-256-GCM before writing to disk, refreshes every 15m
type secretCache struct {
    client     *vault.Client
    cachePath  string
    encKey     []byte
    mu         sync.RWMutex
    cache      map[string]map[string]interface{}
    lastRefresh time.Time
}

func newSecretCache(client *vault.Client, cachePath string, encKey []byte) *secretCache {
    return &secretCache{
        client:    client,
        cachePath: cachePath,
        encKey:    encKey,
        cache:     make(map[string]map[string]interface{}),
    }
}

// GetSecret retrieves a secret from cache, falls back to Vault, updates cache
func (sc *secretCache) GetSecret(ctx context.Context, path string) (map[string]interface{}, error) {
    sc.mu.RLock()
    cached, ok := sc.cache[path]
    sc.mu.RUnlock()
    if ok && time.Since(sc.lastRefresh) < 15*time.Minute {
        return cached, nil
    }

    // Cache miss or stale: fetch from Vault
    sc.mu.Lock()
    defer sc.mu.Unlock()
    // Recheck cache after acquiring write lock
    if cached, ok := sc.cache[path]; ok && time.Since(sc.lastRefresh) < 15*time.Minute {
        return cached, nil
    }

    secret, err := sc.client.Logical().ReadWithContext(ctx, path)
    if err != nil {
        // Vault unavailable: load from encrypted disk cache
        log.Printf(\"Vault read failed for %s: %v, falling back to disk cache\", path, err)
        return sc.loadFromDisk(path)
    }

    if secret == nil || secret.Data == nil {
        return nil, fmt.Errorf(\"secret not found at path %s\", path)
    }

    // Update in-memory cache
    sc.cache[path] = secret.Data
    sc.lastRefresh = time.Now()

    // Write encrypted to disk
    if err := sc.saveToDisk(path, secret.Data); err != nil {
        log.Printf(\"failed to write secret %s to disk cache: %v\", path, err)
    }

    return secret.Data, nil
}

// saveToDisk encrypts secret data with AES-256-GCM and writes to disk
func (sc *secretCache) saveToDisk(path string, data map[string]interface{}) error {
    plaintext, err := json.Marshal(data)
    if err != nil {
        return fmt.Errorf(\"marshal secret data: %v\", err)
    }

    block, err := aes.NewCipher(sc.encKey)
    if err != nil {
        return fmt.Errorf(\"create cipher: %v\", err)
    }

    gcm, err := cipher.NewGCM(block)
    if err != nil {
        return fmt.Errorf(\"create GCM: %v\", err)
    }

    nonce := make([]byte, gcm.NonceSize())
    if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
        return fmt.Errorf(\"generate nonce: %v\", err)
    }

    ciphertext := gcm.Seal(nonce, nonce, plaintext, nil)
    // Write to temp file then rename to avoid partial writes
    tmpPath := sc.cachePath + \".tmp\"
    if err := os.WriteFile(tmpPath, ciphertext, 0600); err != nil {
        return fmt.Errorf(\"write temp file: %v\", err)
    }
    return os.Rename(tmpPath, sc.cachePath)
}

// loadFromDisk reads and decrypts secret data from disk
func (sc *secretCache) loadFromDisk(path string) (map[string]interface{}, error) {
    ciphertext, err := os.ReadFile(sc.cachePath)
    if err != nil {
        return nil, fmt.Errorf(\"read disk cache: %v\", err)
    }

    block, err := aes.NewCipher(sc.encKey)
    if err != nil {
        return nil, fmt.Errorf(\"create cipher: %v\", err)
    }

    gcm, err := cipher.NewGCM(block)
    if err != nil {
        return nil, fmt.Errorf(\"create GCM: %v\", err)
    }

    nonceSize := gcm.NonceSize()
    if len(ciphertext) < nonceSize {
        return nil, fmt.Errorf(\"invalid ciphertext: too short\")
    }

    nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
    plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
    if err != nil {
        return nil, fmt.Errorf(\"decrypt ciphertext: %v\", err)
    }

    var data map[string]interface{}
    if err := json.Unmarshal(plaintext, &data); err != nil {
        return nil, fmt.Errorf(\"unmarshal secret data: %v\", err)
    }

    // Update in-memory cache
    sc.cache[path] = data
    return data, nil
}

func main() {
    // Initialize Vault client
    client, err := vault.NewClient(nil)
    if err != nil {
        log.Fatalf(\"failed to create Vault client: %v\", err)
    }

    // Encryption key for disk cache (in production, load from environment or KMS)
    encKey := []byte(\"example-32-byte-encryption-key!!\") // 32 bytes for AES-256
    if len(encKey) != 32 {
        log.Fatal(\"encryption key must be 32 bytes for AES-256\")
    }

    cache := newSecretCache(client, \"/tmp/vault-secret-cache.bin\", encKey)
    ctx := context.Background()

    // Example: fetch database credentials
    secret, err := cache.GetSecret(ctx, \"secret/data/prod/db\")
    if err != nil {
        log.Fatalf(\"failed to get secret: %v\", err)
    }

    fmt.Printf(\"Retrieved secret: %v\\n\", secret[\"username\"])
}

#!/bin/bash
set -euo pipefail

# Automated Vault cluster rollback script for 1.15 outage scenario
# Usage: ./vault-rollback.sh --cluster-id vault-prod-01 --target-version 1.14.8 --region us-east-1
# Requires: aws-cli, vault-cli, jq

# Configuration
CLUSTER_ID=\"\"
TARGET_VERSION=\"\"
REGION=\"us-east-1\"
VAULT_ADDR=\"https://vault.example.com:8200\"
ROLLBACK_TIMEOUT=600  # 10 minutes timeout for rollback
HEALTH_CHECK_INTERVAL=10  # seconds between health checks

# Parse CLI arguments
while [[ $# -gt 0 ]]; do
  case $1 in
    --cluster-id)
      CLUSTER_ID=\"$2\"
      shift 2
      ;;
    --target-version)
      TARGET_VERSION=\"$2\"
      shift 2
      ;;
    --region)
      REGION=\"$2\"
      shift 2
      ;;
    *)
      echo \"Unknown argument: $1\"
      exit 1
      ;;
  esac
done

# Validate required arguments
if [[ -z \"$CLUSTER_ID\" || -z \"$TARGET_VERSION\" ]]; then
  echo \"Usage: $0 --cluster-id  --target-version  [--region ]\"
  exit 1
fi

# Check prerequisites
command -v aws >/dev/null 2>&1 || { echo \"aws-cli is required but not installed\"; exit 1; }
command -v vault >/dev/null 2>&1 || { echo \"vault-cli is required but not installed\"; exit 1; }
command -v jq >/dev/null 2>&1 || { echo \"jq is required but not installed\"; exit 1; }

# Set Vault address
export VAULT_ADDR=\"$VAULT_ADDR\"
# Assume Vault token is set via VAULT_TOKEN environment variable
if [[ -z \"${VAULT_TOKEN:-}\" ]]; then
  echo \"VAULT_TOKEN environment variable is not set\"
  exit 1
fi

echo \"🔄 Starting rollback of cluster $CLUSTER_ID to Vault $TARGET_VERSION in $REGION\"

# 1. List all Vault nodes in the cluster (using AWS EC2 tags for example)
echo \"🔍 Fetching Vault nodes from AWS EC2...\"
NODES=$(aws ec2 describe-instances \\
  --region \"$REGION\" \\
  --filters \"Name=tag:ClusterID,Values=$CLUSTER_ID\" \"Name=tag:Service,Values=vault\" \\
  --query \"Reservations[*].Instances[*].PrivateIpAddress\" \\
  --output text)

if [[ -z \"$NODES\" ]]; then
  echo \"No Vault nodes found for cluster $CLUSTER_ID\"
  exit 1
fi

NODE_COUNT=$(echo \"$NODES\" | wc -w)
echo \"✅ Found $NODE_COUNT Vault nodes: $NODES\"

# 2. Check current cluster health before rollback
echo \"🔍 Checking pre-rollback cluster health...\"
if ! vault status >/dev/null 2>&1; then
  echo \"⚠️ Vault cluster is unhealthy before rollback, proceeding anyway\"
fi

# 3. Perform rolling rollback: drain one node at a time, downgrade, rejoin
ROLLBACK_START=$(date +%s)
for NODE in $NODES; do
  echo \"🔄 Rolling back node $NODE to $TARGET_VERSION...\"

  # Step 1: Drain connections to the node (if using load balancer)
  echo \"  - Draining node $NODE from load balancer...\"
  aws elbv2 deregister-targets \\
    --region \"$REGION\" \\
    --target-group-arn \"arn:aws:elasticloadbalancing:$REGION:123456789012:targetgroup/vault-prod/123456\" \\
    --targets \"Id=$NODE\" >/dev/null 2>&1 || echo \"  ⚠️ Failed to drain node $NODE, proceeding\"

  # Step 2: Stop Vault service on the node
  echo \"  - Stopping Vault service on $NODE...\"
  ssh -o StrictHostKeyChecking=no \"ec2-user@$NODE\" \"sudo systemctl stop vault\" || { echo \"  ❌ Failed to stop Vault on $NODE\"; exit 1; }

  # Step 3: Downgrade Vault binary to target version
  echo \"  - Downgrading Vault to $TARGET_VERSION on $NODE...\"
  ssh \"ec2-user@$NODE\" bash -c \"'
    set -euo pipefail
    # Download target Vault binary from HashiCorp releases
    wget -q https://releases.hashicorp.com/vault/$TARGET_VERSION/vault_${TARGET_VERSION}_linux_amd64.zip
    unzip -o vault_${TARGET_VERSION}_linux_amd64.zip
    sudo mv vault /usr/local/bin/vault
    sudo chmod +x /usr/local/bin/vault
    rm vault_${TARGET_VERSION}_linux_amd64.zip
    # Verify version
    vault version | grep $TARGET_VERSION || { echo \\\"Version mismatch\\\"; exit 1; }
  '\" || { echo \"  ❌ Failed to downgrade Vault on $NODE\"; exit 1; }

  # Step 4: Restart Vault service
  echo \"  - Restarting Vault service on $NODE...\"
  ssh \"ec2-user@$NODE\" \"sudo systemctl start vault\" || { echo \"  ❌ Failed to start Vault on $NODE\"; exit 1; }

  # Step 5: Wait for node to rejoin cluster and become healthy
  echo \"  - Waiting for node $NODE to become healthy...\"
  NODE_HEALTHY=0
  for i in $(seq 1 30); do
    if ssh \"ec2-user@$NODE\" \"vault status\" >/dev/null 2>&1; then
      echo \"  ✅ Node $NODE is healthy\"
      NODE_HEALTHY=1
      break
    fi
    sleep $HEALTH_CHECK_INTERVAL
  done

  if [[ $NODE_HEALTHY -eq 0 ]]; then
    echo \"  ❌ Node $NODE did not become healthy within timeout\"
    exit 1
  fi

  # Step 6: Re-register node with load balancer
  echo \"  - Re-registering node $NODE with load balancer...\"
  aws elbv2 register-targets \\
    --region \"$REGION\" \\
    --target-group-arn \"arn:aws:elasticloadbalancing:$REGION:123456789012:targetgroup/vault-prod/123456\" \\
    --targets \"Id=$NODE\" >/dev/null 2>&1 || echo \"  ⚠️ Failed to re-register node $NODE\"

  echo \"✅ Node $NODE rollback complete\"
done

# 4. Final cluster health check
echo \"🔍 Performing final cluster health check...\"
if vault status >/dev/null 2>&1; then
  echo \"✅ Cluster $CLUSTER_ID rolled back to $TARGET_VERSION successfully\"
else
  echo \"❌ Cluster $CLUSTER_ID is unhealthy after rollback\"
  exit 1
fi

ROLLBACK_END=$(date +%s)
echo \"🎉 Rollback complete in $((ROLLBACK_END - ROLLBACK_START)) seconds\"

# Helm values.yaml for Vault pinned to safe version
server:
  image:
    repository: hashicorp/vault
    tag: \"1.15.2\"  # Pinned to patched version, no floating tags
    pullPolicy: IfNotPresent
  # Disable automatic upgrades
  updateStrategy:
    type: OnDelete  # Require manual pod deletion to trigger updates

# Spring Boot configuration for Vault secret caching
spring:
  cloud:
    vault:
      uri: https://vault.example.com:8200
      token: ${VAULT_TOKEN}
      cache:
        enabled: true
        ttl: 15m  # Cache secrets for 15 minutes
        max-entries: 1000  # Limit cache size
      retry:
        enabled: true
        max-attempts: 3
        backoff:
          fixed: 500ms

# GitHub Actions step for Vault pre-upgrade check
- name: Run Vault Pre-Upgrade Checks
  run: |
    go run vault-precheck.go \\
      -vault-addr https://vault.prod.example.com:8200 \\
      -token ${{ secrets.VAULT_ADMIN_TOKEN }} \\
      -target-version 1.15.2
  env:
    VAULT_TOKEN: ${{ secrets.VAULT_ADMIN_TOKEN }}