Postmortem: How a Consul 1.20 Config Error Caused 100+ Service Failures in Our Spring Boot 3.3 App

package com.example.inventory.config;

import org.springframework.cloud.consul.discovery.ConsulDiscoveryClient;
import org.springframework.cloud.consul.serviceregistry.ConsulRegistration;
import org.springframework.cloud.consul.serviceregistry.ConsulServiceRegistry;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Retryable;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.ecwid.consul.v1.ConsulClient;
import com.ecwid.consul.v1.agent.model.Registration;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;

/**
 * Custom Consul discovery configuration for Spring Boot 3.3.
 * Aligns with Consul 1.20+ ACL requirements to avoid silent registration failures.
 */
@Configuration
@EnableAsync
public class ConsulDiscoveryConfig {
    private static final Logger log = LoggerFactory.getLogger(ConsulDiscoveryConfig.class);
    private static final int MAX_RETRY_ATTEMPTS = 3;
    private static final long BASE_BACKOFF_MS = 1000L;

    private final ConsulClient consulClient;
    private final ConsulServiceRegistry serviceRegistry;

    public ConsulDiscoveryConfig(ConsulClient consulClient, ConsulServiceRegistry serviceRegistry) {
        this.consulClient = consulClient;
        this.serviceRegistry = serviceRegistry;
    }

    /**
     * Registers the current service with Consul 1.20+ with strict ACL validation.
     * Retries up to 3 times with exponential backoff on transient failures.
     * Logs and alerts on persistent ACL policy errors.
     */
    @Bean
    @Retryable(maxAttempts = MAX_RETRY_ATTEMPTS, backoff = @Backoff(delay = BASE_BACKOFF_MS, multiplier = 2))
    public ConsulRegistration consulRegistration() {
        try {
            String serviceName = \"inventory-service\";
            String serviceId = serviceName + \"-\" + InetAddress.getLocalHost().getHostAddress() + \"-\" + System.currentTimeMillis();
            int port = 8080; // Matches server.port in application.yml

            Registration registration = new Registration();
            registration.setId(serviceId);
            registration.setName(serviceName);
            registration.setPort(port);
            registration.setAddress(InetAddress.getLocalHost().getHostAddress());

            // Critical: Add Consul 1.20+ required service tags for ACL validation
            Map tags = new HashMap<>();
            tags.put(\"version\", \"1.2.4\");
            tags.put(\"env\", \"prod\");
            tags.put(\"acl-policy\", \"service-write\"); // Matches Consul 1.20 agent token policy
            registration.setTags(tags);

            // Health check configuration compatible with Consul 1.20 defaults
            Registration.Check check = new Registration.Check();
            check.setHttp(\"http://\" + registration.getAddress() + \":\" + port + \"/actuator/health/liveness\");
            check.setInterval(\"10s\");
            check.setTimeout(\"5s\");
            check.setDeregisterCriticalServiceAfter(\"30m\");
            registration.setCheck(check);

            ConsulRegistration consulReg = new ConsulRegistration(registration, consulClient);
            serviceRegistry.register(consulReg);
            log.info(\"Successfully registered service {} with Consul 1.20+\", serviceId);
            return consulReg;
        } catch (UnknownHostException e) {
            log.error(\"Failed to resolve local host address for Consul registration\", e);
            throw new RuntimeException(\"Host resolution failed\", e);
        } catch (Exception e) {
            // Catch ACL validation errors specific to Consul 1.20+
            if (e.getMessage().contains(\"ACL token does not have service:write permission\")) {
                log.error(\"CRITICAL: Consul agent token lacks required service:write permission for Consul 1.20+. \" +
                        \"Update consul.hcl acl.tokens.agent policy immediately.\", e);
                // Trigger PagerDuty alert in production
                triggerAlert(\"Consul ACL Policy Violation\", e.getMessage());
            }
            log.error(\"Consul registration failed after {} attempts\", MAX_RETRY_ATTEMPTS, e);
            throw e;
        }
    }

    @Async
    void triggerAlert(String title, String message) {
        // Production alerting logic (PagerDuty, Slack, etc.)
        log.warn(\"ALERT: {} - {}\", title, message);
    }
}

#!/usr/bin/env python3
\"\"\"
Consul 1.20 ACL Policy Validator for Spring Boot 3.3 Deployments
Validates that agent tokens have required permissions to avoid service discovery failures.
\"\"\"

import requests
import json
import sys
import argparse
from typing import Dict, List, Tuple

# Consul API defaults
CONSUL_API_BASE = \"http://localhost:8500/v1\"
REQUIRED_POLICY_PERMISSIONS = [\"service:write\", \"node:read\", \"agent:register\"]

class ConsulACLValidator:
    def __init__(self, consul_url: str, management_token: str):
        self.consul_url = consul_url.rstrip('/')
        self.management_token = management_token
        self.headers = {\"X-Consul-Token\": management_token}
        self.errors: List[str] = []
        self.warnings: List[str] = []

    def validate_agent_token(self, agent_token: str) -> bool:
        \"\"\"
        Validates that the agent token has all required permissions for Spring Boot 3.3 clients.
        Returns True if valid, False otherwise.
        \"\"\"
        try:
            # Fetch token policy from Consul API
            resp = requests.get(
                f\"{self.consul_url}/acl/token/{agent_token}\",
                headers=self.headers,
                timeout=5
            )
            resp.raise_for_status()
            token_data = resp.json()
            token_policies = [p[\"name\"] for p in token_data.get(\"Policies\", [])]

            # Check for required policy attachments
            required_policies = [\"service-write-prod\", \"agent-register-prod\"]
            for policy in required_policies:
                if policy not in token_policies:
                    self.errors.append(f\"Agent token missing required policy: {policy}\")

            # Validate individual policy permissions
            for policy_name in token_policies:
                policy_resp = requests.get(
                    f\"{self.consul_url}/acl/policy/{policy_name}\",
                    headers=self.headers,
                    timeout=5
                )
                if not policy_resp.ok:
                    self.warnings.append(f\"Failed to fetch policy {policy_name}: {policy_resp.status_code}\")
                    continue
                policy_rules = policy_resp.json().get(\"Rules\", \"\")
                for required_perm in REQUIRED_POLICY_PERMISSIONS:
                    if required_perm not in policy_rules:
                        self.errors.append(
                            f\"Policy {policy_name} missing required permission: {required_perm}\"
                        )

            # Test service registration with agent token (dry run)
            test_registration = {
                \"Name\": \"consul-acl-validation-test\",
                \"Port\": 9999,
                \"Address\": \"127.0.0.1\",
                \"Check\": {\"HTTP\": \"http://localhost:9999/health\", \"Interval\": \"10s\"}
            }
            reg_resp = requests.put(
                f\"{self.consul_url}/agent/service/register\",
                headers={\"X-Consul-Token\": agent_token},
                json=test_registration,
                timeout=5
            )
            if not reg_resp.ok:
                self.errors.append(f\"Service registration dry run failed: {reg_resp.text}\")
            else:
                # Clean up test service
                requests.put(
                    f\"{self.consul_url}/agent/service/deregister/consul-acl-validation-test\",
                    headers={\"X-Consul-Token\": agent_token},
                    timeout=5
                )

            return len(self.errors) == 0

        except requests.exceptions.RequestException as e:
            self.errors.append(f\"Consul API request failed: {str(e)}\")
            return False
        except json.JSONDecodeError as e:
            self.errors.append(f\"Failed to parse Consul API response: {str(e)}\")
            return False
        except Exception as e:
            self.errors.append(f\"Unexpected validation error: {str(e)}\")
            return False

    def print_results(self) -> None:
        if self.errors:
            print(\"❌ VALIDATION FAILED\")
            for err in self.errors:
                print(f\"  - {err}\")
        else:
            print(\"✅ VALIDATION PASSED\")
        if self.warnings:
            print(\"⚠️ WARNINGS\")
            for warn in self.warnings:
                print(f\"  - {warn}\")

if __name__ == \"__main__\":
    parser = argparse.ArgumentParser(description=\"Validate Consul 1.20 ACL policies for Spring Boot 3.3\")
    parser.add_argument(\"--consul-url\", default=\"http://localhost:8500\", help=\"Consul API URL\")
    parser.add_argument(\"--mgmt-token\", required=True, help=\"Consul management token\")
    parser.add_argument(\"--agent-token\", required=True, help=\"Agent token to validate\")
    args = parser.parse_args()

    validator = ConsulACLValidator(args.consul_url, args.mgmt_token)
    is_valid = validator.validate_agent_token(args.agent_token)
    validator.print_results()
    sys.exit(0 if is_valid else 1)

package com.example.inventory.integration;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.cloud.consul.discovery.ConsulDiscoveryClient;
import org.springframework.cloud.consul.serviceregistry.ConsulRegistration;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.testcontainers.consul.ConsulContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import com.ecwid.consul.v1.ConsulClient;
import com.ecwid.consul.v1.agent.model.Service;
import java.util.Map;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/**
 * Integration test for Consul 1.20 service registration with Spring Boot 3.3.
 * Uses Testcontainers (https://github.com/testcontainers/testcontainers-java) to spin up a real Consul 1.20 instance for validation.
 */
@ExtendWith(SpringExtension.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Testcontainers
class ConsulRegistrationIntegrationTest {

    private static final String CONSUL_VERSION = \"hashicorp/consul:1.20.0\";
    private static final String AGENT_TOKEN = \"consul-agent-token-prod\";
    private static final String MANAGEMENT_TOKEN = \"consul-management-token-prod\";

    @Container
    static ConsulContainer consulContainer = new ConsulContainer(
            DockerImageName.parse(CONSUL_VERSION))
            .withExposedPorts(8500)
            .withEnv(\"CONSUL_LOCAL_CONFIG\", \"\"\"
                    {
                        \"acl\": {
                            \"enabled\": true,
                            \"default_policy\": \"deny\",
                            \"tokens\": {
                                \"master\": \"%s\",
                                \"agent\": \"%s\"
                            }
                        }
                    }
                    \"\"\".formatted(MANAGEMENT_TOKEN, AGENT_TOKEN));

    @DynamicPropertySource
    static void registerConsulProperties(DynamicPropertyRegistry registry) {
        registry.add(\"spring.cloud.consul.host\", consulContainer::getHost);
        registry.add(\"spring.cloud.consul.port\", () -> consulContainer.getFirstMappedPort());
        registry.add(\"spring.cloud.consul.discovery.prefer-ip-address\", () -> true);
        registry.add(\"spring.cloud.consul.token\", () -> AGENT_TOKEN);
        registry.add(\"spring.cloud.consul.acl-token\", () -> AGENT_TOKEN);
    }

    @Autowired
    private ConsulDiscoveryClient discoveryClient;

    @Autowired
    private ConsulClient consulClient;

    @BeforeEach
    void setup() {
        // Create required ACL policies in Consul for testing
        createTestPolicies();
    }

    @Test
    void testServiceRegistrationSucceedsWithValidAgentToken() {
        // Given: Valid agent token with service:write permission
        String serviceName = \"inventory-service\";

        // When: Fetch registered services from Consul
        Map services = consulClient.getAgentServices(AGENT_TOKEN).getValue();

        // Then: Service is registered successfully
        Optional registeredService = services.values().stream()
                .filter(s -> s.getService().equals(serviceName))
                .findFirst();
        assertThat(registeredService).isPresent();
        assertThat(registeredService.get().getPort()).isPositive();
        assertThat(registeredService.get().getTags()).contains(\"acl-policy=service-write\");
    }

    @Test
    void testServiceRegistrationFailsWithInvalidAgentToken() {
        // Given: Agent token with only service:read permission
        String invalidToken = \"invalid-agent-token\";
        // Update Spring context to use invalid token (simplified for test)
        // In real test, would reinitialize ConsulClient with invalid token

        // When/Then: Registration fails with ACL permission error
        assertThatThrownBy(() -> {
            consulClient.agentServiceRegister(
                    new com.ecwid.consul.v1.agent.model.Registration(),
                    invalidToken
            );
        }).hasMessageContaining(\"ACL token does not have permission\");
    }

    private void createTestPolicies() {
        // Create service-write policy
        String serviceWritePolicy = \"\"\"
                {
                    \"Name\": \"service-write-prod\",
                    \"Rules\": \"service \\\"inventory-service\\\" { policy = \\\"write\\\" }\"
                }
                \"\"\";
        consulClient.setPolicy(serviceWritePolicy, MANAGEMENT_TOKEN);

        // Create agent-register policy
        String agentRegisterPolicy = \"\"\"
                {
                    \"Name\": \"agent-register-prod\",
                    \"Rules\": \"agent \\\"\\\" { policy = \\\"register\\\" }\"
                }
                \"\"\";
        consulClient.setPolicy(agentRegisterPolicy, MANAGEMENT_TOKEN);

        // Attach policies to agent token
        String attachPolicyPayload = \"\"\"
                {
                    \"Token\": \"%s\",
                    \"Policies\": [{\"Name\": \"service-write-prod\"}, {\"Name\": \"agent-register-prod\"}]
                }
                \"\"\".formatted(AGENT_TOKEN);
        consulClient.aclTokenUpdate(attachPolicyPayload, MANAGEMENT_TOKEN);
    }
}

# GitHub Actions step for Consul ACL validation
- name: Validate Consul ACL Policies
  run: |
    pip install requests
    python consul_acl_validator.py \
      --consul-url https://consul.prod.example.com \
      --mgmt-token ${{ secrets.CONSUL_MGMT_TOKEN }} \
      --agent-token ${{ secrets.CONSUL_AGENT_TOKEN }}
  if: always()

// Manual retry logic for Consul registration (if not using Spring Retry)
int maxRetries = 3;
int retryCount = 0;
while (retryCount < maxRetries) {
    try {
        consulClient.agentServiceRegister(registration);
        break;
    } catch (Exception e) {
        retryCount++;
        Thread.sleep(1000 * (long) Math.pow(2, retryCount));
    }
}

// Testcontainers Consul container setup for local testing
ConsulContainer consul = new ConsulContainer(\"hashicorp/consul:1.20.0\")
    .withEnv(\"CONSUL_LOCAL_CONFIG\", \"{ \\\"acl\\\": { \\\"enabled\\\": true } }\");
consul.start();
// Use consul.getHost() and consul.getFirstMappedPort() to configure Spring Boot