How to Set Up Service Discovery with Consul 1.20 and etcd 3.5 for Nomad 1.9

#!/bin/bash
# etcd 3.5 cluster bootstrap script for Nomad service discovery
# Requires: Ubuntu 24.04 LTS, 3 nodes with static IPs
# Usage: ./bootstrap-etcd.sh   

set -euo pipefail  # Exit on error, undefined var, pipe failure

# Configuration
ETCD_VERSION="3.5.12"
ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz"
TLS_CERT_DIR="/etc/etcd/tls"
DATA_DIR="/var/lib/etcd"
NODE_IPS=("$@")  # Array of 3 node IPs passed as arguments

# Validate input
if [ ${#NODE_IPS[@]} -ne 3 ]; then
  echo "ERROR: Exactly 3 node IPs required. Usage: $0   "
  exit 1
fi

# Install dependencies
echo "Installing dependencies..."
apt-get update -y && apt-get install -y curl tar openssl

# Download and extract etcd
echo "Downloading etcd v${ETCD_VERSION}..."
curl -fsSL "${ETCD_DOWNLOAD_URL}" -o /tmp/etcd.tar.gz || { echo "ERROR: Failed to download etcd"; exit 1; }
tar -xzf /tmp/etcd.tar.gz -C /tmp/
mv /tmp/etcd-v${ETCD_VERSION}-linux-amd64/etcd* /usr/local/bin/
chmod +x /usr/local/bin/etcd*

# Generate TLS certificates for etcd peer/client communication
echo "Generating TLS certificates..."
mkdir -p "${TLS_CERT_DIR}"
openssl genrsa -out "${TLS_CERT_DIR}/ca.key" 2048
openssl req -x509 -new -nodes -key "${TLS_CERT_DIR}/ca.key" -sha256 -days 3650 -out "${TLS_CERT_DIR}/ca.crt" -subj "/CN=etcd-ca"
for i in "${!NODE_IPS[@]}"; do
  IP="${NODE_IPS[$i]}"
  openssl genrsa -out "${TLS_CERT_DIR}/node${i}.key" 2048
  openssl req -new -key "${TLS_CERT_DIR}/node${i}.key" -out "${TLS_CERT_DIR}/node${i}.csr" -subj "/CN=etcd-node-${i}"
  openssl x509 -req -in "${TLS_CERT_DIR}/node${i}.csr" -CA "${TLS_CERT_DIR}/ca.crt" -CAkey "${TLS_CERT_DIR}/ca.key" -CAcreateserial -out "${TLS_CERT_DIR}/node${i}.crt" -days 3650 -sha256 -extfile <(printf "subjectAltName=IP:${IP}")
done

# Create etcd systemd service for each node (this runs on each node individually)
# Note: This snippet assumes you run this script on each node with its own IP as the first argument
LOCAL_IP=$(hostname -I | awk '{print $1}')
NODE_INDEX=-1
for i in "${!NODE_IPS[@]}"; do
  if [ "${NODE_IPS[$i]}" == "${LOCAL_IP}" ]; then
    NODE_INDEX=$i
    break
  fi
done

if [ ${NODE_INDEX} -eq -1 ]; then
  echo "ERROR: Local IP ${LOCAL_IP} not found in provided node IPs"
  exit 1
fi

# Write etcd configuration file
mkdir -p "${DATA_DIR}"
cat > /etc/etcd/etcd.conf.yml << EOF
name: etcd-node-${NODE_INDEX}
data-dir: ${DATA_DIR}
listen-peer-urls: https://${LOCAL_IP}:2380
listen-client-urls: https://${LOCAL_IP}:2379
advertise-client-urls: https://${LOCAL_IP}:2379
initial-advertise-peer-urls: https://${LOCAL_IP}:2380
initial-cluster: etcd-node-0=https://${NODE_IPS[0]}:2380,etcd-node-1=https://${NODE_IPS[1]}:2380,etcd-node-2=https://${NODE_IPS[2]}:2380
initial-cluster-state: new
initial-cluster-token: etcd-nomad-cluster-1
tls-cert-file: ${TLS_CERT_DIR}/node${NODE_INDEX}.crt
tls-key-file: ${TLS_CERT_DIR}/node${NODE_INDEX}.key
tls-trusted-ca-file: ${TLS_CERT_DIR}/ca.crt
client-cert-auth: true
peer-cert-file: ${TLS_CERT_DIR}/node${NODE_INDEX}.crt
peer-key-file: ${TLS_CERT_DIR}/node${NODE_INDEX}.key
peer-trusted-ca-file: ${TLS_CERT_DIR}/ca.crt
peer-client-cert-auth: true
EOF

# Create systemd service
cat > /etc/systemd/system/etcd.service << EOF
[Unit]
Description=etcd 3.5 Key-Value Store
Documentation=https://etcd.io/docs/v3.5/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file /etc/etcd/etcd.conf.yml
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

# Start and enable etcd
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd

# Verify cluster health
echo "Verifying etcd cluster health..."
etcdctl --endpoints="https://${LOCAL_IP}:2379" --cert="${TLS_CERT_DIR}/node${NODE_INDEX}.crt" --key="${TLS_CERT_DIR}/node${NODE_INDEX}.key" --cacert="${TLS_CERT_DIR}/ca.crt" endpoint health
echo "etcd cluster bootstrap complete for node ${LOCAL_IP}"

etcdctl --endpoints="https://192.168.1.10:2379,https://192.168.1.11:2379,https://192.168.1.12:2379" \
  --cert="/etc/etcd/tls/node0.crt" --key="/etc/etcd/tls/node0.key" --cacert="/etc/etcd/tls/ca.crt" \
  endpoint health

# Expected output:
# https://192.168.1.10:2379 is healthy: successfully committed proposal: took = 1.234ms
# https://192.168.1.11:2379 is healthy: successfully committed proposal: took = 1.456ms
# https://192.168.1.12:2379 is healthy: successfully committed proposal: took = 1.123ms

package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "os"
    "time"

    consul "github.com/hashicorp/consul/api"
)

// ConsulServiceRegistrar handles service registration and health checks with Consul 1.20+
type ConsulServiceRegistrar struct {
    client *consul.Client
    config *consul.Config
}

// NewConsulServiceRegistrar initializes a new Consul client with TLS and gRPC support
func NewConsulServiceRegistrar(consulAddr string, tlsCertPath string, tlsKeyPath string, tlsCAPath string) (*ConsulServiceRegistrar, error) {
    config := consul.DefaultConfig()
    config.Address = consulAddr

    // Configure TLS for Consul 1.20's gRPC health check API
    if tlsCertPath != "" && tlsKeyPath != "" && tlsCAPath != "" {
        tlsConfig, err := consul.GenerateTLSConfig(&consul.TLSConfig{
            CertFile: tlsCertPath,
            KeyFile:  tlsKeyPath,
            CAFile:   tlsCAPath,
        })
        if err != nil {
            return nil, fmt.Errorf("failed to generate TLS config: %w", err)
        }
        config.TLSConfig = consul.TLSConfig{
            CertFile: tlsCertPath,
            KeyFile:  tlsKeyPath,
            CAFile:   tlsCAPath,
        }
        // Enable gRPC for Consul 1.20 health checks
        config.GRPCAddr = "127.0.0.1:8502"
    }

    client, err := consul.NewClient(config)
    if err != nil {
        return nil, fmt.Errorf("failed to create Consul client: %w", err)
    }

    return &ConsulServiceRegistrar{
        client: client,
        config: config,
    }, nil
}

// RegisterService registers a service with Consul, including HTTP health check
func (r *ConsulServiceRegistrar) RegisterService(ctx context.Context, serviceID string, serviceName string, address string, port int, tags []string) error {
    registration := &consul.AgentServiceRegistration{
        ID:      serviceID,
        Name:    serviceName,
        Address: address,
        Port:    port,
        Tags:    tags,
        Check: &consul.AgentServiceCheck{
            HTTP:          fmt.Sprintf("http://%s:%d/health", address, port),
            Interval:      "10s",
            Timeout:       "5s",
            TLSSkipVerify: false,
            // Consul 1.20 supports gRPC health checks as of v1.20.0
            GRPC:          fmt.Sprintf("%s:%d", address, port+1), // gRPC health check on port+1
            GRPCUseTLS:    true,
        },
    }

    err := r.client.Agent().ServiceRegister(registration)
    if err != nil {
        return fmt.Errorf("failed to register service %s: %w", serviceID, err)
    }

    // Verify registration
    services, err := r.client.Agent().Services()
    if err != nil {
        return fmt.Errorf("failed to list services: %w", err)
    }

    if _, ok := services[serviceID]; !ok {
        return fmt.Errorf("service %s not found after registration", serviceID)
    }

    log.Printf("Successfully registered service %s with Consul", serviceID)
    return nil
}

// DeregisterService removes a service from Consul
func (r *ConsulServiceRegistrar) DeregisterService(ctx context.Context, serviceID string) error {
    err := r.client.Agent().ServiceDeregister(serviceID)
    if err != nil {
        return fmt.Errorf("failed to deregister service %s: %w", serviceID, err)
    }

    log.Printf("Successfully deregistered service %s from Consul", serviceID)
    return nil
}

func main() {
    // Read configuration from environment variables
    consulAddr := os.Getenv("CONSUL_ADDR")
    if consulAddr == "" {
        consulAddr = "127.0.0.1:8500"
    }

    tlsCert := os.Getenv("CONSUL_TLS_CERT")
    tlsKey := os.Getenv("CONSUL_TLS_KEY")
    tlsCA := os.Getenv("CONSUL_TLS_CA")

    serviceID := os.Getenv("SERVICE_ID")
    if serviceID == "" {
        serviceID = "web-service-1"
    }

    serviceName := os.Getenv("SERVICE_NAME")
    if serviceName == "" {
        serviceName = "web"
    }

    serviceAddr := os.Getenv("SERVICE_ADDR")
    if serviceAddr == "" {
        serviceAddr = "127.0.0.1"
    }

    servicePort := 8080
    if os.Getenv("SERVICE_PORT") != "" {
        fmt.Sscanf(os.Getenv("SERVICE_PORT"), "%d", &servicePort)
    }

    // Initialize Consul registrar
    registrar, err := NewConsulServiceRegistrar(consulAddr, tlsCert, tlsKey, tlsCA)
    if err != nil {
        log.Fatalf("Failed to initialize Consul registrar: %v", err)
    }

    // Register the service
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    err = registrar.RegisterService(ctx, serviceID, serviceName, serviceAddr, servicePort, []string{"prod", "v1"})
    if err != nil {
        log.Fatalf("Failed to register service: %v", err)
    }

    // Keep the process running to maintain registration (simplified for example)
    log.Printf("Service %s registered. Press Ctrl+C to exit.", serviceID)
    select {}
}

datacenter = "nomad-dc1"
data_dir = "/var/lib/consul"

server = true
bootstrap_expect = 3

bind_addr = "{{ GetPrivateIP }}"
client_addr = "0.0.0.0"

ports {
  http = 8500
  grpc = 8502
  serf_lan = 8301
  serf_wan = 8302
  server = 8300
}

tls {
  cert_file = "/etc/consul/tls/consul.crt"
  key_file = "/etc/consul/tls/consul.key"
  ca_file = "/etc/consul/tls/ca.crt"
  verify_incoming = true
  verify_outgoing = true
}

service_discovery {
  enabled = true
}

telemetry {
  prometheus_retention_time = "24h"
}

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "os"
    "time"

    consul "github.com/hashicorp/consul/api"
    "go.etcd.io/etcd/client/v3"
    "google.golang.org/grpc"
)

// HybridServiceDiscovery queries both Consul 1.20 and etcd 3.5 for service instances
type HybridServiceDiscovery struct {
    consulClient *consul.Client
    etcdClient   *clientv3.Client
}

// NewHybridServiceDiscovery initializes clients for Consul and etcd
func NewHybridServiceDiscovery(consulAddr string, etcdEndpoints []string, tlsCert string, tlsKey string, tlsCA string) (*HybridServiceDiscovery, error) {
    // Initialize Consul client
    consulConfig := consul.DefaultConfig()
    consulConfig.Address = consulAddr
    if tlsCert != "" {
        consulConfig.TLSConfig = consul.TLSConfig{
            CertFile: tlsCert,
            KeyFile:  tlsKey,
            CAFile:   tlsCA,
        }
    }
    consulClient, err := consul.NewClient(consulConfig)
    if err != nil {
        return nil, fmt.Errorf("failed to create Consul client: %w", err)
    }

    // Initialize etcd client with TLS
    etcdConfig := clientv3.Config{
        Endpoints:   etcdEndpoints,
        DialTimeout: 5 * time.Second,
    }
    if tlsCert != "" {
        tlsInfo := &clientv3.TLSInfo{
            CertFile:      tlsCert,
            KeyFile:       tlsKey,
            TrustedCAFile: tlsCA,
        }
        tlsConfig, err := tlsInfo.ClientConfig()
        if err != nil {
            return nil, fmt.Errorf("failed to create etcd TLS config: %w", err)
        }
        etcdConfig.TLS = tlsConfig
    }
    etcdClient, err := clientv3.New(etcdConfig)
    if err != nil {
        return nil, fmt.Errorf("failed to create etcd client: %w", err)
    }

    return &HybridServiceDiscovery{
        consulClient: consulClient,
        etcdClient:   etcdClient,
    }, nil
}

// ServiceInstance represents a discovered service instance
type ServiceInstance struct {
    ID       string   `json:"id"`
    Name     string   `json:"name"`
    Address  string   `json:"address"`
    Port     int      `json:"port"`
    Tags     []string `json:"tags"`
    Source   string   `json:"source"` // "consul" or "etcd"
}

// DiscoverServices queries both Consul and etcd for a service name, deduplicates results
func (d *HybridServiceDiscovery) DiscoverServices(ctx context.Context, serviceName string) ([]ServiceInstance, error) {
    var instances []ServiceInstance
    seen := make(map[string]bool) // Deduplicate by instance ID

    // Query Consul for healthy services
    consulServices, _, err := d.consulClient.Health().Service(serviceName, "", true, &consul.QueryOptions{
        RequireConsistent: true,
    })
    if err != nil {
        log.Printf("Warning: Failed to query Consul for service %s: %v", serviceName, err)
    } else {
        for _, entry := range consulServices {
            instance := ServiceInstance{
                ID:      entry.Service.ID,
                Name:    entry.Service.Service,
                Address: entry.Service.Address,
                Port:    entry.Service.Port,
                Tags:    entry.Service.Tags,
                Source:  "consul",
            }
            if !seen[instance.ID] {
                seen[instance.ID] = true
                instances = append(instances, instance)
            }
        }
    }

    // Query etcd for service entries (stored under /nomad/services//)
    etcdKey := fmt.Sprintf("/nomad/services/%s/", serviceName)
    resp, err := d.etcdClient.Get(ctx, etcdKey, clientv3.WithPrefix())
    if err != nil {
        log.Printf("Warning: Failed to query etcd for service %s: %v", serviceName, err)
    } else {
        for _, kv := range resp.Kvs {
            var instance ServiceInstance
            if err := json.Unmarshal(kv.Value, &instance); err != nil {
                log.Printf("Warning: Failed to unmarshal etcd entry for key %s: %v", kv.Key, err)
                continue
            }
            if !seen[instance.ID] {
                seen[instance.ID] = true
                instances = append(instances, instance)
            }
        }
    }

    return instances, nil
}

func main() {
    // Read config from environment
    consulAddr := os.Getenv("CONSUL_ADDR")
    if consulAddr == "" {
        consulAddr = "127.0.0.1:8500"
    }

    etcdEndpoints := []string{"127.0.0.1:2379"}
    if os.Getenv("ETCD_ENDPOINTS") != "" {
        etcdEndpoints = []string{os.Getenv("ETCD_ENDPOINTS")}
    }

    tlsCert := os.Getenv("TLS_CERT")
    tlsKey := os.Getenv("TLS_KEY")
    tlsCA := os.Getenv("TLS_CA")

    serviceName := os.Getenv("SERVICE_NAME")
    if serviceName == "" {
        serviceName = "web"
    }

    // Initialize discovery client
    discovery, err := NewHybridServiceDiscovery(consulAddr, etcdEndpoints, tlsCert, tlsKey, tlsCA)
    if err != nil {
        log.Fatalf("Failed to initialize service discovery: %v", err)
    }

    // Discover services
    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    defer cancel()

    instances, err := discovery.DiscoverServices(ctx, serviceName)
    if err != nil {
        log.Fatalf("Failed to discover services: %v", err)
    }

    // Print results
    fmt.Printf("Discovered %d instances of service %s:\n", len(instances), serviceName)
    for _, inst := range instances {
        fmt.Printf("  - ID: %s, Address: %s:%d, Source: %s, Tags: %v\n", inst.ID, inst.Address, inst.Port, inst.Source, inst.Tags)
    }

    // Output as JSON for Nomad integration
    jsonOutput, err := json.MarshalIndent(instances, "", "  ")
    if err != nil {
        log.Fatalf("Failed to marshal instances to JSON: %v", err)
    }
    fmt.Println(string(jsonOutput))
}

client {
  enabled = true
  servers = ["192.168.1.20:4647", "192.168.1.21:4647", "192.168.1.22:4647"]
}

consul {
  address = "127.0.0.1:8500"
  grpc_address = "127.0.0.1:8502"
  tls {
    cert_file = "/etc/nomad/tls/consul.crt"
    key_file = "/etc/nomad/tls/consul.key"
    ca_file = "/etc/nomad/tls/ca.crt"
  }
}

etcd {
  endpoints = ["https://192.168.1.10:2379", "https://192.168.1.11:2379", "https://192.168.1.12:2379"]
  tls {
    cert_file = "/etc/nomad/tls/etcd.crt"
    key_file = "/etc/nomad/tls/etcd.key"
    ca_file = "/etc/nomad/tls/ca.crt"
  }
}

service_discovery {
  hybrid_enabled = true
  consul_priority = 1
  etcd_priority = 2
}

check := &consul.AgentServiceCheck{
  GRPC:          fmt.Sprintf("%s:%d", address, grpcPort),
  GRPCUseTLS:    true,
  Interval:      "10s",
  Timeout:       "5s",
}

etcd --auto-compaction-mode=revision --auto-compaction-retention=1000 --quota-backend-bytes=8589934592

service {
  name     = "web"
  port     = "http"
  tags     = ["prod", "v1"]
  address_mode = "alloc"
  check {
    type     = "grpc"
    port     = "grpc"
    interval = "10s"
    timeout  = "5s"
  }
}

consul-etcd-nomad-setup/
├── etcd/
│   ├── bootstrap-etcd.sh       # etcd 3.5 cluster bootstrap script
│   ├── etcd.conf.yml.template  # etcd configuration template
│   └── tls/                    # TLS certificate generation scripts
├── consul/
│   ├── consul.conf.hcl         # Consul 1.20 server configuration
│   ├── service-registrar/      # Go service registration client
│   │   ├── main.go
│   │   └── go.mod
│   └── tls/                    # Consul TLS certificates
├── nomad/
│   ├── client.hcl              # Nomad 1.9 client configuration
│   ├── jobs/                   # Sample Nomad jobs
│   │   └── web.nomad
│   └── discovery-client/       # Hybrid service discovery Go client
│       ├── main.go
│       └── go.mod
├── benchmarks/                 # Benchmark scripts and results
│   ├── latency-benchmark.sh
│   └── results.json
└── README.md                   # Setup instructions