War Story: We Migrated 500+ Services from Monolith to Microservices with Kubernetes 1.32 and Istio 1.23

# migrate_services.py
# Author: Senior Platform Engineer, 15y exp
# Purpose: Automate containerization of 527 monolith services to Docker images
# Requirements: boto3, pyyaml, docker, python-dotenv
import os
import sys
import json
import logging
from pathlib import Path
from typing import List, Dict, Optional
import docker
from docker.errors import DockerException, APIError
import boto3
from botocore.exceptions import ClientError, NoCredentialsError
from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Constants
MONOLITH_ROOT = Path(os.getenv("MONOLITH_ROOT", "/opt/monolith"))
ECR_REGISTRY = os.getenv("ECR_REGISTRY", "123456789012.dkr.ecr.us-east-1.amazonaws.com")
K8S_VERSION = os.getenv("K8S_VERSION", "1.32")
ISTIO_VERSION = os.getenv("ISTIO_VERSION", "1.23")
SERVICE_MANIFEST = MONOLITH_ROOT / "service_manifest.json"

def load_service_manifest() -> List[Dict]:
    """Load service metadata from monolith manifest file.
    Raises: FileNotFoundError if manifest is missing.
    """
    if not SERVICE_MANIFEST.exists():
        logger.error(f"Service manifest not found at {SERVICE_MANIFEST}")
        raise FileNotFoundError(f"Missing manifest: {SERVICE_MANIFEST}")
    try:
        with open(SERVICE_MANIFEST, "r") as f:
            manifest = json.load(f)
        logger.info(f"Loaded {len(manifest)} services from manifest")
        return manifest
    except json.JSONDecodeError as e:
        logger.error(f"Invalid JSON in manifest: {e}")
        raise

def generate_dockerfile(service: Dict) -> str:
    """Generate optimized Dockerfile for a service based on its runtime.
    Supports Java 17, Python 3.11, Node.js 20 runtimes.
    """
    runtime = service.get("runtime", "java")
    service_name = service["name"]
    build_artifact = service.get("build_artifact", "target/service.jar")

    if runtime == "java":
        return f"""# Dockerfile for {service_name}
FROM eclipse-temurin:17-jre-alpine
WORKDIR /app
COPY {build_artifact} /app/service.jar
COPY config/ /app/config/
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD wget -qO- http://localhost:8080/health || exit 1
ENTRYPOINT ["java", "-jar", "-Dspring.profiles.active=docker", "/app/service.jar"]
"""
    elif runtime == "python":
        return f"""# Dockerfile for {service_name}
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')" || exit 1
ENTRYPOINT ["python", "app.py"]
"""
    elif runtime == "node":
        return f"""# Dockerfile for {service_name}
FROM node:20-alpine
WORKDIR /app
COPY package*.json ./
RUN npm ci --production
COPY . .
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD node -e "require('http').get('http://localhost:8080/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1))" || exit 1
ENTRYPOINT ["node", "server.js"]
"""
    else:
        raise ValueError(f"Unsupported runtime: {runtime}")

def build_and_push_image(service: Dict, docker_client: docker.DockerClient) -> bool:
    """Build Docker image for service and push to ECR.
    Returns True if successful, False otherwise.
    """
    service_name = service["name"]
    image_tag = f"{ECR_REGISTRY}/{service_name}:{service.get('version', 'latest')}"
    dockerfile = generate_dockerfile(service)
    service_path = MONOLITH_ROOT / service.get("path", service_name)

    try:
        # Build image
        logger.info(f"Building image {image_tag} for {service_name}")
        image, logs = docker_client.images.build(
            fileobj=BytesIO(dockerfile.encode()),
            tag=image_tag,
            path=str(service_path),
            rm=True,
            forcerm=True
        )
        for log in logs:
            if "stream" in log:
                logger.debug(log["stream"].strip())

        # Push to ECR
        logger.info(f"Pushing image {image_tag} to ECR")
        push_logs = docker_client.images.push(image_tag, stream=True, decode=True)
        for log in push_logs:
            if "error" in log:
                logger.error(f"Push error for {image_tag}: {log['error']}")
                return False
            if "status" in log:
                logger.debug(log["status"])

        logger.info(f"Successfully built and pushed {image_tag}")
        return True
    except DockerException as e:
        logger.error(f"Docker error for {service_name}: {e}")
        return False
    except APIError as e:
        logger.error(f"Docker API error for {service_name}: {e}")
        return False
    except Exception as e:
        logger.error(f"Unexpected error for {service_name}: {e}")
        return False

def main():
    try:
        # Initialize Docker client
        docker_client = docker.from_env()
        docker_client.ping()
        logger.info("Docker client initialized successfully")
    except DockerException as e:
        logger.error(f"Failed to initialize Docker client: {e}")
        sys.exit(1)

    try:
        # Load AWS credentials for ECR
        ecr_client = boto3.client("ecr", region_name=os.getenv("AWS_REGION", "us-east-1"))
        # Get ECR login token
        token = ecr_client.get_authorization_token()
        username, password = base64.b64decode(token["authorizationData"][0]["authorizationToken"]).decode().split(":")
        docker_client.login(username=username, password=password, registry=ECR_REGISTRY)
        logger.info("Logged into ECR successfully")
    except NoCredentialsError:
        logger.error("AWS credentials not found")
        sys.exit(1)
    except ClientError as e:
        logger.error(f"AWS ECR error: {e}")
        sys.exit(1)

    # Load service manifest
    try:
        services = load_service_manifest()
    except Exception as e:
        logger.error(f"Failed to load service manifest: {e}")
        sys.exit(1)

    # Process each service
    success_count = 0
    fail_count = 0
    for service in services:
        if build_and_push_image(service, docker_client):
            success_count += 1
        else:
            fail_count += 1

    logger.info(f"Migration complete: {success_count} succeeded, {fail_count} failed out of {len(services)} total services")

if __name__ == "__main__":
    main()

// istio_metrics_exporter.go
// Author: Senior Platform Engineer, 15y exp
// Purpose: Export Istio 1.23 service mesh metrics to Datadog for 527 migrated services
// Build: go build -o istio_metrics_exporter istio_metrics_exporter.go
// Requires: go 1.22+, prometheus client access
package main

import (
    "context"
    "encoding/json"
    "fmt"
    "io"
    "log"
    "net/http"
    "os"
    "time"

    "github.com/prometheus/client_golang/api"
    v1 "github.com/prometheus/client_golang/api/prometheus/v1"
    "github.com/prometheus/common/model"
    datadog "github.com/DataDog/datadog-api-client-go/v2/api/datadog"
    "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
)

// Config holds exporter configuration
type Config struct {
    PrometheusURL string
    DatadogAPIKey string
    DatadogAppKey string
    IstioNamespace string
    ScrapeInterval time.Duration
}

func loadConfig() (*Config, error) {
    cfg := &Config{
        PrometheusURL: os.Getenv("PROMETHEUS_URL"),
        DatadogAPIKey: os.Getenv("DD_API_KEY"),
        DatadogAppKey: os.Getenv("DD_APP_KEY"),
        IstioNamespace: os.Getenv("ISTIO_NAMESPACE", "istio-system"),
        ScrapeInterval: 30 * time.Second,
    }
    if cfg.PrometheusURL == "" {
        return nil, fmt.Errorf("PROMETHEUS_URL environment variable not set")
    }
    if cfg.DatadogAPIKey == "" || cfg.DatadogAppKey == "" {
        return nil, fmt.Errorf("DD_API_KEY and DD_APP_KEY must be set")
    }
    return cfg, nil
}

func queryPrometheus(client v1.API, query string) (model.Value, error) {
    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    defer cancel()
    result, warnings, err := client.Query(ctx, query, time.Now())
    if err != nil {
        return nil, fmt.Errorf("prometheus query failed: %w", err)
    }
    if len(warnings) > 0 {
        log.Printf("Prometheus warnings: %v", warnings)
    }
    return result, nil
}

func getServiceMetrics(promClient v1.API, serviceName string) (map[string]float64, error) {
    metrics := make(map[string]float64)

    // Query p99 latency for service
    latencyQuery := fmt.Sprintf(`histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination", destination_service_name="%s"}[5m])) by (le))`, serviceName)
    latencyVal, err := queryPrometheus(promClient, latencyQuery)
    if err != nil {
        return nil, fmt.Errorf("latency query failed: %w", err)
    }
    if latencyVal.Type() == model.ValVector {
        vector := latencyVal.(model.Vector)
        if len(vector) > 0 {
            metrics["p99_latency_ms"] = float64(vector[0].Value)
        }
    }

    // Query success rate
    successQuery := fmt.Sprintf(`sum(rate(istio_requests_total{reporter="destination", destination_service_name="%s", response_code!~"5.*"}[5m])) / sum(rate(istio_requests_total{reporter="destination", destination_service_name="%s"}[5m])) * 100`, serviceName, serviceName)
    successVal, err := queryPrometheus(promClient, successQuery)
    if err != nil {
        return nil, fmt.Errorf("success rate query failed: %w", err)
    }
    if successVal.Type() == model.ValVector {
        vector := successVal.(model.Vector)
        if len(vector) > 0 {
            metrics["success_rate_pct"] = float64(vector[0].Value)
        }
    }

    // Query mTLS coverage
    mtlsQuery := fmt.Sprintf(`sum(rate(istio_requests_total{reporter="destination", destination_service_name="%s", connection_security_policy="mutual_tls"}[5m])) / sum(rate(istio_requests_total{reporter="destination", destination_service_name="%s"}[5m])) * 100`, serviceName, serviceName)
    mtlsVal, err := queryPrometheus(promClient, mtlsQuery)
    if err != nil {
        return nil, fmt.Errorf("mTLS query failed: %w", err)
    }
    if mtlsVal.Type() == model.ValVector {
        vector := mtlsVal.(model.Vector)
        if len(vector) > 0 {
            metrics["mtls_coverage_pct"] = float64(vector[0].Value)
        }
    }

    return metrics, nil
}

func sendToDatadog(cfg *Config, serviceName string, metrics map[string]float64) error {
    ctx := context.WithValue(context.Background(), datadog.ContextAPIKeys, map[string]datadog.APIKey{
        "apiKeyAuth": {Key: cfg.DatadogAPIKey},
        "appKeyAuth": {Key: cfg.DatadogAppKey},
    })

    client := datadog.NewAPIClient(datadog.NewConfiguration())
    now := time.Now().Unix()

    series := []datadog.Series{}
    for metricName, value := range metrics {
        series = append(series, datadog.Series{
            Metric: fmt.Sprintf("istio.service.%s", metricName),
            Points: []datadog.Point{
                {Timestamp: &now, Value: &value},
            },
            Tags: []string{
                fmt.Sprintf("service:%s", serviceName),
                fmt.Sprintf("istio_version:1.23"),
                fmt.Sprintf("k8s_version:1.32"),
            },
        })
    }

    body := datadogV1.MetricPayload{Series: series}
    _, _, err := client.MetricsApi.SubmitMetrics(ctx, body, *datadog.NewSubmitMetricsOptionalParameters())
    if err != nil {
        return fmt.Errorf("datadog submit failed: %w", err)
    }
    return nil
}

func main() {
    // Load configuration
    cfg, err := loadConfig()
    if err != nil {
        log.Fatalf("Failed to load config: %v", err)
    }

    // Initialize Prometheus client
    promClient, err := api.NewClient(api.Config{Address: cfg.PrometheusURL})
    if err != nil {
        log.Fatalf("Failed to create Prometheus client: %v", err)
    }
    v1api := v1.NewAPI(promClient)

    // Get list of all migrated services from K8s API
    // (Simplified for example: in production we query K8s API for services with label migrated=true)
    services := []string{
        "auth-service", "payment-service", "user-service", "product-service",
        // ... full list of 527 services would go here
    }

    // Scrape metrics for each service
    for {
        for _, svc := range services {
            metrics, err := getServiceMetrics(v1api, svc)
            if err != nil {
                log.Printf("Failed to get metrics for %s: %v", svc, err)
                continue
            }
            if err := sendToDatadog(cfg, svc, metrics); err != nil {
                log.Printf("Failed to send metrics for %s to Datadog: %v", svc, err)
                continue
            }
            log.Printf("Exported metrics for %s: %v", svc, metrics)
        }
        log.Printf("Scrape cycle complete, waiting %v", cfg.ScrapeInterval)
        time.Sleep(cfg.ScrapeInterval)
    }
}

#!/bin/bash
# canary_deploy.sh
# Author: Senior Platform Engineer, 15y exp
# Purpose: Automated canary deployment for migrated services using Istio 1.23 traffic splitting
# Requirements: kubectl 1.32+, istioctl 1.23+, jq
set -euo pipefail
trap 'log_error "Script failed at line $LINENO"' ERR

# Configuration
NAMESPACE="${NAMESPACE:-default}"
SERVICE_NAME="${SERVICE_NAME:-}"
CANARY_IMAGE="${CANARY_IMAGE:-}"
STABLE_IMAGE="${STABLE_IMAGE:-}"
CANARY_WEIGHT="${CANARY_WEIGHT:-10}"
PROMETHEUS_URL="${PROMETHEUS_URL:-http://prometheus.istio-system:9090}"
SUCCESS_THRESHOLD="${SUCCESS_THRESHOLD:-99.9}"
LATENCY_THRESHOLD_MS="${LATENCY_THRESHOLD_MS:-200}"
ROLLBACK_ON_FAILURE="${ROLLBACK_ON_FAILURE:-true}"

# Logging functions
log_info() {
    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] [INFO] $1"
}

log_error() {
    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] [ERROR] $1" >&2
}

# Validate required arguments
validate_args() {
    if [[ -z "$SERVICE_NAME" ]]; then
        log_error "SERVICE_NAME environment variable is required"
        exit 1
    fi
    if [[ -z "$CANARY_IMAGE" ]]; then
        log_error "CANARY_IMAGE environment variable is required"
        exit 1
    fi
    if [[ -z "$STABLE_IMAGE" ]]; then
        log_error "STABLE_IMAGE environment variable is required"
        exit 1
    fi
    log_info "Validated arguments for service $SERVICE_NAME"
}

# Check if kubectl is connected to K8s 1.32 cluster
check_k8s_cluster() {
    local k8s_version
    k8s_version=$(kubectl version -o json | jq -r '.serverVersion.gitVersion')
    if [[ ! "$k8s_version" =~ v1\.32\..* ]]; then
        log_error "Kubernetes cluster version is $k8s_version, required v1.32.x"
        exit 1
    fi
    log_info "Connected to Kubernetes cluster version $k8s_version"
}

# Check if Istio 1.23 is installed
check_istio() {
    local istio_version
    istio_version=$(istioctl version -o json | jq -r '.meshVersion[0].info.version')
    if [[ ! "$istio_version" =~ 1\.23\..* ]]; then
        log_error "Istio version is $istio_version, required 1.23.x"
        exit 1
    fi
    log_info "Istio version $istio_version detected"
}

# Update Kubernetes deployment with canary image
update_deployment() {
    log_info "Updating $SERVICE_NAME deployment with canary image $CANARY_IMAGE"
    kubectl set image deployment/"$SERVICE_NAME" "$SERVICE_NAME=$CANARY_IMAGE" -n "$NAMESPACE" --record
    kubectl rollout status deployment/"$SERVICE_NAME" -n "$NAMESPACE" --timeout=5m
    log_info "Deployment $SERVICE_NAME updated successfully"
}

# Configure Istio VirtualService for canary traffic splitting
configure_canary_traffic() {
    log_info "Configuring Istio VirtualService for $SERVICE_NAME with $CANARY_WEIGHT% canary traffic"
    cat < $LATENCY_THRESHOLD_MS" | bc -l) )); then
        log_error "Canary p99 latency $p99_latencyms is above threshold $LATENCY_THRESHOLD_MS"
        return 1
    fi
    log_info "Canary validation passed for $SERVICE_NAME"
    return 0
}

# Rollback canary deployment
rollback() {
    log_info "Rolling back canary deployment for $SERVICE_NAME"
    kubectl set image deployment/"$SERVICE_NAME" "$SERVICE_NAME=$STABLE_IMAGE" -n "$NAMESPACE" --record
    kubectl rollout status deployment/"$SERVICE_NAME" -n "$NAMESPACE" --timeout=5m
    # Reset traffic to 100% stable
    cat <