War Story: We Adopted Chaos Engineering and Cut Outage Frequency by 60%

package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"time\"\n\n\tchaosv1alpha1 \"github.com/chaos-mesh/chaos-mesh/api/v1alpha1\"\n\tchaosclient \"github.com/chaos-mesh/chaos-mesh/pkg/client/clientset/versioned\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n\t\"k8s.io/apimachinery/pkg/runtime/serializer/yaml\"\n)\n\nconst (\n\t// Target namespace for our payment processing workloads\n\ttargetNamespace = \"payment-prod\"\n\t// Redis service name in the target namespace\n\tredisServiceName = \"redis-payment-cache\"\n\t// Max acceptable p99 latency for payment requests during fault injection\n\tmaxAllowedLatency = 500 * time.Millisecond\n\t// Duration to run the chaos experiment\n\texperimentDuration = 5 * time.Minute\n)\n\n// getK8sClients initializes Kubernetes and Chaos Mesh clients\nfunc getK8sClients() (*kubernetes.Clientset, *chaosclient.Clientset, error) {\n\t// Load kubeconfig from default path (~/.kube/config)\n\tkubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(\n\t\tclientcmd.NewDefaultClientConfigLoadingRules(),\n\t\t&clientcmd.ConfigOverrides{},\n\t)\n\trestConfig, err := kubeconfig.ClientConfig()\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"failed to load kubeconfig: %w\", err)\n\t}\n\n\t// Initialize Kubernetes core client\n\tk8sClient, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"failed to create k8s client: %w\", err)\n\t}\n\n\t// Initialize Chaos Mesh client\n\tchaosClient, err := chaosclient.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"failed to create chaos mesh client: %w\", err)\n\t}\n\n\treturn k8sClient, chaosClient, nil\n}\n\n// createRedisDelayExperiment injects a network delay fault into the target Redis service\nfunc createRedisDelayExperiment(chaosClient *chaosclient.Clientset) error {\n\t// Define the network chaos experiment for Redis\n\tnetworkChaos := &chaosv1alpha1.NetworkChaos{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName:      \"redis-payment-delay-5min\",\n\t\t\tNamespace: targetNamespace,\n\t\t},\n\t\tSpec: chaosv1alpha1.NetworkChaosSpec{\n\t\t\tAction: chaosv1alpha1.DelayAction,\n\t\t\tMode:   chaosv1alpha1.OnePodMode, // Target one Redis pod at a time\n\t\t\tSelector: chaosv1alpha1.SelectorSpec{\n\t\t\t\tNamespaces: []string{targetNamespace},\n\t\t\t\tLabelSelectors: map[string]string{\n\t\t\t\t\t\"app\": redisServiceName,\n\t\t\t\t},\n\t\t\t},\n\t\t\tDelay: &chaosv1alpha1.DelaySpec{\n\t\t\t\tLatency: \"300ms\", // Inject 300ms network delay\n\t\t\t\tJitter:  \"50ms\",  // Add up to 50ms jitter\n\t\t\t\tCorrelation: \"100\",\n\t\t\t},\n\t\t\tDuration: &metav1.Duration{Duration: experimentDuration},\n\t\t},\n\t}\n\n\t// Create the experiment in the cluster\n\t_, err := chaosClient.ChaosV1alpha1().NetworkChaos(targetNamespace).Create(\n\t\tcontext.Background(),\n\t\tnetworkChaos,\n\t\tmetav1.CreateOptions{},\n\t)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create network chaos experiment: %w\", err)\n\t}\n\tlog.Printf(\"Successfully created Redis delay experiment: %s\", networkChaos.Name)\n\treturn nil\n}\n\n// verifyPaymentResilience checks if the payment service maintains SLA during the fault\nfunc verifyPaymentResilience(k8sClient *kubernetes.Clientset) error {\n\t// In a real implementation, this would query Prometheus for payment request latency\n\t// For this example, we simulate the check with a mock, but include the actual Prometheus query\n\tpromQuery := `histogram_quantile(0.99, sum(rate(payment_request_duration_seconds_bucket{namespace=\"payment-prod\"}[5m])) by (le))`\n\tlog.Printf(\"Running Prometheus query to check p99 latency: %s\", promQuery)\n\n\t// Simulate waiting for experiment to take effect\n\ttime.Sleep(30 * time.Second)\n\n\t// Mock check: in production, we'd parse Prometheus response\n\tmockP99Latency := 420 * time.Millisecond\n\tif mockP99Latency > maxAllowedLatency {\n\t\treturn fmt.Errorf(\"p99 latency %v exceeds max allowed %v\", mockP99Latency, maxAllowedLatency)\n\t}\n\tlog.Printf(\"Payment service passed resilience check: p99 latency %v\", mockP99Latency)\n\treturn nil\n}\n\nfunc main() {\n\t// Initialize clients\n\tk8sClient, chaosClient, err := getK8sClients()\n\tif err != nil {\n\t\tlog.Fatalf(\"Failed to initialize clients: %v\", err)\n\t}\n\n\t// Create the Redis delay experiment\n\tif err := createRedisDelayExperiment(chaosClient); err != nil {\n\t\tlog.Fatalf(\"Failed to create chaos experiment: %v\", err)\n\t}\n\n\t// Wait for experiment to start\n\ttime.Sleep(10 * time.Second)\n\n\t// Verify payment service resilience\n\tif err := verifyPaymentResilience(k8sClient); err != nil {\n\t\tlog.Fatalf(\"Resilience check failed: %v\", err)\n\t}\n\n\t// Cleanup: delete the experiment after verification\n\terr = chaosClient.ChaosV1alpha1().NetworkChaos(targetNamespace).Delete(\n\t\tcontext.Background(),\n\t\t\"redis-payment-delay-5min\",\n\t\tmetav1.DeleteOptions{},\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"Failed to cleanup chaos experiment: %v\", err)\n\t}\n\tlog.Println(\"Chaos experiment completed successfully, cleanup done\")\n}\n

import time\nimport logging\nfrom litmuschaos import ChaosClient, ChaosExperiment\nfrom kafka import KafkaConsumer, KafkaAdminClient\nfrom kafka.admin import ConfigResource, ConfigResourceType\nfrom kafka.errors import KafkaError\nimport os\nfrom prometheus_api_client import PrometheusConnect\n\n# Configure logging\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n# Configuration constants\nLITMUS_PROJECT_ID = \"payment-chaos\"\nLITMUS_EXPERIMENT_NAME = \"kafka-payment-lag-inject\"\nKAFKA_BROKERS = [\"kafka-broker-0:9092\", \"kafka-broker-1:9092\", \"kafka-broker-2:9092\"]\nKAFKA_TOPIC = \"payment-events\"\nCONSUMER_GROUP = \"payment-processor-group\"\nMAX_ALLOWED_LAG = 1000  # Max allowed consumer lag per partition\nEXPERIMENT_DURATION = 300  # 5 minutes in seconds\nPROMETHEUS_URL = \"http://prometheus.monitoring:9090\"\n\ndef init_litmus_client():\n    \"\"\"Initialize LitmusChaos client with API token from environment\"\"\"\n    api_token = os.getenv(\"LITMUS_API_TOKEN\")\n    if not api_token:\n        raise ValueError(\"LITMUS_API_TOKEN environment variable not set\")\n    \n    try:\n        client = ChaosClient(\n            project_id=LITMUS_PROJECT_ID,\n            api_token=api_token,\n            endpoint=\"https://litmus-chaos-api.prod:443\"\n        )\n        logger.info(\"Successfully initialized LitmusChaos client\")\n        return client\n    except Exception as e:\n        raise RuntimeError(f\"Failed to initialize Litmus client: {e}\")\n\ndef init_kafka_consumer():\n    \"\"\"Initialize Kafka consumer to check lag\"\"\"\n    try:\n        consumer = KafkaConsumer(\n            bootstrap_servers=KAFKA_BROKERS,\n            group_id=CONSUMER_GROUP,\n            enable_auto_commit=False,\n            auto_offset_reset=\"latest\"\n        )\n        logger.info(\"Successfully initialized Kafka consumer\")\n        return consumer\n    except KafkaError as e:\n        raise RuntimeError(f\"Failed to initialize Kafka consumer: {e}\")\n\ndef create_kafka_fault_experiment(litmus_client):\n    \"\"\"Create a LitmusChaos experiment to pause Kafka topic partitions\"\"\"\n    experiment = ChaosExperiment(\n        name=LITMUS_EXPERIMENT_NAME,\n        description=\"Inject partition pause on payment-events topic to test consumer lag handling\",\n        fault_type=\"kafka-fault\",\n        experiment_args={\n            \"kafkaEndpoint\": \",\".join(KAFKA_BROKERS),\n            \"topic\": KAFKA_TOPIC,\n            \"partition\": \"0,1,2\",  # Target first 3 partitions of the topic\n            \"faultType\": \"pausePartition\",\n            \"duration\": str(EXPERIMENT_DURATION),\n            \"consumerGroup\": CONSUMER_GROUP\n        }\n    )\n\n    try:\n        # Schedule the experiment to run immediately\n        response = litmus_client.schedule_experiment(experiment)\n        logger.info(f\"Scheduled Kafka fault experiment: {response['experimentId']}\")\n        return response[\"experimentId\"]\n    except Exception as e:\n        raise RuntimeError(f\"Failed to create Kafka fault experiment: {e}\")\n\ndef check_consumer_lag():\n    \"\"\"Check current consumer lag for the payment processor group\"\"\"\n    try:\n        admin_client = KafkaAdminClient(bootstrap_servers=KAFKA_BROKERS)\n        consumer = init_kafka_consumer()\n\n        # Get topic partition offsets\n        topic_partitions = consumer.assignment()\n        if not topic_partitions:\n            consumer.poll(timeout_ms=1000)\n            topic_partitions = consumer.assignment()\n\n        total_lag = 0\n        for tp in topic_partitions:\n            # Get committed offset for the consumer group\n            committed = consumer.committed(tp)\n            # Get end offset for the partition\n            end_offset = admin_client.list_consumer_group_offsets(\n                group_id=CONSUMER_GROUP,\n                partitions=[tp]\n            ).get(tp, None)\n\n            if committed and end_offset:\n                lag = end_offset.offset - committed\n                total_lag += lag\n                logger.debug(f\"Partition {tp.partition} lag: {lag}\")\n\n        logger.info(f\"Total consumer lag for group {CONSUMER_GROUP}: {total_lag}\")\n        return total_lag\n    except KafkaError as e:\n        raise RuntimeError(f\"Failed to check consumer lag: {e}\")\n\ndef verify_kafka_resilience(experiment_id, litmus_client):\n    \"\"\"Verify that consumer lag stays within acceptable limits during fault\"\"\"\n    start_time = time.time()\n    while time.time() - start_time < EXPERIMENT_DURATION:\n        lag = check_consumer_lag()\n        if lag > MAX_ALLOWED_LAG:\n            # Abort the experiment if lag exceeds threshold\n            litmus_client.stop_experiment(experiment_id)\n            raise RuntimeError(f\"Consumer lag {lag} exceeds max allowed {MAX_ALLOWED_LAG}\")\n        time.sleep(30)  # Check every 30 seconds\n\n    logger.info(\"Kafka consumer lag stayed within acceptable limits during fault injection\")\n    return True\n\ndef main():\n    try:\n        # Initialize clients\n        litmus_client = init_litmus_client()\n        init_kafka_consumer()\n\n        # Create and schedule the fault experiment\n        experiment_id = create_kafka_fault_experiment(litmus_client)\n\n        # Wait for experiment to start\n        time.sleep(10)\n        logger.info(f\"Kafka fault experiment {experiment_id} is now running\")\n\n        # Verify resilience\n        verify_kafka_resilience(experiment_id, litmus_client)\n\n        # Cleanup: stop experiment if still running\n        litmus_client.stop_experiment(experiment_id)\n        logger.info(\"Kafka chaos experiment completed successfully\")\n\n    except Exception as e:\n        logger.error(f\"Chaos experiment failed: {e}\")\n        raise\n\nif __name__ == \"__main__\":\n    main()\n

name: Chaos Engineering PR Check\n\non:\n  pull_request:\n    paths:\n      - \"services/payment/**\"  # Run only when payment service code changes\n    branches:\n      - main\n      - release/*\n\nenv:\n  KUBECONFIG: ${{ secrets.KUBECONFIG }}\n  CHAOS_MESH_VERSION: v2.6.3\n  PAYMENT_SERVICE_NAMESPACE: payment-prod\n  MAX_ALLOWED_ERROR_RATE: 0.1%  # Max 0.1% error rate during chaos\n\njobs:\n  run-chaos-experiment:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read\n      pull-requests: write\n\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@v4\n        with:\n          fetch-depth: 0  # Fetch full history for blame info\n\n      - name: Set up Go 1.21\n        uses: actions/setup-go@v5\n        with:\n          go-version: 1.21\n          cache: true\n\n      - name: Install kubectl\n        uses: azure/setup-kubectl@v3\n        with:\n          version: v1.28.0\n\n      - name: Verify kubeconfig is valid\n        run: |\n          kubectl cluster-info\n          kubectl auth can-i create networkchaos.chaos-mesh.io -n $PAYMENT_SERVICE_NAMESPACE\n\n      - name: Install Chaos Mesh CLI (chaosctl)\n        run: |\n          curl -sSL https://github.com/chaos-mesh/chaos-mesh/releases/download/$CHAOS_MESH_VERSION/chaosctl-linux-amd64 -o /usr/local/bin/chaosctl\n          chmod +x /usr/local/bin/chaosctl\n          chaosctl version\n\n      - name: Build payment service chaos test binary\n        run: |\n          cd services/payment\n          go build -o payment-chaos-test ./cmd/chaos-test\n          chmod +x payment-chaos-test\n\n      - name: Run Redis delay chaos experiment\n        id: redis-chaos\n        run: |\n          # Create a unique experiment name for this PR\n          EXPERIMENT_NAME=\"pr-${{ github.event.pull_request.number }}-redis-delay\"\n          \n          # Apply the network chaos manifest for Redis\n          cat <