Postmortem: How a Bug in CircleCI 7.0 Caused 50+ Failed Mobile App Builds

# Broken CircleCI 7.0 Config for iOS App (v2.1 schema)
# Failed for 53 builds between March 12-15 2024 due to executor regression
version: 2.1

# Define executors to reuse configuration across jobs
executors:
  ios-executor:
    macos:
      xcode: "15.2" # Fixed Xcode version to match project requirements
      # BUG: CircleCI 7.0 ignores resource_class for macOS executors with custom signing
      # This line was silently dropped, causing resource contention
      resource_class: "medium" # 4 vCPU, 8GB RAM - required for Swift compile times
    environment:
      DEVELOPER_DIR: "/Applications/Xcode_15.2.app/Contents/Developer"
      MATCH_PASSWORD: "<>" # Encrypted signing cert password

# Define jobs to run in pipeline
jobs:
  build-and-sign-ios:
    executor: ios-executor
    steps:
      - checkout:
          path: ~/project # Explicit checkout path to avoid permission issues
      # Install CocoaPods dependencies with error handling
      - run:
          name: "Install CocoaPods Dependencies"
          command: |
            set -e # Exit immediately if any command fails
            cd ~/project/ios
            pod install --repo-update
            if [ $? -ne 0 ]; then
              echo "ERROR: CocoaPods install failed with exit code $?"
              exit 1
            fi
          no_output_timeout: 10m # Timeout for slow dependency fetches
      # Build iOS app with signing (failing step)
      - run:
          name: "Build and Sign iOS App"
          command: |
            set -e
            cd ~/project/ios
            # BUG TRIGGER: CircleCI 7.0 executor drops resource allocation, causing code signing to OOM
            xcodebuild -workspace MyApp.xcworkspace -scheme MyApp -configuration Release \
              -sdk iphoneos -archivePath ~/project/build/MyApp.xcarchive \
              CODE_SIGN_IDENTITY="iPhone Distribution: My Corp (ABC123XYZ)" \
              PROVISIONING_PROFILE_SPECIFIER="MyApp_AdHoc" \
              clean archive
            if [ $? -ne 0 ]; then
              echo "ERROR: xcodebuild archive failed with exit code $?"
              # Capture diagnostic logs for postmortem
              cp ~/Library/Logs/xcodebuild.xcactivitylog ~/project/build/
              exit 1
            fi
          no_output_timeout: 30m
      # Archive and upload build artifacts
      - run:
          name: "Export IPA and Upload Artifacts"
          command: |
            set -e
            cd ~/project/build
            xcodebuild -exportArchive -archivePath MyApp.xcarchive \
              -exportOptionsPlist ~/project/ios/exportOptions.plist \
              -exportPath ~/project/build/ipa
            # Upload to TestFlight via Fastlane (simplified for example)
            fastlane pilot upload --ipa ~/project/build/ipa/MyApp.ipa
          no_output_timeout: 15m
      - store_artifacts:
          path: ~/project/build
          destination: build-output

# Define workflow to trigger jobs
workflows:
  ios-release-workflow:
    jobs:
      - build-and-sign-ios:
          filters:
            branches:
              only: /release/.*/ # Only run on release branches

#!/usr/bin/env python3
"""
CircleCI 7.0 Executor Bug Detection Script
Queries CircleCI API v2 to check for macOS executor resource allocation failures
Corresponding to the March 2024 mobile build outage
"""

import os
import sys
import json
import time
import logging
from datetime import datetime, timedelta
import requests
from requests.exceptions import RequestException

# Configure logging for audit trails
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

# Constants for CircleCI API
CIRCLECI_API_BASE = "https://circleci.com/api/v2"
CIRCLECI_TOKEN = os.environ.get("CIRCLECI_TOKEN") # Token with read scope
LOOKBACK_DAYS = 3 # Check last 3 days of builds
RESOURCE_CLASS = "medium" # Affected resource class
EXECUTOR_TYPE = "macos" # Affected executor type

def validate_env_vars():
    """Validate required environment variables are set"""
    if not CIRCLECI_TOKEN:
        logger.error("Missing required environment variable: CIRCLECI_TOKEN")
        sys.exit(1)

def get_failed_builds(project_slug: str) -> list:
    """
    Fetch failed builds for a given project in the lookback window
    Args:
        project_slug: CircleCI project slug (e.g., gh/org/repo)
    Returns:
        List of failed build objects
    """
    validate_env_vars()
    headers = {
        "Circle-Token": CIRCLECI_TOKEN,
        "Accept": "application/json"
    }
    # Calculate start time for lookback
    start_time = datetime.utcnow() - timedelta(days=LOOKBACK_DAYS)
    start_iso = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
    url = f"{CIRCLECI_API_BASE}/project/{project_slug}/pipeline"
    params = {
        "start_date": start_iso,
        "status": "failed",
        "limit": 100 # Max limit per page
    }
    failed_builds = []
    try:
        while url:
            response = requests.get(url, headers=headers, params=params)
            response.raise_for_status() # Raise HTTPError for bad status codes
            data = response.json()
            failed_builds.extend(data.get("items", []))
            # Handle pagination
            url = data.get("next_page")
            params = {} # Params only needed for first request
            time.sleep(0.5) # Rate limit avoidance
    except RequestException as e:
        logger.error(f"Failed to fetch builds for {project_slug}: {e}")
        return []
    return failed_builds

def check_executor_regression(build: dict) -> bool:
    """
    Check if a failed build matches the CircleCI 7.0 executor regression pattern
    Args:
        build: Build object from CircleCI API
    Returns:
        True if build failed due to the regression, False otherwise
    """
    # Check if build uses macOS executor with medium resource class
    executor_info = build.get("executor", {})
    if executor_info.get("type") != EXECUTOR_TYPE:
        return False
    if executor_info.get("resource_class") != RESOURCE_CLASS:
        return False
    # Check if failure is due to OOM (exit code 137) or signing failure
    failure_message = build.get("failure_message", "")
    return any(keyword in failure_message for keyword in ["exit code 137", "code signing failed", "resource allocation"])

def main():
    """Main entry point for bug detection script"""
    if len(sys.argv) < 2:
        logger.error("Usage: python detect_bug.py  [project_slug2] ...")
        sys.exit(1)
    project_slugs = sys.argv[1:]
    total_affected = 0
    for slug in project_slugs:
        logger.info(f"Checking project: {slug}")
        failed_builds = get_failed_builds(slug)
        affected_builds = [b for b in failed_builds if check_executor_regression(b)]
        if affected_builds:
            logger.warning(f"Found {len(affected_builds)} builds affected by CircleCI 7.0 regression in {slug}")
            total_affected += len(affected_builds)
            # Write affected build IDs to output file
            output_file = f"affected_builds_{slug.replace('/', '_')}.json"
            with open(output_file, "w") as f:
                json.dump(affected_builds, f, indent=2)
            logger.info(f"Affected build details written to {output_file}")
        else:
            logger.info(f"No affected builds found in {slug}")
    logger.info(f"Total affected builds across all projects: {total_affected}")
    # Exit with non-zero code if any affected builds found
    sys.exit(1 if total_affected > 0 else 0)

if __name__ == "__main__":
    main()

package executor

import (
    "context"
    "testing"
    "time"

    "github.com/stretchr/testify/assert"
    "github.com/stretchr/testify/require"
    "go.uber.org/zap/zaptest"
)

// TestMacOSExecutorResourceAllocation_7_0_Regression tests the fix for the
// CircleCI 7.0 bug where resource_class was ignored for macOS executors with custom signing
func TestMacOSExecutorResourceAllocation_7_0_Regression(t *testing.T) {
    // Initialize test logger
    logger := zaptest.NewLogger(t)
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Test case 1: Verify medium resource class is allocated when specified
    t.Run("MediumResourceClassAllocated", func(t *testing.T) {
        // Mock executor config matching CircleCI 7.0 broken behavior
        config := ExecutorConfig{
            ExecutorType:  "macos",
            XcodeVersion:  "15.2",
            ResourceClass: "medium", // 4 vCPU, 8GB RAM
            SigningConfig: &SigningConfig{
                UseCustomCert: true,
                CertPath:      "/tmp/test_cert.p12",
            },
        }
        // Create executor with test dependencies
        executor, err := NewMacOSExecutor(ctx, config, logger)
        require.NoError(t, err, "Failed to create macOS executor")
        defer executor.Cleanup()

        // Allocate resources and verify allocation matches config
        allocated, err := executor.AllocateResources(ctx)
        require.NoError(t, err, "Resource allocation failed")
        assert.Equal(t, "medium", allocated.ResourceClass, "Resource class mismatch")
        assert.Equal(t, 4, allocated.VCPU, "vCPU count mismatch")
        assert.Equal(t, 8192, allocated.MemoryMB, "Memory allocation mismatch")

        // Verify resource allocation is not dropped when custom signing is enabled
        assert.True(t, allocated.SigningEnabled, "Custom signing should be enabled")
        assert.Equal(t, config.SigningConfig.CertPath, allocated.SigningCertPath, "Signing cert path mismatch")
    })

    // Test case 2: Verify regression where resource_class was ignored does not occur
    t.Run("Regression_ResourceClassNotDropped", func(t *testing.T) {
        // Config that triggered the original bug: custom signing + resource class
        config := ExecutorConfig{
            ExecutorType:  "macos",
            XcodeVersion:  "15.2",
            ResourceClass: "medium",
            SigningConfig: &SigningConfig{
                UseCustomCert: true,
            },
        }
        // Run allocation 100 times to check for flakiness
        for i := 0; i < 100; i++ {
            executor, err := NewMacOSExecutor(ctx, config, logger)
            require.NoError(t, err, "Iteration %d: Failed to create executor", i)
            allocated, err := executor.AllocateResources(ctx)
            require.NoError(t, err, "Iteration %d: Resource allocation failed", i)
            assert.Equal(t, "medium", allocated.ResourceClass, "Iteration %d: Resource class dropped", i)
            executor.Cleanup()
        }
    })

    // Test case 3: Verify large resource class works for heavy Swift compiles
    t.Run("LargeResourceClassAllocation", func(t *testing.T) {
        config := ExecutorConfig{
            ExecutorType:  "macos",
            XcodeVersion:  "15.2",
            ResourceClass: "large", // 8 vCPU, 16GB RAM
            SigningConfig: &SigningConfig{
                UseCustomCert: true,
            },
        }
        executor, err := NewMacOSExecutor(ctx, config, logger)
        require.NoError(t, err)
        defer executor.Cleanup()

        allocated, err := executor.AllocateResources(ctx)
        require.NoError(t, err)
        assert.Equal(t, "large", allocated.ResourceClass)
        assert.Equal(t, 8, allocated.VCPU)
        assert.Equal(t, 16384, allocated.MemoryMB)
    })
}

# Short snippet for executor validation step
- run:
    name: "Validate Executor Resource Allocation"
    command: |
      set -e
      python3 detect_bug.py "gh/myorg/myrepo" || {
        echo "ERROR: Executor regression detected, aborting build"
        exit 1
      }

# Short snippet for canary workflow parameter
version: 2.1
parameters:
  executor-version:
    type: string
    default: "6.2.0" # Canary: set to 7.0.0 for 10% of builds
jobs:
  build:
    executor:
      macos:
        xcode: "15.2"
        resource_class: "medium"
    environment:
      CIRCLECI_EXECUTOR_VERSION: << pipeline.parameters.executor-version >>

# Short snippet for OOM alerting via Prometheus
- name: "Export Build Metrics to Prometheus"
  command: |
    set -e
    curl -X POST http://prometheus-pushgateway:9091/metrics/job/circleci-build \
      --data "build_exit_code{project=\"myapp-ios\"} $EXIT_CODE" \
      --data "build_duration_seconds $BUILD_DURATION"