Retrospective: Implementing GitOps with ArgoCD and Pulumi Across 3 Cloud Providers – What Worked and What Didn't

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
import * as gcp from "@pulumi/gcp";
import * as azure from "@pulumi/azure-native";
import * as k8s from "@pulumi/kubernetes";
import { Vpc } from "./vpc"; // Local module for shared VPC config

// Configuration constants - pulled from Pulumi..yaml
const config = new pulumi.Config();
const stack = pulumi.getStack();
const project = pulumi.getProject();
const awsRegion = config.require("aws:region") || "us-east-1";
const gcpRegion = config.require("gcp:region") || "us-central1";
const azureRegion = config.require("azure:location") || "eastus";
const clusterVersion = config.require("clusterVersion") || "1.28";

// Error handler for cloud resource provisioning
const handleProvisionError = (cloud: string, err: Error) => {
    pulumi.log.error(`Failed to provision ${cloud} resources: ${err.message}`);
    // Alert on-call via Pulumi webhook integration (configured in stack settings)
    if (config.requireBoolean("enableAlerts")) {
        // In production, this would call PagerDuty/Slack API via secret webhook URL
        pulumi.log.warn(`Alert triggered for ${cloud} provisioning failure`);
    }
    throw err; // Fail stack deployment on critical errors
};

// AWS EKS Cluster Provisioning
let eksCluster: aws.eks.Cluster;
try {
    const awsVpc = new Vpc("aws-vpc", { region: awsRegion });
    eksCluster = new aws.eks.Cluster("multi-cloud-eks", {
        roleArn: awsVpc.eksRoleArn,
        vpcConfig: {
            subnetIds: awsVpc.privateSubnetIds,
            endpointPrivateAccess: true,
            endpointPublicAccess: false, // Private endpoint only for compliance
        },
        version: clusterVersion,
        tags: {
            Project: project,
            Stack: stack,
            ManagedBy: "pulumi",
            CloudProvider: "aws",
        },
    });

    // Node group with spot instances for cost savings
    new aws.eks.NodeGroup("eks-spot-nodes", {
        clusterName: eksCluster.name,
        nodeRoleArn: awsVpc.eksNodeRoleArn,
        subnetIds: awsVpc.privateSubnetIds,
        scalingConfig: {
            desiredSize: 2,
            maxSize: 10,
            minSize: 1,
        },
        instanceTypes: ["t3.large", "t3a.large"], // Spot eligible instances
        capacityType: "SPOT",
        labels: { "node-type": "spot", "cloud": "aws" },
        tags: { CloudProvider: "aws" },
    });
} catch (err) {
    handleProvisionError("AWS", err as Error);
}

// GCP GKE Cluster Provisioning
let gkeCluster: gcp.container.Cluster;
try {
    const gcpVpc = new Vpc("gcp-vpc", { region: gcpRegion, cloud: "gcp" });
    gkeCluster = new gcp.container.Cluster("multi-cloud-gke", {
        location: gcpRegion,
        initialNodeCount: 1,
        minMasterVersion: clusterVersion,
        nodeConfig: {
            machineType: "e2-standard-4",
            preemptible: true, // GCP spot equivalent
            labels: { "cloud": "gcp", "node-type": "spot" },
            oauthScopes: ["https://www.googleapis.com/auth/cloud-platform"],
        },
        network: gcpVpc.vpcId,
        subnetwork: gcpVpc.subnetId,
        privateClusterConfig: {
            enablePrivateNodes: true,
            masterIpv4CidrBlock: "172.16.0.0/28",
        },
        resourceLabels: {
            project: project,
            stack: stack,
            managed_by: "pulumi",
        },
    });
} catch (err) {
    handleProvisionError("GCP", err as Error);
}

// Azure AKS Cluster Provisioning
let aksCluster: azure.containerservice.ManagedCluster;
try {
    const azureVnet = new Vpc("azure-vnet", { region: azureRegion, cloud: "azure" });
    aksCluster = new azure.containerservice.ManagedCluster("multi-cloud-aks", {
        resourceGroupName: azureVnet.resourceGroupName,
        location: azureRegion,
        kubernetesVersion: clusterVersion,
        dnsPrefix: `${project}-${stack}-aks`,
        agentPoolProfiles: [{
            name: "spotpool",
            count: 2,
            vmSize: "Standard_D4s_v3",
            type: "VirtualMachineScaleSets",
            scaleSetPriority: "Spot",
            scaleSetEvictionPolicy: "Delete",
            mode: "System",
        }],
        networkProfile: {
            networkPlugin: "azure",
            vnetSubnetId: azureVnet.subnetId,
        },
        identity: { type: "SystemAssigned" },
        tags: {
            Project: project,
            Stack: stack,
            ManagedBy: "pulumi",
            CloudProvider: "azure",
        },
    });
} catch (err) {
    handleProvisionError("Azure", err as Error);
}

// Export cluster endpoints for ArgoCD configuration
export const eksEndpoint = eksCluster.endpoint;
export const gkeEndpoint = gkeCluster.endpoint.apply(e => `https://${e}`);
export const aksEndpoint = aksCluster.fqdn.apply(fqdn => `https://${fqdn}`);

# ArgoCD ApplicationSet for multi-cloud guestbook app deployment
# Valid for ArgoCD v2.8.4+, requires clusters to be pre-registered in ArgoCD
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: guestbook-multi-cloud
  namespace: argocd
  labels:
    app.kubernetes.io/name: guestbook
    app.kubernetes.io/managed-by: argocd
spec:
  # Generator to target all 3 registered clusters with cloud label
  generators:
  - clusters:
      selector:
        matchLabels:
          cloud: "aws" # Matches EKS cluster
  - clusters:
      selector:
        matchLabels:
          cloud: "gcp" # Matches GKE cluster
  - clusters:
      selector:
        matchLabels:
          cloud: "azure" # Matches AKS cluster
  # Template for each Application instance
  template:
    metadata:
      name: guestbook-{{.metadata.labels.cloud}}
      namespace: argocd
      labels:
        cloud: "{{.metadata.labels.cloud}}"
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/multi-cloud-guestbook # Canonical GitHub URL per rules
        targetRevision: main
        path: k8s/overlays/{{.metadata.labels.cloud}} # Cloud-specific kustomize overlay
        kustomize:
          images:
          - guestbook:latest=guestbook:{{.metadata.labels.cloud}}-{{git.commit}}
      destination:
        server: "{{.metadata.annotations.argocd\.argoproj\.io/server-url}}" # Dynamic cluster URL
        namespace: guestbook
      syncPolicy:
        automated:
          prune: true # Delete resources removed from git
          selfHeal: true # Correct drift automatically
          allowEmpty: false # Fail if no resources to sync
        syncOptions:
        - CreateNamespace=true # Create guestbook namespace if missing
        - PrunePropagationPolicy=foreground # Wait for resource deletion before proceeding
        - RespectIgnoreDifferences=true # Honor .argocdignore rules
        retry:
          limit: 5 # Retry failed syncs up to 5 times
          backoff:
            duration: 30s # Initial retry delay
            factor: 2 # Exponential backoff multiplier
            maxDuration: 5m # Max retry delay
      # Health check configuration to prevent broken deployments
      ignoreDifferences:
      - group: apps
        kind: Deployment
        name: guestbook
        jsonPointers:
        - /spec/replicas # Ignore replica count drift (handled by HPA)
      - group: ""
        kind: Service
        name: guestbook-svc
        jsonPointers:
        - /spec/ports/0/nodePort # Ignore auto-assigned node ports
  # Error handling: alert on sync failure via ArgoCD notifications
  notifications:
    subscriptions:
    - recipients:
      - slack:infra-alerts
      triggers:
      - on-sync-failed
      - on-sync-retry-exceeded
    templates:
    - name: multi-cloud-sync-failure
      template: |
        argocd app sync failed for {{.app.metadata.name}} on {{.app.spec.destination.server}}
        Error: {{.error}}
        Commit: {{.commit}}

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "os"
    "time"

    argocd "github.com/argoproj/argo-cd/v2/pkg/apiclient"
    "github.com/argoproj/argo-cd/v2/pkg/apiclient/application"
    "github.com/pulumi/pulumi/sdk/v3/go/pulumi"
    "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
)

// DriftReport represents a single app's drift status
type DriftReport struct {
    AppName      string    `json:"appName"`
    Cloud        string    `json:"cloud"`
    Drifted      bool      `json:"drifted"`
    DriftPercent float64   `json:"driftPercent"`
    LastSync     time.Time `json:"lastSync"`
}

func main() {
    // Initialize Pulumi context for config
    ctx, err := pulumi.NewContext(context.Background())
    if err != nil {
        log.Fatalf("Failed to create Pulumi context: %v", err)
    }
    defer ctx.Close()

    // Load configuration from Pulumi..yaml
    cfg := config.New(ctx)
    argoCDURL := cfg.Require("argocd:url")
    argoCDToken := cfg.RequireSecret("argocd:token").Get(ctx) // Encrypted secret
    reportPath := cfg.Require("drift:reportPath")

    // Create ArgoCD API client
    clientOpts := argocd.ClientOptions{
        ServerAddr: argoCDURL,
        AuthToken:  argoCDToken,
        Insecure:   false, // Use TLS in production
    }
    client, err := argocd.NewClient(clientOpts)
    if err != nil {
        log.Fatalf("Failed to create ArgoCD client: %v", err)
    }
    defer client.Close()

    // List all applications in ArgoCD
    appClient := client.ApplicationClient()
    listResp, err := appClient.List(context.Background(), &application.ApplicationQuery{})
    if err != nil {
        log.Fatalf("Failed to list ArgoCD applications: %v", err)
    }

    // Generate drift report for each app
    var reports []DriftReport
    for _, app := range listResp.Items {
        cloud := app.Labels["cloud"]
        if cloud == "" {
            log.Printf("Skipping app %s: no cloud label", app.Metadata.Name)
            continue
        }

        // Calculate drift percentage (simplified: 100% if status is OutOfSync)
        drifted := app.Status.Sync.Status == "OutOfSync"
        driftPercent := 0.0
        if drifted {
            // In production, this would compare resource hashes from Pulumi state
            driftPercent = 100.0
        }

        reports = append(reports, DriftReport{
            AppName:      app.Metadata.Name,
            Cloud:        cloud,
            Drifted:      drifted,
            DriftPercent: driftPercent,
            LastSync:     app.Status.OperationState.FinishedAt.Time,
        })
    }

    // Write report to file
    reportJSON, err := json.MarshalIndent(reports, "", "  ")
    if err != nil {
        log.Fatalf("Failed to marshal drift report: %v", err)
    }
    if err := os.WriteFile(reportPath, reportJSON, 0644); err != nil {
        log.Fatalf("Failed to write drift report: %v", err)
    }

    // Print summary
    fmt.Printf("Drift report generated: %s\n", reportPath)
    fmt.Printf("Total apps scanned: %d\n", len(reports))
    fmt.Printf("Drifted apps: %d\n", countDrifted(reports))
}

// countDrifted returns the number of drifted apps in the report
func countDrifted(reports []DriftReport) int {
    count := 0
    for _, r := range reports {
        if r.Drifted {
            count++
        }
    }
    return count
}

// Pin Pulumi SDK versions in package.json
{
  "dependencies": {
    "@pulumi/pulumi": "3.77.1",
    "@pulumi/aws": "6.32.0",
    "@pulumi/gcp": "7.18.0",
    "@pulumi/azure-native": "2.54.0"
  }
}

// Create a new GKE cluster using shared Vpc module
const gcpVpc = new Vpc("gcp-vpc", { cloud: "gcp", region: "us-central1" });
const gkeCluster = new GkeCluster("checkout-gke", {
  vpcId: gcpVpc.vpcId,
  subnetId: gcpVpc.subnetId,
  nodeCount: 3,
});

# Enable native drift detection in ArgoCD Application
spec:
  syncPolicy:
    automated:
      selfHeal: true
  ignoreDifferences:
  - group: apps
    kind: Deployment
    jsonPointers:
    - /spec/replicas