Code Story: How Netflix Built Their Canary Deployment Pipeline with Argo Rollouts 2.12 in 2026

package netflixmetricprovider

import (
    "context"
    "fmt"
    "io"
    "net/http"
    "time"

    rolloutsv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
    "github.com/argoproj/argo-rollouts/pkg/metricproviders"
    promapi "github.com/prometheus/client_golang/api"
    promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
    "github.com/prometheus/common/model"
    "golang.org/x/exp/slog"
)

const (
    defaultPrometheusTimeout = 10 * time.Second
    netflixPlaybackNamespace = "netflix-playback"
)

// NetflixPlaybackMetricProvider implements metricproviders.MetricProvider for Netflix's playback service canary metrics
type NetflixPlaybackMetricProvider struct {
    promClient promv1.API
    logger     *slog.Logger
}

// NewNetflixPlaybackMetricProvider initializes a new metric provider with a Prometheus client
func NewNetflixPlaybackMetricProvider(promURL string, logger *slog.Logger) (*NetflixPlaybackMetricProvider, error) {
    if promURL == "" {
        return nil, fmt.Errorf("prometheus URL cannot be empty")
    }
    client, err := promapi.NewClient(promapi.Config{Address: promURL})
    if err != nil {
        return nil, fmt.Errorf("failed to create Prometheus client: %w", err)
    }
    return &NetflixPlaybackMetricProvider{
        promClient: promv1.NewAPI(client),
        logger:     logger.With("provider", "netflix-playback-metric"),
    }, nil
}

// GetMetrics fetches canary success rate from Prometheus for the given rollout
func (p *NetflixPlaybackMetricProvider) GetMetrics(ctx context.Context, rollout *rolloutsv1alpha1.Rollout, metric rolloutsv1alpha1.Metric) ([]metricproviders.MetricResult, error) {
    ctx, cancel := context.WithTimeout(ctx, defaultPrometheusTimeout)
    defer cancel()

    // Construct Prometheus query for playback success rate: ratio of 2xx responses to total requests for canary pods
    query := fmt.Sprintf(`sum(rate(http_requests_total{namespace="%s", rollout="%s", pod=~"canary-.+", status!~"5.."}[5m])) / sum(rate(http_requests_total{namespace="%s", rollout="%s", pod=~"canary-.+"}[5m])) * 100`,
        netflixPlaybackNamespace, rollout.Name, netflixPlaybackNamespace, rollout.Name)

    p.logger.Info("fetching canary success rate", "query", query, "rollout", rollout.Name)

    // Execute Prometheus query with error handling
    results, warnings, err := p.promClient.Query(ctx, query, time.Now())
    if err != nil {
        return nil, fmt.Errorf("prometheus query failed: %w", err)
    }
    if len(warnings) > 0 {
        p.logger.Warn("prometheus query warnings", "warnings", warnings)
    }

    // Validate result type
    vector, ok := results.(model.Vector)
    if !ok {
        return nil, fmt.Errorf("unexpected prometheus result type: %T", results)
    }
    if len(vector) == 0 {
        return nil, fmt.Errorf("no metrics returned for query: %s", query)
    }

    // Extract success rate value
    successRate := float64(vector[0].Value)
    p.logger.Info("fetched canary success rate", "rollout", rollout.Name, "success_rate", successRate)

    return []metricproviders.MetricResult{{
        Value:  successRate,
        Status: metricproviders.MetricStatusSuccess,
    }}, nil
}

// ValidateMetric checks if the metric configuration is valid for Netflix's use case
func (p *NetflixPlaybackMetricProvider) ValidateMetric(metric rolloutsv1alpha1.Metric) error {
    if metric.Name == "" {
        return fmt.Errorf("metric name cannot be empty")
    }
    if metric.Type != rolloutsv1alpha1.MetricTypePrometheus {
        return fmt.Errorf("unsupported metric type: %s, only prometheus is supported", metric.Type)
    }
    return nil
}

# Terraform configuration for deploying Argo Rollouts 2.12 controller to Netflix's production EKS cluster
# Provider configuration for AWS and Kubernetes
tterraform {
  required_version = ">= 1.6.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.10"
    }
  }
}

# Configure AWS provider for us-east-1 (Netflix primary region)
provider "aws" {
  region = "us-east-1"
}

# Fetch EKS cluster details for Netflix's playback cluster
data "aws_eks_cluster" "playback_cluster" {
  name = "netflix-playback-prod-eks-2026"
}

data "aws_eks_cluster_auth" "playback_cluster_auth" {
  name = data.aws_eks_cluster.playback_cluster.name
}

# Configure Kubernetes provider with EKS cluster credentials
provider "kubernetes" {
  host                   = data.aws_eks_cluster.playback_cluster.endpoint
  cluster_ca_certificate = base64decode(data.aws_eks_cluster.playback_cluster.certificate_authority[0].data)
  token                  = data.aws_eks_cluster_auth.playback_cluster_auth.token
}

# Configure Helm provider for Kubernetes
provider "helm" {
  kubernetes {
    host                   = data.aws_eks_cluster.playback_cluster.endpoint
    cluster_ca_certificate = base64decode(data.aws_eks_cluster.playback_cluster.certificate_authority[0].data)
    token                  = data.aws_eks_cluster_auth.playback_cluster_auth.token
  }
}

# Deploy Argo Rollouts 2.12 controller via Helm
resource "helm_release" "argo_rollouts" {
  name             = "argo-rollouts"
  repository       = "https://argoproj.github.io/argo-helm"
  chart            = "argo-rollouts"
  version          = "2.12.0" # Pinned to Argo Rollouts 2.12.0 as used by Netflix
  namespace        = "argo-rollouts"
  create_namespace = true

  # Netflix-specific values for production hardening
  set {
    name  = "controller.replicas"
    value = "3" # High availability for 15M daily deployments
  }

  set {
    name  = "controller.metrics.enabled"
    value = "true"
  }

  set {
    name  = "controller.metrics.serviceMonitor.enabled"
    value = "true" # Integrate with Netflix's Prometheus stack
  }

  set {
    name  = "controller.extraArgs"
    value = "{--experiment-requeue-time=10s,--canary-requeue-time=5s}" # Tune for Netflix's traffic patterns
  }

  # Error handling: wait for rollout to complete before marking resource as created
  wait          = true
  wait_for_jobs = true
  timeout       = 300 # 5 minute timeout for controller deployment

  # Validate Helm chart version exists
  lifecycle {
    precondition {
      condition     = can(regex("^2\.12\.\d+$", self.version))
      error_message = "Argo Rollouts version must be 2.12.x as used by Netflix production"
    }
  }
}

# Deploy Netflix's custom metric provider plugin for Argo Rollouts
resource "kubernetes_deployment" "netflix_metric_provider" {
  metadata {
    name      = "netflix-metric-provider"
    namespace = "argo-rollouts"
    labels = {
      app = "netflix-metric-provider"
    }
  }

  spec {
    replicas = 2

    selector {
      match_labels = {
        app = "netflix-metric-provider"
      }
    }

    template {
      metadata {
        labels = {
          app = "netflix-metric-provider"
        }
      }

      spec {
        container {
          name  = "metric-provider"
          image = "netflix-docker.jfrog.io/argo-rollouts/netflix-metric-provider:v2.12.0"
          port {
            container_port = 8080
          }

          env {
            name = "PROMETHEUS_URL"
            value = "http://prometheus-k8s.monitoring.svc:9090"
          }

          # Health checks for production readiness
          liveness_probe {
            http_get {
              path = "/healthz"
              port = 8080
            }
            initial_delay_seconds = 5
            period_seconds        = 10
          }

          readiness_probe {
            http_get {
              path = "/readyz"
              port = 8080
            }
            initial_delay_seconds = 3
            period_seconds        = 5
          }
        }
      }
    }
  }

  # Wait for deployment to roll out successfully
  wait_for_rollout = true
}

package netflixmetricprovider

import (
    "context"
    "errors"
    "testing"
    "time"

    rolloutsv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
    "github.com/argoproj/argo-rollouts/pkg/metricproviders"
    promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
    "github.com/prometheus/common/model"
    "golang.org/x/exp/slog"
    "golang.org/x/exp/slog/slogtest"
)

// MockPrometheusAPI mocks the Prometheus API for testing
type MockPrometheusAPI struct {
    queryResult model.Value
    queryErr    error
    warnings    promv1.Warnings
}

func (m *MockPrometheusAPI) Query(ctx context.Context, query string, ts time.Time) (model.Value, promv1.Warnings, error) {
    return m.queryResult, m.warnings, m.queryErr
}

func (m *MockPrometheusAPI) QueryRange(ctx context.Context, query string, r promv1.Range) (model.Value, promv1.Warnings, error) {
    return nil, nil, errors.New("query range not implemented")
}

func (m *MockPrometheusAPI) AlertManagers(ctx context.Context) (promv1.AlertManagersResult, error) {
    return promv1.AlertManagersResult{}, errors.New("not implemented")
}

func TestNetflixPlaybackMetricProvider_GetMetrics_Success(t *testing.T) {
    // Initialize test logger
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()
    var logBuf slogtest.Logs
    logger := slog.New(slogtest.NewHandler(&logBuf))

    // Set up mock Prometheus API to return 99.9% success rate
    mockAPI := &MockPrometheusAPI{
        queryResult: model.Vector{
            &model.Sample{Value: 99.9},
        },
        queryErr: nil,
    }

    // Create metric provider with mock API
    provider := &NetflixPlaybackMetricProvider{
        promClient: mockAPI,
        logger:     logger,
    }

    // Create test rollout object
    rollout := &rolloutsv1alpha1.Rollout{
        Name:      "playback-canary-test",
        Namespace: netflixPlaybackNamespace,
    }

    // Create test metric
    metric := rolloutsv1alpha1.Metric{
        Name: "playback-success-rate",
        Type: rolloutsv1alpha1.MetricTypePrometheus,
    }

    // Execute GetMetrics
    results, err := provider.GetMetrics(ctx, rollout, metric)
    if err != nil {
        t.Fatalf("unexpected error: %v", err)
    }

    // Validate results
    if len(results) != 1 {
        t.Fatalf("expected 1 result, got %d", len(results))
    }
    if results[0].Status != metricproviders.MetricStatusSuccess {
        t.Fatalf("expected success status, got %v", results[0].Status)
    }
    if results[0].Value != 99.9 {
        t.Fatalf("expected 99.9 success rate, got %f", results[0].Value)
    }
}

func TestNetflixPlaybackMetricProvider_GetMetrics_PrometheusError(t *testing.T) {
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()
    var logBuf slogtest.Logs
    logger := slog.New(slogtest.NewHandler(&logBuf))

    // Set up mock to return error
    mockAPI := &MockPrometheusAPI{
        queryErr: errors.New("prometheus unreachable"),
    }

    provider := &NetflixPlaybackMetricProvider{
        promClient: mockAPI,
        logger:     logger,
    }

    rollout := &rolloutsv1alpha1.Rollout{
        Name:      "playback-canary-test",
        Namespace: netflixPlaybackNamespace,
    }

    metric := rolloutsv1alpha1.Metric{
        Name: "playback-success-rate",
        Type: rolloutsv1alpha1.MetricTypePrometheus,
    }

    _, err := provider.GetMetrics(ctx, rollout, metric)
    if err == nil {
        t.Fatal("expected error when Prometheus is unreachable")
    }
    if !errors.Is(err, mockAPI.queryErr) {
        t.Fatalf("expected prometheus error, got %v", err)
    }
}

func TestNetflixPlaybackMetricProvider_GetMetrics_EmptyResults(t *testing.T) {
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()
    var logBuf slogtest.Logs
    logger := slog.New(slogtest.NewHandler(&logBuf))

    // Set up mock to return empty vector
    mockAPI := &MockPrometheusAPI{
        queryResult: model.Vector{},
    }

    provider := &NetflixPlaybackMetricProvider{
        promClient: mockAPI,
        logger:     logger,
    }

    rollout := &rolloutsv1alpha1.Rollout{
        Name:      "playback-canary-test",
        Namespace: netflixPlaybackNamespace,
    }

    metric := rolloutsv1alpha1.Metric{
        Name: "playback-success-rate",
        Type: rolloutsv1alpha1.MetricTypePrometheus,
    }

    _, err := provider.GetMetrics(ctx, rollout, metric)
    if err == nil {
        t.Fatal("expected error when no metrics are returned")
    }
}

func TestNewNetflixPlaybackMetricProvider_EmptyURL(t *testing.T) {
    var logBuf slogtest.Logs
    logger := slog.New(slogtest.NewHandler(&logBuf))

    _, err := NewNetflixPlaybackMetricProvider("", logger)
    if err == nil {
        t.Fatal("expected error when Prometheus URL is empty")
    }
}

resource "helm_release" "argo_rollouts" {
  name     = "argo-rollouts"
  version  = "2.12.0" # Explicitly pinned, no ~> or >=
  # ... other config
}

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: playback-canary
spec:
  strategy:
    canary:
      steps:
        - setWeight: 5
        - pause: {duration: 3m}
        - setWeight: 20
        - pause: {duration: 3m}
        # ... additional steps

spec:
  strategy:
    canary:
      metrics:
        - name: playback-success-rate
          type: Prometheus
          provider:
            netflix:
              query: "sum(rate(playback_success_total[5m])) / sum(rate(playback_requests_total[5m])) * 100"