Retrospective: 3 Years Building and Scaling a 100-Engineer Team at a Unicorn

// deploy-validator/main.go
// Validates production deployment eligibility against CI status, feature flags, and rollout quotas.
// Requires GITHUB_TOKEN, LAUNCHDARKLY_SDK_KEY, and SLACK_WEBHOOK_URL environment variables.
// Uses https://github.com/google/go-github v58.0.0 for GitHub API interactions.
// Uses https://github.com/launchdarkly/go-server-sdk/v6 v6.0.0 for feature flag checks.
package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "os"
    "time"

    "github.com/google/go-github/v58/github"
    ld "github.com/launchdarkly/go-server-sdk/v6"
    "github.com/launchdarkly/go-server-sdk/v6/ldcomponents"
)

// DeploymentRequest represents a pending production deployment
type DeploymentRequest struct {
    Service     string `json:"service"`
    SHA         string `json:"sha"`
    Environment string `json:"environment"`
    RolloutPct  int    `json:"rollout_pct"`
}

// ValidationResult holds the outcome of a deployment validation
type ValidationResult struct {
    Eligible bool   `json:"eligible"`
    Reason   string `json:"reason,omitempty"`
    Service  string `json:"service"`
    SHA      string `json:"sha"`
}

func main() {
    // Load required environment variables
    ghToken := os.Getenv("GITHUB_TOKEN")
    if ghToken == "" {
        log.Fatal("GITHUB_TOKEN environment variable is required")
    }
    ldKey := os.Getenv("LAUNCHDARKLY_SDK_KEY")
    if ldKey == "" {
        log.Fatal("LAUNCHDARKLY_SDK_KEY environment variable is required")
    }
    slackWebhook := os.Getenv("SLACK_WEBHOOK_URL")
    if slackWebhook == "" {
        log.Fatal("SLACK_WEBHOOK_URL environment variable is required")
    }

    // Initialize GitHub client
    ghClient := github.NewClient(nil).WithAuthToken(ghToken)

    // Initialize LaunchDarkly client
    ldClient, err := ld.MakeCustomClient(ldKey, ld.Config{
        Logging: ldcomponents.Logging().MinLevel(ld.LogError),
    }, 5*time.Second)
    if err != nil {
        log.Fatalf("failed to initialize LaunchDarkly client: %v", err)
    }
    defer ldClient.Close()

    // Read deployment request from stdin (piped from CI pipeline)
    var req DeploymentRequest
    decoder := json.NewDecoder(os.Stdin)
    if err := decoder.Decode(&req); err != nil {
        log.Fatalf("failed to decode deployment request: %v", err)
    }

    // Validate deployment
    result := validateDeployment(context.Background(), ghClient, ldClient, req)

    // Output result as JSON
    output, err := json.MarshalIndent(result, "", "  ")
    if err != nil {
        log.Fatalf("failed to marshal validation result: %v", err)
    }
    fmt.Println(string(output))

    // Send Slack alert if deployment is rejected
    if !result.Eligible {
        alertBody := map[string]string{
            "text": fmt.Sprintf("🚨 Deployment rejected for %s (SHA: %s): %s", req.Service, req.SHA, result.Reason),
        }
        alertJSON, _ := json.Marshal(alertBody)
        http.Post(slackWebhook, "application/json", nil) // Simplified for brevity, add error handling in prod
    }
}

// validateDeployment checks all eligibility criteria for a production deployment
func validateDeployment(ctx context.Context, ghClient *github.Client, ldClient *ld.LDClient, req DeploymentRequest) ValidationResult {
    // 1. Check that the SHA has a passing CI status
    ciPassing, err := checkCIPassing(ctx, ghClient, req.SHA)
    if err != nil {
        return ValidationResult{Eligible: false, Reason: fmt.Sprintf("failed to check CI status: %v", err), Service: req.Service, SHA: req.SHA}
    }
    if !ciPassing {
        return ValidationResult{Eligible: false, Reason: "CI checks failed for SHA", Service: req.Service, SHA: req.SHA}
    }

    // 2. Check that required feature flags are enabled for the service
    flagsEnabled, err := checkFeatureFlags(ctx, ldClient, req.Service, req.Environment)
    if err != nil {
        return ValidationResult{Eligible: false, Reason: fmt.Sprintf("failed to check feature flags: %v", err), Service: req.Service, SHA: req.SHA}
    }
    if !flagsEnabled {
        return ValidationResult{Eligible: false, Reason: "Required feature flags not enabled", Service: req.Service, SHA: req.SHA}
    }

    // 3. Check rollout percentage quota (max 10% per hour for production)
    rolloutAllowed, err := checkRolloutQuota(ctx, req.Service, req.RolloutPct)
    if err != nil {
        return ValidationResult{Eligible: false, Reason: fmt.Sprintf("failed to check rollout quota: %v", err), Service: req.Service, SHA: req.SHA}
    }
    if !rolloutAllowed {
        return ValidationResult{Eligible: false, Reason: fmt.Sprintf("Rollout percentage %d exceeds quota", req.RolloutPct), Service: req.Service, SHA: req.SHA}
    }

    return ValidationResult{Eligible: true, Service: req.Service, SHA: req.SHA}
}

// checkCIPassing verifies that all required CI checks passed for a given SHA
func checkCIPassing(ctx context.Context, ghClient *github.Client, sha string) (bool, error) {
    // We use the internal github org "unicorn-eng" and repo "core-services" for all deployments
    owner, repo := "unicorn-eng", "core-services"
    checks, _, err := ghClient.Checks.ListCheckRunsForRef(ctx, owner, repo, sha, nil)
    if err != nil {
        return false, fmt.Errorf("failed to list check runs: %w", err)
    }
    for _, check := range checks.CheckRuns {
        if check.Status == nil || *check.Status != "completed" {
            return false, nil
        }
        if check.Conclusion == nil || *check.Conclusion != "success" {
            return false, nil
        }
    }
    return true, nil
}

// checkFeatureFlags verifies that all required flags for a service/environment are enabled
func checkFeatureFlags(ctx context.Context, ldClient *ld.LDClient, service, environment string) (bool, error) {
    // Required flags per service: we check the "deploy-eligible" flag for the given environment
    flagKey := fmt.Sprintf("%s-deploy-eligible-%s", service, environment)
    user := ld.NewUserBuilder("deploy-validator").Anonymous(true).Build()
    value, err := ldClient.BoolVariation(flagKey, user, false)
    if err != nil {
        return false, fmt.Errorf("failed to evaluate flag %s: %w", flagKey, err)
    }
    return value, nil
}

// checkRolloutQuota ensures we don't exceed 10% rollout per hour for a service
func checkRolloutQuota(ctx context.Context, service string, rolloutPct int) (bool, error) {
    // In production, this queries our Redis cluster for recent rollout percentages
    // For brevity, we return true if rolloutPct <= 10
    return rolloutPct <= 10, nil
}

# velocity-analyzer/analyzer.py
# Analyzes 3 years of sprint data to identify engineering velocity trends and bottlenecks.
# Requires JIRA_API_TOKEN, JIRA_EMAIL, and JIRA_URL environment variables.
# Uses https://github.com/pycontribs/jira v3.6.0 for Jira API interactions.
# Outputs a CSV report and matplotlib chart of velocity over time.
import os
import csv
import logging
from datetime import datetime, timedelta
from typing import List, Dict

from jira import JIRA, JIRAError
import matplotlib.pyplot as plt

# Configure logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)

# Constants
SPRINT_BOARD_ID = 123 # Internal unicorn-eng sprint board ID
VELOCITY_WINDOW_DAYS = 90 # Lookback window for rolling velocity
MAX_SPRINTS = 78 # Analyze last 3 years (26 sprints/year * 3 years)

class SprintMetrics:
    def __init__(self, sprint_id: int, name: str, start_date: datetime, end_date: datetime):
        self.sprint_id = sprint_id
        self.name = name
        self.start_date = start_date
        self.end_date = end_date
        self.story_points_completed = 0
        self.tickets_completed = 0
        self.cycle_time_days = 0.0
        self.bottleneck_category = ""

    def calculate_cycle_time(self, tickets: List[Dict]) -> None:
        """Calculate average cycle time for completed tickets in the sprint."""
        if not tickets:
            self.cycle_time_days = 0.0
            return
        total_cycle_time = 0.0
        for ticket in tickets:
            if "completed_date" not in ticket or "created_date" not in ticket:
                continue
            cycle_time = (ticket["completed_date"] - ticket["created_date"]).days
            total_cycle_time += cycle_time
        self.cycle_time_days = total_cycle_time / len(tickets)

    def identify_bottleneck(self) -> None:
        """Categorize the primary bottleneck for the sprint based on cycle time and ticket type."""
        if self.cycle_time_days > 7:
            self.bottleneck_category = "Code Review"
        elif self.cycle_time_days > 4:
            self.bottleneck_category = "QA Testing"
        else:
            self.bottleneck_category = "Development"

def get_jira_client() -> JIRA:
    """Initialize and return an authenticated Jira client."""
    api_token = os.getenv("JIRA_API_TOKEN")
    email = os.getenv("JIRA_EMAIL")
    url = os.getenv("JIRA_URL")
    if not all([api_token, email, url]):
        raise ValueError("JIRA_API_TOKEN, JIRA_EMAIL, and JIRA_URL must be set")
    try:
        return JIRA(server=url, basic_auth=(email, api_token))
    except JIRAError as e:
        logger.error(f"Failed to connect to Jira: {e}")
        raise

def fetch_sprints(jira_client: JIRA) -> List[Dict]:
    """Fetch all sprints from the sprint board, filtered to completed sprints in the last 3 years."""
    three_years_ago = datetime.now() - timedelta(days=365*3)
    try:
        sprints = jira_client.sprints(SPRINT_BOARD_ID)
    except JIRAError as e:
        logger.error(f"Failed to fetch sprints: {e}")
        raise
    valid_sprints = []
    for sprint in sprints:
        if sprint.state != "closed":
            continue
        if sprint.endDate < three_years_ago:
            continue
        valid_sprints.append({
            "id": sprint.id,
            "name": sprint.name,
            "startDate": sprint.startDate,
            "endDate": sprint.endDate
        })
    # Sort sprints by start date ascending
    valid_sprints.sort(key=lambda x: x["startDate"])
    return valid_sprints[:MAX_SPRINTS] # Limit to last 78 sprints (3 years)

def fetch_sprint_tickets(jira_client: JIRA, sprint_id: int) -> List[Dict]:
    """Fetch all completed tickets for a given sprint, with story points and dates."""
    jql = f"sprint = {sprint_id} AND status = Done AND type in (Story, Bug, Task)"
    try:
        issues = jira_client.search_issues(jql, maxResults=0, fields=["storyPoints", "created", "resolutiondate", "labels"])
    except JIRAError as e:
        logger.error(f"Failed to fetch tickets for sprint {sprint_id}: {e}")
        return []
    tickets = []
    for issue in issues:
        story_points = 0
        if hasattr(issue.fields, "storyPoints") and issue.fields.storyPoints:
            story_points = issue.fields.storyPoints
        created_date = datetime.strptime(issue.fields.created, "%Y-%m-%dT%H:%M:%S.%f%z")
        completed_date = None
        if issue.fields.resolutiondate:
            completed_date = datetime.strptime(issue.fields.resolutiondate, "%Y-%m-%dT%H:%M:%S.%f%z")
        tickets.append({
            "key": issue.key,
            "story_points": story_points,
            "created_date": created_date,
            "completed_date": completed_date,
            "labels": issue.fields.labels
        })
    return tickets

def generate_report(sprints: List[SprintMetrics], output_csv: str) -> None:
    """Write sprint metrics to a CSV file."""
    with open(output_csv, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["Sprint ID", "Sprint Name", "Start Date", "End Date", "Story Points Completed", "Tickets Completed", "Avg Cycle Time (Days)", "Bottleneck"])
        for sprint in sprints:
            writer.writerow([
                sprint.sprint_id,
                sprint.name,
                sprint.start_date.strftime("%Y-%m-%d"),
                sprint.end_date.strftime("%Y-%m-%d"),
                sprint.story_points_completed,
                sprint.tickets_completed,
                f"{sprint.cycle_time_days:.2f}",
                sprint.bottleneck_category
            ])
    logger.info(f"Report written to {output_csv}")

def plot_velocity(sprints: List[SprintMetrics], output_png: str) -> None:
    """Generate a matplotlib chart of velocity over time."""
    sprint_names = [s.name for s in sprints]
    velocity = [s.story_points_completed for s in sprints]
    plt.figure(figsize=(12, 6))
    plt.plot(sprint_names, velocity, marker="o")
    plt.xticks(rotation=45, ha="right")
    plt.xlabel("Sprint")
    plt.ylabel("Story Points Completed")
    plt.title("Engineering Velocity (Last 3 Years)")
    plt.tight_layout()
    plt.savefig(output_png)
    logger.info(f"Chart saved to {output_png}")

def main():
    try:
        jira_client = get_jira_client()
        logger.info("Connected to Jira successfully")
        sprints = fetch_sprints(jira_client)
        logger.info(f"Fetched {len(sprints)} sprints to analyze")
        sprint_metrics = []
        for sprint_data in sprints:
            sprint = SprintMetrics(
                sprint_id=sprint_data["id"],
                name=sprint_data["name"],
                start_date=sprint_data["startDate"],
                end_date=sprint_data["endDate"]
            )
            tickets = fetch_sprint_tickets(jira_client, sprint.sprint_id)
            sprint.story_points_completed = sum(t["story_points"] for t in tickets)
            sprint.tickets_completed = len(tickets)
            sprint.calculate_cycle_time(tickets)
            sprint.identify_bottleneck()
            sprint_metrics.append(sprint)
            logger.info(f"Processed sprint {sprint.name}: {sprint.story_points_completed} SP, {sprint.tickets_completed} tickets")
        generate_report(sprint_metrics, "velocity_report.csv")
        plot_velocity(sprint_metrics, "velocity_chart.png")
        logger.info("Analysis complete")
    except Exception as e:
        logger.error(f"Analysis failed: {e}")
        raise

if __name__ == "__main__":
    main()

# terraform-k8s-namespace/main.tf
# Terraform module to provision Kubernetes namespaces with resource quotas for team isolation.
# Requires kubectl configured with cluster admin access, or KUBE_CONFIG environment variable set.
# Uses https://github.com/hashicorp/terraform v1.6.0 and https://github.com/hashicorp/terraform-provider-kubernetes v2.23.0.
terraform {
  required_version = ">= 1.6.0"
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.23.0"
    }
  }
}

variable "team_name" {
  type        = string
  description = "Name of the engineering team (e.g., 'core-services', 'mobile-eng')"
  validation {
    condition     = can(regex("^[a-z0-9-]+$", var.team_name))
    error_message = "Team name must be lowercase alphanumeric with hyphens only."
  }
}

variable "environment" {
  type        = string
  description = "Deployment environment (e.g., 'staging', 'production')"
  validation {
    condition     = contains(["staging", "production", "development"], var.environment)
    error_message = "Environment must be one of: staging, production, development."
  }
}

variable "resource_quotas" {
  type = object({
    cpu_limit       = string # e.g., "10" (10 cores)
    memory_limit    = string # e.g., "32Gi"
    pod_limit       = number # e.g., 50
    storage_limit   = string # e.g., "100Gi"
  })
  description = "Resource quotas for the namespace"
  default = {
    cpu_limit     = "4"
    memory_limit  = "16Gi"
    pod_limit     = 20
    storage_limit = "50Gi"
  }
}

variable "labels" {
  type        = map(string)
  description = "Additional labels to apply to the namespace"
  default     = {}
}

# Provision Kubernetes namespace
resource "kubernetes_namespace" "team_namespace" {
  metadata {
    name = "${var.team_name}-${var.environment}"
    labels = merge({
      "team"         = var.team_name
      "environment"  = var.environment
      "managed-by"   = "terraform"
      "cost-center"  = "engineering"
    }, var.labels)
  }
}

# Apply resource quota to namespace
resource "kubernetes_resource_quota" "team_quota" {
  metadata {
    name      = "${var.team_name}-${var.environment}-quota"
    namespace = kubernetes_namespace.team_namespace.metadata[0].name
  }
  spec {
    hard = {
      "limits.cpu"          = var.resource_quotas.cpu_limit
      "limits.memory"       = var.resource_quotas.memory_limit
      "pods"                = var.resource_quotas.pod_limit
      "requests.storage"    = var.resource_quotas.storage_limit
    }
  }
}

# Apply network policy to restrict ingress to namespace (only allow from same team)
resource "kubernetes_network_policy" "team_ingress" {
  metadata {
    name      = "${var.team_name}-${var.environment}-ingress"
    namespace = kubernetes_namespace.team_namespace.metadata[0].name
  }
  spec {
    pod_selector {}
    ingress {
      from {
        namespace_selector {
          match_labels = {
            "team" = var.team_name
          }
        }
      }
    }
    policy_types = ["Ingress"]
  }
}

# Output namespace details
output "namespace_name" {
  value       = kubernetes_namespace.team_namespace.metadata[0].name
  description = "Name of the provisioned Kubernetes namespace"
}

output "namespace_resource_quota" {
  value = {
    cpu_limit     = var.resource_quotas.cpu_limit
    memory_limit  = var.resource_quotas.memory_limit
    pod_limit     = var.resource_quotas.pod_limit
    storage_limit = var.resource_quotas.storage_limit
  }
  description = "Resource quotas applied to the namespace"
}

output "namespace_labels" {
  value       = kubernetes_namespace.team_namespace.metadata[0].labels
  description = "Labels applied to the namespace"
}

// .github/workflows/pr-size-check.yml
name: PR Size Check
on: [pull_request]
jobs:
  check-size:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Check PR line count
        run: |
          LINES=$(git diff --stat origin/main | tail -1 | awk '{print $1}')
          if [ $LINES -gt 400 ]; then
            echo "PR exceeds 400 line limit. Break into smaller PRs."
            exit 1
          fi