War Story: We Built a DevOps CLI With Go 1.26 and Cobra 1.8 Used by 1k Engineers in 2026

package main\n\nimport (\n\t\"errors\",\n\t\"fmt\",\n\t\"os\",\n\t\"path/filepath\",\n\t\"time\",\n\n\t\"github.com/spf13/cobra\",\n\t\"github.com/spf13/viper\",\n)\n\n// CliConfig holds all tunable parameters for the devops CLI\ntype CliConfig struct {\n\tAPIEndpoint string        `mapstructure:\"api_endpoint\"`\n\tTimeout     time.Duration `mapstructure:\"timeout\"`\n\tVerbose     bool          `mapstructure:\"verbose\"`\n\tAuthToken   string        `mapstructure:\"auth_token\"`\n}\n\nvar (\n\tcfgFile string\n\trootCmd = &cobra.Command{\n\t\tUse:   \"devopsctl\",\n\t\tShort: \"Internal DevOps CLI for deployment, monitoring, and incident response\",\n\t\tLong: `devopsctl is the official internal CLI for our engineering organization.\nIt replaces manual kubectl, terraform, and custom script workflows with a unified,\nvalidated interface used by 1000+ engineers across 42 product teams.`,\n\t\tPersistentPreRunE: func(cmd *cobra.Command, args []string) error {\n\t\t\t// Initialize config before any subcommand runs\n\t\t\tif err := initConfig(); err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to load config: %w\", err)\n\t\t\t}\n\t\t\t// Validate required auth token for non-help commands\n\t\t\tif !cmd.HasSubCommands() && viper.GetString(\"auth_token\") == \"\" {\n\t\t\t\treturn errors.New(\"auth_token is required: set via DEVOPSCTL_AUTH_TOKEN env or config file\")\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t\tRun: func(cmd *cobra.Command, args []string) {\n\t\t\t// Default to help if no subcommand is provided\n\t\t\t_ = cmd.Help()\n\t\t},\n\t}\n)\n\n// initConfig loads configuration from file, env vars, and flags\nfunc initConfig() error {\n\tif cfgFile != \"\" {\n\t\t// Use config file from flag\n\t\tviper.SetConfigFile(cfgFile)\n\t} else {\n\t\t// Search for config in standard paths\n\t\thome, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to get home directory: %w\", err)\n\t\t}\n\t\tviper.AddConfigPath(filepath.Join(home, \".devopsctl\"))\n\t\tviper.AddConfigPath(\"/etc/devopsctl\")\n\t\tviper.SetConfigType(\"yaml\")\n\t\tviper.SetConfigName(\"config\")\n\t}\n\n\t// Bind environment variables with prefix\n\tviper.SetEnvPrefix(\"DEVOPSCTL\")\n\tviper.AutomaticEnv()\n\n\t// Set defaults\n\tviper.SetDefault(\"api_endpoint\", \"https://api.devops.internal:8443\")\n\tviper.SetDefault(\"timeout\", 30*time.Second)\n\tviper.SetDefault(\"verbose\", false)\n\n\t// Read config file\n\tif err := viper.ReadInConfig(); err != nil {\n\t\tvar viperErr viper.ConfigFileNotFoundError\n\t\tif !errors.As(err, &viperErr) {\n\t\t\treturn fmt.Errorf(\"failed to read config: %w\", err)\n\t\t}\n\t\t// Config file not found is acceptable, use defaults\n\t}\n\n\treturn nil\n}\n\nfunc main() {\n\t// Add persistent flags\n\trootCmd.PersistentFlags().StringVar(&cfgFile, \"config\", \"\", \"Config file path (default: ~/.devopsctl/config.yaml)\")\n\trootCmd.PersistentFlags().BoolP(\"verbose\", \"v\", false, \"Enable verbose logging\")\n\trootCmd.PersistentFlags().DurationP(\"timeout\", \"t\", 30*time.Second, \"API request timeout\")\n\n\t// Bind persistent flags to viper\n\tif err := viper.BindPFlag(\"verbose\", rootCmd.PersistentFlags().Lookup(\"verbose\")); err != nil {\n\t\tfmt.Fprintf(os.Stderr, \"failed to bind verbose flag: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tif err := viper.BindPFlag(\"timeout\", rootCmd.PersistentFlags().Lookup(\"timeout\")); err != nil {\n\t\tfmt.Fprintf(os.Stderr, \"failed to bind timeout flag: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\t// Add subcommands (deploy, monitor, incident)\n\trootCmd.AddCommand(newDeployCmd())\n\trootCmd.AddCommand(newMonitorCmd())\n\trootCmd.AddCommand(newIncidentCmd())\n\n\t// Execute root command\n\tif err := rootCmd.Execute(); err != nil {\n\t\tfmt.Fprintf(os.Stderr, \"error executing command: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n}\n

package main\n\nimport (\n\t\"errors\",\n\t\"fmt\",\n\t\"regexp\",\n\n\t\"github.com/spf13/cobra\",\n\tappsv1 \"k8s.io/api/apps/v1\",\n\tcorev1 \"k8s.io/api/core/v1\",\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\",\n)\n\n// newDeployCmd creates the deploy subcommand with full validation and rollback logic\nfunc newDeployCmd() *cobra.Command {\n\tvar (\n\t\timageTag         string\n\t\tnamespace        string\n\t\treplicas         int\n\t\tdryRun           bool\n\t\trollbackOnFail   bool\n\t)\n\n\tcmd := &cobra.Command{\n\t\tUse:   \"deploy [service-name]\",\n\t\tShort: \"Deploy a service to a Kubernetes namespace\",\n\t\tLong: `Deploy a containerized service to a target Kubernetes namespace.\nSupports rolling updates, automatic rollback on failure, and dry-run validation.\nRequires cluster admin access for the target namespace.`,\n\t\tArgs: cobra.ExactArgs(1), // Require exactly one service name argument\n\t\tPreRunE: func(cmd *cobra.Command, args []string) error {\n\t\t\t// Validate required flags\n\t\t\tif imageTag == \"\" {\n\t\t\t\treturn errors.New(\"--image-tag is required\")\n\t\t\t}\n\t\t\tif namespace == \"\" {\n\t\t\t\treturn errors.New(\"--namespace is required\")\n\t\t\t}\n\t\t\tif replicas < 1 {\n\t\t\t\treturn errors.New(\"--replicas must be at least 1\")\n\t\t\t}\n\t\t\t// Validate image tag format (semver or git SHA)\n\t\t\tif !isValidImageTag(imageTag) {\n\t\t\t\treturn fmt.Errorf(\"invalid image tag %q: must be semver (v1.2.3) or 7+ char git SHA\", imageTag)\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\t\tserviceName := args[0]\n\t\t\tif dryRun {\n\t\t\t\tfmt.Printf(\"DRY RUN: Would deploy %s:%s to %s with %d replicas\\n\", serviceName, imageTag, namespace, replicas)\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\t// Load kubeconfig (uses in-cluster config if running in pod)\n\t\t\tconfig, err := getKubeConfig()\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to load kubeconfig: %w\", err)\n\t\t\t}\n\n\t\t\t// Create deployment object\n\t\t\tdeploy := &appsv1.Deployment{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName:      serviceName,\n\t\t\t\t\tNamespace: namespace,\n\t\t\t\t},\n\t\t\t\tSpec: appsv1.DeploymentSpec{\n\t\t\t\t\tReplicas: &replicas,\n\t\t\t\t\tSelector: &metav1.LabelSelector{\n\t\t\t\t\t\tMatchLabels: map[string]string{\"app\": serviceName},\n\t\t\t\t\t},\n\t\t\t\t\tTemplate: corev1.PodTemplateSpec{\n\t\t\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\t\t\tLabels: map[string]string{\"app\": serviceName},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tSpec: corev1.PodSpec{\n\t\t\t\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tName:  serviceName,\n\t\t\t\t\t\t\t\t\tImage: fmt.Sprintf(\"registry.internal/%s:%s\", serviceName, imageTag),\n\t\t\t\t\t\t\t\t\tPorts: []corev1.ContainerPort{{ContainerPort: 8080}},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\n\t\t\t// Apply deployment with rollback logic\n\t\t\tif err := applyDeployment(config, deploy, rollbackOnFail); err != nil {\n\t\t\t\treturn fmt.Errorf(\"deployment failed: %w\", err)\n\t\t\t}\n\n\t\t\tfmt.Printf(\"Successfully deployed %s:%s to %s (replicas: %d)\\n\", serviceName, imageTag, namespace, replicas)\n\t\t\treturn nil\n\t\t},\n\t}\n\n\t// Add flags for deploy command\n\tcmd.Flags().StringVar(&imageTag, \"image-tag\", \"\", \"Container image tag (semver or git SHA) (required)\")\n\tcmd.Flags().StringVar(&namespace, \"namespace\", \"default\", \"Target Kubernetes namespace\")\n\tcmd.Flags().IntVar(&replicas, \"replicas\", 3, \"Number of deployment replicas\")\n\tcmd.Flags().BoolVar(&dryRun, \"dry-run\", false, \"Validate deployment without applying\")\n\tcmd.Flags().BoolVar(&rollbackOnFail, \"rollback-on-fail\", true, \"Automatically rollback on deployment failure\")\n\n\t// Mark required flags\n\t_ = cmd.MarkFlagRequired(\"image-tag\")\n\t_ = cmd.MarkFlagRequired(\"namespace\")\n\n\treturn cmd\n}\n\n// isValidImageTag validates image tag format (semver or 7+ char git SHA)\nfunc isValidImageTag(tag string) bool {\n\t// Semver check: vX.Y.Z where X,Y,Z are digits\n\tsemverRegex := regexp.MustCompile(`^v\d+\.\d+\.\d+$`)\n\tif semverRegex.MatchString(tag) {\n\t\treturn true\n\t}\n\t// Git SHA check: 7-40 hex characters\n\tshaRegex := regexp.MustCompile(`^[0-9a-f]{7,40}$`)\n\treturn shaRegex.MatchString(tag)\n}\n\n// getKubeConfig loads kubeconfig from default paths or in-cluster config\nfunc getKubeConfig() (*rest.Config, error) {\n\t// Implementation omitted for brevity, uses client-go's config loading\n\treturn clientcmd.BuildConfigFromFlags(\"\", \"\")\n}\n\n// applyDeployment applies the deployment to the cluster with optional rollback\nfunc applyDeployment(config *rest.Config, deploy *appsv1.Deployment, rollbackOnFail bool) error {\n\t// Implementation omitted for brevity, uses k8s client-go to apply deployment\n\treturn nil\n}\n

package main\n\nimport (\n\t\"crypto/sha256\",\n\t\"encoding/hex\",\n\t\"fmt\",\n\t\"io\",\n\t\"log\",\n\t\"net/http\",\n\t\"os\",\n\t\"path/filepath\",\n\t\"time\",\n\n\t\"github.com/spf13/cobra\",\n)\n\n// UpdateServer serves CLI binaries and checksums for self-hosted updates\ntype UpdateServer struct {\n\tBinaryDir string\n\tPort      int\n}\n\n// NewUpdateServer creates a new update server instance\nfunc NewUpdateServer(binaryDir string, port int) *UpdateServer {\n\treturn &UpdateServer{\n\t\tBinaryDir: binaryDir,\n\t\tPort:      port,\n\t}\n}\n\n// Start starts the HTTP update server with checksum validation\nfunc (s *UpdateServer) Start() error {\n\thttp.HandleFunc(\"/update/check\", s.handleUpdateCheck)\n\thttp.HandleFunc(\"/update/download\", s.handleDownload)\n\thttp.HandleFunc(\"/health\", s.handleHealth)\n\n\taddr := fmt.Sprintf(\":%d\", s.Port)\n\tlog.Printf(\"Starting update server on %s, serving binaries from %s\", addr, s.BinaryDir)\n\treturn http.ListenAndServe(addr, nil)\n}\n\n// handleUpdateCheck returns the latest version and checksum for a target OS/arch\nfunc (s *UpdateServer) handleUpdateCheck(w http.ResponseWriter, r *http.Request) {\n\tos := r.URL.Query().Get(\"os\")\n\tarch := r.URL.Query().Get(\"arch\")\n\tif os == \"\" || arch == \"\" {\n\t\thttp.Error(w, \"missing os or arch query parameter\", http.StatusBadRequest)\n\t\treturn\n\t}\n\n\t// Find latest version binary for target OS/arch\n\tlatestBin, err := s.findLatestBinary(os, arch)\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"no binary found for %s/%s: %v\", os, arch, err), http.StatusNotFound)\n\t\treturn\n\t}\n\n\t// Calculate checksum\n\tchecksum, err := s.calculateChecksum(latestBin)\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"failed to calculate checksum: %v\", err), http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\t// Return version info as JSON\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tfmt.Fprintf(w, `{\"version\": \"%s\", \"checksum\": \"%s\", \"url\": \"/update/download?os=%s&arch=%s\"}`,\n\t\tfilepath.Base(latestBin), checksum, os, arch)\n}\n\n// handleDownload serves the binary file for a target OS/arch\nfunc (s *UpdateServer) handleDownload(w http.ResponseWriter, r *http.Request) {\n\tos := r.URL.Query().Get(\"os\")\n\tarch := r.URL.Query().Get(\"arch\")\n\tif os == \"\" || arch == \"\" {\n\t\thttp.Error(w, \"missing os or arch query parameter\", http.StatusBadRequest)\n\t\treturn\n\t}\n\n\tlatestBin, err := s.findLatestBinary(os, arch)\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"no binary found for %s/%s: %v\", os, arch, err), http.StatusNotFound)\n\t\treturn\n\t}\n\n\t// Set headers for binary download\n\tw.Header().Set(\"Content-Type\", \"application/octet-stream\")\n\tw.Header().Set(\"Content-Disposition\", fmt.Sprintf(\"attachment; filename=\\\"%s\\\"\", filepath.Base(latestBin)))\n\thttp.ServeFile(w, r, latestBin)\n}\n\n// handleHealth returns 200 OK for health checks\nfunc (s *UpdateServer) handleHealth(w http.ResponseWriter, r *http.Request) {\n\tw.WriteHeader(http.StatusOK)\n\tfmt.Fprint(w, \"OK\")\n}\n\n// findLatestBinary finds the latest version binary for a target OS/arch\nfunc (s *UpdateServer) findLatestBinary(os, arch string) (string, error) {\n\tpattern := filepath.Join(s.BinaryDir, fmt.Sprintf(\"devopsctl_%s_%s_*\", os, arch))\n\tmatches, err := filepath.Glob(pattern)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"glob failed: %w\", err)\n\t}\n\tif len(matches) == 0 {\n\t\treturn \"\", fmt.Errorf(\"no binaries matching pattern %s\", pattern)\n\t}\n\n\t// Sort by modification time, return newest\n\tvar newest string\n\tvar newestTime time.Time\n\tfor _, match := range matches {\n\t\tinfo, err := os.Stat(match)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif info.ModTime().After(newestTime) {\n\t\t\tnewestTime = info.ModTime()\n\t\t\tnewest = match\n\t\t}\n\t}\n\n\tif newest == \"\" {\n\t\treturn \"\", fmt.Errorf(\"no valid binaries found\")\n\t}\n\treturn newest, nil\n}\n\n// calculateChecksum computes SHA256 checksum of a file\nfunc (s *UpdateServer) calculateChecksum(path string) (string, error) {\n\tfile, err := os.Open(path)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to open file: %w\", err)\n\t}\n\tdefer file.Close()\n\n\thash := sha256.New()\n\tif _, err := io.Copy(hash, file); err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to hash file: %w\", err)\n\t}\n\n\treturn hex.EncodeToString(hash.Sum(nil)), nil\n}\n\nfunc main() {\n\tvar (\n\t\tbinaryDir string\n\t\tport      int\n\t)\n\n\tvar rootCmd = &cobra.Command{\n\t\tUse:   \"update-server\",\n\t\tShort: \"Self-hosted update server for devopsctl CLI\",\n\t\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\t\tserver := NewUpdateServer(binaryDir, port)\n\t\t\treturn server.Start()\n\t\t},\n\t}\n\n\trootCmd.Flags().StringVar(&binaryDir, \"binary-dir\", \"/var/lib/devopsctl/binaries\", \"Directory containing CLI binaries\")\n\trootCmd.Flags().IntVar(&port, \"port\", 8080, \"Port to listen on\")\n\n\tif err := rootCmd.Execute(); err != nil {\n\t\tlog.Fatalf(\"Server failed: %v\", err)\n\t}\n}\n

// Annotate namespace flag with enum validation for allowed namespaces\ncmd.Flags().StringP(\"namespace\", \"n\", \"default\", \"Target Kubernetes namespace\")\n_ = cmd.Flags().SetAnnotation(\"namespace\", cobra.BashCompOneRequiredFlag, []string{\"true\"})\n_ = cmd.Flags().SetAnnotation(\"namespace\", cobra.FlagAnnotationKey, []string{\n\t\"validate:enum=default,prod,staging,test\",\n})

# Build optimized binary for Linux amd64\nGOOS=linux GOARCH=amd64 GOFLAGS=\"-trimpath\" go build \\\n  -ldflags=\"-s -w -X main.version=$(git describe --tags)\" \\\n  -o devopsctl_linux_amd64 main.go

// Check for updates from self-hosted server\nfunc checkForUpdates(currentVersion string) (string, error) {\n\tresp, err := http.Get(fmt.Sprintf(\"https://updates.devops.internal/update/check?os=%s&arch=%s\", runtime.GOOS, runtime.GOARCH))\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"update check failed: %w\", err)\n\t}\n\tdefer resp.Body.Close()\n\t// Parse response and compare versions\n\treturn \"\", nil\n}