Postmortem: How a Lack of CKS-Certified Engineers Caused a Security Breach in Our Cluster

// cksMisconfigChecker is a CKS-focused Kubernetes misconfiguration scanner
// that validates pods against 8 core CKS curriculum controls:
// 1. No privileged containers
// 2. No hostPath volumes
// 3. Must run as non-root
// 4. No hostNetwork/hostPID/hostIPC
// 5. Dropped ALL capabilities
// 6. Read-only root filesystem
// 7. No allowedHostPorts
// 8. Secrets not mounted as environment variables
package main

import (
    "context"
    "flag"
    "fmt"
    "os"
    "path/filepath"

    "github.com/google/go-cmp/cmp"
    v1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/client-go/util/homedir"
)

const (
    exitCodeSuccess = 0
    exitCodeMisconfig = 1
    exitCodeError = 2
)

func main() {
    var kubeconfig *string
    if home := homedir.HomeDir(); home != "" {
        kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
    } else {
        kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
    }
    flag.Parse()

    // Validate kubeconfig path exists if provided
    if *kubeconfig != "" {
        if _, err := os.Stat(*kubeconfig); os.IsNotExist(err) {
            fmt.Fprintf(os.Stderr, "Error: kubeconfig file %s does not exist\n", *kubeconfig)
            os.Exit(exitCodeError)
        }
    }

    // Create clientset
    config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Error building kubeconfig: %v\n", err)
        os.Exit(exitCodeError)
    }
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Error creating clientset: %v\n", err)
        os.Exit(exitCodeError)
    }

    ctx := context.Background()
    // List all pods across all namespaces
    pods, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{})
    if err != nil {
        fmt.Fprintf(os.Stderr, "Error listing pods: %v\n", err)
        os.Exit(exitCodeError)
    }

    fmt.Printf("Scanning %d pods across all namespaces for CKS misconfigurations...\n", len(pods.Items))
    hasMisconfig := false

    for _, pod := range pods.Items {
        ns := pod.Namespace
        podName := pod.Name
        for _, container := range pod.Spec.Containers {
            // Check 1: Privileged containers (CKS Domain 3: Pod Security)
            if container.SecurityContext != nil && container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Container: %s - Privileged container detected (CKS Control 3.1)\n", ns, podName, container.Name)
                hasMisconfig = true
            }

            // Check 2: HostPath volumes (CKS Domain 3: Pod Security)
            for _, volume := range pod.Spec.Volumes {
                if volume.HostPath != nil {
                    fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Volume: %s - HostPath volume detected (CKS Control 3.2)\n", ns, podName, volume.Name)
                    hasMisconfig = true
                }
            }

            // Check 3: Run as non-root (CKS Domain 3: Pod Security)
            if container.SecurityContext != nil && container.SecurityContext.RunAsNonRoot != nil && !*container.SecurityContext.RunAsNonRoot {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Container: %s - Running as root (CKS Control 3.3)\n", ns, podName, container.Name)
                hasMisconfig = true
            }

            // Check 4: Host network/PID/IPC (CKS Domain 3: Pod Security)
            if pod.Spec.HostNetwork {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s - Host network enabled (CKS Control 3.4)\n", ns, podName)
                hasMisconfig = true
            }
            if pod.Spec.HostPID {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s - Host PID enabled (CKS Control 3.5)\n", ns, podName)
                hasMisconfig = true
            }
            if pod.Spec.HostIPC {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s - Host IPC enabled (CKS Control 3.6)\n", ns, podName)
                hasMisconfig = true
            }

            // Check 5: Capabilities (CKS Domain 3: Pod Security)
            if container.SecurityContext != nil && len(container.SecurityContext.Capabilities.Add) > 0 {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Container: %s - Added capabilities: %v (CKS Control 3.7)\n", ns, podName, container.Name, container.SecurityContext.Capabilities.Add)
                hasMisconfig = true
            }

            // Check 6: Read-only root filesystem (CKS Domain 3: Pod Security)
            if container.SecurityContext != nil && container.SecurityContext.ReadOnlyRootFilesystem != nil && !*container.SecurityContext.ReadOnlyRootFilesystem {
                fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Container: %s - Writable root filesystem (CKS Control 3.8)\n", ns, podName, container.Name)
                hasMisconfig = true
            }

            // Check 7: Allowed host ports (CKS Domain 3: Pod Security)
            for _, port := range container.Ports {
                if port.HostPort != 0 {
                    fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Container: %s - Host port %d exposed (CKS Control 3.9)\n", ns, podName, container.Name, port.HostPort)
                    hasMisconfig = true
                }
            }

            // Check 8: Secrets as env vars (CKS Domain 4: Secrets Management)
            for _, env := range container.Env {
                if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil {
                    fmt.Fprintf(os.Stderr, "[MISCONFIG] Namespace: %s, Pod: %s, Container: %s - Secret %s mounted as env var (CKS Control 4.1)\n", ns, podName, container.Name, env.ValueFrom.SecretKeyRef.Name)
                    hasMisconfig = true
                }
            }
        }
    }

    if hasMisconfig {
        fmt.Fprintf(os.Stderr, "Scan complete: CKS misconfigurations detected\n")
        os.Exit(exitCodeMisconfig)
    }
    fmt.Println("Scan complete: No CKS misconfigurations detected")
    os.Exit(exitCodeSuccess)
}

// cks_pod_policy.rego: OPA Gatekeeper policy enforcing CKS curriculum controls
// Package name must match Gatekeeper template
package cks.pod_security

// Import future keywords for cleaner syntax
import future.keywords.if
import future.keywords.in
import future.keywords.contains

// Reference: https://github.com/open-policy-agent/gatekeeper
// CKS Domain 3: Pod Security controls enforced below

// Rule 1: No privileged containers (CKS Control 3.1)
deny_privileged[msg] {
    container := input.review.object.spec.containers[_]
    container.securityContext.privileged == true
    msg := sprintf("Privileged container %s in pod %s is not allowed (CKS Control 3.1)", [container.name, input.review.object.metadata.name])
}

// Rule 2: No hostPath volumes (CKS Control 3.2)
deny_hostpath[msg] {
    volume := input.review.object.spec.volumes[_]
    volume.hostPath != null
    msg := sprintf("HostPath volume %s in pod %s is not allowed (CKS Control 3.2)", [volume.name, input.review.object.metadata.name])
}

// Rule 3: Must run as non-root (CKS Control 3.3)
deny_runasroot[msg] {
    container := input.review.object.spec.containers[_]
    not container.securityContext.runAsNonRoot == true
    msg := sprintf("Container %s in pod %s must run as non-root (CKS Control 3.3)", [container.name, input.review.object.metadata.name])
}

// Rule 4: No host network (CKS Control 3.4)
deny_hostnetwork[msg] {
    input.review.object.spec.hostNetwork == true
    msg := sprintf("Pod %s uses host network which is not allowed (CKS Control 3.4)", [input.review.object.metadata.name])
}

// Rule 5: No host PID (CKS Control 3.5)
deny_hostpid[msg] {
    input.review.object.spec.hostPID == true
    msg := sprintf("Pod %s uses host PID which is not allowed (CKS Control 3.5)", [input.review.object.metadata.name])
}

// Rule 6: No host IPC (CKS Control 3.6)
deny_hostipc[msg] {
    input.review.object.spec.hostIPC == true
    msg := sprintf("Pod %s uses host IPC which is not allowed (CKS Control 3.6)", [input.review.object.metadata.name])
}

// Rule 7: No added capabilities (CKS Control 3.7)
deny_capabilities[msg] {
    container := input.review.object.spec.containers[_]
    capability := container.securityContext.capabilities.add[_]
    msg := sprintf("Container %s in pod %s has added capability %s which is not allowed (CKS Control 3.7)", [container.name, input.review.object.metadata.name, capability])
}

// Rule 8: Read-only root filesystem required (CKS Control 3.8)
deny_writableroot[msg] {
    container := input.review.object.spec.containers[_]
    not container.securityContext.readOnlyRootFilesystem == true
    msg := sprintf("Container %s in pod %s must use read-only root filesystem (CKS Control 3.8)", [container.name, input.review.object.metadata.name])
}

// Rule 9: No host ports (CKS Control 3.9)
deny_hostport[msg] {
    container := input.review.object.spec.containers[_]
    port := container.ports[_]
    port.hostPort > 0
    msg := sprintf("Container %s in pod %s exposes host port %d which is not allowed (CKS Control 3.9)", [container.name, input.review.object.metadata.name, port.hostPort])
}

// Rule 10: Secrets must not be mounted as environment variables (CKS Control 4.1)
deny_secret_env[msg] {
    container := input.review.object.spec.containers[_]
    env := container.env[_]
    env.valueFrom.secretKeyRef != null
    msg := sprintf("Container %s in pod %s uses secret %s as environment variable which is not allowed (CKS Control 4.1)", [container.name, input.review.object.metadata.name, env.valueFrom.secretKeyRef.name])
}

// Rule 11: Minimum 3 replicas for critical workloads (CKS Domain 5: Cluster Maintenance)
deny_replicas[msg] {
    input.review.object.metadata.labels.critical == "true"
    input.review.object.spec.replicas < 3
    msg := sprintf("Critical pod %s must have at least 3 replicas (CKS Control 5.1)", [input.review.object.metadata.name])
}

// Aggregate all deny messages
deny[msg] {
    msg := deny_privileged[_]
} else {
    msg := deny_hostpath[_]
} else {
    msg := deny_runasroot[_]
} else {
    msg := deny_hostnetwork[_]
} else {
    msg := deny_hostpid[_]
} else {
    msg := deny_hostipc[_]
} else {
    msg := deny_capabilities[_]
} else {
    msg := deny_writableroot[_]
} else {
    msg := deny_hostport[_]
} else {
    msg := deny_secret_env[_]
} else {
    msg := deny_replicas[_]
}

"""
cks_audit_report.py: Generates a CKS-focused misconfiguration report using kubeaudit
Requires: kubeaudit v0.22.0+, Python 3.9+
CKS controls mapped: https://github.com/kubernetes-security/cks
"""

import subprocess
import json
import sys
from datetime import datetime
import os

# Configuration
KUBEAUDIT_PATH = "/usr/local/bin/kubeaudit"
REPORT_PATH = f"cks_audit_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
CKS_CONTROLS = {
    "PrivilegedContainer": "CKS-3.1",
    "HostPathVolume": "CKS-3.2",
    "RunAsRoot": "CKS-3.3",
    "HostNetwork": "CKS-3.4",
    "HostPID": "CKS-3.5",
    "HostIPC": "CKS-3.6",
    "AddedCapabilities": "CKS-3.7",
    "WritableRootFs": "CKS-3.8",
    "HostPort": "CKS-3.9",
    "SecretAsEnv": "CKS-4.1"
}

def check_kubeaudit_installed():
    """Verify kubeaudit is installed and accessible"""
    try:
        subprocess.run([KUBEAUDIT_PATH, "version"], capture_output=True, check=True)
        return True
    except (subprocess.CalledProcessError, FileNotFoundError):
        print(f"Error: kubeaudit not found at {KUBEAUDIT_PATH}. Install from https://github.com/Shopify/kubeaudit", file=sys.stderr)
        return False

def run_kubeaudit_scan():
    """Run kubeaudit all scanner and return JSON output"""
    try:
        result = subprocess.run(
            [KUBEAUDIT_PATH, "all", "-f", "json"],
            capture_output=True,
            text=True,
            check=True
        )
        return json.loads(result.stdout)
    except subprocess.CalledProcessError as e:
        print(f"Error running kubeaudit scan: {e.stderr}", file=sys.stderr)
        sys.exit(1)

def filter_cks_findings(scan_results):
    """Filter scan results to only include CKS-mapped controls"""
    cks_findings = []
    for finding in scan_results.get("findings", []):
        control_name = finding.get("name")
        if control_name in CKS_CONTROLS:
            finding["cks_control"] = CKS_CONTROLS[control_name]
            cks_findings.append(finding)
    return cks_findings

def generate_report(cks_findings):
    """Generate HTML and JSON reports"""
    # JSON report
    with open(REPORT_PATH, "w") as f:
        json.dump({
            "scan_time": datetime.now().isoformat(),
            "total_cks_findings": len(cks_findings),
            "findings": cks_findings
        }, f, indent=2)
    print(f"JSON report saved to {REPORT_PATH}")

    # Summary output
    print(f"\nCKS Audit Summary:")
    print(f"Total CKS misconfigurations: {len(cks_findings)}")
    control_counts = {}
    for finding in cks_findings:
        control = finding["cks_control"]
        control_counts[control] = control_counts.get(control, 0) + 1
    for control, count in control_counts.items():
        print(f"  {control}: {count} findings")

if __name__ == "__main__":
    print("Starting CKS audit scan...")
    if not check_kubeaudit_installed():
        sys.exit(1)
    print("Running kubeaudit scan...")
    scan_results = run_kubeaudit_scan()
    print("Filtering CKS-related findings...")
    cks_findings = filter_cks_findings(scan_results)
    generate_report(cks_findings)
    if len(cks_findings) > 0:
        print("CKS misconfigurations detected. Review report for remediation steps.")
        sys.exit(1)
    print("No CKS misconfigurations detected.")
    sys.exit(0)

kubectl get pods -A -o json | jq '.items[] | select(.spec.containers[].securityContext.privileged == true) | {namespace: .metadata.namespace, pod: .metadata.name, container: .spec.containers[].name}'

- name: Run CKS Kubescape Scan
  uses: kubescape/github-action@v0.3.0
  with:
    fail-threshold: 80
    frameworks: cks
    args: --format json --output kubescape-results.json

apiVersion: v1
kind: Pod
metadata:
  name: least-priv-pod
spec:
  securityContext:
    runAsNonRoot: true
    runAsUser: 1001
    readOnlyRootFilesystem: true
  containers:
  - name: app
    image: nginx:1.25.3
    securityContext:
      capabilities:
        drop: ["ALL"]
      readOnlyRootFilesystem: true
    volumeMounts:
    - name: tmp
      mountPath: /tmp
  volumes:
  - name: tmp
    emptyDir: {}