Postmortem: Cryptojacking Attack on Kubernetes 1.34 Nodes: How We Added gVisor 1.0 and Falco 0.38

#!/bin/bash
# install-gvisor-1.0.sh
# Deploys gVisor 1.0 as a Kubernetes DaemonSet on K8s 1.34 worker nodes
# Includes pre-flight checks, error handling, and rollback capability
set -euo pipefail
trap 'echo "Error occurred at line $LINENO. Rolling back..."; rollback' ERR

# Configuration
GVISOR_VERSION="1.0.0"
KUBERNETES_VERSION="1.34"
NAMESPACE="kube-system"
DAEMONSET_NAME="gvisor-installer"
LOG_FILE="/var/log/gvisor-install-$(date +%s).log"

# Redirect all output to log file and stdout
exec > >(tee -a "$LOG_FILE") 2>&1

rollback() {
    echo "Rolling back gVisor installation..."
    kubectl delete daemonset "$DAEMONSET_NAME" -n "$NAMESPACE" --ignore-not-found=true
    echo "Rollback complete. Check $LOG_FILE for details."
    exit 1
}

preflight_checks() {
    echo "Running pre-flight checks..."
    # Check kubectl connectivity
    if ! kubectl cluster-info > /dev/null 2>&1; then
        echo "ERROR: Unable to connect to Kubernetes cluster"
        exit 1
    fi
    # Check K8s version
    local k8s_version
    k8s_version=$(kubectl version -o json | jq -r '.serverVersion.gitVersion' | cut -d'v' -f2 | cut -d'.' -f1,2)
    if [[ "$k8s_version" != "$KUBERNETES_VERSION" ]]; then
        echo "ERROR: Expected K8s version $KUBERNETES_VERSION, found $k8s_version"
        exit 1
    fi
    # Check if gVisor is already installed
    if kubectl get daemonset "$DAEMONSET_NAME" -n "$NAMESPACE" > /dev/null 2>&1; then
        echo "ERROR: gVisor DaemonSet $DAEMONSET_NAME already exists in $NAMESPACE"
        exit 1
    fi
    echo "Pre-flight checks passed."
}

generate_daemonset() {
    cat <

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "os"
    "os/exec"
    "strings"
    "time"
)

// FalcoRule represents a single Falco rule definition
type FalcoRule struct {
    Rule        string   `json:"rule"`
    Description string   `json:"description"`
    Condition   string   `json:"condition"`
    Output      string   `json:"output"`
    Priority    string   `json:"priority"`
}

// FalcoEvent represents a simulated cryptojacking event
type FalcoEvent struct {
    Time     string `json:"time"`
    Rule     string `json:"rule"`
    Priority string `json:"priority"`
    Output   string `json:"output"`
}

const (
    falcoAPIURL = "http://localhost:8765" // Default Falco gRPC API endpoint
    rulePath    = "/etc/falco/falco_rules.yaml"
)

func main() {
    fmt.Println("Starting Falco 0.38 rule validator for cryptojacking detection...")
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Step 1: Validate Falco rules file exists
    if err := validateRulesFile(); err != nil {
        fmt.Printf("ERROR: Rule validation failed: %v\n", err)
        os.Exit(1)
    }

    // Step 2: Load and parse cryptojacking rules
    rules, err := loadCryptojackingRules()
    if err != nil {
        fmt.Printf("ERROR: Failed to load rules: %v\n", err)
        os.Exit(1)
    }
    fmt.Printf("Loaded %d cryptojacking-specific rules\n", len(rules))

    // Step 3: Simulate cryptojacking events and check detection
    simulateEvents(ctx, rules)
}

// validateRulesFile checks if the Falco rules file exists and is valid YAML
func validateRulesFile() error {
    fmt.Printf("Validating Falco rules file at %s...\n", rulePath)
    if _, err := os.Stat(rulePath); os.IsNotExist(err) {
        return fmt.Errorf("rules file not found at %s", rulePath)
    }
    // Use falco CLI to validate rules
    cmd := exec.Command("falco", "--validate", rulePath)
    output, err := cmd.CombinedOutput()
    if err != nil {
        return fmt.Errorf("falco validation failed: %s\nOutput: %s", err, output)
    }
    fmt.Println("Rules file validation passed.")
    return nil
}

// loadCryptojackingRules parses the Falco rules file to extract cryptojacking-related rules
func loadCryptojackingRules() ([]FalcoRule, error) {
    fmt.Println("Loading cryptojacking rules from Falco config...")
    // In production, we'd parse YAML properly, but for brevity use grep + jq
    cmd := exec.Command("bash", "-c", fmt.Sprintf("grep -A 10 'cryptojack' %s | jq -s '.'", rulePath))
    output, err := cmd.CombinedOutput()
    if err != nil {
        return nil, fmt.Errorf("failed to parse rules: %s\nOutput: %s", err, output)
    }
    var rules []FalcoRule
    if err := json.Unmarshal(output, &rules); err != nil {
        return nil, fmt.Errorf("failed to unmarshal rules: %v", err)
    }
    // Filter for high priority rules
    var filtered []FalcoRule
    for _, r := range rules {
        if strings.Contains(strings.ToLower(r.Description), "cryptojack") {
            filtered = append(filtered, r)
        }
    }
    return filtered, nil
}

// simulateEvents sends simulated cryptojacking events to Falco and checks for alerts
func simulateEvents(ctx context.Context, rules []FalcoRule) {
    fmt.Println("Simulating cryptojacking events...")
    for _, rule := range rules {
        fmt.Printf("\nTesting rule: %s\n", rule.Rule)
        // Simulate event by triggering the condition (simplified for example)
        event := FalcoEvent{
            Time:     time.Now().Format(time.RFC3339),
            Rule:     rule.Rule,
            Priority: rule.Priority,
            Output:   fmt.Sprintf("Simulated cryptojacking event for rule %s", rule.Rule),
        }
        // Send event to Falco API (simplified, real implementation would use gRPC)
        eventJSON, _ := json.Marshal(event)
        req, err := http.NewRequestWithContext(ctx, "POST", falcoAPIURL+"/events", strings.NewReader(string(eventJSON)))
        if err != nil {
            fmt.Printf("ERROR: Failed to create request: %v\n", err)
            continue
        }
        resp, err := http.DefaultClient.Do(req)
        if err != nil {
            fmt.Printf("ERROR: Failed to send event: %v\n", err)
            continue
        }
        defer resp.Body.Close()
        body, _ := io.ReadAll(resp.Body)
        if resp.StatusCode == 200 {
            fmt.Printf("SUCCESS: Rule %s detected event\n", rule.Rule)
        } else {
            fmt.Printf("WARNING: Rule %s did not detect event. Status: %d, Body: %s\n", rule.Rule, resp.StatusCode, body)
        }
    }
}

package main

import (
    "encoding/csv"
    "fmt"
    "io"
    "os"
    "sort"
    "time"
)

// MetricSnapshot represents a single metric data point
type MetricSnapshot struct {
    Timestamp              time.Time
    CPUUsage               float64 // Percentage
    MemoryUsage            float64 // Percentage
    CryptojackingDetections int
    NodeCount              int
}

// IncidentReport summarizes pre and post mitigation metrics
type IncidentReport struct {
    PreMitigation  MetricSnapshot
    PostMitigation MetricSnapshot
    CostSaved      float64 // USD per hour
    AttackDuration time.Duration
}

func main() {
    fmt.Println("Generating cryptojacking incident postmortem report...")
    // Load pre-mitigation metrics (during attack)
    preMetrics, err := loadMetrics("pre_mitigation_metrics.csv")
    if err != nil {
        fmt.Printf("ERROR: Failed to load pre-mitigation metrics: %v\n", err)
        os.Exit(1)
    }
    // Load post-mitigation metrics (after gVisor + Falco)
    postMetrics, err := loadMetrics("post_mitigation_metrics.csv")
    if err != nil {
        fmt.Printf("ERROR: Failed to load post-mitigation metrics: %v\n", err)
        os.Exit(1)
    }
    // Aggregate metrics
    preAgg := aggregateMetrics(preMetrics)
    postAgg := aggregateMetrics(postMetrics)
    // Calculate cost savings (assume $4.20 per vCPU hour, 4 vCPUs per node)
    costSaved := calculateCostSavings(preAgg, postAgg, 4.20, 4)
    // Generate report
    report := IncidentReport{
        PreMitigation:  preAgg,
        PostMitigation: postAgg,
        CostSaved:      costSaved,
        AttackDuration: 72 * time.Hour, // From incident timeline
    }
    printReport(report)
}

// loadMetrics reads metric data from a CSV file
func loadMetrics(filePath string) ([]MetricSnapshot, error) {
    fmt.Printf("Loading metrics from %s...\n", filePath)
    file, err := os.Open(filePath)
    if err != nil {
        return nil, fmt.Errorf("failed to open file: %v", err)
    }
    defer file.Close()
    reader := csv.NewReader(file)
    // Skip header
    _, err = reader.Read()
    if err != nil {
        return nil, fmt.Errorf("failed to read header: %v", err)
    }
    var metrics []MetricSnapshot
    for {
        row, err := reader.Read()
        if err == io.EOF {
            break
        }
        if err != nil {
            return nil, fmt.Errorf("failed to read row: %v", err)
        }
        // Parse row: timestamp, cpu_usage, memory_usage, detections, node_count
        if len(row) < 5 {
            fmt.Printf("WARNING: Skipping invalid row: %v\n", row)
            continue
        }
        ts, err := time.Parse(time.RFC3339, row[0])
        if err != nil {
            fmt.Printf("WARNING: Invalid timestamp %s: %v\n", row[0], err)
            continue
        }
        var cpu, mem float64
        var detections, nodes int
        fmt.Sscanf(row[1], "%f", &cpu)
        fmt.Sscanf(row[2], "%f", &mem)
        fmt.Sscanf(row[3], "%d", &detections)
        fmt.Sscanf(row[4], "%d", &nodes)
        metrics = append(metrics, MetricSnapshot{
            Timestamp:              ts,
            CPUUsage:               cpu,
            MemoryUsage:            mem,
            CryptojackingDetections: detections,
            NodeCount:              nodes,
        })
    }
    sort.Slice(metrics, func(i, j int) bool {
        return metrics[i].Timestamp.Before(metrics[j].Timestamp)
    })
    fmt.Printf("Loaded %d metric snapshots\n", len(metrics))
    return metrics, nil
}

// aggregateMetrics calculates average metrics from a slice of snapshots
func aggregateMetrics(metrics []MetricSnapshot) MetricSnapshot {
    if len(metrics) == 0 {
        return MetricSnapshot{}
    }
    var totalCPU, totalMem float64
    var totalDetections, totalNodes int
    for _, m := range metrics {
        totalCPU += m.CPUUsage
        totalMem += m.MemoryUsage
        totalDetections += m.CryptojackingDetections
        totalNodes += m.NodeCount
    }
    return MetricSnapshot{
        CPUUsage:                totalCPU / float64(len(metrics)),
        MemoryUsage:             totalMem / float64(len(metrics)),
        CryptojackingDetections: totalDetections / len(metrics), // Average per snapshot
        NodeCount:               totalNodes / len(metrics),
    }
}

// calculateCostSavings computes hourly cost savings from reduced CPU usage
func calculateCostSavings(pre, post MetricSnapshot, costPerVCPUHour float64, vCPUsPerNode int) float64 {
    // Average CPU reduction per node
    cpuReduction := pre.CPUUsage - post.CPUUsage
    if cpuReduction < 0 {
        cpuReduction = 0
    }
    // Total vCPUs across all nodes
    totalVCPUs := float64(post.NodeCount * vCPUsPerNode)
    // Cost saved per hour: (cpuReduction / 100) * totalVCPUs * costPerVCPUHour
    return (cpuReduction / 100) * totalVCPUs * costPerVCPUHour
}

// printReport outputs the incident report to stdout
func printReport(report IncidentReport) {
    fmt.Println("\n=== Cryptojacking Incident Postmortem Report ===")
    fmt.Printf("Attack Duration: %s\n", report.AttackDuration)
    fmt.Printf("Pre-Mitigation Avg CPU Usage: %.2f%%\n", report.PreMitigation.CPUUsage)
    fmt.Printf("Post-Mitigation Avg CPU Usage: %.2f%%\n", report.PostMitigation.CPUUsage)
    fmt.Printf("CPU Usage Reduction: %.2f%%\n", report.PreMitigation.CPUUsage-report.PostMitigation.CPUUsage)
    fmt.Printf("Pre-Mitigation Avg Memory Usage: %.2f%%\n", report.PreMitigation.MemoryUsage)
    fmt.Printf("Post-Mitigation Avg Memory Usage: %.2f%%\n", report.PostMitigation.MemoryUsage)
    fmt.Printf("Hourly Cost Savings: $%.2f\n", report.CostSaved)
    fmt.Printf("Total Cost Saved (72h Attack Window): $%.2f\n", report.CostSaved*72)
    fmt.Println("===============================================\n")
}

kubectl get runtimeclass
# Output should include:
# NAME      HANDLER   AGE
# gvisor    runsc     3d
# patch default runtime class for untrusted workloads
kubectl patch deployment untrusted-workload -n default --type merge -p '{"spec": {"template": {"spec": {"runtimeClassName": "gvisor"}}}}'

falco --validate -r /etc/falco/falco_rules.yaml -r /etc/falco/falco_cryptojacking_rules.yaml
# Simulate a cryptojacking event to test detection
falco --simulate -e "spawn xmrig -o pool.example.com -u wallet" -r /etc/falco/falco_cryptojacking_rules.yaml
# Output should include:
# Rule: Cryptojacking Miner Detected
# Priority: Critical
# Output: Cryptojacking miner process xmrig detected (user=root command=xmrig -o pool.example.com -u wallet)

curl -X POST -H "Content-Type: application/json" \
  -d '{"text": "🚨 CRITICAL: Falco Alert - Cryptojacking Detected\nNode: ip-10-0-1-12.ec2.internal\nProcess: xmrig\nCPU Usage: 99%\nLink: https://grafana.example.com/d/node-metrics/ip-10-0-1-12"}' \
  https://hooks.slack.com/services/your-slack-webhook-url
# Verify the alert appears in your Slack channel
# You should also configure automatic node isolation via kubectl:
kubectl taint nodes ip-10-0-1-12.ec2.internal cryptojacking=true:NoSchedule