War Story: Debugging a Race Condition in Go 1.24 Microservices with Delve 1.23

// payment_processor_broken.go
// Demonstrates the race condition that caused production outages in Go 1.24
// The shared in-memory cache is accessed concurrently without proper synchronization
package main

import (
    \"context\"
    \"encoding/json\"
    \"errors\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"sync\"
    \"time\"
)

// PaymentRequest represents an incoming payment request
type PaymentRequest struct {
    UserID string  `json:\"user_id\"`
    Amount float64 `json:\"amount\"`
}

// PaymentResponse is the API response for payment processing
type PaymentResponse struct {
    Status  string  `json:\"status\"`
    Message string  `json:\"message\"`
    Amount  float64 `json:\"amount,omitempty\"`
}

// Shared in-memory cache (RACE CONDITION SOURCE: no proper sync)
// In Go 1.24, the scheduler prioritizes goroutines differently, making this race more likely
var paymentCache = make(map[string]PaymentResponse)
var cacheMutex sync.Mutex // Added later during debugging, originally missing

// getCachedPayment retrieves a cached payment if it exists
// BUG: Original implementation did not use cacheMutex, leading to concurrent map read/write
func getCachedPayment(userID string) (PaymentResponse, bool) {
    // Original broken code:
    // resp, ok := paymentCache[userID]
    // return resp, ok

    // Temporary workaround during debugging (still broken, for demonstration)
    // Note: This still has a race, as map reads are not atomic
    resp, ok := paymentCache[userID]
    return resp, ok
}

// processPayment handles the core payment logic, simulates third-party API call
func processPayment(ctx context.Context, req PaymentRequest) (PaymentResponse, error) {
    // Simulate 100-300ms latency for payment gateway
    select {
    case <-ctx.Done():
        return PaymentResponse{}, ctx.Err()
    case <-time.After(time.Millisecond * time.Duration(100+time.Now().UnixNano()%200)):
        // Simulate 0.5% failure rate
        if time.Now().UnixNano()%200 == 0 {
            return PaymentResponse{}, errors.New(\"payment gateway timeout\")
        }
        return PaymentResponse{
            Status:  \"success\",
            Message: \"payment processed\",
            Amount:  req.Amount,
        }, nil
    }
}

// paymentHandler is the HTTP handler for /process-payment
func paymentHandler(w http.ResponseWriter, r *http.Request) {
    if r.Method != http.MethodPost {
        http.Error(w, \"method not allowed\", http.StatusMethodNotAllowed)
        return
    }

    var req PaymentRequest
    if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
        http.Error(w, \"invalid request body\", http.StatusBadRequest)
        return
    }
    if req.UserID == \"\" || req.Amount <= 0 {
        http.Error(w, \"invalid user_id or amount\", http.StatusBadRequest)
        return
    }

    // Check cache first (RACE CONDITION HERE)
    if cached, ok := getCachedPayment(req.UserID); ok {
        w.Header().Set(\"Content-Type\", \"application/json\")
        json.NewEncoder(w).Encode(cached)
        return
    }

    // Process payment if not cached
    ctx, cancel := context.WithTimeout(r.Context(), time.Second*2)
    defer cancel()

    resp, err := processPayment(ctx, req)
    if err != nil {
        log.Printf(\"payment failed for user %s: %v\", req.UserID, err)
        http.Error(w, \"payment processing failed\", http.StatusInternalServerError)
        return
    }

    // Write to cache (RACE CONDITION HERE: concurrent map write)
    // Original broken code did not lock here
    paymentCache[req.UserID] = resp

    // Return response
    w.Header().Set(\"Content-Type\", \"application/json\")
    w.WriteHeader(http.StatusOK)
    json.NewEncoder(w).Encode(resp)
}

func main() {
    // Setup logging
    log.SetOutput(os.Stdout)
    log.SetFlags(log.Lshortfile | log.LstdFlags)

    // Register handler
    http.HandleFunc(\"/process-payment\", paymentHandler)

    // Start server
    port := os.Getenv(\"PORT\")
    if port == \"\" {
        port = \"8080\"
    }
    log.Printf(\"starting payment processor on :%s\", port)
    if err := http.ListenAndServe(fmt.Sprintf(\":%s\", port), nil); err != nil {
        log.Fatalf(\"server failed: %v\", err)
    }
}

// load_test.go
// Reproduces the race condition in payment_processor_broken.go under load
// Requires the broken processor running on :8080 to execute
package main

import (
    \"bytes\"
    \"context\"
    \"encoding/json\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"sync\"
    \"sync/atomic\"
    \"time\"
)

// TestResult tracks load test metrics
type TestResult struct {
    TotalRequests  uint64
    SuccessCount   uint64
    ErrorCount     uint64
    P50Latency     time.Duration
    P99Latency     time.Duration
    RaceErrors     uint64
}

func main() {
    // Configuration
    targetURL := \"http://localhost:8080/process-payment\"
    concurrency := 100 // Simulate 100 concurrent users
    totalRequests := 10000
    requestTimeout := time.Second * 5

    // Validate target is reachable
    _, err := http.Get(targetURL)
    if err != nil {
        log.Fatalf(\"target %s is not reachable: %v\", targetURL, err)
    }

    // Prepare request body template
    reqBody := PaymentRequest{UserID: \"test-user\", Amount: 49.99}
    bodyBytes, _ := json.Marshal(reqBody)

    // Metrics
    var result TestResult
    latencies := make(chan time.Duration, totalRequests)

    // Wait group for goroutines
    var wg sync.WaitGroup
    reqPerGoroutine := totalRequests / concurrency

    log.Printf(\"starting load test: %d total requests, %d concurrency\", totalRequests, concurrency)

    for i := 0; i < concurrency; i++ {
        wg.Add(1)
        go func(goroutineID int) {
            defer wg.Done()
            client := &http.Client{Timeout: requestTimeout}
            for j := 0; j < reqPerGoroutine; j++ {
                start := time.Now()

                // Vary user ID to hit cache and trigger race
                userID := fmt.Sprintf(\"user-%d-%d\", goroutineID, j%10) // Reuse 10 user IDs to trigger cache writes
                reqBody.UserID = userID
                body, _ := json.Marshal(reqBody)

                req, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer(body))
                if err != nil {
                    atomic.AddUint64(&result.ErrorCount, 1)
                    continue
                }
                req.Header.Set(\"Content-Type\", \"application/json\")

                resp, err := client.Do(req)
                if err != nil {
                    atomic.AddUint64(&result.ErrorCount, 1)
                    continue
                }
                resp.Body.Close()

                // Track success/error
                if resp.StatusCode == http.StatusOK {
                    atomic.AddUint64(&result.SuccessCount, 1)
                } else {
                    atomic.AddUint64(&result.ErrorCount, 1)
                }

                atomic.AddUint64(&result.TotalRequests, 1)
                latencies <- time.Since(start)
            }
        }(i)
    }

    // Wait for all requests to finish
    wg.Wait()
    close(latencies)

    // Calculate latency percentiles
    var latencySlice []time.Duration
    for lat := range latencies {
        latencySlice = append(latencySlice, lat)
    }
    // Sort latencies for percentile calculation
    sort.Slice(latencySlice, func(i, j int) bool { return latencySlice[i] < latencySlice[j] })
    if len(latencySlice) > 0 {
        result.P50Latency = latencySlice[len(latencySlice)*50/100]
        result.P99Latency = latencySlice[len(latencySlice)*99/100]
    }

    log.Printf(\"load test complete: total=%d, success=%d, errors=%d, p99=%v\",
        atomic.LoadUint64(&result.TotalRequests),
        atomic.LoadUint64(&result.SuccessCount),
        atomic.LoadUint64(&result.ErrorCount),
        result.P99Latency)

    // Check for race condition symptoms (500 errors under load)
    if atomic.LoadUint64(&result.ErrorCount) > uint64(totalRequests)*5/100 { // >5% error rate
        fmt.Println(\"RACE CONDITION REPRODUCED: Error rate exceeds 5% threshold\")
        os.Exit(1)
    } else {
        fmt.Println(\"No race condition symptoms detected\")
        os.Exit(0)
    }
}

// PaymentRequest matches the processor's request struct
type PaymentRequest struct {
    UserID string  `json:\"user_id\"`
    Amount float64 `json:\"amount\"`
}

// Required for sort.Slice
import \"sort\"

// payment_processor_fixed.go
// Fixed version of the payment processor, resolves the race condition
// Uses sync.RWMutex for cache access, adds proper error handling
package main

import (
    \"context\"
    \"encoding/json\"
    \"errors\"
    \"fmt\"
    \"log\"
    \"net/http\"
    \"os\"
    \"sync\"
    \"time\"
)

// PaymentRequest represents an incoming payment request
type PaymentRequest struct {
    UserID string  `json:\"user_id\"`
    Amount float64 `json:\"amount\"`
}

// PaymentResponse is the API response for payment processing
type PaymentResponse struct {
    Status  string  `json:\"status\"`
    Message string  `json:\"message\"`
    Amount  float64 `json:\"amount,omitempty\"`
}

// Thread-safe payment cache with RWMutex
type PaymentCache struct {
    mu    sync.RWMutex
    store map[string]PaymentResponse
}

// NewPaymentCache initializes a new thread-safe cache
func NewPaymentCache() *PaymentCache {
    return &PaymentCache{
        store: make(map[string]PaymentResponse),
    }
}

// Get retrieves a cached payment, uses read lock for concurrent access
func (c *PaymentCache) Get(userID string) (PaymentResponse, bool) {
    c.mu.RLock()
    defer c.mu.RUnlock()
    resp, ok := c.store[userID]
    return resp, ok
}

// Set writes a payment to the cache, uses write lock for exclusive access
func (c *PaymentCache) Set(userID string, resp PaymentResponse) {
    c.mu.Lock()
    defer c.mu.Unlock()
    c.store[userID] = resp
}

// Global cache instance (initialized at startup)
var paymentCache = NewPaymentCache()

// processPayment handles the core payment logic, simulates third-party API call
func processPayment(ctx context.Context, req PaymentRequest) (PaymentResponse, error) {
    // Simulate 100-300ms latency for payment gateway
    select {
    case <-ctx.Done():
        return PaymentResponse{}, ctx.Err()
    case <-time.After(time.Millisecond * time.Duration(100+time.Now().UnixNano()%200)):
        // Simulate 0.5% failure rate
        if time.Now().UnixNano()%200 == 0 {
            return PaymentResponse{}, errors.New(\"payment gateway timeout\")
        }
        return PaymentResponse{
            Status:  \"success\",
            Message: \"payment processed\",
            Amount:  req.Amount,
        }, nil
    }
}

// paymentHandler is the HTTP handler for /process-payment
func paymentHandler(w http.ResponseWriter, r *http.Request) {
    if r.Method != http.MethodPost {
        http.Error(w, \"method not allowed\", http.StatusMethodNotAllowed)
        return
    }

    var req PaymentRequest
    if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
        http.Error(w, \"invalid request body\", http.StatusBadRequest)
        return
    }
    if req.UserID == \"\" || req.Amount <= 0 {
        http.Error(w, \"invalid user_id or amount\", http.StatusBadRequest)
        return
    }

    // Check cache first (thread-safe read)
    if cached, ok := paymentCache.Get(req.UserID); ok {
        w.Header().Set(\"Content-Type\", \"application/json\")
        json.NewEncoder(w).Encode(cached)
        return
    }

    // Process payment if not cached
    ctx, cancel := context.WithTimeout(r.Context(), time.Second*2)
    defer cancel()

    resp, err := processPayment(ctx, req)
    if err != nil {
        log.Printf(\"payment failed for user %s: %v\", req.UserID, err)
        http.Error(w, \"payment processing failed\", http.StatusInternalServerError)
        return
    }

    // Write to cache (thread-safe write)
    paymentCache.Set(req.UserID, resp)

    // Return response
    w.Header().Set(\"Content-Type\", \"application/json\")
    w.WriteHeader(http.StatusOK)
    json.NewEncoder(w).Encode(resp)
}

func main() {
    // Setup logging
    log.SetOutput(os.Stdout)
    log.SetFlags(log.Lshortfile | log.LstdFlags)

    // Register handler
    http.HandleFunc(\"/process-payment\", paymentHandler)

    // Start server
    port := os.Getenv(\"PORT\")
    if port == \"\" {
        port = \"8080\"
    }
    log.Printf(\"starting fixed payment processor on :%s\", port)
    if err := http.ListenAndServe(fmt.Sprintf(\":%s\", port), nil); err != nil {
        log.Fatalf(\"server failed: %v\", err)
    }
}

type PaymentCache struct {
    mu    sync.RWMutex
    store map[string]PaymentResponse
}

func (c *PaymentCache) Get(userID string) (PaymentResponse, bool) {
    c.mu.RLock()
    defer c.mu.RUnlock()
    return c.store[userID]
}

func (c *PaymentCache) Set(userID string, resp PaymentResponse) {
    c.mu.Lock()
    defer c.mu.Unlock()
    c.store[userID] = resp
}