Postmortem: A Rate Limit Exceeded on OpenAI API Caused Our App to Go Down for 1 Hour

// Package openai implements a faulty OpenAI API client that caused the outage
// Original implementation deployed on Mar 10, 2024, without rate limit safeguards
package main

import (
    "context"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "os"
    "time"

    "github.com/openai/openai-go/v1" // https://github.com/openai/openai-go
    "github.com/openai/openai-go/option"
)

// FaultyChatClient is the original client with no rate limit controls
type FaultyChatClient struct {
    apiKey string
    model  string
    client *openai.Client
}

// NewFaultyChatClient initializes a client with no retry or rate limit config
func NewFaultyChatClient(apiKey, model string) *FaultyChatClient {
    // BUG: No rate limit configuration, default SDK retry is disabled
    client := openai.NewClient(
        option.WithAPIKey(apiKey),
        // Misconfiguration: RateLimiter is set to nil, no throttling
        option.WithRateLimiter(nil),
    )
    return &FaultyChatClient{
        apiKey: apiKey,
        model:  model,
        client: client,
    }
}

// GenerateResponse sends a chat request with no backoff or rate limit checks
func (c *FaultyChatClient) GenerateResponse(ctx context.Context, prompt string) (string, error) {
    // BUG: No token bucket or rate limit pre-check
    // BUG: Retry logic is disabled, so 429 errors are returned immediately
    resp, err := c.client.Chat.Completions.New(ctx, openai.ChatCompletionNewParams{
        Model: openai.F(c.model),
        Messages: openai.F([]openai.ChatCompletionMessageParamUnion{
            openai.UserMessage(prompt),
        }),
    })
    if err != nil {
        // Original error handling: no retry, just return error
        return "", fmt.Errorf("chat completion failed: %w", err)
    }
    if len(resp.Choices) == 0 {
        return "", fmt.Errorf("no choices returned from API")
    }
    return resp.Choices[0].Message.Content, nil
}

func main() {
    apiKey := os.Getenv("OPENAI_API_KEY")
    if apiKey == "" {
        fmt.Fprintf(os.Stderr, "OPENAI_API_KEY environment variable is required\n")
        os.Exit(1)
    }

    client := NewFaultyChatClient(apiKey, "gpt-3.5-turbo")

    // Simulate high traffic: 15k requests per minute, 5x our tier limit
    concurrency := 500
    results := make(chan error, concurrency)

    for i := 0; i < concurrency; i++ {
        go func(id int) {
            ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
            defer cancel()
            _, err := client.GenerateResponse(ctx, fmt.Sprintf("Test prompt %d", id))
            results <- err
        }(i)
    }

    // Count 429 errors
    var rateLimitErrors int
    for i := 0; i < concurrency; i++ {
        err := <-results
        if err != nil {
            if apiErr, ok := err.(*openai.Error); ok && apiErr.StatusCode == 429 {
                rateLimitErrors++
            }
        }
    }
    fmt.Printf("Total 429 Rate Limit Errors: %d/%d requests\n", rateLimitErrors, concurrency)
}

// Package openai implements a fixed OpenAI API client with rate limiting and circuit breaking
// Deployed on Mar 12, 2024 post-outage, eliminates rate limit cascading failures
package main

import (
    "context"
    "fmt"
    "net/http"
    "os"
    "sync"
    "time"

    "github.com/openai/openai-go/v1" // https://github.com/openai/openai-go
    "github.com/openai/openai-go/option"
    "github.com/uber-go/ratelimit" // https://github.com/uber-go/ratelimit v0.3.0
    "github.com/sony/gobreaker"   // https://github.com/sony/gobreaker v0.5.0
)

// RateLimitedChatClient is the fixed client with token bucket and circuit breaker
type RateLimitedChatClient struct {
    client      *openai.Client
    model       string
    rateLimiter ratelimit.Limiter
    breaker     *gobreaker.CircuitBreaker
}

// NewRateLimitedChatClient initializes a client with proper rate limits and circuit breaking
func NewRateLimitedChatClient(apiKey, model string, rpmLimit int) *RateLimitedChatClient {
    // Configure rate limiter: 10% under tier limit to avoid bursts
    // For 10k RPM tier, set to 9k RPM = 150 RPS
    ratePerSecond := float64(rpmLimit) / 60.0 * 0.9
    limiter := ratelimit.New(ratePerSecond, ratelimit.WithSlack(10))

    // Configure circuit breaker: trip after 5 consecutive 429 errors
    breakerSettings := gobreaker.Settings{
        Name:        "openai-api",
        MaxRequests: 1,
        Interval:    30 * time.Second,
        Timeout:     60 * time.Second,
        ReadyToTrip: func(counts gobreaker.Counts) bool {
            return counts.ConsecutiveFailures > 5
        },
        OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) {
            fmt.Printf("Circuit breaker %s state changed from %s to %s\n", name, from, to)
        },
    }
    breaker := gobreaker.NewCircuitBreaker(breakerSettings)

    // Configure OpenAI client with default retry (3 retries, exponential backoff)
    client := openai.NewClient(
        option.WithAPIKey(apiKey),
        option.WithRequestTimeout(30*time.Second),
    )

    return &RateLimitedChatClient{
        client:      client,
        model:       model,
        rateLimiter: limiter,
        breaker:     breaker,
    }
}

// GenerateResponse sends a chat request with rate limiting and circuit breaking
func (c *RateLimitedChatClient) GenerateResponse(ctx context.Context, prompt string) (string, error) {
    // Apply rate limiting: block until we have capacity
    c.rateLimiter.Take()

    // Execute request through circuit breaker
    result, err := c.breaker.Execute(func() (interface{}, error) {
        resp, err := c.client.Chat.Completions.New(ctx, openai.ChatCompletionNewParams{
            Model: openai.F(c.model),
            Messages: openai.F([]openai.ChatCompletionMessageParamUnion{
                openai.UserMessage(prompt),
            }),
        })
        if err != nil {
            // Wrap error to preserve status code for circuit breaker
            return nil, fmt.Errorf("api request failed: %w", err)
        }
        if len(resp.Choices) == 0 {
            return nil, fmt.Errorf("no choices returned from API")
        }
        return resp.Choices[0].Message.Content, nil
    })

    if err != nil {
        return "", fmt.Errorf("chat completion failed: %w", err)
    }
    return result.(string), nil
}

func main() {
    apiKey := os.Getenv("OPENAI_API_KEY")
    if apiKey == "" {
        fmt.Fprintf(os.Stderr, "OPENAI_API_KEY environment variable is required\n")
        os.Exit(1)
    }

    // Initialize with 10k RPM tier limit (our actual OpenAI tier)
    client := NewRateLimitedChatClient(apiKey, "gpt-3.5-turbo", 10000)

    // Simulate same high traffic as original outage
    concurrency := 500
    results := make(chan error, concurrency)
    start := time.Now()

    for i := 0; i < concurrency; i++ {
        go func(id int) {
            ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
            defer cancel()
            _, err := client.GenerateResponse(ctx, fmt.Sprintf("Test prompt %d", id))
            results <- err
        }(i)
    }

    var rateLimitErrors int
    var successCount int
    for i := 0; i < concurrency; i++ {
        err := <-results
        if err != nil {
            if apiErr, ok := err.(*openai.Error); ok && apiErr.StatusCode == 429 {
                rateLimitErrors++
            }
        } else {
            successCount++
        }
    }
    elapsed := time.Since(start)
    fmt.Printf("Results: %d success, %d rate limit errors\n", successCount, rateLimitErrors)
    fmt.Printf("Elapsed time: %v (throughput: %.2f RPS)\n", elapsed, float64(concurrency)/elapsed.Seconds())
}

"""
OpenAI Rate Limit Monitor
Deployed as a sidecar container post-outage to track quota usage in real time
Requires: openai==1.13.3 (https://github.com/openai/openai-python), prometheus-client==0.19.0
"""

import os
import time
import logging
from typing import Dict, Optional

import openai
from openai import OpenAI
from prometheus_client import Gauge, Counter, start_http_server, REGISTRY
import requests

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger("openai-rate-monitor")

# Prometheus metrics
OPENAI_QUOTA_REMAINING = Gauge(
    "openai_api_quota_remaining",
    "Remaining requests in current rate limit window",
    ["model", "tier"]
)
OPENAI_QUOTA_LIMIT = Gauge(
    "openai_api_quota_limit",
    "Total requests allowed in current rate limit window",
    ["model", "tier"]
)
OPENAI_429_COUNT = Counter(
    "openai_api_rate_limit_errors_total",
    "Total 429 rate limit errors",
    ["model", "tier"]
)
OPENAI_REQUEST_LATENCY = Gauge(
    "openai_api_request_latency_seconds",
    "Latency of last OpenAI API request",
    ["model", "tier"]
)

class RateLimitMonitor:
    """Monitors OpenAI API rate limits and exports metrics to Prometheus."""

    def __init__(self, api_key: str, model: str, tier: str, check_interval: int = 10):
        self.client = OpenAI(api_key=api_key)
        self.model = model
        self.tier = tier
        self.check_interval = check_interval
        self.last_quota_remaining = 0
        self.last_quota_limit = 0

        # Validate API key on startup
        try:
            self.client.models.list()
            logger.info("Successfully authenticated with OpenAI API")
        except openai.AuthenticationError as e:
            logger.error(f"Authentication failed: {e}")
            raise
        except Exception as e:
            logger.error(f"Failed to initialize monitor: {e}")
            raise

    def _parse_rate_limit_headers(self, response: requests.Response) -> Optional[Dict]:
        """Parse rate limit headers from OpenAI API response."""
        headers = response.headers
        try:
            remaining = int(headers.get("x-ratelimit-remaining-requests", 0))
            limit = int(headers.get("x-ratelimit-limit-requests", 0))
            reset_at = headers.get("x-ratelimit-reset-requests", "")
            return {
                "remaining": remaining,
                "limit": limit,
                "reset_at": reset_at
            }
        except (ValueError, TypeError) as e:
            logger.warning(f"Failed to parse rate limit headers: {e}")
            return None

    def check_rate_limits(self) -> None:
        """Send a test request to check current rate limit status."""
        start_time = time.time()
        try:
            # Send a minimal chat completion request to get rate limit headers
            response = self.client.chat.completions.create(
                model=self.model,
                messages=[{"role": "user", "content": "ping"}],
                max_tokens=1
            )
            latency = time.time() - start_time
            OPENAI_REQUEST_LATENCY.labels(model=self.model, tier=self.tier).set(latency)

            # Parse rate limit headers from response
            # Note: OpenAI Python SDK doesn't expose raw response headers directly,
            # so we use a custom http client to capture them
            # (Simplified for example: in production we use a custom transport)
            logger.info(f"Rate limit check successful: latency {latency:.2f}s")

        except openai.RateLimitError as e:
            latency = time.time() - start_time
            OPENAI_REQUEST_LATENCY.labels(model=self.model, tier=self.tier).set(latency)
            OPENAI_429_COUNT.labels(model=self.model, tier=self.tier).inc()
            logger.error(f"Rate limit exceeded: {e}")

            # Parse retry-after header if present
            retry_after = e.headers.get("retry-after", 60)
            logger.info(f"Retrying after {retry_after} seconds")
            time.sleep(int(retry_after))

        except Exception as e:
            latency = time.time() - start_time
            OPENAI_REQUEST_LATENCY.labels(model=self.model, tier=self.tier).set(latency)
            logger.error(f"Unexpected error checking rate limits: {e}")

    def run(self) -> None:
        """Run the monitor loop indefinitely."""
        logger.info(f"Starting rate limit monitor for model {self.model}, tier {self.tier}")
        while True:
            self.check_rate_limits()
            time.sleep(self.check_interval)

if __name__ == "__main__":
    api_key = os.getenv("OPENAI_API_KEY")
    if not api_key:
        raise ValueError("OPENAI_API_KEY environment variable is required")

    model = os.getenv("OPENAI_MODEL", "gpt-3.5-turbo")
    tier = os.getenv("OPENAI_TIER", "10k_rpm")
    check_interval = int(os.getenv("CHECK_INTERVAL", "10"))

    # Start Prometheus metrics server on port 9090
    start_http_server(9090)
    logger.info("Prometheus metrics server started on :9090")

    monitor = RateLimitMonitor(api_key, model, tier, check_interval)
    monitor.run()

// Configure rate limiter to 90% of 10k RPM tier limit
// 10k RPM = 166.67 RPS, 90% = 150 RPS
ratePerSecond := float64(10000) / 60.0 * 0.9
limiter := ratelimit.New(ratePerSecond, ratelimit.WithSlack(10))

breakerSettings := gobreaker.Settings{
    Name:        "openai-api",
    MaxRequests: 1,
    Interval:    30 * time.Second,
    Timeout:     60 * time.Second,
    ReadyToTrip: func(counts gobreaker.Counts) bool {
        return counts.ConsecutiveFailures > 5
    },
}

OPENAI_QUOTA_REMAINING = Gauge(
    "openai_api_quota_remaining",
    "Remaining requests in current rate limit window",
    ["model", "tier"]
)
OPENAI_429_COUNT = Counter(
    "openai_api_rate_limit_errors_total",
    "Total 429 rate limit errors",
    ["model", "tier"]
)