Benchmark: Keycloak 22.0 vs. AWS Cognito 2026 for 10k Concurrent User Authentication

package main

import (
    "context"
    "crypto/tls"
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "os"
    "sync"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

// Config holds benchmark configuration
type Config struct {
    TargetURL     string
    ClientID      string
    ClientSecret  string
    Username      string
    Password      string
    Concurrency   int
    Duration      time.Duration
    AuthType      string // "keycloak" or "cognito"
}

// MetricCollector tracks latency and error metrics
type MetricCollector struct {
    latencyMs []float64
    errors    int
    mu        sync.Mutex
}

// AuthRequest represents a single auth request payload
type AuthRequest struct {
    GrantType string `json:"grant_type"`
    ClientID  string `json:"client_id"`
    Username  string `json:"username"`
    Password  string `json:"password"`
    Scope     string `json:"scope,omitempty"`
}

// AuthResponse represents a successful auth response
type AuthResponse struct {
    AccessToken  string `json:"access_token"`
    RefreshToken string `json:"refresh_token"`
    ExpiresIn    int    `json:"expires_in"`
}

var (
    latencyHistogram = promauto.NewHistogram(prometheus.HistogramOpts{
        Name:    "auth_latency_ms",
        Help:    "Auth request latency in milliseconds",
        Buckets: prometheus.DefBuckets,
    })
    errorCounter = promauto.NewCounter(prometheus.CounterOpts{
        Name: "auth_errors_total",
        Help: "Total auth request errors",
    })
)

func main() {
    // Parse command line flags (simplified for brevity)
    cfg := Config{
        TargetURL:     "https://keycloak.example.com/realms/master/protocol/openid-connect/token",
        ClientID:      "benchmark-client",
        ClientSecret:  "secret",
        Username:      "testuser",
        Password:      "testpass123",
        Concurrency:   10000,
        Duration:      10 * time.Minute,
        AuthType:      "keycloak",
    }

    // Start Prometheus metrics server
    go func() {
        http.Handle("/metrics", promhttp.Handler())
        log.Fatal(http.ListenAndServe(":9090", nil))
    }()

    collector := &MetricCollector{}
    var wg sync.WaitGroup

    // Start concurrent workers
    startTime := time.Now()
    for i := 0; i < cfg.Concurrency; i++ {
        wg.Add(1)
        go func(workerID int) {
            defer wg.Done()
            client := &http.Client{
                Timeout: 5 * time.Second,
                Transport: &http.Transport{
                    TLSClientConfig:     &tls.Config{InsecureSkipVerify: false},
                    MaxIdleConns:        1000,
                    MaxIdleConnsPerHost: 1000,
                    IdleConnTimeout:     90 * time.Second,
                },
            }

            // Each worker sends auth requests until test duration elapses
            for time.Since(startTime) < cfg.Duration {
                reqStart := time.Now()
                err := sendAuthRequest(client, cfg, collector)
                elapsed := time.Since(reqStart).Milliseconds()

                if err != nil {
                    collector.mu.Lock()
                    collector.errors++
                    collector.mu.Unlock()
                    errorCounter.Inc()
                    log.Printf("Worker %d error: %v", workerID, err)
                    // Retry on rate limit (429)
                    if apiErr, ok := err.(interface{ StatusCode() int }); ok && apiErr.StatusCode() == 429 {
                        time.Sleep(1 * time.Second)
                    }
                    continue
                }

                latencyHistogram.Observe(float64(elapsed))
                collector.mu.Lock()
                collector.latencyMs = append(collector.latencyMs, float64(elapsed))
                collector.mu.Unlock()
            }
        }(i)
    }

    wg.Wait()
    // Calculate and print final metrics
    collector.mu.Lock()
    defer collector.mu.Unlock()
    // Sort latency slice for percentile calculation (simplified)
    // ... percentile calculation logic ...
    fmt.Printf("Test complete. Throughput: %.2f auth/sec\n", float64(len(collector.latencyMs))/cfg.Duration.Seconds())
    fmt.Printf("Total errors: %d\n", collector.errors)
}

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "os"
    "time"
)

// KeycloakClient wraps the Keycloak Admin API
type KeycloakClient struct {
    BaseURL string
    Token   string
    Client  *http.Client
}

// Realm represents a Keycloak realm configuration
type Realm struct {
    ID          string `json:"id"`
    Realm       string `json:"realm"`
    Enabled     bool   `json:"enabled"`
    DisplayName string `json:"displayName"`
}

// Client represents a Keycloak client configuration
type Client struct {
    ClientID     string   `json:"clientId"`
    Name         string   `json:"name"`
    Enabled      bool     `json:"enabled"`
    ClientAuthenticatorType string `json:"clientAuthenticatorType"`
    Secret       string   `json:"secret"`
    RedirectURIs []string `json:"redirectUris"`
    Protocol     string   `json:"protocol"`
}

// User represents a Keycloak user
type User struct {
    Username   string `json:"username"`
    Enabled    bool   `json:"enabled"`
    Email      string `json:"email"`
    FirstName  string `json:"firstName"`
    LastName   string `json:"lastName"`
    Credentials []Credential `json:"credentials"`
}

// Credential represents a user password credential
type Credential struct {
    Type      string `json:"type"`
    Value     string `json:"value"`
    Temporary bool   `json:"temporary"`
}

func NewKeycloakClient(baseURL, adminUser, adminPass string) (*KeycloakClient, error) {
    client := &http.Client{Timeout: 10 * time.Second}
    // Get admin token
    tokenURL := fmt.Sprintf("%s/realms/master/protocol/openid-connect/token", baseURL)
    req, err := http.NewRequest("POST", tokenURL, nil)
    if err != nil {
        return nil, fmt.Errorf("create token request: %w", err)
    }
    req.SetBasicAuth(adminUser, adminPass)
    req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
    // Send form data for password grant
    // ... form data logic ...
    resp, err := client.Do(req)
    if err != nil {
        return nil, fmt.Errorf("get admin token: %w", err)
    }
    defer resp.Body.Close()

    var tokenResp struct {
        AccessToken string `json:"access_token"`
    }
    if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
        return nil, fmt.Errorf("decode token response: %w", err)
    }

    return &KeycloakClient{
        BaseURL: baseURL,
        Token:   tokenResp.AccessToken,
        Client:  client,
    }, nil
}

func (kc *KeycloakClient) CreateRealm(ctx context.Context, realm Realm) error {
    url := fmt.Sprintf("%s/admin/realms", kc.BaseURL)
    body, err := json.Marshal(realm)
    if err != nil {
        return fmt.Errorf("marshal realm: %w", err)
    }
    req, err := http.NewRequestWithContext(ctx, "POST", url, body)
    if err != nil {
        return fmt.Errorf("create realm request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", kc.Token))
    req.Header.Set("Content-Type", "application/json")

    resp, err := kc.Client.Do(req)
    if err != nil {
        return fmt.Errorf("create realm: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusCreated {
        return fmt.Errorf("create realm failed: status %d", resp.StatusCode)
    }
    return nil
}

func (kc *KeycloakClient) CreateClient(ctx context.Context, realm string, client Client) error {
    url := fmt.Sprintf("%s/admin/realms/%s/clients", kc.BaseURL, realm)
    body, err := json.Marshal(client)
    if err != nil {
        return fmt.Errorf("marshal client: %w", err)
    }
    req, err := http.NewRequestWithContext(ctx, "POST", url, body)
    if err != nil {
        return fmt.Errorf("create client request: %w", err)
    }
    req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", kc.Token))
    req.Header.Set("Content-Type", "application/json")

    resp, err := kc.Client.Do(req)
    if err != nil {
        return fmt.Errorf("create client: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusCreated {
        return fmt.Errorf("create client failed: status %d", resp.StatusCode)
    }
    return nil
}

func main() {
    kc, err := NewKeycloakClient("https://keycloak.example.com", "admin", "admin")
    if err != nil {
        log.Fatalf("Failed to create Keycloak client: %v", err)
    }

    // Create benchmark realm
    realm := Realm{
        ID:          "benchmark-realm",
        Realm:       "benchmark-realm",
        Enabled:     true,
        DisplayName: "Benchmark Realm",
    }
    if err := kc.CreateRealm(context.Background(), realm); err != nil {
        log.Fatalf("Failed to create realm: %v", err)
    }

    // Create benchmark client
    client := Client{
        ClientID:     "benchmark-client",
        Name:         "Benchmark Client",
        Enabled:      true,
        ClientAuthenticatorType: "client-secret",
        Secret:       "benchmark-secret",
        RedirectURIs: []string{"*"},
        Protocol:     "openid-connect",
    }
    if err := kc.CreateClient(context.Background(), "benchmark-realm", client); err != nil {
        log.Fatalf("Failed to create client: %v", err)
    }

    fmt.Println("Keycloak benchmark realm and client created successfully")
}

package main

import (
    "context"
    "fmt"
    "log"
    "time"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider"
    "github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider/types"
)

// CognitoSetup holds Cognito configuration
type CognitoSetup struct {
    UserPoolID string
    ClientID   string
    Region     string
}

func CreateCognitoUserPool(ctx context.Context, region string) (*CognitoSetup, error) {
    cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region))
    if err != nil {
        return nil, fmt.Errorf("load AWS config: %w", err)
    }

    client := cognitoidentityprovider.NewFromConfig(cfg)

    // Create user pool
    userPoolName := fmt.Sprintf("benchmark-pool-%d", time.Now().Unix())
    createPoolInput := &cognitoidentityprovider.CreateUserPoolInput{
        PoolName: aws.String(userPoolName),
        Policies: &types.UserPoolPolicyType{
            PasswordPolicy: &types.PasswordPolicyType{
                MinimumLength:   aws.Int32(8),
                RequireUppercase: aws.Bool(false),
                RequireLowercase: aws.Bool(false),
                RequireNumbers:   aws.Bool(false),
                RequireSymbols:   aws.Bool(false),
            },
        },
        AutoVerifiedAttributes: []types.VerifiedAttributeType{types.VerifiedAttributeTypeEmail},
    }

    poolResp, err := client.CreateUserPool(ctx, createPoolInput)
    if err != nil {
        return nil, fmt.Errorf("create user pool: %w", err)
    }

    // Create app client
    createClientInput := &cognitoidentityprovider.CreateUserPoolClientInput{
        UserPoolId: poolResp.UserPool.Id,
        ClientName: aws.String("benchmark-client"),
        ExplicitAuthFlows: []types.ExplicitAuthFlowsType{
            types.ExplicitAuthFlowsTypeUserPasswordAuth,
        },
        AllowedOAuthFlows: []types.OAuthFlowType{types.OAuthFlowTypeImplicit},
        AllowedOAuthScopes: []string{"openid", "email", "profile"},
        CallbackURLs: []string{"https://example.com/callback"},
    }

    clientResp, err := client.CreateUserPoolClient(ctx, createClientInput)
    if err != nil {
        return nil, fmt.Errorf("create app client: %w", err)
    }

    // Create test user
    createUserInput := &cognitoidentityprovider.AdminCreateUserInput{
        UserPoolId: poolResp.UserPool.Id,
        Username:   aws.String("benchmark-user"),
        TemporaryPassword: aws.String("TempPass123!"),
        MessageAction: types.MessageActionTypeSuppress,
    }

    _, err = client.AdminCreateUser(ctx, createUserInput)
    if err != nil {
        return nil, fmt.Errorf("create test user: %w", err)
    }

    // Set permanent password
    setPassInput := &cognitoidentityprovider.AdminSetUserPasswordInput{
        UserPoolId: poolResp.UserPool.Id,
        Username:   aws.String("benchmark-user"),
        Password:   aws.String("TestPass123!"),
        Permanent:  aws.Bool(true),
    }

    _, err = client.AdminSetUserPassword(ctx, setPassInput)
    if err != nil {
        return nil, fmt.Errorf("set user password: %w", err)
    }

    return &CognitoSetup{
        UserPoolID: *poolResp.UserPool.Id,
        ClientID:   *clientResp.UserPoolClient.ClientId,
        Region:     region,
    }, nil
}

func main() {
    setup, err := CreateCognitoUserPool(context.Background(), "us-east-1")
    if err != nil {
        log.Fatalf("Failed to create Cognito resources: %v", err)
    }

    fmt.Printf("Cognito User Pool ID: %s\n", setup.UserPoolID)
    fmt.Printf("App Client ID: %s\n", setup.ClientID)
    fmt.Printf("Endpoint: https://cognito-idp.%s.amazonaws.com/%s\n", setup.Region, setup.UserPoolID)
}

import (
    "context"
    "time"
    "github.com/dgraph-io/ristretto"
)

var tokenCache *ristretto.Cache

func init() {
    cache, err := ristretto.NewCache(&ristretto.Config{
        NumCounters: 1e6,     // Track frequency for 1M keys
        MaxCost:     1 << 30, // 1GB max cache size
        BufferItems: 64,      // Balance throughput and consistency
    })
    if err != nil {
        panic(fmt.Sprintf("failed to create token cache: %v", err))
    }
    tokenCache = cache
}

func GetCachedToken(ctx context.Context, userID string) (string, bool) {
    val, found := tokenCache.Get(userID)
    if !found {
        return "", false
    }
    token := val.(string)
    // In production, validate expiration via JWT claims
    return token, true
}

func CacheToken(userID string, token string, expiresIn time.Duration) {
    // Cost is the size of the token in bytes
    cost := int64(len(token))
    if !tokenCache.Set(userID, token, cost) {
        log.Printf("failed to cache token for user %s", userID)
    }
    // Evict token after expiration
    time.AfterFunc(expiresIn, func() {
        tokenCache.Del(userID)
    })
}

import http from 'k6/http';
import { check, sleep } from 'k6';
import { Trend } from 'k6/metrics';

export const options = {
  vus: 10000,
  duration: '10m',
  thresholds: {
    'http_req_duration': ['p(99)<200'], // Fail if p99 > 200ms
    'http_req_failed': ['rate<0.01'],   // Fail if error rate > 1%
  },
};

const cognitoEndpoint = 'https://cognito-idp.us-east-1.amazonaws.com/';
const clientID = 'your-client-id';
const username = 'benchmark-user';
const password = 'TestPass123!';

export default function () {
  const url = cognitoEndpoint;
  const payload = JSON.stringify({
    AuthFlow: 'USER_PASSWORD_AUTH',
    ClientId: clientID,
    AuthParameters: {
      USERNAME: username,
      PASSWORD: password,
    },
  });

  const params = {
    headers: {
      'Content-Type': 'application/x-amz-json-1.1',
      'X-Amz-Target': 'AWSCognitoIdentityProviderService.InitiateAuth',
    },
  };

  const res = http.post(url, payload, params);
  check(res, {
    'status is 200': (r) => r.status === 200,
    'has access token': (r) => JSON.parse(r.body).AccessToken !== undefined,
  });
}