Benchmark: Jaeger 1.50 vs. Honeycomb 2.0 vs. Datadog APM 7.0 for 1M Spans per Second

package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "sync"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
    "go.opentelemetry.io/otel/trace"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

const (
    targetThroughput = 1_000_000 // spans per second
    spanBatchSize    = 512       // match benchmark methodology
    numWorkers       = 16        // match c6g.4xlarge vCPU count
)

func main() {
    // Validate environment variables for OTLP endpoint
    otlpEndpoint := os.Getenv("OTLP_ENDPOINT")
    if otlpEndpoint == "" {
        log.Fatal("OTLP_ENDPOINT environment variable must be set")
    }

    // Initialize OTLP gRPC exporter with benchmark-matched settings
    ctx := context.Background()
    conn, err := grpc.DialContext(ctx, otlpEndpoint,
        grpc.WithTransportCredentials(insecure.NewCredentials()),
        grpc.WithBlock(),
        grpc.WithTimeout(5*time.Second), // match benchmark 5s timeout
    )
    if err != nil {
        log.Fatalf("failed to dial OTLP endpoint: %v", err)
    }
    defer conn.Close()

    exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))
    if err != nil {
        log.Fatalf("failed to create OTLP trace exporter: %v", err)
    }

    // Configure resource with benchmark-matched attributes
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName("benchmark-generator"),
            semconv.ServiceVersion("1.0.0"),
            attribute.String("benchmark.id", "1m-span-2024"),
        ),
    )
    if err != nil {
        log.Fatalf("failed to create resource: %v", err)
    }

    // Initialize tracer provider with batch span processor
    bsp := sdktrace.NewBatchSpanProcessor(
        exporter,
        sdktrace.WithMaxQueueSize(spanBatchSize*100), // buffer for throughput
        sdktrace.WithBatchTimeout(5*time.Second),    // match benchmark
        sdktrace.WithMaxExportBatchSize(spanBatchSize),
    )
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithResource(res),
        sdktrace.WithSpanProcessor(bsp),
    )
    defer func() { _ = tp.Shutdown(ctx) }()
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))

    tracer := tp.Tracer("benchmark-tracer")

    // Calculate spans per worker per second to hit target throughput
    spansPerWorker := targetThroughput / numWorkers
    rateLimiter := time.NewTicker(time.Second / time.Duration(spansPerWorker))
    defer rateLimiter.Stop()

    // Start worker goroutines to generate spans
    var done sync.WaitGroup
    done.Add(numWorkers)
    for i := 0; i < numWorkers; i++ {
        go func(workerID int) {
            defer done.Done()
            for {
                select {
                case <-rateLimiter.C:
                    // Generate span with benchmark-matched attributes: 12 total (4 string, 4 int, 4 bool)
                    _, span := tracer.Start(ctx, fmt.Sprintf("benchmark-span-worker-%d", workerID))
                    span.SetAttributes(
                        // 4 string attributes
                        attribute.String("http.method", "GET"),
                        attribute.String("http.url", "/api/v1/users"),
                        attribute.String("service.name", "user-service"),
                        attribute.String("worker.id", fmt.Sprintf("%d", workerID)),
                        // 4 int attributes
                        attribute.Int("http.status_code", 200),
                        attribute.Int("span.worker_id", workerID),
                        attribute.Int("span.batch_id", 0),
                        attribute.Int("span.size_bytes", 1024),
                        // 4 bool attributes
                        attribute.Bool("span.sampled", true),
                        attribute.Bool("span.has_error", false),
                        attribute.Bool("worker.active", true),
                        attribute.Bool("benchmark.run", true),
                    )
                    // Add 1 span event as per benchmark methodology
                    span.AddEvent("span.created", trace.WithAttributes(attribute.Int("event.time_ms", time.Now().UnixMilli())))
                    span.End()
                }
            }
        }(i)
    }

    // Run for 24 hours to match benchmark duration
    log.Printf("Starting span generation at %d spans/sec for 24 hours", targetThroughput)
    time.Sleep(24 * time.Hour)
    done.Wait()
}

// Jaeger 1.50 self-hosted deployment on AWS matching benchmark hardware
// Requires Terraform 1.5+ and AWS CLI configured

terraform {
  required_version = ">= 1.5.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
}

variable "region" {
  type        = string
  default     = "us-east-1"
  description = "AWS region to deploy Jaeger cluster"
}

variable "jaeger_version" {
  type        = string
  default     = "1.50.0"
  description = "Jaeger release version to deploy"
  validation {
    condition     = can(regex("^1\\.50\\.[0-9]+$", var.jaeger_version))
    error_message = "Must be a valid Jaeger 1.50.x release version."
  }
}

variable "instance_type" {
  type        = string
  default     = "c6g.4xlarge"
  description = "EC2 instance type for Jaeger nodes (matches benchmark hardware)"
  validation {
    condition     = var.instance_type == "c6g.4xlarge"
    error_message = "Must use c6g.4xlarge to match benchmark hardware."
  }
}

variable "cluster_size" {
  type        = number
  default     = 8
  description = "Number of Jaeger nodes (matches benchmark 8x instances)"
  validation {
    condition     = var.cluster_size == 8
    error_message = "Must use 8 nodes to match benchmark configuration."
  }
}

provider "aws" {
  region = var.region
}

// Security group allowing OTLP gRPC (4317) and Jaeger UI (16686)
resource "aws_security_group" "jaeger_sg" {
  name        = "jaeger-1-50-bench-sg"
  description = "Allow Jaeger traffic for benchmark cluster"

  ingress {
    from_port   = 4317
    to_port     = 4317
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"] // Restrict in production!
    description = "OTLP gRPC ingest"
  }

  ingress {
    from_port   = 16686
    to_port     = 16686
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"] // Restrict in production!
    description = "Jaeger UI"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow all outbound traffic"
  }
}

// IAM role for EC2 instances to access S3 for span storage
resource "aws_iam_role" "jaeger_role" {
  name = "jaeger-1-50-bench-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })
}

resource "aws_iam_role_policy_attachment" "s3_access" {
  role       = aws_iam_role.jaeger_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess" // Restrict to specific bucket in production
}

resource "aws_iam_instance_profile" "jaeger_profile" {
  name = "jaeger-1-50-bench-profile"
  role = aws_iam_role.jaeger_role.name
}

// EC2 instances for Jaeger cluster
resource "aws_instance" "jaeger_node" {
  count                  = var.cluster_size
  ami                    = "ami-0c7217cdde317cfec" // Ubuntu 22.04 ARM64 us-east-1
  instance_type          = var.instance_type
  vpc_security_group_ids = [aws_security_group.jaeger_sg.id]
  iam_instance_profile   = aws_iam_instance_profile.jaeger_profile.name
  user_data = templatefile("${path.module}/jaeger-init.sh", {
    jaeger_version = var.jaeger_version
    node_id        = count.index
    cluster_size   = var.cluster_size
  })

  tags = {
    Name    = "jaeger-1-50-bench-node-${count.index}"
    Benchmark = "1m-span-2024"
  }
}

// Output Jaeger UI endpoint
output "jaeger_ui_endpoint" {
  value       = "http://${aws_instance.jaeger_node[0].public_ip}:16686"
  description = "Public endpoint for Jaeger UI"
}

// Output OTLP gRPC endpoint
output "otlp_grpc_endpoint" {
  value       = "${aws_instance.jaeger_node[0].public_ip}:4317"
  description = "OTLP gRPC endpoint for span ingest"
}

// Honeycomb 2.0 API query to fetch p99 latency for a service
// Requires HONEYCOMB_API_KEY and HONEYCOMB_DATASET environment variables set

package main

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "log"
    "net/http"
    "os"
    "time"
)

const (
    honeycombAPIBase = "https://api.honeycomb.io/1/queries"
    queryTimeout     = 30 * time.Second
)

type HoneycombQueryRequest struct {
    Calculations []Calculation `json:"calculations"`
    Filter       Filter       `json:"filter"`
    TimeRange    int64        `json:"time_range"` // seconds
    Granularity  int64        `json:"granularity"` // seconds
}

type Calculation struct {
    Op     string `json:"op"`     // e.g., "P99"
    Column string `json:"column"` // e.g., "duration_ms"
}

type Filter struct {
    Op     string `json:"op"`     // e.g., "="
    Column string `json:"column"` // e.g., "service.name"
    Value  string `json:"value"`  // e.g., "user-service"
}

type HoneycombQueryResponse struct {
    ID      string   `json:"id"`
    Status  string   `json:"status"`
    Results []Result `json:"results"`
    Error   string   `json:"error"`
}

type Result struct {
    Time       int64   `json:"time"`
    P99Latency float64 `json:"p99_duration_ms"`
}

func main() {
    // Validate environment variables
    apiKey := os.Getenv("HONEYCOMB_API_KEY")
    if apiKey == "" {
        log.Fatal("HONEYCOMB_API_KEY environment variable must be set")
    }
    dataset := os.Getenv("HONEYCOMB_DATASET")
    if dataset == "" {
        log.Fatal("HONEYCOMB_DATASET environment variable must be set")
    }

    // Construct query request for p99 latency of user-service over 10 minutes
    reqBody := HoneycombQueryRequest{
        Calculations: []Calculation{
            {
                Op:     "P99",
                Column: "duration_ms",
            },
        },
        Filter: Filter{
            Op:     "=",
            Column: "service.name",
            Value:  "user-service",
        },
        TimeRange:   600,   // 10 minutes as per benchmark query latency test
        Granularity: 60,    // 1 minute granularity
    }

    jsonBody, err := json.Marshal(reqBody)
    if err != nil {
        log.Fatalf("failed to marshal query request: %v", err)
    }

    // Create HTTP request with timeout
    ctx, cancel := context.WithTimeout(context.Background(), queryTimeout)
    defer cancel()

    url := fmt.Sprintf("%s/%s", honeycombAPIBase, dataset)
    httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonBody))
    if err != nil {
        log.Fatalf("failed to create HTTP request: %v", err)
    }
    httpReq.Header.Set("X-Honeycomb-Team", apiKey)
    httpReq.Header.Set("Content-Type", "application/json")

    // Execute query
    client := &http.Client{}
    resp, err := client.Do(httpReq)
    if err != nil {
        log.Fatalf("failed to execute query: %v", err)
    }
    defer resp.Body.Close()

    // Check response status
    if resp.StatusCode != http.StatusOK {
        body, _ := io.ReadAll(resp.Body)
        log.Fatalf("query failed with status %d: %s", resp.StatusCode, string(body))
    }

    // Parse response
    body, err := io.ReadAll(resp.Body)
    if err != nil {
        log.Fatalf("failed to read response body: %v", err)
    }

    var queryResp HoneycombQueryResponse
    if err := json.Unmarshal(body, &queryResp); err != nil {
        log.Fatalf("failed to unmarshal response: %v", err)
    }

    if queryResp.Error != "" {
        log.Fatalf("honeycomb query error: %s", queryResp.Error)
    }

    // Print results
    fmt.Printf("Honeycomb 2.0 P99 Latency for user-service (10min range):\n")
    for _, res := range queryResp.Results {
        t := time.Unix(res.Time, 0).UTC()
        fmt.Printf("%s: %.2f ms\n", t.Format("15:04:05"), res.P99Latency)
    }
}

// Optimal OpenTelemetry batch config for Jaeger 1.50 at 1M spans/sec
bsp := sdktrace.NewBatchSpanProcessor(
    exporter,
    sdktrace.WithMaxQueueSize(51200),       // 512 spans * 100 queue slots
    sdktrace.WithBatchTimeout(5 * time.Second),
    sdktrace.WithMaxExportBatchSize(512),   // Match Jaeger's default max message size
    sdktrace.WithExportTimeout(5 * time.Second),
)

# OpenTelemetry Collector sampling config for Jaeger 1.50
processors:
  sampling:
    rules:
      - name: drop-health-checks
        rule: 'http.url == "/health"'
        probability: 0.5
      - name: keep-errors
        rule: 'http.status_code >= 400'
        probability: 1.0
service:
  pipelines:
    traces:
      processors: [sampling]

# Prometheus alert for Jaeger 1.50 span drop rate
- alert: JaegerHighSpanDropRate
  expr: rate(jaeger_ingester_spans_dropped_total[5m]) / rate(jaeger_ingester_spans_received_total[5m]) > 0.01
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: "Jaeger drop rate exceeds 1%"
    description: "Jaeger node {{ $labels.instance }} has a span drop rate of {{ $value | humanizePercentage }}"