Prometheus ve Grafana'yı Derinlemesine Anlamak — TSDB, PromQL ve Custom Exporter

cpu_usage{instance="server-1"}  →  [(10:00, 45), (10:15, 52), (10:30, 61), (10:45, 48)]

prometheus-lab/
├── docker-compose.yml
├── prometheus/
│   ├── prometheus.yml
│   └── rules.yml
├── grafana/
│   └── provisioning/
│       └── datasources/
│           └── prometheus.yml
└── custom-exporter/
    ├── exporter.py
    └── Dockerfile

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "rules.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'custom-exporter'
    static_configs:
      - targets: ['custom-exporter:8000']

version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=15d'
      - '--web.enable-lifecycle'

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin

  custom-exporter:
    build: ./custom-exporter
    ports:
      - "8000:8000"

volumes:
  prometheus_data:
  grafana_data:

docker exec -it prometheus sh
ls /prometheus

chunks_head/
wal/
01XXXXXXXXXXXXXXXXXXXXXX/   ← 2 saatlik block
01YYYYYYYYYYYYYYYYYYYYYY/

cat /prometheus/01XXXX.../meta.json

{
  "minTime": 1700000000000,
  "maxTime": 1700007200000,
  "stats": {
    "numSamples": 15420,
    "numSeries": 312
  },
  "compactionLevel": 1
}

from prometheus_client import start_http_server, Counter, Gauge, Histogram
import time, random

REQUEST_COUNT = Counter(
    'app_requests_total',
    'Toplam istek sayisi',
    ['method', 'endpoint', 'status']
)

ACTIVE_USERS = Gauge(
    'app_active_users',
    'Anlik aktif kullanici sayisi'
)

REQUEST_DURATION = Histogram(
    'app_request_duration_seconds',
    'Istek suresi dagilimi',
    ['endpoint'],
    buckets=[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5]
)

def simulate_traffic():
    endpoints = ['/api/users', '/api/orders', '/api/products', '/health']

    while True:
        endpoint = random.choice(endpoints)
        status = random.choice(['200', '200', '200', '404', '500'])
        duration = random.uniform(0.01, 0.3)

        REQUEST_COUNT.labels(
            method='GET',
            endpoint=endpoint,
            status=status
        ).inc()

        REQUEST_DURATION.labels(endpoint=endpoint).observe(duration)
        ACTIVE_USERS.set(random.randint(10, 150))
        time.sleep(0.5)

if __name__ == '__main__':
    start_http_server(8000)
    simulate_traffic()

# TYPE app_requests_total counter
app_requests_total{endpoint="/api/users",method="GET",status="200"} 142.0

# TYPE app_active_users gauge
app_active_users 87.0

# TYPE app_request_duration_seconds histogram
app_request_duration_seconds_bucket{endpoint="/api/users",le="0.1"} 89.0
app_request_duration_seconds_bucket{endpoint="/api/users",le="0.25"} 134.0
app_request_duration_seconds_sum{endpoint="/api/users"} 18.4
app_request_duration_seconds_count{endpoint="/api/users"} 142.0

# Saniyedeki istek hızı
rate(app_requests_total[5m])

# Endpoint bazında grupla
sum by (endpoint) (rate(app_requests_total[1m]))

# Hata oranı
sum(rate(app_requests_total{status="500"}[2m]))
/
sum(rate(app_requests_total[2m]))
* 100

histogram_quantile(0.95,
  sum by (endpoint, le) (
    rate(app_request_duration_seconds_bucket[2m])
  )
)

predict_linear(prometheus_tsdb_storage_blocks_bytes[1h], 3600)

absent(up{job="custom-exporter"})

groups:
  - name: app.rules
    rules:

      - alert: HighErrorRate
        expr: |
          sum(rate(app_requests_total{status="500"}[2m]))
          /
          sum(rate(app_requests_total[2m]))
          > 0.10
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Hata orani yuksek"
          description: "Hata orani %{{ $value | humanizePercentage }} ulastu"

      - alert: HighLatency
        expr: |
          histogram_quantile(0.95,
            sum by (endpoint, le) (
              rate(app_request_duration_seconds_bucket[2m])
            )
          ) > 0.5
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Yuksek latency: {{ $labels.endpoint }}"

      - alert: ExporterDown
        expr: up{job="custom-exporter"} == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "Custom exporter down"

Koşul doğru → pending → (1 dakika geçerse) → firing
Koşul yanlış → resolved