Postmortem: How a Jira 11.0 Outage Caused 4 Hour Delay for Our 1000+ Ticket Sprint

import os
import json
import time
import logging
import smtplib
from flask import Flask, request, jsonify
from datetime import datetime
from email.mime.text import MIMEText

# Configure structured logging for audit trails
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s | %(levelname)s | %(message)s',
    handlers=[logging.FileHandler('jira_outage_detector.log'), logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

app = Flask(__name__)

# Threshold for failed Jira API checks before triggering alert
FAILURE_THRESHOLD = 3
# Jira 11.0 health check endpoint (new in 11.0, deprecated in 10.x)
JIRA_HEALTH_ENDPOINT = os.getenv('JIRA_HEALTH_URL', 'https://jira.example.com/rest/api/11/health')
# Alert config
SMTP_SERVER = os.getenv('SMTP_SERVER', 'smtp.example.com')
ALERT_RECIPIENTS = os.getenv('ALERT_RECIPIENTS', 'oncall@example.com').split(',')

# In-memory failure counter (use Redis in prod)
failure_count = 0
last_alert_time = 0
ALERT_COOLDOWN = 300  # 5 minutes between alerts

def send_alert(subject, body):
    """Send email alert to oncall team with error context"""
    try:
        msg = MIMEText(body)
        msg['Subject'] = subject
        msg['From'] = 'jira-monitor@example.com'
        msg['To'] = ', '.join(ALERT_RECIPIENTS)

        with smtplib.SMTP(SMTP_SERVER, 587) as server:
            server.starttls()
            server.login(os.getenv('SMTP_USER'), os.getenv('SMTP_PASS'))
            server.send_message(msg)
        logger.info(f"Sent alert: {subject}")
    except Exception as e:
        logger.error(f"Failed to send alert: {str(e)}")

def check_jira_health():
    """Poll Jira 11.0 health endpoint and return status"""
    import requests
    try:
        response = requests.get(
            JIRA_HEALTH_ENDPOINT,
            timeout=5,
            headers={'Accept': 'application/json'}
        )
        # Jira 11.0 returns 200 with {"status": "UP"} on healthy instances
        if response.status_code == 200:
            data = response.json()
            return data.get('status') == 'UP'
        return False
    except requests.exceptions.RequestException as e:
        logger.warning(f"Health check failed: {str(e)}")
        return False

@app.route('/webhook/jira-status', methods=['POST'])
def handle_jira_webhook():
    """Handle Jira status change webhooks (sent by Atlassian when version updates occur)"""
    global failure_count, last_alert_time
    try:
        payload = request.get_json()
        if not payload:
            return jsonify({'error': 'Invalid payload'}), 400

        event_type = payload.get('eventType')
        if event_type == 'jira.version.updated':
            new_version = payload.get('newVersion')
            logger.info(f"Detected Jira version update to {new_version}")
            # Reset failure count on version change to avoid stale alerts
            failure_count = 0

        # Run health check
        is_healthy = check_jira_health()
        if not is_healthy:
            failure_count += 1
            logger.warning(f"Jira health check failed ({failure_count}/{FAILURE_THRESHOLD})")

            if failure_count >= FAILURE_THRESHOLD:
                current_time = time.time()
                if current_time - last_alert_time > ALERT_COOLDOWN:
                    alert_body = f"""
                    Jira Outage Detected
                    Time: {datetime.now().isoformat()}
                    Failure Count: {failure_count}
                    Last Health Check: {is_healthy}
                    Version: {payload.get('newVersion', 'unknown')}
                    """
                    send_alert(f"CRITICAL: Jira 11.0 Outage Detected", alert_body)
                    last_alert_time = current_time
        else:
            failure_count = 0  # Reset on successful check

        return jsonify({'status': 'processed'}), 200
    except Exception as e:
        logger.error(f"Webhook handler error: {str(e)}")
        return jsonify({'error': 'Internal server error'}), 500

if __name__ == '__main__':
    logger.info("Starting Jira 11.0 outage detector on port 5000")
    app.run(host='0.0.0.0', port=5000, debug=False)

import os
import json
import time
import logging
from datetime import datetime, timedelta
from typing import List, Dict, Optional

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s | %(levelname)s | %(message)s'
)
logger = logging.getLogger(__name__)

class JiraReindexer:
    """Client to reindex Jira 11.0 tickets post-outage, with retry logic and batch processing"""

    def __init__(self, base_url: str, api_token: str, project_key: str):
        self.base_url = base_url.rstrip('/')
        self.api_token = api_token
        self.project_key = project_key
        self.session = self._init_session()
        self.batch_size = 50  # Jira 11.0 max batch reindex size
        self.max_retries = 5

    def _init_session(self) -> requests.Session:
        """Initialize session with retry logic for transient errors"""
        session = requests.Session()
        retry_strategy = Retry(
            total=self.max_retries,
            backoff_factor=1,
            status_forcelist=[429, 500, 502, 503, 504],
            allowed_methods=["GET", "POST"]
        )
        adapter = HTTPAdapter(max_retries=retry_strategy)
        session.mount("https://", adapter)
        session.headers.update({
            'Authorization': f'Bearer {self.api_token}',
            'Accept': 'application/json',
            'Content-Type': 'application/json'
        })
        return session

    def get_unindexed_tickets(self, last_updated: Optional[datetime] = None) -> List[str]:
        """Fetch ticket keys for tickets updated since last outage, not yet reindexed"""
        jql = f'project={self.project_key} AND updated >= "{last_updated.isoformat() if last_updated else "2024-10-12T00:00:00Z"}" AND status != "Reindexed"'
        url = f'{self.base_url}/rest/api/11/search'
        params = {
            'jql': jql,
            'fields': 'key',
            'maxResults': 1000  # Paginate if >1000, but we'll handle first page for example
        }

        try:
            response = self.session.get(url, params=params, timeout=10)
            response.raise_for_status()
            data = response.json()
            return [issue['key'] for issue in data.get('issues', [])]
        except requests.exceptions.HTTPError as e:
            logger.error(f"Failed to fetch unindexed tickets: {str(e)}")
            if e.response.status_code == 400:
                logger.error(f"Invalid JQL: {jql}")
            return []
        except Exception as e:
            logger.error(f"Unexpected error fetching tickets: {str(e)}")
            return []

    def reindex_batch(self, ticket_keys: List[str]) -> bool:
        """Reindex a batch of tickets using Jira 11.0 batch reindex endpoint"""
        if not ticket_keys:
            return True

        url = f'{self.base_url}/rest/api/11/reindex/batch'
        payload = {
            'issueKeys': ticket_keys[:self.batch_size],  # Truncate to max batch size
            'type': 'FULL'  # Jira 11.0 supports FULL or INCREMENTAL
        }

        try:
            response = self.session.post(url, json=payload, timeout=30)
            response.raise_for_status()
            task_id = response.json().get('taskId')
            logger.info(f"Started reindex task {task_id} for {len(ticket_keys[:self.batch_size])} tickets")
            return self._poll_task_status(task_id)
        except requests.exceptions.HTTPError as e:
            logger.error(f"Reindex batch failed: {str(e)}")
            return False
        except Exception as e:
            logger.error(f"Unexpected reindex error: {str(e)}")
            return False

    def _poll_task_status(self, task_id: str, poll_interval: int = 10, max_polls: int = 30) -> bool:
        """Poll reindex task status until complete or timeout"""
        url = f'{self.base_url}/rest/api/11/task/{task_id}'
        for _ in range(max_polls):
            try:
                response = self.session.get(url, timeout=10)
                response.raise_for_status()
                status = response.json().get('status')
                if status == 'COMPLETE':
                    logger.info(f"Reindex task {task_id} completed successfully")
                    return True
                elif status == 'FAILED':
                    logger.error(f"Reindex task {task_id} failed: {response.json().get('error')}")
                    return False
                logger.info(f"Task {task_id} status: {status}, polling again in {poll_interval}s")
                time.sleep(poll_interval)
            except Exception as e:
                logger.error(f"Error polling task {task_id}: {str(e)}")
                return False
        logger.error(f"Reindex task {task_id} timed out after {max_polls * poll_interval}s")
        return False

    def run_reindex(self, last_updated: Optional[datetime] = None):
        """Main entry point to reindex all unindexed tickets in batches"""
        logger.info(f"Starting reindex for project {self.project_key}")
        ticket_keys = self.get_unindexed_tickets(last_updated)
        if not ticket_keys:
            logger.info("No unindexed tickets found")
            return

        logger.info(f"Found {len(ticket_keys)} unindexed tickets")
        for i in range(0, len(ticket_keys), self.batch_size):
            batch = ticket_keys[i:i+self.batch_size]
            logger.info(f"Processing batch {i//self.batch_size + 1}: {len(batch)} tickets")
            success = self.reindex_batch(batch)
            if not success:
                logger.error(f"Batch {i//self.batch_size + 1} failed, retrying after 60s")
                time.sleep(60)
                self.reindex_batch(batch)
            time.sleep(5)  # Rate limit to avoid Jira 11.0 throttling

        logger.info(f"Reindex complete for project {self.project_key}")

if __name__ == '__main__':
    # Load config from env vars
    required_vars = ['JIRA_BASE_URL', 'JIRA_API_TOKEN', 'JIRA_PROJECT_KEY']
    missing = [var for var in required_vars if not os.getenv(var)]
    if missing:
        logger.error(f"Missing required env vars: {missing}")
        exit(1)

    reindexer = JiraReindexer(
        base_url=os.getenv('JIRA_BASE_URL'),
        api_token=os.getenv('JIRA_API_TOKEN'),
        project_key=os.getenv('JIRA_PROJECT_KEY')
    )

    # Reindex tickets updated since outage start (Oct 12 2024 09:00 UTC)
    outage_start = datetime(2024, 10, 12, 9, 0, 0)
    reindexer.run_reindex(last_updated=outage_start)

import time
import logging
from datetime import datetime, timedelta
from functools import wraps
from typing import Callable, Any, Optional

import requests
from requests.exceptions import RequestException

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s | %(levelname)s | %(message)s'
)
logger = logging.getLogger(__name__)

class JiraCircuitBreaker:
    """Circuit breaker implementation for Jira 11.0 API calls to prevent cascading failures"""

    def __init__(
        self,
        failure_threshold: int = 5,
        recovery_timeout: int = 60,
        half_open_max_calls: int = 3
    ):
        self.failure_threshold = failure_threshold  # Number of failures before opening circuit
        self.recovery_timeout = recovery_timeout    # Seconds to wait before trying half-open
        self.half_open_max_calls = half_open_max_calls  # Max calls in half-open state

        self.failure_count = 0
        self.last_failure_time = None
        self.state = 'CLOSED'  # CLOSED, OPEN, HALF_OPEN
        self.half_open_calls = 0

    def __call__(self, func: Callable) -> Callable:
        """Decorator to wrap Jira API call functions with circuit breaker logic"""
        @wraps(func)
        def wrapper(*args, **kwargs) -> Any:
            if self.state == 'OPEN':
                if self._should_attempt_recovery():
                    self.state = 'HALF_OPEN'
                    self.half_open_calls = 0
                    logger.info("Circuit breaker moving to HALF_OPEN state")
                else:
                    raise CircuitBreakerOpenError("Jira circuit breaker is OPEN, failing fast")

            if self.state == 'HALF_OPEN':
                if self.half_open_calls >= self.half_open_max_calls:
                    raise CircuitBreakerOpenError("Max half-open calls reached, circuit remains OPEN")
                self.half_open_calls += 1

            try:
                result = func(*args, **kwargs)
                self._on_success()
                return result
            except Exception as e:
                self._on_failure()
                raise

        return wrapper

    def _should_attempt_recovery(self) -> bool:
        """Check if enough time has passed since last failure to attempt recovery"""
        if not self.last_failure_time:
            return True
        return (datetime.now() - self.last_failure_time).seconds >= self.recovery_timeout

    def _on_success(self):
        """Reset circuit breaker on successful call"""
        if self.state == 'HALF_OPEN':
            logger.info("Circuit breaker success in HALF_OPEN, moving to CLOSED")
            self.state = 'CLOSED'
        self.failure_count = 0
        self.half_open_calls = 0

    def _on_failure(self):
        """Increment failure count and open circuit if threshold reached"""
        self.failure_count += 1
        self.last_failure_time = datetime.now()
        logger.warning(f"Circuit breaker failure {self.failure_count}/{self.failure_threshold}")

        if self.failure_count >= self.failure_threshold:
            self.state = 'OPEN'
            logger.error(f"Circuit breaker opened after {self.failure_count} failures")

class CircuitBreakerOpenError(Exception):
    """Custom exception raised when circuit breaker is open"""
    pass

# Example usage: Wrapped Jira API call to fetch sprint details
@JiraCircuitBreaker(failure_threshold=3, recovery_timeout=30)
def fetch_sprint_details(sprint_id: str, jira_url: str, api_token: str) -> Optional[Dict]:
    """Fetch sprint details from Jira 11.0 API with circuit breaker protection"""
    url = f'{jira_url}/rest/api/11/sprint/{sprint_id}'
    headers = {
        'Authorization': f'Bearer {api_token}',
        'Accept': 'application/json'
    }

    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        return response.json()
    except RequestException as e:
        logger.error(f"Failed to fetch sprint {sprint_id}: {str(e)}")
        raise
    except Exception as e:
        logger.error(f"Unexpected error fetching sprint {sprint_id}: {str(e)}")
        raise

if __name__ == '__main__':
    # Test the circuit breaker with a mock failing Jira endpoint
    test_jira_url = 'https://jira-failing.example.com'
    test_token = 'invalid-token'

    for i in range(10):
        try:
            sprint = fetch_sprint_details('123', test_jira_url, test_token)
            print(f"Call {i+1}: Success")
        except CircuitBreakerOpenError as e:
            print(f"Call {i+1}: {str(e)}")
            time.sleep(5)
        except Exception as e:
            print(f"Call {i+1}: API error: {str(e)}")
            time.sleep(1)

resource "kubernetes_deployment" "jira" {
  metadata {
    name = "jira"
    namespace = "atlassian"
  }
  spec {
    replicas = 2
    selector {
      match_labels = {
        app = "jira"
      }
    }
    template {
      metadata {
        labels = {
          app = "jira"
        }
      }
      spec {
        container {
          # Pin to N-1 stable, never use latest
          image = "atlassian/jira-software:10.4.2-jdk17"
          name  = "jira"
          port {
            container_port = 8080
          }
        }
      }
    }
  }
}

CircuitBreaker circuitBreaker = CircuitBreaker.ofDefaults("jira-api");
Supplier decoratedSupplier = CircuitBreaker.decorateSupplier(circuitBreaker, () -> {
    // Call Jira API here
    return fetchJiraData();
});

// Execute with fallback
try {
    String result = decoratedSupplier.get();
} catch (CircuitBreakerOpenException e) {
    // Return cached data
    return redisClient.get("jira:sprint:123");
}

scrape_configs:
  - job_name: 'jira-health'
    metrics_path: '/rest/api/11/health'
    params:
      format: ['prometheus']
    static_configs:
      - targets: ['jira.example.com:8080']
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance