vs Support Blockers What You Need to Know

import os
import json
import time
import logging
from datetime import datetime, timedelta
from typing import Dict, List, Optional

import requests
from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# GitHub API configuration
GITHUB_API_BASE = "https://api.github.com"
REPO_OWNER = "your-org"
REPO_NAME = "your-repo"
SUPPORT_BLOCKER_LABEL = "support-blocker"
PRIORITY_THRESHOLDS = {
    "critical": 2,  # Issues with >=2 comments in first hour
    "high": 1,      # Issues with >=1 comment in first 4 hours
    "medium": 0     # All other support blockers
}

class GitHubTriageBot:
    def __init__(self, token: str, on_call_slack_id: str):
        self.token = token
        self.on_call_slack_id = on_call_slack_id
        self.headers = {
            "Authorization": f"token {self.token}",
            "Accept": "application/vnd.github.v3+json",
            "User-Agent": "SupportBlockerTriageBot/1.0"
        }
        self.rate_limit_remaining = 5000  # Default GitHub rate limit
        self.rate_limit_reset = 0

    def _handle_rate_limit(self, response: requests.Response) -> None:
        """Check for rate limit headers and sleep if needed."""
        if "X-RateLimit-Remaining" in response.headers:
            self.rate_limit_remaining = int(response.headers["X-RateLimit-Remaining"])
        if "X-RateLimit-Reset" in response.headers:
            self.rate_limit_reset = int(response.headers["X-RateLimit-Reset"])

        if response.status_code == 403 and self.rate_limit_remaining == 0:
            sleep_time = self.rate_limit_reset - time.time()
            if sleep_time > 0:
                logger.warning(f"Rate limit exceeded. Sleeping for {sleep_time:.0f} seconds")
                time.sleep(sleep_time)
            return True
        return False

    def _make_request(self, method: str, endpoint: str, **kwargs) -> Optional[requests.Response]:
        """Make a GitHub API request with rate limit and error handling."""
        url = f"{GITHUB_API_BASE}{endpoint}"
        retries = 3
        for attempt in range(retries):
            try:
                response = requests.request(method, url, headers=self.headers, **kwargs)
                if self._handle_rate_limit(response):
                    continue  # Retry after rate limit sleep
                if response.status_code >= 400:
                    logger.error(f"Request failed: {response.status_code} - {response.text}")
                    if attempt < retries -1:
                        time.sleep(2 ** attempt)  # Exponential backoff
                        continue
                    return None
                return response
            except requests.exceptions.RequestException as e:
                logger.error(f"Request exception: {e}")
                if attempt < retries -1:
                    time.sleep(2 ** attempt)
                    continue
                return None
        return None

    def get_support_blocker_issues(self) -> List[Dict]:
        """Fetch all open issues with the support-blocker label."""
        endpoint = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues?labels={SUPPORT_BLOCKER_LABEL}&state=open"
        response = self._make_request("GET", endpoint)
        if not response:
            return []
        return response.json()

    def calculate_priority(self, issue: Dict) -> str:
        """Determine issue priority based on age and comment count."""
        created_at = datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ")
        age_hours = (datetime.utcnow() - created_at).total_seconds() / 3600
        comment_count = issue["comments"]

        if comment_count >= PRIORITY_THRESHOLDS["critical"] and age_hours <= 1:
            return "priority-critical"
        elif comment_count >= PRIORITY_THRESHOLDS["high"] and age_hours <= 4:
            return "priority-high"
        else:
            return "priority-medium"

    def assign_issue(self, issue_number: int, assignee: str) -> bool:
        """Assign an issue to a specific user."""
        endpoint = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}"
        data = {"assignees": [assignee]}
        response = self._make_request("PATCH", endpoint, json=data)
        return response is not None and response.status_code == 200

    def add_label(self, issue_number: int, label: str) -> bool:
        """Add a label to an issue."""
        endpoint = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/labels"
        data = {"labels": [label]}
        response = self._make_request("POST", endpoint, json=data)
        return response is not None and response.status_code == 200

    def post_templated_comment(self, issue_number: int, priority: str) -> bool:
        """Post a templated SLA comment based on priority."""
        sla_map = {
            "priority-critical": "4 hours",
            "priority-high": "24 hours",
            "priority-medium": "72 hours"
        }
        sla = sla_map.get(priority, "72 hours")
        body = f"## Support Blocker Triage\n\nThis issue has been automatically triaged as {priority}. Our SLA for resolution is {sla}. Assigned to on-call engineer: @{self.on_call_slack_id}\n\nPlease provide any additional context in the comments."
        endpoint = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments"
        data = {"body": body}
        response = self._make_request("POST", endpoint, json=data)
        return response is not None and response.status_code == 201

    def run_triage(self) -> None:
        """Main triage loop."""
        logger.info("Starting support blocker triage run")
        issues = self.get_support_blocker_issues()
        logger.info(f"Found {len(issues)} open support blocker issues")

        for issue in issues:
            issue_number = issue["number"]
            assignee = issue.get("assignee")

            # Skip already assigned issues
            if assignee:
                logger.info(f"Issue #{issue_number} already assigned to {assignee['login']}, skipping")
                continue

            # Calculate priority and add label
            priority = self.calculate_priority(issue)
            if not self.add_label(issue_number, priority):
                logger.error(f"Failed to add label {priority} to issue #{issue_number}")
                continue

            # Assign to on-call engineer
            if not self.assign_issue(issue_number, self.on_call_slack_id):
                logger.error(f"Failed to assign issue #{issue_number} to {self.on_call_slack_id}")
                continue

            # Post templated comment
            if not self.post_templated_comment(issue_number, priority):
                logger.error(f"Failed to post comment to issue #{issue_number}")
                continue

            logger.info(f"Successfully triaged issue #{issue_number} as {priority}")

if __name__ == "__main__":
    token = os.getenv("GITHUB_TOKEN")
    on_call = os.getenv("ON_CALL_SLACK_ID")

    if not token:
        logger.error("Missing GITHUB_TOKEN environment variable")
        exit(1)
    if not on_call:
        logger.error("Missing ON_CALL_SLACK_ID environment variable")
        exit(1)

    bot = GitHubTriageBot(token, on_call)
    bot.run_triage()

package main

import (
    "bufio"
    "context"
    "fmt"
    "log"
    "net/http"
    "os"
    "os/signal"
    "regexp"
    "strings"
    "sync"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "github.com/slack-go/slack"
)

// Configuration
const (
    defaultLogPath       = "/var/log/app/app.log"
    defaultSlackWebhook  = ""
    defaultPrometheusPort = "9090"
    alertCooldown        = 1 * time.Hour
    slidingWindow       = 1 * time.Hour
)

// Support blocker pattern definitions
var supportBlockerPatterns = map[string]*regexp.Regexp{
    "500_error":       regexp.MustCompile(`\b500\b.*Internal Server Error`),
    "db_timeout":      regexp.MustCompile(`database connection timeout|could not connect to database`),
    "third_party_api": regexp.MustCompile(`third-party API (timeout|error|unavailable)`),
}

// Metrics
var (
    blockerCounter = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "support_blocker_total",
            Help: "Total number of detected support blocker events",
        },
        []string{"pattern"},
    )
    blockerRate = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "support_blocker_rate_per_min",
            Help: "Rate of support blocker events per minute over sliding window",
        },
        []string{"pattern"},
    )
)

// BlockerTracker tracks occurrence counts over a sliding window
type BlockerTracker struct {
    mu        sync.RWMutex
    events    map[string][]time.Time // pattern -> list of event timestamps
    thresholds map[string]int       // pattern -> alert threshold per sliding window
}

func NewBlockerTracker(thresholds map[string]int) *BlockerTracker {
    return &BlockerTracker{
        events:    make(map[string][]time.Time),
        thresholds: thresholds,
    }
}

func (bt *BlockerTracker) AddEvent(pattern string) {
    bt.mu.Lock()
    defer bt.mu.Unlock()
    bt.events[pattern] = append(bt.events[pattern], time.Now())
}

func (bt *BlockerTracker) CleanupOldEvents() {
    bt.mu.Lock()
    defer bt.mu.Unlock()
    cutoff := time.Now().Add(-slidingWindow)
    for pattern, timestamps := range bt.events {
        var valid []time.Time
        for _, ts := range timestamps {
            if ts.After(cutoff) {
                valid = append(valid, ts)
            }
        }
        bt.events[pattern] = valid
    }
}

func (bt *BlockerTracker) GetRate(pattern string) float64 {
    bt.mu.RLock()
    defer bt.mu.RUnlock()
    count := len(bt.events[pattern])
    return float64(count) / slidingWindow.Minutes()
}

func (bt *BlockerTracker) CheckThreshold(pattern string) bool {
    bt.mu.RLock()
    defer bt.mu.RUnlock()
    count := len(bt.events[pattern])
    threshold, exists := bt.thresholds[pattern]
    if !exists {
        return false
    }
    return count >= threshold
}

func main() {
    // Load configuration from environment
    logPath := os.Getenv("LOG_PATH")
    if logPath == "" {
        logPath = defaultLogPath
        log.Printf("LOG_PATH not set, using default: %s", logPath)
    }

    slackWebhook := os.Getenv("SLACK_WEBHOOK_URL")
    prometheusPort := os.Getenv("PROMETHEUS_PORT")
    if prometheusPort == "" {
        prometheusPort = defaultPrometheusPort
    }

    // Initialize metrics
    prometheus.MustRegister(blockerCounter)
    prometheus.MustRegister(blockerRate)

    // Initialize tracker with thresholds (10 events per sliding window)
    thresholds := make(map[string]int)
    for pattern := range supportBlockerPatterns {
        thresholds[pattern] = 10
    }
    tracker := NewBlockerTracker(thresholds)

    // Start Prometheus metrics server
    go func() {
        http.Handle("/metrics", promhttp.Handler())
        log.Printf("Prometheus metrics listening on :%s", prometheusPort)
        if err := http.ListenAndServe(fmt.Sprintf(":%s", prometheusPort), nil); err != nil {
            log.Fatalf("Failed to start Prometheus server: %v", err)
        }
    }()

    // Initialize Slack client if webhook is set
    var slackClient *slack.Client
    if slackWebhook != "" {
        slackClient = slack.New(slackWebhook)
    }

    // Context for graceful shutdown
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    // Handle OS signals for graceful shutdown
    sigChan := make(chan os.Signal, 1)
    signal.Notify(sigChan, os.Interrupt)
    go func() {
        <-sigChan
        log.Println("Shutting down...")
        cancel()
    }()

    // Start log file monitoring
    go monitorLogs(ctx, logPath, tracker, slackClient)

    // Start periodic cleanup and metric update
    ticker := time.NewTicker(1 * time.Minute)
    defer ticker.Stop()
    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker.C:
            tracker.CleanupOldEvents()
            // Update rate metrics
            for pattern := range supportBlockerPatterns {
                rate := tracker.GetRate(pattern)
                blockerRate.WithLabelValues(pattern).Set(rate)
            }
        }
    }
}

func monitorLogs(ctx context.Context, logPath string, tracker *BlockerTracker, slackClient *slack.Client) {
    file, err := os.Open(logPath)
    if err != nil {
        log.Fatalf("Failed to open log file %s: %v", logPath, err)
    }
    defer file.Close()

    // Seek to end of file for new logs only
    _, err = file.Seek(0, 2)
    if err != nil {
        log.Fatalf("Failed to seek to end of log file: %v", err)
    }

    scanner := bufio.NewScanner(file)
    for {
        select {
        case <-ctx.Done():
            return
        default:
            if scanner.Scan() {
                line := scanner.Text()
                // Check line against all patterns
                for patternName, re := range supportBlockerPatterns {
                    if re.MatchString(line) {
                        // Update metrics and tracker
                        blockerCounter.WithLabelValues(patternName).Inc()
                        tracker.AddEvent(patternName)
                        log.Printf("Detected support blocker: %s", patternName)

                        // Check if threshold exceeded
                        if tracker.CheckThreshold(patternName) {
                            // Send Slack alert if client exists
                            if slackClient != nil {
                                msg := fmt.Sprintf("🚨 Support Blocker Alert: %s exceeded threshold (10 events in 1 hour)", patternName)
                                _, _, err := slackClient.PostMessageContext(ctx, "support-alerts", slack.MsgOptionText(msg, false))
                                if err != nil {
                                    log.Printf("Failed to send Slack alert: %v", err)
                                } else {
                                    log.Printf("Sent Slack alert for %s", patternName)
                                }
                            }
                        }
                    }
                }
            } else {
                // No new line, sleep briefly
                time.Sleep(100 * time.Millisecond)
            }
        }
    }
}

import { App, LogLevel } from "@slack/bolt";
import PagerDuty from "pagerduty-api";
import { DateTime } from "luxon";
import { promises as fs } from "fs";
import path from "path";

// Configuration
const SUPPORT_BLOCKER_THRESHOLD = 5; // 5 blockers in 1 hour triggers incident
const SLIDING_WINDOW_HOURS = 1;
const DAILY_SUMMARY_TIME = "09:00"; // Daily summary at 9 AM UTC

// Initialize Slack app
const app = new App({
  token: process.env.SLACK_BOT_TOKEN,
  signingSecret: process.env.SLACK_SIGNING_SECRET,
  logLevel: LogLevel.INFO,
});

// Initialize PagerDuty client
const pagerduty = new PagerDuty({
  apiKey: process.env.PAGERDUTY_API_KEY,
  serviceId: process.env.PAGERDUTY_SERVICE_ID,
});

// In-memory store for support blocker events (replace with Redis in production)
const blockerEvents: Array<{ timestamp: DateTime; pattern: string }> = [];
const dataFilePath = path.join(__dirname, "blocker_events.json");

// Load persisted events from file
async function loadEvents(): Promise {
  try {
    const data = await fs.readFile(dataFilePath, "utf-8");
    const parsed = JSON.parse(data);
    blockerEvents.push(
      ...parsed.map((e: any) => ({
        timestamp: DateTime.fromISO(e.timestamp),
        pattern: e.pattern,
      }))
    );
    console.log(`Loaded ${blockerEvents.length} persisted events`);
  } catch (err: any) {
    if (err.code === "ENOENT") {
      console.log("No persisted events found, starting fresh");
      return;
    }
    console.error("Failed to load persisted events:", err);
  }
}

// Persist events to file
async function persistEvents(): Promise {
  try {
    const data = JSON.stringify(
      blockerEvents.map((e) => ({
        timestamp: e.timestamp.toISO(),
        pattern: e.pattern,
      }))
    );
    await fs.writeFile(dataFilePath, data, "utf-8");
  } catch (err) {
    console.error("Failed to persist events:", err);
  }
}

// Clean up old events outside sliding window
function cleanupOldEvents(): void {
  const cutoff = DateTime.utc().minus({ hours: SLIDING_WINDOW_HOURS });
  const initialLength = blockerEvents.length;
  for (let i = blockerEvents.length - 1; i >= 0; i--) {
    if (blockerEvents[i].timestamp < cutoff) {
      blockerEvents.splice(i, 1);
    }
  }
  console.log(`Cleaned up ${initialLength - blockerEvents.length} old events`);
}

// Check if threshold is exceeded
function isThresholdExceeded(): boolean {
  cleanupOldEvents();
  return blockerEvents.length >= SUPPORT_BLOCKER_THRESHOLD;
}

// Get top blocker patterns in last 24 hours
function getTopPatterns(): Array<{ pattern: string; count: number }> {
  const cutoff = DateTime.utc().minus({ hours: 24 });
  const patternCounts: Record = {};
  blockerEvents.forEach((event) => {
    if (event.timestamp >= cutoff) {
      patternCounts[event.pattern] = (patternCounts[event.pattern] || 0) + 1;
    }
  });
  return Object.entries(patternCounts)
    .map(([pattern, count]) => ({ pattern, count }))
    .sort((a, b) => b.count - a.count)
    .slice(0, 5);
}

// Trigger PagerDuty incident
async function triggerIncident(): Promise {
  try {
    const response = await pagerduty.incidents.create({
      incident: {
        type: "incident",
        title: "Support Blocker Spike Detected",
        service: { id: process.env.PAGERDUTY_SERVICE_ID, type: "service_reference" },
        urgency: "high",
        body: {
          type: "incident_body",
          details: `Detected ${blockerEvents.length} support blockers in the last hour, exceeding threshold of ${SUPPORT_BLOCKER_THRESHOLD}`,
        },
      },
    });
    console.log(`Triggered PagerDuty incident: ${response.incident.id}`);
    // Post to Slack channel
    await app.client.chat.postMessage({
      channel: process.env.SUPPORT_CHANNEL_ID!,
      text: `🚨 Triggered PagerDuty incident for support blocker spike: ${response.incident.html_url}`,
    });
  } catch (err) {
    console.error("Failed to trigger PagerDuty incident:", err);
  }
}

// Generate daily summary
async function generateDailySummary(): Promise {
  cleanupOldEvents();
  const topPatterns = getTopPatterns();
  const total24h = blockerEvents.filter(
    (e) => e.timestamp >= DateTime.utc().minus({ hours: 24 })
  ).length;
  const summary = [
    `*Daily Support Blocker Summary (${DateTime.utc().toFormat("yyyy-MM-dd")})*` ,
    `Total blockers in last 24 hours: ${total24h}`,
    `Top 5 patterns:`,
    ...topPatterns.map((p, i) => `${i + 1}. ${p.pattern}: ${p.count} occurrences`),
  ].join("\n");
  try {
    await app.client.chat.postMessage({
      channel: process.env.SUPPORT_CHANNEL_ID!,
      text: summary,
    });
    console.log("Posted daily summary to Slack");
  } catch (err) {
    console.error("Failed to post daily summary:", err);
  }
}

// Listen for support blocker events from external system (e.g., webhook from triage bot)
app.message(/support-blocker/i, async ({ message, say }) => {
  try {
    const pattern = (message as any).text.match(/pattern: (\w+)/)?.[1] || "unknown";
    blockerEvents.push({
      timestamp: DateTime.utc(),
      pattern,
    });
    await persistEvents();
    console.log(`Recorded support blocker event: ${pattern}`);

    // Check threshold
    if (isThresholdExceeded()) {
      await triggerIncident();
    }
  } catch (err) {
    console.error("Error handling support blocker message:", err);
  }
});

// Schedule daily summary
setInterval(async () => {
  const now = DateTime.utc();
  if (now.toFormat("HH:mm") === DAILY_SUMMARY_TIME) {
    await generateDailySummary();
  }
}, 60 * 1000); // Check every minute

// Start app
(async () => {
  await loadEvents();
  const port = process.env.PORT || 3000;
  await app.start(port);
  console.log(`Slack bot listening on port ${port}`);
})();

# .pre-commit-config.yaml
repos:
  - repo: https://github.com/bridgecrewio/checkov
    rev: 3.2.14
    hooks:
      - id: checkov
        args: [--directory, terraform/, --framework, terraform, --soft-fail]

# .github/ISSUE_TEMPLATE/support-blocker.md
name: Support Blocker
description: Report a bug or issue affecting production
labels: ["support-blocker"]
body:
  - type: input
    id: user-id
    attributes:
      label: User ID
      description: The ID of the user experiencing the issue
      placeholder: usr_12345
    validations:
      required: true
  - type: input
    id: request-id
    attributes:
      label: Request ID
      description: The X-Request-ID header from the API response
      placeholder: req_67890
    validations:
      required: true
  - type: textarea
    id: steps
    attributes:
      label: Steps to Reproduce
      description: Detailed steps to reproduce the issue
      placeholder: |
        1. Go to /api/v1/checkout
        2. Enter payment details
        3. Click submit
    validations:
      required: true
  - type: textarea
    id: expected
    attributes:
      label: Expected Behavior
      description: What should have happened
    validations:
      required: true
  - type: textarea
    id: actual
    attributes:
      label: Actual Behavior
      description: What actually happened
    validations:
      required: true

import requests
from datetime import datetime

def link_issue_to_deploy(issue_number: int, repo_owner: str, repo_name: str, token: str):
    headers = {"Authorization": f"token {token}"}
    # Get issue creation time
    issue_resp = requests.get(
        f"https://api.github.com/repos/{repo_owner}/{repo_name}/issues/{issue_number}",
        headers=headers
    )
    issue_data = issue_resp.json()
    issue_created = datetime.strptime(issue_data["created_at"], "%Y-%m-%dT%H:%M:%SZ")

    # Get recent deploys
    deploys_resp = requests.get(
        f"https://api.github.com/repos/{repo_owner}/{repo_name}/deployments",
        headers=headers
    )
    deploys = deploys_resp.json()

    # Find most recent deploy before issue creation
    matching_deploy = None
    for deploy in deploys:
        deploy_time = datetime.strptime(deploy["created_at"], "%Y-%m-%dT%H:%M:%SZ")
        if deploy_time < issue_created:
            matching_deploy = deploy
            break

    if matching_deploy:
        # Add comment linking to deploy
        comment = f"Linked to deploy: {matching_deploy['html_url']}"
        requests.post(
            f"https://api.github.com/repos/{repo_owner}/{repo_name}/issues/{issue_number}/comments",
            headers=headers,
            json={"body": comment}
        )