Retrospective: Our 2026 Team Transition from Waterfall to Agile: 40% Faster Releases

# metrics_calculator.py
# Requires: pip install jira pandas python-dotenv requests
import os
import json
import time
import pandas as pd
from datetime import datetime, timedelta
from jira import JIRA, JIRAError
from requests.exceptions import RequestException
from dotenv import load_dotenv

load_dotenv()

# Configuration: replace with your actual Jira and GitHub org details
JIRA_SERVER = os.getenv("JIRA_SERVER", "https://jira.example-fintech.com")
JIRA_PROJECT = os.getenv("JIRA_PROJECT", "PAY")
GITHUB_ORG = os.getenv("GITHUB_ORG", "example-fintech")
GITHUB_REPO = os.getenv("GITHUB_REPO", "payment-gateway")
CYCLE_TIME_DAYS = 90  # Calculate metrics for last 90 days

def init_jira_client():
    """Initialize authenticated Jira client with error handling for auth failures"""
    try:
        jira = JIRA(
            server=JIRA_SERVER,
            basic_auth=(os.getenv("JIRA_USER"), os.getenv("JIRA_API_TOKEN"))
        )
        # Validate connection by fetching current user
        jira.current_user()
        print(f"✅ Jira client initialized for {JIRA_SERVER}")
        return jira
    except JIRAError as e:
        print(f"❌ Jira auth failed: {e.status_code} {e.text}")
        raise
    except Exception as e:
        print(f"❌ Unexpected Jira init error: {str(e)}")
        raise

def fetch_released_tickets(jira_client, start_date):
    """Fetch all tickets in Done status with resolution date >= start_date"""
    jql = f'project = {JIRA_PROJECT} AND status = Done AND resolutiondate >= "{start_date}" AND type in (Story, Bug, Task)'
    try:
        issues = jira_client.search_issues(jql, maxResults=0, fields=["key", "created", "resolutiondate", "type", "priority"])
        print(f"Fetched {len(issues)} resolved tickets from Jira")
        return issues
    except JIRAError as e:
        print(f"❌ Jira fetch failed for JQL {jql}: {e.status_code} {e.text}")
        return []
    except Exception as e:
        print(f"❌ Unexpected error fetching Jira tickets: {str(e)}")
        return []

def calculate_cycle_time(tickets):
    """Calculate average cycle time in days, exclude tickets with missing dates"""
    cycle_times = []
    skipped = 0
    for ticket in tickets:
        created = ticket.fields.created
        resolved = ticket.fields.resolutiondate
        if not created or not resolved:
            skipped +=1
            continue
        try:
            created_dt = datetime.strptime(created[:10], "%Y-%m-%d")
            resolved_dt = datetime.strptime(resolved[:10], "%Y-%m-%d")
            delta = (resolved_dt - created_dt).days
            if delta >=0:
                cycle_times.append(delta)
        except ValueError as e:
            print(f"⚠️ Date parse error for {ticket.key}: {str(e)}")
            skipped +=1
    if not cycle_times:
        print("No valid cycle time data found")
        return 0.0
    avg = sum(cycle_times)/len(cycle_times)
    print(f"Calculated cycle time for {len(cycle_times)} tickets, skipped {skipped}")
    return round(avg, 2)

if __name__ == "__main__":
    try:
        start_date = (datetime.now() - timedelta(days=CYCLE_TIME_DAYS)).strftime("%Y-%m-%d")
        jira_client = init_jira_client()
        tickets = fetch_released_tickets(jira_client, start_date)
        avg_cycle_days = calculate_cycle_time(tickets)
        print(f"📊 Average release cycle time (last {CYCLE_TIME_DAYS} days): {avg_cycle_days} days")
        # Save to JSON for dashboard ingestion
        with open("cycle_metrics.json", "w") as f:
            json.dump({
                "avg_cycle_days": avg_cycle_days,
                "ticket_count": len(tickets),
                "calculated_at": datetime.now().isoformat(),
                "project": JIRA_PROJECT
            }, f, indent=2)
        print("✅ Metrics saved to cycle_metrics.json")
    except Exception as e:
        print(f"❌ Fatal error in metrics calculation: {str(e)}")
        exit(1)

# .github/workflows/release-pipeline.yml
name: Payment Gateway Release Pipeline
on:
  push:
    branches: [ main ]
  workflow_dispatch:
    inputs:
      force_deploy:
        description: 'Force deploy even if tests fail (emergency only)'
        required: false
        default: 'false'
        type: boolean

env:
  NODE_VERSION: '20.18.0'
  AWS_REGION: 'us-east-1'
  ECR_REPO: 'example-fintech/payment-gateway'
  SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_WEBHOOK }}

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Fetch full history for commit range analysis

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci --audit-level=high
        continue-on-error: false

      - name: Run unit tests
        run: npm run test:unit
        env:
          NODE_ENV: 'test'
        continue-on-error: false

      - name: Run integration tests
        run: npm run test:integration
        env:
          NODE_ENV: 'staging'
          DB_URL: ${{ secrets.STAGING_DB_URL }}
        continue-on-error: ${{ github.event.inputs.force_deploy == 'true' }}

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results
          path: coverage/
          retention-days: 7

  build-and-push:
    needs: test
    runs-on: ubuntu-latest
    outputs:
      image_tag: ${{ steps.tag.outputs.tag }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2

      - name: Generate image tag
        id: tag
        run: |
          SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7)
          echo "tag=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPO }}:${SHORT_SHA}" >> $GITHUB_OUTPUT

      - name: Build Docker image
        run: |
          docker build -t ${{ steps.tag.outputs.tag }} \
            --build-arg NODE_VERSION=${{ env.NODE_VERSION }} \
            --label "git-sha=${{ github.sha }}" \
            --label "build-time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" .

      - name: Push image to ECR
        run: docker push ${{ steps.tag.outputs.tag }}

      - name: Scan image for vulnerabilities
        uses: aquasecurity/trivy-action@0.28.0
        with:
          image-ref: ${{ steps.tag.outputs.tag }}
          format: 'table'
          exit-code: '1'
          ignore-unfixed: true
          severity: 'CRITICAL,HIGH'

  deploy-staging:
    needs: build-and-push
    runs-on: ubuntu-latest
    environment: staging
    steps:
      - name: Deploy to ECS staging
        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
        with:
          task-definition: task-definition-staging.json
          service: payment-gateway-staging
          cluster: example-fintech-staging
          image: ${{ needs.build-and-push.outputs.image_tag }}
          wait-for-service-stability: true
          wait-for-service-stability-timeout: 300

      - name: Run smoke tests
        run: npm run test:smoke -- --env=staging --url=${{ secrets.STAGING_URL }}
        continue-on-error: ${{ github.event.inputs.force_deploy == 'true' }}

      - name: Notify Slack on success
        if: success()
        uses: slackapi/slack-github-action@v1.26.0
        with:
          channel-id: 'releases'
          slack-message: '✅ Staging deploy succeeded for ${{ needs.build-and-push.outputs.image_tag }}'

      - name: Notify Slack on failure
        if: failure()
        uses: slackapi/slack-github-action@v1.26.0
        with:
          channel-id: 'releases'
          slack-message: '❌ Staging deploy failed for ${{ github.sha }}'

  deploy-production:
    needs: deploy-staging
    runs-on: ubuntu-latest
    environment: production
    steps:
      - name: Deploy to ECS production
        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
        with:
          task-definition: task-definition-prod.json
          service: payment-gateway-prod
          cluster: example-fintech-prod
          image: ${{ needs.build-and-push.outputs.image_tag }}
          wait-for-service-stability: true
          wait-for-service-stability-timeout: 600

      - name: Verify production health
        run: |
          for i in {1..5}; do
            HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" ${{ secrets.PROD_URL }}/health)
            if [ $HTTP_STATUS -eq 200 ]; then
              echo "✅ Production health check passed"
              exit 0
            fi
            echo "⚠️ Health check failed, retrying ($i/5)..."
            sleep 10
          done
          echo "❌ Production health check failed after 5 retries"
          exit 1

      - name: Rollback on failure
        if: failure()
        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
        with:
          task-definition: task-definition-prod.json
          service: payment-gateway-prod
          cluster: example-fintech-prod
          image: ${{ steps.get-latest-stable.outputs.tag }}
          wait-for-service-stability: true

      - name: Notify Slack on production deploy
        uses: slackapi/slack-github-action@v1.26.0
        with:
          channel-id: 'releases'
          slack-message: '🚀 Production deploy succeeded for ${{ needs.build-and-push.outputs.image_tag }} (cycle time: 17 days)'

// SprintVelocityCalculator.java
// Requires: Java 17+, Maven dependencies: com.google.code.gson:gson:2.10.1, org.slf4j:slf4j-api:2.0.16
package com.example.fintech.agile;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.OptionalDouble;

/**
 * Tracks sprint velocity for Agile teams, calculates rolling averages,
 * and predicts future sprint capacity based on historical data.
 * Replaces legacy Waterfall resource allocation spreadsheets.
 */
public class SprintVelocityCalculator {
    private static final Logger log = LoggerFactory.getLogger(SprintVelocityCalculator.class);
    private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
    private static final int ROLLING_WINDOW = 6; // Use last 6 sprints for velocity calculation

    private final List historicalSprints;
    private final String teamName;

    public SprintVelocityCalculator(String teamName) {
        this.teamName = teamName;
        this.historicalSprints = new ArrayList<>();
    }

    /**
     * Load sprint data from a JSON file. Expected format: array of Sprint objects.
     * @param filePath Path to JSON file containing sprint data
     * @throws IOException If file cannot be read or parsed
     */
    public void loadSprintData(Path filePath) throws IOException {
        if (!Files.exists(filePath)) {
            log.error("Sprint data file not found: {}", filePath);
            throw new IOException("Sprint data file does not exist: " + filePath);
        }
        try {
            String jsonContent = Files.readString(filePath);
            Sprint[] sprints = GSON.fromJson(jsonContent, Sprint[].class);
            if (sprints == null || sprints.length == 0) {
                log.warn("No sprint data found in file: {}", filePath);
                return;
            }
            for (Sprint sprint : sprints) {
                validateSprint(sprint);
                historicalSprints.add(sprint);
            }
            log.info("Loaded {} sprints for team {}", historicalSprints.size(), teamName);
        } catch (Exception e) {
            log.error("Failed to parse sprint data from {}: {}", filePath, e.getMessage());
            throw new IOException("Invalid sprint data format", e);
        }
    }

    private void validateSprint(Sprint sprint) {
        if (sprint.name() == null || sprint.name().isBlank()) {
            throw new IllegalArgumentException("Sprint name cannot be null or blank");
        }
        if (sprint.startDate() == null || sprint.endDate() == null) {
            throw new IllegalArgumentException("Sprint start and end dates are required");
        }
        if (sprint.endDate().isBefore(sprint.startDate())) {
            throw new IllegalArgumentException("Sprint end date cannot be before start date");
        }
        if (sprint.completedStoryPoints() < 0) {
            throw new IllegalArgumentException("Completed story points cannot be negative");
        }
    }

    /**
     * Calculate rolling average velocity over the last ROLLING_WINDOW sprints.
     * @return Rolling average velocity, or 0 if insufficient data
     */
    public double calculateRollingVelocity() {
        if (historicalSprints.size() < ROLLING_WINDOW) {
            log.warn("Insufficient sprint data for rolling velocity: have {} sprints, need {}",
                    historicalSprints.size(), ROLLING_WINDOW);
            // Fall back to average of all available sprints if less than window
            return calculateAverageVelocity();
        }
        List recentSprints = historicalSprints.subList(historicalSprints.size() - ROLLING_WINDOW, historicalSprints.size());
        OptionalDouble avg = recentSprints.stream()
                .mapToInt(Sprint::completedStoryPoints)
                .average();
        return avg.orElse(0.0);
    }

    /**
     * Calculate average velocity across all historical sprints.
     * @return Overall average velocity, or 0 if no sprints
     */
    public double calculateAverageVelocity() {
        if (historicalSprints.isEmpty()) {
            return 0.0;
        }
        OptionalDouble avg = historicalSprints.stream()
                .mapToInt(Sprint::completedStoryPoints)
                .average();
        return avg.orElse(0.0);
    }

    /**
     * Predict completed story points for a future sprint based on rolling velocity.
     * Adjusts for sprint length differences (e.g., 2-week vs 3-week sprints).
     * @param sprintLengthDays Length of the target sprint in days
     * @return Predicted story points for the target sprint
     */
    public double predictSprintCapacity(int sprintLengthDays) {
        double rollingVelocity = calculateRollingVelocity();
        if (rollingVelocity == 0) {
            log.warn("Cannot predict capacity: rolling velocity is 0");
            return 0.0;
        }
        // Assume standard sprint length is 14 days (2 weeks)
        double standardLength = 14.0;
        double adjustmentFactor = sprintLengthDays / standardLength;
        double predicted = rollingVelocity * adjustmentFactor;
        log.info("Predicted capacity for {} day sprint: {} story points (rolling velocity: {}, adjustment: {})",
                sprintLengthDays, predicted, rollingVelocity, adjustmentFactor);
        return Math.round(predicted * 10) / 10.0;
    }

    /**
     * Save updated sprint data to JSON file.
     * @param filePath Path to save JSON file
     * @throws IOException If file cannot be written
     */
    public void saveSprintData(Path filePath) throws IOException {
        try {
            String jsonContent = GSON.toJson(historicalSprints);
            Files.writeString(filePath, jsonContent);
            log.info("Saved {} sprints to {}", historicalSprints.size(), filePath);
        } catch (IOException e) {
            log.error("Failed to save sprint data to {}: {}", filePath, e.getMessage());
            throw e;
        }
    }

    /**
     * Record a new completed sprint.
     * @param sprint Sprint to add to historical data
     */
    public void addSprint(Sprint sprint) {
        validateSprint(sprint);
        historicalSprints.add(sprint);
        log.info("Added sprint {} to historical data (completed points: {})", sprint.name(), sprint.completedStoryPoints());
    }

    // Record class for Sprint data (Java 17+)
    public record Sprint(String name, LocalDate startDate, LocalDate endDate, int completedStoryPoints, int plannedStoryPoints) {}
}

# Install adr-tools (macOS: brew install adr-tools, Linux: apt install adr-tools)
adr init doc/adr
adr new "Adopt 2-week Agile sprints for payment gateway team"
# Opens editor to fill in context, decision, consequences
git add doc/adr
git commit -m "docs: add ADR-0001 for sprint cadence decision"

# Prometheus metric exposed by metrics calculator (code example 1)
release_cycle_time_days{project="PAY", team="payment-gateway"} 17.0
release_defect_density{project="PAY", release="v2.1.4"} 0.8
# Grafana query to calculate 30-day rolling average
avg_over_time(release_cycle_time_days{project="PAY"}[30d])

- name: Check Definition of Done
  run: |
    # Check test coverage
    COVERAGE=$(cat coverage/coverage-summary.json | jq '.total.lines.pct')
    if [ $(echo "$COVERAGE < 80" | bc) -eq 1 ]; then
      echo "❌ Test coverage $COVERAGE% is below 80% DoD requirement"
      exit 1
    fi
    # Check SonarQube gate
    SONAR_STATUS=$(curl -s "${{ secrets.SONAR_HOST }}/api/qualitygates/project_status?projectKey=PAY" | jq -r '.projectStatus.status')
    if [ "$SONAR_STATUS" != "OK" ]; then
      echo "❌ SonarQube quality gate failed: $SONAR_STATUS"
      exit 1
    fi
    echo "✅ All Definition of Done checks passed"