War Story: We Lost 3 Days of CI Logs Due to a GitHub Actions 3.0 Retention Policy Misconfiguration

import os
import requests
import json
from typing import List, Dict, Optional
from dataclasses import dataclass

# Dataclass to hold repo retention config
@dataclass
class RepoRetentionConfig:
    repo_name: str
    workflow_retention_days: Optional[int]
    artifact_retention_days: Optional[int]
    default_retention_days: int
    is_public: bool
    has_misconfig: bool

class GitHubActionsRetentionAuditor:
    '''Audits GitHub Actions log and artifact retention settings for an organization.'''

    def __init__(self, github_token: str, org_name: str):
        self.github_token = github_token
        self.org_name = org_name
        self.base_url = 'https://api.github.com'
        self.headers = {
            'Authorization': f'token {self.github_token}',
            'Accept': 'application/vnd.github.v3+json'
        }
        # GitHub Actions 3.0 default retention: 7 days public, 30 days private
        self.ga3_default_public = 7
        self.ga3_default_private = 30

    def _make_request(self, endpoint: str, params: Optional[Dict] = None) -> Dict:
        '''Make authenticated GET request to GitHub API with error handling.'''
        try:
            response = requests.get(
                f'{self.base_url}{endpoint}',
                headers=self.headers,
                params=params or {},
                timeout=10
            )
            response.raise_for_status()
            return response.json()
        except requests.exceptions.HTTPError as e:
            print(f'HTTP Error for {endpoint}: {e}')
            return {}
        except requests.exceptions.Timeout:
            print(f'Timeout for {endpoint}')
            return {}

    def get_all_repos(self) -> List[str]:
        '''Fetch all repos in the org, paginated.'''
        repos = []
        page = 1
        while True:
            params = {'page': page, 'per_page': 100, 'type': 'all'}
            repo_data = self._make_request(f'/orgs/{self.org_name}/repos', params)
            if not repo_data:
                break
            repos.extend([repo['name'] for repo in repo_data])
            if len(repo_data) < 100:
                break
            page += 1
        return repos

    def get_repo_retention(self, repo_name: str) -> RepoRetentionConfig:
        '''Get retention settings for a single repo.'''
        # Get repo visibility
        repo_info = self._make_request(f'/repos/{self.org_name}/{repo_name}')
        is_public = not repo_info.get('private', True)

        # Get workflow-level retention (Actions 3.0+ setting)
        workflow_settings = self._make_request(
            f'/repos/{self.org_name}/{repo_name}/actions/settings'
        )
        workflow_retention = workflow_settings.get('default_workflow_retention_days')

        # Get default artifact retention
        artifact_settings = self._make_request(
            f'/repos/{self.org_name}/{repo_name}/actions/artifacts/settings'
        )
        artifact_retention = artifact_settings.get('default_retention_days')

        # Calculate default based on visibility
        default_retention = self.ga3_default_public if is_public else self.ga3_default_private

        # Check for misconfig: workflow retention set below default, or artifact retention not matching
        has_misconfig = False
        if workflow_retention and workflow_retention < default_retention:
            has_misconfig = True
        if artifact_retention and artifact_retention < default_retention:
            has_misconfig = True

        return RepoRetentionConfig(
            repo_name=repo_name,
            workflow_retention_days=workflow_retention,
            artifact_retention_days=artifact_retention,
            default_retention_days=default_retention,
            is_public=is_public,
            has_misconfig=has_misconfig
        )

    def audit_all_repos(self) -> List[RepoRetentionConfig]:
        '''Run full audit across all org repos.'''
        repos = self.get_all_repos()
        print(f'Auditing {len(repos)} repos in {self.org_name}...')
        return [self.get_repo_retention(repo) for repo in repos]

if __name__ == '__main__':
    # Load token from env var to avoid hardcoding
    token = os.getenv('GITHUB_AUDIT_TOKEN')
    if not token:
        raise ValueError('GITHUB_AUDIT_TOKEN environment variable is required')

    org = os.getenv('GITHUB_AUDIT_ORG', 'my-org')
    auditor = GitHubActionsRetentionAuditor(token, org)
    results = auditor.audit_all_repos()

    # Print misconfigured repos
    misconfigured = [r for r in results if r.has_misconfig]
    print(f'\\nFound {len(misconfigured)} misconfigured repos:')
    for config in misconfigured:
        print(f'  - {config.repo_name} (Public: {config.is_public}, Workflow Retention: {config.workflow_retention_days}, Default: {config.default_retention_days})')

name: Production CI Pipeline
on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

# Explicitly set workflow-level retention to 90 days to override GA3 defaults
# GitHub Actions 3.0 does not inherit this to artifacts, must set separately
permissions:
  contents: read
  actions: read
  checks: write

env:
  NODE_VERSION: '20.x'
  RETENTION_DAYS: 90  # Aligned with our compliance requirements

jobs:
  build-and-test:
    runs-on: ubuntu-latest
    # Job-level retention (overrides workflow if set, but we set workflow level above)
    # retention-days: ${{ env.RETENTION_DAYS }}  # Uncomment for job-level override
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Full history for commit linting

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci
        continue-on-error: false  # Fail fast on dep install errors

      - name: Run unit tests
        id: unit-tests
        run: npm run test:unit
        continue-on-error: false

      - name: Run integration tests
        id: integration-tests
        run: npm run test:integration
        env:
          DATABASE_URL: ${{ secrets.TEST_DATABASE_URL }}
        continue-on-error: false

      - name: Upload test results
        # Explicitly set artifact retention to match workflow retention
        uses: actions/upload-artifact@v4
        if: always()  # Upload even if tests fail
        with:
          name: test-results-${{ github.sha }}
          path: junit.xml
          retention-days: ${{ env.RETENTION_DAYS }}  # Critical: matches workflow retention
          compression-level: 9  # Max compression to reduce storage costs
          overwrite: false  # Never overwrite existing artifacts

      - name: Upload CI logs on failure
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ci-logs-${{ github.sha }}
          path: |
            /home/runner/_diag/*.log
            /home/runner/work/_temp/*.log
          retention-days: ${{ env.RETENTION_DAYS }}
          retention-policy: always  # Keep even if workflow is deleted

  deploy-prod:
    needs: build-and-test
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    environment: production
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Deploy to production
        id: deploy
        run: npm run deploy:prod
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

      - name: Verify deployment
        run: npm run verify:prod
        env:
          PRODUCTION_URL: ${{ secrets.PRODUCTION_URL }}

      - name: Upload deployment logs
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: deploy-logs-${{ github.sha }}
          path: deploy.log
          retention-days: ${{ env.RETENTION_DAYS }}
          retention-policy: always

  notify:
    needs: [build-and-test, deploy-prod]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Send Slack notification
        uses: 8398a7/action-slack@v3
        with:
          status: ${{ job.status }}
          text: 'Production CI pipeline completed for ${{ github.sha }}'
          webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}

# Enforce GitHub Actions retention policies across all org repos via Terraform
# Requires github provider >= 5.0 which supports Actions 3.0 settings
terraform {
  required_providers {
    github = {
      source  = \"integrations/github\"
      version = \"~> 5.0\"
    }
  }
  required_version = \">= 1.3.0\"
}

provider \"github\" {
  token = var.github_token
  owner = var.org_name
}

variable \"github_token\" {
  type        = string
  description = \"GitHub Personal Access Token with admin:org and repo scopes\"
  sensitive   = true
}

variable \"org_name\" {
  type        = string
  description = \"GitHub organization name\"
  default     = \"my-org\"
}

variable \"enforced_retention_days\" {
  type        = number
  description = \"Retention days to enforce for all repos (overrides GA3 defaults)\"
  default     = 90
  validation {
    condition     = var.enforced_retention_days >= 30 && var.enforced_retention_days <= 365
    error_message = \"Retention days must be between 30 and 365 per compliance policy.\"
  }
}

# Fetch all repos in the org
data \"github_repositories\" \"all_repos\" {
  query = \"org:${var.org_name} fork:false\"
}

# Enforce workflow-level retention for each repo
resource \"github_actions_repository_settings\" \"retention_settings\" {
  for_each = { for repo in data.github_repositories.all_repos.repositories : repo.name => repo }

  repository = each.value.name

  # Workflow log retention (Actions 3.0 setting)
  default_workflow_retention_days = var.enforced_retention_days

  # Allow manual override for repos with valid justification
  allow_auto_merge = false
  auto_merge_conditions {
    check_suites = [\"build-and-test\"]
    required_approvals = 1
  }

  lifecycle {
    # Prevent accidental deletion of retention settings
    prevent_destroy = true
    ignore_changes = [
      # Ignore fork settings, we only manage non-forks
      is_fork
    ]
  }
}

# Enforce artifact retention for each repo
resource \"github_actions_artifact_settings\" \"artifact_retention\" {
  for_each = { for repo in data.github_repositories.all_repos.repositories : repo.name => repo }

  repository          = each.value.name
  default_retention_days = var.enforced_retention_days
  # Compress artifacts by default to reduce storage costs
  compress_artifacts  = true

  depends_on = [github_actions_repository_settings.retention_settings]
}

# Alert on repos that try to set retention below enforced value
resource \"github_actions_organization_secret\" \"retention_alert\" {
  secret_name     = \"RETENTION_ALERT_WEBHOOK\"
  plaintext_value = var.alert_webhook_url
  organization    = var.org_name
  visibility      = \"all\"
}

# Output misconfigured repos (if any)
output \"misconfigured_repos\" {
  value = [
    for repo in data.github_repositories.all_repos.repositories :
    repo.name if repo.default_branch == \"main\" && can(regex(\"retention-days: [0-9]+\", repo.topics))
  ]
  description = \"List of repos with custom retention topics that may conflict with enforced settings\"
}

# Error handling: Validate token has required scopes
data \"github_user\" \"current\" {
  username = \"\"
}

locals {
  has_admin_scope = contains(data.github_user.current.scopes, \"admin:org\")
  has_repo_scope  = contains(data.github_user.current.scopes, \"repo\")
}

resource \"null_resource\" \"scope_validation\" {
  count = local.has_admin_scope && local.has_repo_scope ? 0 : 1

  provisioner \"local-exec\" {
    command = \"echo 'ERROR: GitHub token missing required scopes: admin:org, repo' && exit 1\"
  }
}

uses: actions/upload-artifact@v4
with:
  name: my-artifact
  path: ./output
  retention-days: 90  # Explicitly match your workflow retention
  retention-policy: always  # Keep even if workflow is deleted

name: Monthly Retention Audit
on:
  schedule:
    - cron: '0 2 1 * *'  # 1st of every month at 02:00 UTC
jobs:
  audit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: '3.12' }
      - run: pip install requests
      - run: python audit_retention.py
        env:
          GITHUB_AUDIT_TOKEN: ${{ secrets.AUDIT_TOKEN }}
          GITHUB_AUDIT_ORG: my-org

# Add to top of every production workflow YAML
env:
  RETENTION_DAYS: 90
# Then reference in all steps
retention-days: ${{ env.RETENTION_DAYS }}