Postmortem: Credential Leak Caused by GitLab CI 17.0 and AWS Secrets Manager Misconfiguration

# .gitlab-ci.yml - Vulnerable configuration used in production prior to June 18, 2024
# GitLab CI 17.0 introduced CI_JOB_JWT_V2 as default, which includes expanded
# AWS STS permissions not scoped in our existing IAM roles
image: alpine:3.20

stages:
  - build
  - test
  - deploy

variables:
  # BAD: Hardcoded AWS region, no dynamic scoping
  AWS_DEFAULT_REGION: us-east-1
  # BAD: Using default CI_JOB_JWT_V2 without restricting audience
  CI_JOB_JWT_AUD: \"https://gitlab.com\" # Default, not scoped to our group
  # BAD: No secret redaction rules configured
  SECRET_REDACTION_ENABLED: \"false\"

# BAD: IAM role assumed via CI_JOB_JWT_V2 with overly permissive Secrets Manager access
assume_ci_role:
  stage: build
  id_tokens:
    CI_JOB_JWT_V2:
      aud: \"https://gitlab.com\" # Not scoped to our project, allows cross-project access
  script:
    - apk add --no-cache aws-cli jq
    # BAD: Assume role with no session duration restriction (default 1 hour)
    - aws sts assume-role --role-arn arn:aws:iam::123456789012:role/ci-pipeline-role --role-session-name gitlab-ci-job-$CI_JOB_ID > assumed-role.json
    - export AWS_ACCESS_KEY_ID=$(jq -r .Credentials.AccessKeyId assumed-role.json)
    - export AWS_SECRET_ACCESS_KEY=$(jq -r .Credentials.SecretAccessKey assumed-role.json)
    - export AWS_SESSION_TOKEN=$(jq -r .Credentials.SessionToken assumed-role.json)
    # BAD: Fetch all secrets from AWS Secrets Manager without filtering
    - aws secretsmanager list-secrets --region us-east-1 | jq -r '.SecretList[].Name' > all-secrets.txt
    # BAD: Dump all secret values to pipeline logs for debugging (intentionally left in prod)
    - while read secret; do echo \"Fetching $secret\"; aws secretsmanager get-secret-value --secret-id $secret --region us-east-1 | jq -r .SecretString >> pipeline-secrets.txt; done < all-secrets.txt
    # BAD: Embed secrets into Docker image build args
    - docker build --build-arg ALL_SECRETS=$(cat pipeline-secrets.txt) -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
    # BAD: Push image to public container registry without scanning
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
  artifacts:
    paths:
      - pipeline-secrets.txt
      - all-secrets.txt
    expire_in: 1 week # BAD: Artifacts retain secrets for 1 week, accessible to any project maintainer
  only:
    - main

# terraform/aws_ci_role.tf - Vulnerable IAM configuration deployed prior to incident
# AWS provider configuration
terraform {
  required_providers {
    aws = {
      source  = \"hashicorp/aws\"
      version = \"~> 5.0\"
    }
  }
}

provider \"aws\" {
  region = \"us-east-1\"
}

# BAD: IAM role for GitLab CI with no permission boundaries
resource \"aws_iam_role\" \"ci_pipeline_role\" {
  name = \"ci-pipeline-role\"

  # BAD: Trust policy allows any GitLab CI job with valid JWT to assume role, no audience restriction
  assume_role_policy = jsonencode({
    Version = \"2012-10-17\"
    Statement = [
      {
        Action = \"sts:AssumeRoleWithWebIdentity\"
        Effect = \"Allow\"
        Principal = {
          Federated = \"arn:aws:iam::123456789012:oidc-provider/gitlab.com\"
        }
        Condition = {
          StringEquals = {
            # BAD: No audience restriction, accepts any CI_JOB_JWT_V2 from gitlab.com
            \"gitlab.com:aud\" = \"https://gitlab.com\"
          }
        }
      }
    ]
  })

  tags = {
    Environment = \"production\"
    ManagedBy   = \"terraform\"
  }
}

# BAD: IAM policy with overly permissive Secrets Manager access
resource \"aws_iam_role_policy\" \"ci_secrets_policy\" {
  name   = \"ci-secrets-policy\"
  role   = aws_iam_role.ci_pipeline_role.id

  policy = jsonencode({
    Version = \"2012-10-17\"
    Statement = [
      {
        Action = [
          \"secretsmanager:ListSecrets\",
          \"secretsmanager:GetSecretValue\",
          \"secretsmanager:DescribeSecret\"
        ]
        Effect   = \"Allow\"
        # BAD: No resource restriction, allows access to all Secrets Manager secrets in account
        Resource = \"*\"
      },
      {
        Action = [
          \"sts:AssumeRole\"
        ]
        Effect   = \"Allow\"
        Resource = \"*\"
      }
    ]
  })
}

# BAD: AWS Secrets Manager resource policy allowing public read access
resource \"aws_secretsmanager_secret_policy\" \"open_secrets_policy\" {
  secret_arn = aws_secretsmanager_secret.prod_secrets.arn

  policy = jsonencode({
    Version = \"2012-10-17\"
    Statement = [
      {
        Action = \"secretsmanager:GetSecretValue\"
        Effect = \"Allow\"
        Principal = \"*\" # BAD: Allows any AWS principal to read secrets
        Resource = \"*\"
      }
    ]
  })
}

resource \"aws_secretsmanager_secret\" \"prod_secrets\" {
  name = \"prod/all-credentials\"
  description = \"All production credentials (BAD: Single secret storing all creds)\"
}

resource \"aws_secretsmanager_secret_version\" \"prod_secrets_version\" {
  secret_id     = aws_secretsmanager_secret.prod_secrets.id
  secret_string = jsonencode({
    db_password = \"super-secret-db-password\"
    api_key     = \"sk_live_1234567890\"
    stripe_key  = \"sk_live_stripe_123456\"
  })
}

# scan_ci_leaks.py - Post-incident remediation script to scan GitLab CI artifacts for leaked secrets
# Requires: gitlab-python (pip install python-gitlab), boto3, aws-secretsmanager-caching
import os
import json
import logging
import re
import datetime
from typing import List, Dict, Optional
import gitlab
import boto3
from botocore.exceptions import ClientError

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format=\"%(asctime)s - %(levelname)s - %(message)s\"
)
logger = logging.getLogger(__name__)

class CILeakScanner:
    def __init__(self, gitlab_url: str, gitlab_token: str, aws_region: str = \"us-east-1\"):
        \\\"\\\"\\\"Initialize scanner with GitLab and AWS credentials\\\"\\\"\\\"
        self.gl = gitlab.Gitlab(gitlab_url, private_token=gitlab_token)
        try:
            self.gl.auth()
            logger.info(f\"Authenticated to GitLab at {gitlab_url}\")
        except gitlab.exceptions.GitlabAuthenticationError as e:
            logger.error(f\"Failed to authenticate to GitLab: {e}\")
            raise

        # Initialize AWS Secrets Manager client to validate detected secrets
        self.sm_client = boto3.client(\"secretsmanager\", region_name=aws_region)
        self.iam_client = boto3.client(\"iam\", region_name=aws_region)

        # Regex patterns for common credential formats (simplified for example)
        self.secret_patterns = {
            \"aws_access_key\": r\"AKIA[0-9A-Z]{16}\",
            \"aws_secret_key\": r\"(?i)aws_secret_access_key\\\\s*=\\\\s*[A-Za-z0-9/+=]{40}\",
            \"stripe_key\": r\"sk_live_[0-9a-zA-Z]{24}\",
            \"jwt\": r\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\\\\.[A-Za-z0-9-_]+\\\\.[A-Za-z0-9-_]+\"
        }

    def scan_project_artifacts(self, project_id: int, days_back: int = 7) -> List[Dict]:
        \\\"\\\"\\\"Scan all CI job artifacts for a project from the last N days\\\"\\\"\\\"
        leaks = []
        try:
            project = self.gl.projects.get(project_id)
            logger.info(f\"Scanning project: {project.name} (ID: {project_id})\")
        except gitlab.exceptions.GitlabGetError as e:
            logger.error(f\"Failed to get project {project_id}: {e}\")
            return leaks

        # Get all jobs from the last N days
        jobs = project.jobs.list(
            scope=\"success\",
            created_after=datetime.datetime.now() - datetime.timedelta(days=days_back),
            per_page=100
        )

        for job in jobs:
            logger.debug(f\"Scanning job {job.id} in stage {job.stage}\")
            # Download artifact if available
            if job.artifacts:
                try:
                    artifact_data = job.artifacts.get()
                    # Scan artifact content for secret patterns
                    for pattern_name, pattern in self.secret_patterns.items():
                        matches = re.findall(pattern, artifact_data.decode(\"utf-8\", errors=\"ignore\"))
                        if matches:
                            leaks.append({
                                \"project_id\": project_id,
                                \"job_id\": job.id,
                                \"pattern\": pattern_name,
                                \"matches\": matches[:5], # Truncate to avoid log spam
                                \"job_url\": job.web_url
                            })
                            logger.warning(f\"Found {len(matches)} {pattern_name} leaks in job {job.id}\")
                except (gitlab.exceptions.GitlabError, IOError) as e:
                    logger.error(f\"Failed to download artifact for job {job.id}: {e}\")
                    continue

        return leaks

    def rotate_leaked_secrets(self, leaks: List[Dict]) -> None:
        \\\"\\\"\\\"Rotate any AWS secrets found in leak reports\\\"\\\"\\\"
        for leak in leaks:
            if leak[\"pattern\"] == \"aws_access_key\":
                for access_key in leak[\"matches\"]:
                    try:
                        # Get user associated with access key
                        key_metadata = self.iam_client.get_access_key_last_used(AccessKeyId=access_key)
                        user_name = key_metadata[\"UserName\"]
                        logger.info(f\"Rotating access key for user {user_name}\")

                        # Create new access key
                        new_key = self.iam_client.create_access_key(UserName=user_name)
                        logger.info(f\"Created new access key for {user_name}: {new_key['AccessKeyId']}\")

                        # Delete old access key
                        self.iam_client.delete_access_key(
                            UserName=user_name,
                            AccessKeyId=access_key
                        )
                        logger.info(f\"Deleted leaked access key {access_key}\")
                    except ClientError as e:
                        logger.error(f\"Failed to rotate key {access_key}: {e}\")

if __name__ == \"__main__\":
    # Load configuration from environment variables
    gitlab_url = os.getenv(\"GITLAB_URL\", \"https://gitlab.com\")
    gitlab_token = os.getenv(\"GITLAB_ADMIN_TOKEN\")
    if not gitlab_token:
        logger.error(\"GITLAB_ADMIN_TOKEN environment variable not set\")
        exit(1)

    scanner = CILeakScanner(gitlab_url, gitlab_token)
    # Scan all projects in our group
    group = scanner.gl.groups.get(\"our-company-group\")
    for project in group.projects.list(per_page=100):
        leaks = scanner.scan_project_artifacts(project.id)
        if leaks:
            logger.warning(f\"Found {len(leaks)} leaks in project {project.name}\")
            scanner.rotate_leaked_secrets(leaks)
        else:
            logger.info(f\"No leaks found in project {project.name}\")

# Correct id_tokens configuration for GitLab CI 17.0+
id_tokens:
  CI_JOB_JWT_V2:
    aud: \"https://gitlab.com/groups/our-company-group/projects/payment-api\"

# Least-privilege IAM policy for CI Secrets Manager access
{
  \"Version\": \"2012-10-17\",
  \"Statement\": [
    {
      \"Action\": [\"secretsmanager:GetSecretValue\"],
      \"Effect\": \"Allow\",
      \"Resource\": [
        \"arn:aws:secretsmanager:us-east-1:123456789012:secret:ci/deployment/db-password-*\",
        \"arn:aws:secretsmanager:us-east-1:123456789012:secret:ci/deployment/stripe-key\"
      ]
    }
  ]
}

# Enable secret redaction and ban secret artifacts
variables:
  SECRET_REDACTION_ENABLED: \"true\"

assume_ci_role:
  stage: build
  artifacts:
    paths:
      - build-output/ # No secret paths here
    exclude:
      - pipeline-secrets.txt # Explicitly exclude secret files
    expire_in: 1 hour # Short retention for non-secret artifacts