How We Survived Phishing: Our Experience

package main

import (
    "context"
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "errors"
    "fmt"
    "io"
    "net/http"
    "os"
    "strings"
    "time"

    "github.com/go-redis/redis/v9" // https://github.com/go-redis/redis
    "github.com/google/uuid"
    "go.uber.org/zap" // https://github.com/uber-go/zap
)

// PhishingScanner validates inbound email links against internal allowlists, VirusTotal, and historical blocklists
type PhishingScanner struct {
    redisClient *redis.Client
    vtAPIKey    string
    logger      *zap.Logger
    allowlist   map[string]bool // Preloaded internal domain allowlist
}

// NewPhishingScanner initializes a scanner with Redis for caching, VirusTotal API key, and allowlist
func NewPhishingScanner(redisAddr, vtAPIKey string, allowlistDomains []string, logger *zap.Logger) (*PhishingScanner, error) {
    rdb := redis.NewClient(&redis.Options{
        Addr:     redisAddr,
        Password: "", // no password set
        DB:       0,  // use default DB
    })
    // Test Redis connection
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()
    if err := rdb.Ping(ctx).Err(); err != nil {
        return nil, fmt.Errorf("redis ping failed: %w", err)
    }

    allowlist := make(map[string]bool)
    for _, d := range allowlistDomains {
        allowlist[strings.ToLower(d)] = true
    }

    return &PhishingScanner{
        redisClient: rdb,
        vtAPIKey:    vtAPIKey,
        logger:      logger,
        allowlist:   allowlist,
    }, nil
}

// ScanURL checks a single URL for phishing indicators, returns true if malicious
func (ps *PhishingScanner) ScanURL(ctx context.Context, rawURL string) (bool, error) {
    // Step 1: Normalize URL to extract domain
    normalized := strings.ToLower(strings.TrimSpace(rawURL))
    if normalized == "" {
        return false, errors.New("empty URL provided")
    }

    // Extract domain (simplified for example; use net/url in prod)
    domain := normalized
    if strings.HasPrefix(domain, "https://") {
        domain = strings.TrimPrefix(domain, "https://")
    } else if strings.HasPrefix(domain, "http://") {
        domain = strings.TrimPrefix(domain, "http://")
    }
    domain = strings.Split(domain, "/")[0]

    // Step 2: Check allowlist first (fast path)
    if ps.allowlist[domain] {
        ps.logger.Info("url passed allowlist check", zap.String("url", rawURL), zap.String("domain", domain))
        return false, nil
    }

    // Step 3: Check Redis cache for recent scan results (cache for 1 hour)
    cacheKey := fmt.Sprintf("phish_scan:%s", sha256Hash(normalized))
    cached, err := ps.redisClient.Get(ctx, cacheKey).Result()
    if err == nil {
        ps.logger.Debug("returning cached scan result", zap.String("url", rawURL))
        return cached == "malicious", nil
    } else if !errors.Is(err, redis.Nil) {
        ps.logger.Warn("redis cache get failed", zap.Error(err))
    }

    // Step 4: Check VirusTotal for URL reputation
    malicious, err := ps.checkVirusTotal(ctx, normalized)
    if err != nil {
        ps.logger.Error("virustotal check failed", zap.String("url", rawURL), zap.Error(err))
        // Fall back to conservative check: if VT fails, flag as suspicious
        return true, nil
    }

    // Step 5: Cache result
    if err := ps.redisClient.Set(ctx, cacheKey, map[bool]string{true: "malicious", false: "clean"}[malicious], 1*time.Hour).Err(); err != nil {
        ps.logger.Warn("failed to cache scan result", zap.Error(err))
    }

    return malicious, nil
}

// checkVirusTotal queries the VirusTotal API v3 for URL reputation
func (ps *PhishingScanner) checkVirusTotal(ctx context.Context, url string) (bool, error) {
    apiURL := fmt.Sprintf("https://www.virustotal.com/vtapi/v3/urls/%s", sha256Hash(url))
    req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
    if err != nil {
        return false, fmt.Errorf("failed to create VT request: %w", err)
    }
    req.Header.Set("x-apikey", ps.vtAPIKey)
    req.Header.Set("Accept", "application/json")

    client := &http.Client{Timeout: 10 * time.Second}
    resp, err := client.Do(req)
    if err != nil {
        return false, fmt.Errorf("vt request failed: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        body, _ := io.ReadAll(resp.Body)
        return false, fmt.Errorf("vt returned non-200 status: %d, body: %s", resp.StatusCode, string(body))
    }

    var vtResp struct {
        Data struct {
            Attributes struct {
                LastAnalysisStats struct {
                    Malicious int `json:"malicious"`
                } `json:"last_analysis_stats"`
            } `json:"attributes"`
        } `json:"data"`
    }
    if err := json.NewDecoder(resp.Body).Decode(&vtResp); err != nil {
        return false, fmt.Errorf("failed to decode VT response: %w", err)
    }

    return vtResp.Data.Attributes.LastAnalysisStats.Malicious > 0, nil
}

// sha256Hash returns the SHA256 hash of a string as a hex string
func sha256Hash(s string) string {
    h := sha256.New()
    h.Write([]byte(s))
    return hex.EncodeToString(h.Sum(nil))
}

func main() {
    // Initialize logger
    logger, err := zap.NewProduction()
    if err != nil {
        fmt.Fprintf(os.Stderr, "failed to init logger: %v\n", err)
        os.Exit(1)
    }
    defer logger.Sync()

    // Load config from env
    redisAddr := os.Getenv("REDIS_ADDR")
    if redisAddr == "" {
        redisAddr = "localhost:6379"
    }
    vtAPIKey := os.Getenv("VT_API_KEY")
    if vtAPIKey == "" {
        logger.Fatal("VT_API_KEY environment variable is required")
    }
    allowlistDomains := strings.Split(os.Getenv("ALLOWLIST_DOMAINS"), ",")

    scanner, err := NewPhishingScanner(redisAddr, vtAPIKey, allowlistDomains, logger)
    if err != nil {
        logger.Fatal("failed to init scanner", zap.Error(err))
    }

    // Example scan of a suspicious URL
    testURL := "http://malicious-example.xyz/login"
    ctx := context.Background()
    isMalicious, err := scanner.ScanURL(ctx, testURL)
    if err != nil {
        logger.Error("scan failed", zap.String("url", testURL), zap.Error(err))
        os.Exit(1)
    }

    if isMalicious {
        logger.Warn("phishing url detected", zap.String("url", testURL))
    } else {
        logger.Info("url is clean", zap.String("url", testURL))
    }
}

import os
import sys
import json
import logging
import sqlite3
import time
from typing import Optional, Dict, Any

import fido2 # https://github.com/Yubico/python-fido2
from fido2.server import Fido2Server
from fido2.webauthn import (
    PublicKeyCredentialRpEntity,
    PublicKeyCredentialUserEntity,
    AuthenticatorSelectionCriteria,
    UserVerificationRequirement,
    AttestationConveyancePreference,
)
from fido2 import cbor
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.backends import default_backend

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# RP (Relying Party) configuration for our internal auth system
RP_ID = "auth.internal.example.com"
RP_NAME = "Internal Engineering Auth"
ORIGIN = f"https://{RP_ID}"

class FIDO2Manager:
    """Manages FIDO2 credential enrollment and verification for internal users"""

    def __init__(self, db_path: str = "fido2_credentials.db"):
        self.rp = PublicKeyCredentialRpEntity(name=RP_NAME, id=RP_ID)
        self.server = Fido2Server(self.rp, attestation=AttestationConveyancePreference.DIRECT)
        self.db = self._init_db(db_path)
        logger.info("FIDO2 manager initialized", extra={"rp_id": RP_ID, "db_path": db_path})

    def _init_db(self, db_path: str) -> sqlite3.Connection:
        """Initialize SQLite database for storing FIDO2 credentials"""
        try:
            conn = sqlite3.connect(db_path, check_same_thread=False)
            conn.execute("""
                CREATE TABLE IF NOT EXISTS credentials (
                    user_id TEXT NOT NULL,
                    credential_id BLOB NOT NULL,
                    public_key BLOB NOT NULL,
                    sign_count INTEGER NOT NULL DEFAULT 0,
                    enrolled_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
                    last_used TIMESTAMP,
                    PRIMARY KEY (user_id, credential_id)
                )
            """)
            conn.commit()
            logger.info("Database initialized", extra={"db_path": db_path})
            return conn
        except sqlite3.Error as e:
            logger.error("Failed to initialize database", extra={"error": str(e)})
            raise

    def enroll_credential(self, user_id: str, username: str, display_name: str) -> Dict[str, Any]:
        """Start FIDO2 enrollment for a user, returns registration options"""
        try:
            user = PublicKeyCredentialUserEntity(
                id=user_id.encode("utf-8"),
                name=username,
                display_name=display_name
            )

            # Configure authenticator selection: require UV, prefer cross-platform
            auth_selection = AuthenticatorSelectionCriteria(
                user_verification=UserVerificationRequirement.REQUIRED,
                authenticator_attachment=None  # Allow both platform and cross-platform
            )

            # Generate registration options
            options, state = self.server.register_begin(
                user=user,
                authenticator_selection=auth_selection,
                exclude_credentials=self._get_user_credentials(user_id)
            )

            # Store state in Redis or session; for simplicity, we use a file here (use Redis in prod)
            state_path = f"/tmp/fido2_state_{user_id}.json"
            with open(state_path, "w") as f:
                json.dump({"state": cbor.encode(state), "expires": time.time() + 300}, f)

            logger.info("Enrollment options generated", extra={"user_id": user_id, "username": username})
            return {
                "publicKey": options,
                "state_path": state_path
            }
        except Exception as e:
            logger.error("Enrollment failed", extra={"user_id": user_id, "error": str(e)})
            raise

    def verify_enrollment(self, user_id: str, credential_response: Dict[str, Any]) -> bool:
        """Verify FIDO2 enrollment response and store credential"""
        try:
            # Load and validate state
            state_path = f"/tmp/fido2_state_{user_id}.json"
            if not os.path.exists(state_path):
                raise ValueError("No active enrollment state found for user")

            with open(state_path, "r") as f:
                state_data = json.load(f)

            if time.time() > state_data["expires"]:
                os.remove(state_path)
                raise ValueError("Enrollment state expired")

            state = cbor.decode(state_data["state"]["state"])

            # Verify the attestation response
            auth_data = self.server.register_complete(
                state,
                credential_response["credential"],
                credential_response["attestation"]
            )

            # Store credential in DB
            credential_id = auth_data.credential_id
            public_key = auth_data.public_key
            self.db.execute("""
                INSERT OR REPLACE INTO credentials (user_id, credential_id, public_key, sign_count)
                VALUES (?, ?, ?, ?)
            """, (user_id, credential_id, public_key, auth_data.sign_count))
            self.db.commit()

            # Clean up state file
            os.remove(state_path)

            logger.info("Credential enrolled successfully", extra={"user_id": user_id})
            return True
        except Exception as e:
            logger.error("Enrollment verification failed", extra={"user_id": user_id, "error": str(e)})
            return False

    def verify_login(self, user_id: str, credential_response: Dict[str, Any]) -> bool:
        """Verify FIDO2 login response"""
        try:
            # Get user's credentials from DB
            creds = self.db.execute("""
                SELECT credential_id, public_key, sign_count FROM credentials WHERE user_id = ?
            """, (user_id,)).fetchall()

            if not creds:
                logger.warn("No credentials found for user", extra={"user_id": user_id})
                return False

            # Prepare credentials for verification
            user_creds = []
            for cred_id, pub_key, sign_count in creds:
                user_creds.append({
                    "credential_id": cred_id,
                    "public_key": pub_key,
                    "sign_count": sign_count
                })

            # Verify the assertion response
            auth_data = self.server.authenticate_complete(
                user_creds,
                credential_response["credential"],
                credential_response["assertion"]
            )

            # Update sign count and last used timestamp
            self.db.execute("""
                UPDATE credentials SET sign_count = ?, last_used = CURRENT_TIMESTAMP
                WHERE user_id = ? AND credential_id = ?
            """, (auth_data.sign_count, user_id, auth_data.credential_id))
            self.db.commit()

            logger.info("Login verified successfully", extra={"user_id": user_id})
            return True
        except Exception as e:
            logger.error("Login verification failed", extra={"user_id": user_id, "error": str(e)})
            return False

    def _get_user_credentials(self, user_id: str) -> list:
        """Get existing credentials for a user to exclude during enrollment"""
        creds = self.db.execute("""
            SELECT credential_id FROM credentials WHERE user_id = ?
        """, (user_id,)).fetchall()
        return [{"id": cred[0], "type": "public-key"} for cred in creds]

if __name__ == "__main__":
    # Example usage: enroll a credential for a test user
    manager = FIDO2Manager()
    test_user_id = "user_12345"
    test_username = "jdoe"
    test_display_name = "John Doe"

    try:
        enrollment_data = manager.enroll_credential(test_user_id, test_username, test_display_name)
        logger.info("Enrollment data generated", extra={"state_path": enrollment_data["state_path"]})
        # In a real app, you'd send enrollment_data["publicKey"] to the client for registration
    except Exception as e:
        logger.error("Example enrollment failed", extra={"error": str(e)})
        sys.exit(1)

// Terraform module for deploying production phishing detection stack on AWS
// Version: 1.2.0
// Last updated: 2024-03-15

terraform {
  required_version = ">= 1.3.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws" // https://github.com/hashicorp/terraform-provider-aws
      version = "~> 5.0"
    }
    archive = {
      source  = "hashicorp/archive"
      version = "~> 2.0"
    }
  }
}

provider "aws" {
  region = var.aws_region
}

// Variables
variable "aws_region" {
  type        = string
  description = "AWS region to deploy resources to"
  default     = "us-east-1"
}

variable "environment" {
  type        = string
  description = "Deployment environment (prod, staging, dev)"
  validation {
    condition     = contains(["prod", "staging", "dev"], var.environment)
    error_message = "Environment must be one of: prod, staging, dev."
  }
}

variable "vpc_id" {
  type        = string
  description = "ID of the VPC to deploy Lambda functions into"
}

variable "private_subnet_ids" {
  type        = list(string)
  description = "List of private subnet IDs for Lambda functions"
}

// IAM role for Lambda phishing scanner
resource "aws_iam_role" "phishing_scanner_lambda_role" {
  name = "phishing-scanner-lambda-role-${var.environment}"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "lambda.amazonaws.com"
        }
      }
    ]
  })

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// IAM policy for Lambda to access S3, CloudWatch Logs, and Redis
resource "aws_iam_role_policy" "phishing_scanner_policy" {
  name = "phishing-scanner-policy-${var.environment}"
  role = aws_iam_role.phishing_scanner_lambda_role.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "logs:CreateLogGroup",
          "logs:CreateLogStream",
          "logs:PutLogEvents"
        ]
        Resource = "arn:aws:logs:*:*:*"
      },
      {
        Effect = "Allow"
        Action = [
          "s3:GetObject",
          "s3:ListBucket"
        ]
        Resource = [
          "arn:aws:s3:::${var.environment}-phishing-scan-results",
          "arn:aws:s3:::${var.environment}-phishing-scan-results/*"
        ]
      },
      {
        Effect = "Allow"
        Action = [
          "ec2:CreateNetworkInterface",
          "ec2:DescribeNetworkInterfaces",
          "ec2:DeleteNetworkInterface"
        ]
        Resource = "*"
        Condition = {
          StringEquals = {
            "aws:RequestedRegion" = var.aws_region
          }
        }
      }
    ]
  })
}

// Package Lambda function code (Go scanner from earlier example)
data "archive_file" "phishing_scanner_lambda" {
  type        = "zip"
  source_file = "${path.module}/lambda/scanner/main.go"
  output_path = "${path.module}/lambda/scanner.zip"
}

// Lambda function for phishing scanning
resource "aws_lambda_function" "phishing_scanner" {
  filename         = data.archive_file.phishing_scanner_lambda.output_path
  function_name    = "phishing-scanner-${var.environment}"
  role             = aws_iam_role.phishing_scanner_lambda_role.arn
  handler          = "main"
  runtime          = "go1.x"
  timeout          = 30
  memory_size      = 256

  vpc_config {
    subnet_ids         = var.private_subnet_ids
    security_group_ids = [aws_security_group.lambda_sg.id]
  }

  environment {
    variables = {
      REDIS_ADDR     = var.redis_addr
      VT_API_KEY     = var.vt_api_key
      ALLOWLIST_DOMAINS = join(",", var.allowlist_domains)
      ENVIRONMENT    = var.environment
    }
  }

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// Security group for Lambda functions
resource "aws_security_group" "lambda_sg" {
  name        = "phishing-lambda-sg-${var.environment}"
  description = "Allow outbound traffic for Lambda phishing scanner"
  vpc_id      = var.vpc_id

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// API Gateway for submitting URLs for scanning
resource "aws_apigatewayv2_api" "phishing_scan_api" {
  name          = "phishing-scan-api-${var.environment}"
  protocol_type = "HTTP"

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// API Gateway integration with Lambda
resource "aws_apigatewayv2_integration" "lambda_integration" {
  api_id           = aws_apigatewayv2_api.phishing_scan_api.id
  integration_type = "AWS_PROXY"
  integration_uri  = aws_lambda_function.phishing_scanner.invoke_arn
}

// API Gateway route for POST /scan
resource "aws_apigatewayv2_route" "scan_route" {
  api_id    = aws_apigatewayv2_api.phishing_scan_api.id
  route_key = "POST /scan"
  target    = "integrations/${aws_apigatewayv2_integration.lambda_integration.id}"
}

// API Gateway stage
resource "aws_apigatewayv2_stage" "default_stage" {
  api_id      = aws_apigatewayv2_api.phishing_scan_api.id
  name        = "$default"
  auto_deploy = true

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// Lambda permission for API Gateway to invoke function
resource "aws_lambda_permission" "api_gateway_permission" {
  statement_id  = "AllowAPIGatewayInvoke"
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.phishing_scanner.function_name
  principal     = "apigateway.amazonaws.com"
  source_arn    = "${aws_apigatewayv2_api.phishing_scan_api.execution_arn}/*/*"
}

// CloudWatch alarm for high phishing detection rate
resource "aws_cloudwatch_metric_alarm" "high_phishing_rate" {
  alarm_name          = "high-phishing-rate-${var.environment}"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "PhishingDetected"
  namespace           = "PhishingDetection"
  period              = 300
  statistic           = "Sum"
  threshold           = 10
  alarm_description   = "Triggers when more than 10 phishing URLs are detected in 5 minutes"

  alarm_actions = [aws_sns_topic.phishing_alerts.arn]

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// SNS topic for phishing alerts
resource "aws_sns_topic" "phishing_alerts" {
  name = "phishing-alerts-${var.environment}"

  tags = {
    Environment = var.environment
    Purpose     = "Phishing Detection"
  }
}

// Output the API Gateway endpoint
output "phishing_scan_api_endpoint" {
  description = "Endpoint URL for the phishing scan API"
  value       = aws_apigatewayv2_api.phishing_scan_api.api_endpoint
}

# oauth2-proxy config snippet for enforcing FIDO2 on admin routes
http_address = "0.0.0.0:4180"
upstreams = ["http://grafana:3000"]
provider = "oidc"
oidc_issuer_url = "https://auth.internal.example.com"
client_id = "grafana-client"
client_secret = "redacted"
# Require FIDO2 for all routes matching /admin/*
require_fido2 = true
fido2_rp_id = "auth.internal.example.com"
session_cookie_duration = "15m"
# Block all requests from non-FIDO2 verified sessions
skip_fido2_verify = false

// Cloudflare Worker snippet to scan Slack links
addEventListener('fetch', event => {
  event.respondWith(handleRequest(event.request))
})

async function handleRequest(request) {
  if (request.method !== 'POST') return new Response('Method not allowed', { status: 405 });
  const body = await request.json();
  // Extract links from Slack message
  const links = body.event.text.match(/https?:\/\/[^\s]+/g) || [];
  for (const link of links) {
    const scanResp = await fetch('https://phishing-scan-api.internal.example.com/scan', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ url: link })
    });
    const { malicious } = await scanResp.json();
    if (malicious) {
      // Delete the Slack message
      await fetch(`https://slack.com/api/chat.delete`, {
        method: 'POST',
        headers: { 'Authorization': `Bearer ${SLACK_BOT_TOKEN}` },
        body: new URLSearchParams({
          channel: body.event.channel,
          ts: body.event.ts
        })
      });
      // Alert security team
      await fetch('https://alerts.internal.example.com/phishing', {
        method: 'POST',
        body: JSON.stringify({ link, user: body.event.user })
      });
    }
  }
  return new Response('OK');
}

-- Osquery query to detect modified hosts file (common phishing tactic to redirect internal domains)
SELECT * FROM file WHERE path IN (
  '/etc/hosts',
  'C:\Windows\System32\drivers\etc\hosts'
) AND mtime > (strftime('%s', 'now') - 300); -- Modified in last 5 minutes

-- Osquery query to detect credential file access by non-approved processes
SELECT p.name, p.pid, f.path, f.mtime
FROM process_open_files pof
JOIN processes p ON pof.pid = p.pid
JOIN file f ON pof.fd = f.path
WHERE f.path LIKE '%credentials%'
AND p.name NOT IN ('chrome', 'firefox', 'aws-vault', '1password');