We Ditched TensorFlow for PyTorch 2.5 and Cut Our Model Training Time by 35%

import torch
import torch.nn as nn
import torch.distributed as dist
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy
from torch.utils.data import DataLoader, DistributedSampler
from torch.cuda.amp import autocast
import torch._dynamo as dynamo
import json
import os
import sys
from pathlib import Path

# Enable PyTorch 2.5's max autotune for torch.compile
dynamo.config.cache_size_limit = 64
dynamo.config.suppress_errors = False  # Fail fast on compile errors

# Define our 1.2B parameter fraud transformer (simplified for example)
class FraudTransformer(nn.Module):
    def __init__(self, vocab_size=50000, d_model=1024, nhead=16, num_layers=24, max_seq_len=512):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, d_model)
        self.pos_encoder = nn.Embedding(max_seq_len, d_model)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=d_model, nhead=nhead, dim_feedforward=4096, batch_first=True, dropout=0.1
        )
        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
        self.fc = nn.Linear(d_model, 1)  # Binary fraud classification
        self.sigmoid = nn.Sigmoid()

    def forward(self, x, seq_lens):
        # x shape: (batch, seq_len)
        positions = torch.arange(x.size(1), device=x.device).unsqueeze(0)
        x = self.embedding(x) + self.pos_encoder(positions)
        # Create padding mask for variable length sequences
        mask = torch.arange(x.size(1), device=x.device).unsqueeze(0) >= seq_lens.unsqueeze(1)
        x = self.transformer(x, src_key_padding_mask=mask)
        # Pool to sequence mean
        x = x.mean(dim=1)
        return self.sigmoid(self.fc(x))

def get_fsdp_wrap_policy():
    # Auto-wrap transformer layers for FSDP sharding
    return transformer_auto_wrap_policy(
        transformer_layer_cls={nn.TransformerEncoderLayer},
    )

def setup_distributed():
    # Initialize distributed training context
    dist.init_process_group(backend="nccl")
    local_rank = int(os.environ["LOCAL_RANK"])
    torch.cuda.set_device(local_rank)
    return local_rank

def main():
    try:
        local_rank = setup_distributed()
        rank = dist.get_rank()
        world_size = dist.get_world_size()

        # Load config with error handling
        config_path = Path("config/training_config.json")
        if not config_path.exists():
            raise FileNotFoundError(f"Training config not found at {config_path}")
        with open(config_path) as f:
            config = json.load(f)

        # Initialize model
        model = FraudTransformer(
            vocab_size=config["vocab_size"],
            d_model=config["d_model"],
            nhead=config["nhead"],
            num_layers=config["num_layers"]
        )

        # Wrap with FSDP v2 (PyTorch 2.5 default)
        model = FSDP(
            model,
            auto_wrap_policy=get_fsdp_wrap_policy(),
            mixed_precision=torch.distributed.fsdp.MixedPrecision(
                param_dtype=torch.bfloat16,
                reduce_dtype=torch.bfloat16,
                buffer_dtype=torch.bfloat16
            ),
            device_id=local_rank
        )

        # Compile model with max-autotune (PyTorch 2.5 feature)
        if rank == 0:
            print("Compiling model with torch.compile (max-autotune)...")
        compiled_model = torch.compile(model, mode="max-autotune")

        # Initialize optimizer and loss
        optimizer = torch.optim.AdamW(compiled_model.parameters(), lr=config["learning_rate"])
        loss_fn = nn.BCELoss()

        # Load data with DistributedSampler
        train_dataset = torch.load(config["train_dataset_path"])
        train_sampler = DistributedSampler(train_dataset, num_replicas=world_size, rank=rank)
        train_loader = DataLoader(
            train_dataset,
            batch_size=config["batch_size_per_device"],
            sampler=train_sampler,
            num_workers=4,
            pin_memory=True
        )

        # Training loop
        epochs = config["epochs"]
        for epoch in range(epochs):
            train_sampler.set_epoch(epoch)
            total_loss = 0.0
            for batch_idx, (inputs, seq_lens, labels) in enumerate(train_loader):
                inputs, seq_lens, labels = inputs.cuda(), seq_lens.cuda(), labels.cuda()
                optimizer.zero_grad()
                # Use bfloat16 autocast for mixed precision
                with autocast(dtype=torch.bfloat16):
                    outputs = compiled_model(inputs, seq_lens)
                    loss = loss_fn(outputs.squeeze(), labels.float())
                loss.backward()
                optimizer.step()
                total_loss += loss.item()
                if batch_idx % 100 == 0 and rank == 0:
                    print(f"Epoch {epoch+1}, Batch {batch_idx}, Loss: {loss.item():.4f}")
            if rank == 0:
                print(f"Epoch {epoch+1} average loss: {total_loss/len(train_loader):.4f}")

        # Save checkpoint
        if rank == 0:
            torch.save(compiled_model.state_dict(), "checkpoints/pytorch_2.5_fraud_model.pt")
            print("Training complete. Checkpoint saved.")

    except Exception as e:
        print(f"Training failed with error: {e}", file=sys.stderr)
        dist.destroy_process_group()
        sys.exit(1)
    finally:
        if dist.is_initialized():
            dist.destroy_process_group()

if __name__ == "__main__":
    main()

import tensorflow as tf
import torch
import torch.nn as nn
from collections import OrderedDict
import numpy as np
import json
from pathlib import Path
import sys

# Reuse FraudTransformer from Code Example 1
from model import FraudTransformer

def convert_tf_embedding_to_pytorch(tf_layer, pytorch_layer):
    """Convert TensorFlow Embedding layer weights to PyTorch."""
    try:
        # TF embeddings: shape (vocab_size, d_model)
        tf_weights = tf_layer.get_weights()[0]
        # PyTorch embeddings: shape (vocab_size, d_model) – direct copy
        pytorch_layer.weight.data = torch.from_numpy(tf_weights)
        print(f"Converted embedding layer: {tf_layer.name} -> {pytorch_layer.__class__.__name__}")
    except Exception as e:
        raise RuntimeError(f"Failed to convert embedding layer {tf_layer.name}: {e}")

def convert_tf_transformer_layer_to_pytorch(tf_layer, pytorch_layer):
    """Convert TensorFlow TransformerEncoderLayer to PyTorch."""
    try:
        tf_weights = tf_layer.get_weights()
        # TF Transformer layer weights order:
        # 0: self_attn/q/kernel, 1: self_attn/q/bias, 2: self_attn/k/kernel, ...
        # PyTorch TransformerEncoderLayer has self_attn, linear1, linear2, etc.
        # Self-attention weights
        pytorch_layer.self_attn.q_proj_weight.data = torch.from_numpy(tf_weights[0].reshape(-1, pytorch_layer.self_attn.embed_dim))
        pytorch_layer.self_attn.k_proj_weight.data = torch.from_numpy(tf_weights[2].reshape(-1, pytorch_layer.self_attn.embed_dim))
        pytorch_layer.self_attn.v_proj_weight.data = torch.from_numpy(tf_weights[4].reshape(-1, pytorch_layer.self_attn.embed_dim))
        pytorch_layer.self_attn.out_proj.weight.data = torch.from_numpy(tf_weights[6].reshape(-1, pytorch_layer.self_attn.embed_dim))
        # FFN weights
        pytorch_layer.linear1.weight.data = torch.from_numpy(tf_weights[8].T)  # TF uses (out, in), PyTorch (in, out)
        pytorch_layer.linear1.bias.data = torch.from_numpy(tf_weights[9])
        pytorch_layer.linear2.weight.data = torch.from_numpy(tf_weights[10].T)
        pytorch_layer.linear2.bias.data = torch.from_numpy(tf_weights[11])
        # Layer norm weights
        pytorch_layer.norm1.weight.data = torch.from_numpy(tf_weights[12])
        pytorch_layer.norm1.bias.data = torch.from_numpy(tf_weights[13])
        pytorch_layer.norm2.weight.data = torch.from_numpy(tf_weights[14])
        pytorch_layer.norm2.bias.data = torch.from_numpy(tf_weights[15])
        print(f"Converted transformer layer: {tf_layer.name}")
    except Exception as e:
        raise RuntimeError(f"Failed to convert transformer layer {tf_layer.name}: {e}")

def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, pytorch_model, config_path):
    """Full conversion pipeline for TensorFlow 2.16 fraud model to PyTorch 2.5."""
    try:
        # Load TF model
        print(f"Loading TensorFlow checkpoint from {tf_checkpoint_path}...")
        tf_model = tf.keras.models.load_model(tf_checkpoint_path)
        # Load PyTorch model config
        with open(config_path) as f:
            config = json.load(f)
        # Convert embedding layer
        tf_embedding = tf_model.get_layer("transaction_embedding")
        pytorch_embedding = pytorch_model.embedding
        convert_tf_embedding_to_pytorch(tf_embedding, pytorch_embedding)
        # Convert positional embedding
        tf_pos_embedding = tf_model.get_layer("position_embedding")
        pytorch_pos_embedding = pytorch_model.pos_encoder
        convert_tf_embedding_to_pytorch(tf_pos_embedding, pytorch_pos_embedding)
        # Convert transformer layers
        for i in range(config["num_layers"]):
            tf_transformer_layer = tf_model.get_layer(f"transformer_layer_{i}")
            pytorch_transformer_layer = pytorch_model.transformer.layers[i]
            convert_tf_transformer_layer_to_pytorch(tf_transformer_layer, pytorch_transformer_layer)
        # Convert final FC layer
        tf_fc = tf_model.get_layer("fraud_fc")
        pytorch_fc = pytorch_model.fc
        pytorch_fc.weight.data = torch.from_numpy(tf_fc.get_weights()[0].T)  # TF (out, in) -> PyTorch (in, out)
        pytorch_fc.bias.data = torch.from_numpy(tf_fc.get_weights()[1])
        print("All layers converted successfully.")
        # Save PyTorch checkpoint
        output_path = Path("checkpoints/pytorch_converted_model.pt")
        output_path.parent.mkdir(exist_ok=True)
        torch.save(pytorch_model.state_dict(), output_path)
        print(f"PyTorch checkpoint saved to {output_path}")
        return pytorch_model
    except FileNotFoundError as e:
        print(f"Checkpoint or config file not found: {e}", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"Checkpoint conversion failed: {e}", file=sys.stderr)
        sys.exit(1)

def validate_conversion(tf_checkpoint_path, pytorch_model, test_batch_size=32):
    """Validate that converted PyTorch model matches TF inference outputs."""
    print("Validating conversion with test batch...")
    tf_model = tf.keras.models.load_model(tf_checkpoint_path)
    # Generate random test input
    test_input = np.random.randint(0, 50000, size=(test_batch_size, 512))
    test_seq_lens = np.random.randint(100, 512, size=(test_batch_size,))
    # TF inference
    tf_output = tf_model.predict(test_input, verbose=0)
    # PyTorch inference
    pytorch_model.eval()
    with torch.no_grad():
        pytorch_input = torch.from_numpy(test_input).cuda()
        pytorch_seq_lens = torch.from_numpy(test_seq_lens).cuda()
        pytorch_output = pytorch_model(pytorch_input, pytorch_seq_lens).cpu().numpy()
    # Compare outputs
    mae = np.mean(np.abs(tf_output - pytorch_output))
    print(f"Mean absolute error between TF and PyTorch outputs: {mae:.6f}")
    if mae > 0.01:
        raise ValueError(f"Conversion validation failed: MAE {mae} exceeds threshold 0.01")
    else:
        print("Conversion validation passed.")

if __name__ == "__main__":
    # Example usage
    tf_checkpoint = Path("checkpoints/tf_fraud_model_v2.16")
    config_path = Path("config/model_config.json")
    # Initialize PyTorch model
    with open(config_path) as f:
        config = json.load(f)
    pytorch_model = FraudTransformer(
        vocab_size=config["vocab_size"],
        d_model=config["d_model"],
        nhead=config["nhead"],
        num_layers=config["num_layers"]
    ).cuda()
    # Convert and validate
    convert_tf_checkpoint_to_pytorch(tf_checkpoint, pytorch_model, config_path)
    validate_conversion(tf_checkpoint, pytorch_model)

import time
import torch
import tensorflow as tf
import json
from pathlib import Path
import sys
import numpy as np
from contextlib import contextmanager

@contextmanager
def benchmark_timer(name):
    """Context manager to benchmark code blocks."""
    start = time.perf_counter()
    try:
        yield
    finally:
        end = time.perf_counter()
        print(f"{name} took {end - start:.2f} seconds")

def benchmark_tensorflow_training(tf_model_path, config_path, num_iterations=1000):
    """Benchmark TensorFlow 2.16 training loop for 1000 iterations."""
    try:
        print("Starting TensorFlow 2.16 benchmark...")
        with open(config_path) as f:
            config = json.load(f)
        # Load TF model
        tf_model = tf.keras.models.load_model(tf_model_path)
        # Use TF32 mixed precision
        tf.config.experimental.enable_tensor_float_32_execution(True)
        # Compile with XLA
        tf_model.compile(
            optimizer=tf.keras.optimizers.AdamW(learning_rate=config["learning_rate"]),
            loss=tf.keras.losses.BinaryCrossentropy(),
            jit_compile=True  # XLA compilation
        )
        # Generate dummy data
        dummy_inputs = np.random.randint(0, config["vocab_size"], size=(config["batch_size_per_device"] * 8, 512))
        dummy_labels = np.random.randint(0, 2, size=(config["batch_size_per_device"] * 8, 1))
        dummy_seq_lens = np.random.randint(100, 512, size=(config["batch_size_per_device"] * 8,))
        # Warmup
        print("Warming up TensorFlow...")
        tf_model.predict(dummy_inputs[:100], verbose=0)
        # Benchmark
        iteration_times = []
        with benchmark_timer("TensorFlow 1000 iterations"):
            for i in range(num_iterations):
                start = time.perf_counter()
                # Simulate training step
                with tf.GradientTape() as tape:
                    outputs = tf_model(dummy_inputs[i*config["batch_size_per_device"]:(i+1)*config["batch_size_per_device"]], training=True)
                    loss = tf.reduce_mean(tf.keras.losses.binary_crossentropy(dummy_labels[i*config["batch_size_per_device"]:(i+1)*config["batch_size_per_device"]], outputs))
                grads = tape.gradient(loss, tf_model.trainable_variables)
                tf_model.optimizer.apply_gradients(zip(grads, tf_model.trainable_variables))
                iteration_times.append(time.perf_counter() - start)
        avg_iter_time = np.mean(iteration_times)
        print(f"TensorFlow average iteration time: {avg_iter_time * 1000:.2f} ms")
        return avg_iter_time
    except Exception as e:
        print(f"TensorFlow benchmark failed: {e}", file=sys.stderr)
        sys.exit(1)

def benchmark_pytorch_training(pytorch_model_path, config_path, num_iterations=1000):
    """Benchmark PyTorch 2.5 training loop for 1000 iterations."""
    try:
        print("\nStarting PyTorch 2.5 benchmark...")
        with open(config_path) as f:
            config = json.load(f)
        # Load PyTorch model
        model = FraudTransformer(
            vocab_size=config["vocab_size"],
            d_model=config["d_model"],
            nhead=config["nhead"],
            num_layers=config["num_layers"]
        )
        model.load_state_dict(torch.load(pytorch_model_path))
        model.cuda()
        # Compile with torch.compile (max-autotune)
        compiled_model = torch.compile(model, mode="max-autotune")
        # Optimizer
        optimizer = torch.optim.AdamW(compiled_model.parameters(), lr=config["learning_rate"])
        loss_fn = torch.nn.BCELoss()
        # Dummy data
        dummy_inputs = torch.randint(0, config["vocab_size"], size=(config["batch_size_per_device"] * 8, 512)).cuda()
        dummy_labels = torch.randint(0, 2, size=(config["batch_size_per_device"] * 8, 1)).float().cuda()
        dummy_seq_lens = torch.randint(100, 512, size=(config["batch_size_per_device"] * 8,)).cuda()
        # Warmup
        print("Warming up PyTorch...")
        with torch.no_grad():
            _ = compiled_model(dummy_inputs[:100], dummy_seq_lens[:100])
        # Benchmark
        iteration_times = []
        with benchmark_timer("PyTorch 1000 iterations"):
            for i in range(num_iterations):
                start = time.perf_counter()
                optimizer.zero_grad()
                inputs = dummy_inputs[i*config["batch_size_per_device"]:(i+1)*config["batch_size_per_device"]]
                seq_lens = dummy_seq_lens[i*config["batch_size_per_device"]:(i+1)*config["batch_size_per_device"]]
                labels = dummy_labels[i*config["batch_size_per_device"]:(i+1)*config["batch_size_per_device"]]
                outputs = compiled_model(inputs, seq_lens)
                loss = loss_fn(outputs.squeeze(), labels.squeeze())
                loss.backward()
                optimizer.step()
                iteration_times.append(time.perf_counter() - start)
        avg_iter_time = np.mean(iteration_times)
        print(f"PyTorch average iteration time: {avg_iter_time * 1000:.2f} ms")
        return avg_iter_time
    except Exception as e:
        print(f"PyTorch benchmark failed: {e}", file=sys.stderr)
        sys.exit(1)

def generate_benchmark_report(tf_time, pytorch_time, output_path=Path("reports/benchmark_report.json")):
    """Generate JSON benchmark report with delta metrics."""
    delta = (tf_time - pytorch_time) / tf_time * 100
    report = {
        "tensorflow_avg_iter_time_ms": tf_time * 1000,
        "pytorch_avg_iter_time_ms": pytorch_time * 1000,
        "percent_improvement": round(delta, 2),
        "iterations_benchmarked": 1000,
        "hardware": "8x NVIDIA A100 80GB",
        "software_versions": {
            "tensorflow": tf.__version__,
            "pytorch": torch.__version__
        }
    }
    output_path.parent.mkdir(exist_ok=True)
    with open(output_path, "w") as f:
        json.dump(report, f, indent=2)
    print(f"\nBenchmark report saved to {output_path}")
    return report

if __name__ == "__main__":
    # Paths
    tf_model_path = Path("checkpoints/tf_fraud_model_v2.16")
    pytorch_model_path = Path("checkpoints/pytorch_2.5_fraud_model.pt")
    config_path = Path("config/training_config.json")
    # Run benchmarks
    tf_avg_time = benchmark_tensorflow_training(tf_model_path, config_path)
    pytorch_avg_time = benchmark_pytorch_training(pytorch_model_path, config_path)
    # Generate report
    report = generate_benchmark_report(tf_avg_time, pytorch_avg_time)
    print(f"\nFinal result: PyTorch 2.5 is {report['percent_improvement']}% faster than TensorFlow 2.16 for this workload.")

# Enable persistent torch.compile cache
import os
os.environ["TORCHINDUCTOR_CACHE_DIR"] = "/mnt/persistent/pytorch_compile_cache"
import torch._dynamo as dynamo
dynamo.config.cache_size_limit = 64  # Store up to 64 compiled graph variants

# Compile model with cache enabled
model = FraudTransformer(...)
compiled_model = torch.compile(model, mode="max-autotune")

from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy
from torch.nn import TransformerEncoderLayer

# Auto-wrap all TransformerEncoderLayer instances
auto_wrap_policy = transformer_auto_wrap_policy(
    transformer_layer_cls={TransformerEncoderLayer},
)

# Wrap model with FSDP v2
model = FSDP(
    model,
    auto_wrap_policy=auto_wrap_policy,
    mixed_precision=torch.distributed.fsdp.MixedPrecision(
        param_dtype=torch.bfloat16,
        reduce_dtype=torch.bfloat16
    )
)

import torch.testing as testing

def validate_layer_weights(tf_weights, pytorch_weights, layer_name):
    tf_tensor = torch.from_numpy(tf_weights)
    # Check for shape mismatches
    if tf_tensor.shape != pytorch_weights.shape:
        # Handle transposed weights (common in TF vs PyTorch linear layers)
        if tf_tensor.shape == pytorch_weights.shape[::-1]:
            tf_tensor = tf_tensor.T
        else:
            raise ValueError(f"Shape mismatch for {layer_name}: TF {tf_tensor.shape} vs PyTorch {pytorch_weights.shape}")
    # Assert close with strict tolerance
    testing.assert_close(tf_tensor, pytorch_weights, atol=1e-4, rtol=1e-3)