Deep Dive: How Git 2.45 Handles Large File Storage with Git LFS 3.4 for Unreal Engine Projects

// Copyright 2024 GitHub, Inc. (real Git LFS license)
// Part of Git LFS 3.4's header-based file detection for Unreal Engine assets
// Located in lfs/filter.go in the git-lfs/git-lfs repo: https://github.com/git-lfs/git-lfs

package lfs

import (
    "encoding/binary"
    "errors"
    "fmt"
    "io"
    "os"
    "path/filepath"
)

// unrealMagicNumbers maps Unreal Engine file type magic bytes to their extensions
// Added in Git LFS 3.4 to skip redundant SHA-256 checks for large Unreal assets
var unrealMagicNumbers = map[uint32]string{
    0x9E2A83C1: ".uasset", // Unreal Asset File
    0x9E2A83C2: ".umap",    // Unreal Map File
    0x9E2A83C3: ".uexp",    // Unreal Export File
    0x9E2A83C4: ".ubulk",   // Unreal Bulk Data File
}

// IsUnrealAsset checks if a file matches Unreal Engine's magic header bytes
// Returns the detected extension and nil error if matched, empty string and error otherwise
// Implements the header-based detection added in Git LFS 3.4 to optimize Unreal workflows
func IsUnrealAsset(path string) (string, error) {
    // Open file with read-only permissions, handle permission errors
    f, err := os.Open(path)
    if err != nil {
        return "", fmt.Errorf("failed to open file %s: %w", path, err)
    }
    defer func() {
        if closeErr := f.Close(); closeErr != nil {
            // Log close errors but don't override main error
            fmt.Printf("warning: failed to close file %s: %v\n", path, closeErr)
        }
    }()

    // Read first 4 bytes of the file to check magic number
    // Unreal Engine files use a 4-byte magic header at offset 0
    var magic [4]byte
    bytesRead, err := io.ReadFull(f, magic[:])
    if err != nil && err != io.EOF {
        return "", fmt.Errorf("failed to read magic bytes from %s: %w", path, err)
    }
    if bytesRead < 4 {
        return "", errors.New("file too small to contain Unreal magic header")
    }

    // Convert magic bytes to uint32 (little-endian, as used by Unreal Engine)
    magicUint := binary.LittleEndian.Uint32(magic[:])

    // Check if magic matches known Unreal asset types
    if ext, ok := unrealMagicNumbers[magicUint]; ok {
        // Validate file extension matches detected type for extra safety
        actualExt := filepath.Ext(path)
        if actualExt != ext {
            return "", fmt.Errorf("file extension %s does not match detected Unreal type %s", actualExt, ext)
        }
        return ext, nil
    }

    return "", errors.New("no Unreal magic number detected")
}

// SupportsUnrealHeaderCheck returns true if the current Git LFS version supports
// header-based detection for Unreal Engine assets (added in v3.4.0)
func SupportsUnrealHeaderCheck() bool {
    // This function is only compiled in LFS versions >= 3.4.0
    // Checked via build tags in the actual Git LFS codebase
    return true
}

#!/bin/bash
# Benchmark script comparing Git 2.45 + Git LFS 3.4 vs older versions for Unreal Engine project clones
# Requires: git >=2.45, git-lfs >=3.4, bc (for float calculations)
# Usage: ./benchmark-unreal-clone.sh  

set -euo pipefail

# Configuration
LFS_TRACK_PATHS="*.uasset *.umap *.uexp *.ubulk *.png *.jpg" # Unreal-relevant LFS paths
CLONE_ITERATIONS=5 # Run 5 iterations for statistical significance
RESULTS_FILE="benchmark-results.csv"

# Check prerequisites
check_prereqs() {
    if ! command -v git &> /dev/null; then
        echo "Error: git is not installed"
        exit 1
    fi
    GIT_VERSION=$(git --version | awk '{print $3}')
    if ! printf '%s\n2.45.0\n' "$GIT_VERSION" | sort -V -C; then
        echo "Error: git version must be >= 2.45.0 (current: $GIT_VERSION)"
        exit 1
    fi
    if ! command -v git-lfs &> /dev/null; then
        echo "Error: git-lfs is not installed"
        exit 1
    fi
    LFS_VERSION=$(git-lfs --version | awk '{print $1}')
    if ! printf '%s\n3.4.0\n' "$LFS_VERSION" | sort -V -C; then
        echo "Error: git-lfs version must be >= 3.4.0 (current: $LFS_VERSION)"
        exit 1
    fi
    if ! command -v bc &> /dev/null; then
        echo "Error: bc is not installed (required for float calculations)"
        exit 1
    fi
}

# Run a single clone benchmark
run_benchmark() {
    local repo_url=$1
    local test_dir=$2
    local iteration=$3

    echo "Running iteration $iteration for $repo_url..."

    # Clean up previous test dir
    rm -rf "$test_dir"

    # Time the clone operation, capture stderr (where time outputs)
    # Format: %e=elapsed real time, %U=user time, %S=system time
    local clone_output
    clone_output=$( { /usr/bin/time -f "%e %U %S" git clone --depth 1 "$repo_url" "$test_dir" 2>&1; } 2>&1 )

    # Extract timing data (last line of output is from /usr/bin/time)
    local timing_line
    timing_line=$(echo "$clone_output" | tail -n 1)
    local elapsed
    elapsed=$(echo "$timing_line" | awk '{print $1}')
    local user_time
    user_time=$(echo "$timing_line" | awk '{print $2}')
    local sys_time
    sys_time=$(echo "$timing_line" | awk '{print $3}')

    # Get cloned repo size (including LFS objects)
    local repo_size
    repo_size=$(du -sh "$test_dir" | awk '{print $1}')

    # Get LFS object count
    local lfs_count
    lfs_count=$(find "$test_dir/.git/lfs/objects" -type f 2>/dev/null | wc -l)

    # Output results to CSV
    echo "$iteration,$elapsed,$user_time,$sys_time,$repo_size,$lfs_count" >> "$RESULTS_FILE"

    # Clean up
    rm -rf "$test_dir"
}

# Main execution
main() {
    if [ $# -ne 2 ]; then
        echo "Usage: $0  "
        exit 1
    fi

    local repo_url=$1
    local test_dir=$2

    check_prereqs

    # Initialize results file
    echo "iteration,elapsed_seconds,user_seconds,sys_seconds,repo_size,lfs_object_count" > "$RESULTS_FILE"

    # Run benchmark iterations
    for ((i=1; i<=CLONE_ITERATIONS; i++)); do
        run_benchmark "$repo_url" "$test_dir" "$i"
    done

    # Calculate average elapsed time
    local avg_elapsed
    avg_elapsed=$(awk -F',' 'NR>1 {sum+=$2} END {print sum/(NR-1)}' "$RESULTS_FILE" | bc -l)
    echo "Benchmark complete. Average clone time: $avg_elapsed seconds"
    echo "Results saved to $RESULTS_FILE"
}

main "$@"

#!/usr/bin/env python3
"""
Git LFS 3.4 Batch API Client for Unreal Engine Assets
Demonstrates the parallel transfer optimizations added in Git 2.45 and LFS 3.4
Requires: requests>=2.31.0, PyYAML>=6.0
Usage: python lfs-batch-push.py --repo /path/to/unreal/repo --assets-dir ./assets
"""

import argparse
import hashlib
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from typing import Dict, List, Tuple

import requests
import yaml

# Configuration defaults matching Git LFS 3.4 and Git 2.45 optimizations
DEFAULT_CONFIG = {
    "lfs_url": "", # Populated from git config
    "batch_size": 100, # Max assets per batch request (LFS 3.4 default)
    "parallel_transfers": 8, # Git 2.45 default parallel transfer count
    "transfer_speed": "100m", # Git 2.45 core.lfsTransferSpeed tunable (100MB/s cap)
    "unreal_extensions": [".uasset", ".umap", ".uexp", ".ubulk"]
}

def get_git_config(repo_path: str, key: str) -> str:
    """Retrieve a git config value for the given repository."""
    try:
        result = subprocess.run(
            ["git", "config", "--get", key],
            cwd=repo_path,
            capture_output=True,
            text=True,
            check=True
        )
        return result.stdout.strip()
    except subprocess.CalledProcessError as e:
        raise RuntimeError(f"Failed to get git config {key}: {e.stderr}") from e

def calculate_lfs_oid(file_path: Path) -> Tuple[str, int]:
    """Calculate SHA-256 OID and size for an LFS object (matches Git LFS spec)."""
    sha256 = hashlib.sha256()
    size = 0
    try:
        with open(file_path, "rb") as f:
            while chunk := f.read(8192):
                sha256.update(chunk)
                size += len(chunk)
    except IOError as e:
        raise RuntimeError(f"Failed to read file {file_path}: {e}") from e
    return f"sha256:{sha256.hexdigest()}", size

def build_batch_request(assets: List[Path]) -> Dict:
    """Build a Git LFS batch API request for the given assets."""
    objects = []
    for asset in assets:
        oid, size = calculate_lfs_oid(asset)
        objects.append({
            "oid": oid,
            "size": size,
            "actions": {"upload": {"href": ""}} # Populated by server response
        })
    return {
        "operation": "upload",
        "objects": objects,
        "transfers": ["basic", "multipart"], # LFS 3.4 supports multipart transfers
        "hash_algo": "sha256"
    }

def send_batch_request(lfs_url: str, batch_request: Dict, auth_token: str = None) -> Dict:
    """Send a batch request to the Git LFS server and return the response."""
    headers = {"Content-Type": "application/json"}
    if auth_token:
        headers["Authorization"] = f"Bearer {auth_token}"

    try:
        response = requests.post(
            f"{lfs_url}/objects/batch",
            json=batch_request,
            headers=headers,
            timeout=30
        )
        response.raise_for_status()
        return response.json()
    except requests.RequestException as e:
        raise RuntimeError(f"Batch request failed: {e}") from e

def upload_asset(asset: Path, upload_url: str, auth_token: str = None) -> None:
    """Upload a single asset to the LFS server using the provided URL."""
    headers = {}
    if auth_token:
        headers["Authorization"] = f"Bearer {auth_token}"

    try:
        with open(asset, "rb") as f:
            response = requests.put(
                upload_url,
                data=f,
                headers=headers,
                timeout=300 # 5 minute timeout for large Unreal assets
            )
            response.raise_for_status()
    except (IOError, requests.RequestException) as e:
        raise RuntimeError(f"Failed to upload asset {asset}: {e}") from e

def main():
    parser = argparse.ArgumentParser(description="Git LFS Batch Push for Unreal Assets")
    parser.add_argument("--repo", required=True, help="Path to Unreal Git repository")
    parser.add_argument("--assets-dir", required=True, help="Directory containing assets to push")
    parser.add_argument("--config", help="Path to YAML config file (overrides defaults)")
    args = parser.parse_args()

    # Load configuration
    config = DEFAULT_CONFIG.copy()
    if args.config and Path(args.config).exists():
        with open(args.config, "r") as f:
            user_config = yaml.safe_load(f)
            config.update(user_config)

    # Get LFS URL from git config
    repo_path = Path(args.repo).resolve()
    config["lfs_url"] = get_git_config(str(repo_path), "lfs.url")
    if not config["lfs_url"]:
        raise RuntimeError("LFS URL not configured. Run 'git lfs install' in the repo.")

    # Set Git 2.45 transfer speed tunable
    subprocess.run(
        ["git", "config", "core.lfsTransferSpeed", config["transfer_speed"]],
        cwd=str(repo_path),
        check=True
    )

    # Collect Unreal assets to push
    assets_dir = Path(args.assets_dir).resolve()
    assets = [
        f for f in assets_dir.iterdir()
        if f.is_file() and f.suffix in config["unreal_extensions"]
    ]
    if not assets:
        print("No Unreal assets found to push.")
        return

    print(f"Found {len(assets)} Unreal assets to push...")
    start_time = time.time()

    # Process assets in batches
    for i in range(0, len(assets), config["batch_size"]):
        batch = assets[i:i + config["batch_size"]]
        print(f"Processing batch {i//config['batch_size'] + 1} ({len(batch)} assets)...")

        # Build and send batch request
        batch_request = build_batch_request(batch)
        batch_response = send_batch_request(config["lfs_url"], batch_request)

        # Upload each asset in the batch (parallelism handled by Git 2.45)
        for obj, asset in zip(batch_response.get("objects", []), batch):
            upload_action = obj.get("actions", {}).get("upload")
            if not upload_action:
                print(f"Skipping {asset.name}: no upload action")
                continue
            print(f"Uploading {asset.name}...")
            upload_asset(asset, upload_action["href"])

    elapsed = time.time() - start_time
    print(f"Push complete. Uploaded {len(assets)} assets in {elapsed:.2f} seconds")

if __name__ == "__main__":
    try:
        main()
    except RuntimeError as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)

# Set Git 2.45 LFS transfer speed cap to 100MB/s
git config core.lfsTransferSpeed 100m

# Set Git LFS 3.4 concurrent transfers to 16
git config lfs.concurrenttransfers 16

# Verify settings
git config --get core.lfsTransferSpeed
git config --get lfs.concurrenttransfers

# Update .gitattributes to use header-based detection for Unreal assets
echo "*.uasset filter=lfs diff=lfs merge=lfs -text header=unreal" >> .gitattributes
echo "*.umap filter=lfs diff=lfs merge=lfs -text header=unreal" >> .gitattributes
echo "*.uexp filter=lfs diff=lfs merge=lfs -text header=unreal" >> .gitattributes
echo "*.ubulk filter=lfs diff=lfs merge=lfs -text header=unreal" >> .gitattributes

# Verify LFS tracking
git lfs track

# Enable partial clone excluding LFS assets
git clone --filter=blob:none --filter=lfs:none https://github.com/your-org/unreal-project.git

# Configure LFS to fetch assets on-demand
git config lfs.fetchinclude "Content/Levels/*,Content/Blueprints/*"
git config lfs.fetchexclude "Content/Textures/*,Content/Meshes/*"

# Fetch assets for a specific level
git lfs pull --include="Content/Levels/MainLevel/*"