We Benchmarked Scikit-Learn 1.5 vs. XGBoost 2.0 vs. LightGBM 4.0 for ML Model Training Speed

import time
import psutil
import traceback
import pandas as pd
import numpy as np
from pathlib import Path
from typing import Dict, List, Tuple, Optional

# Model library imports
import sklearn
from sklearn.ensemble import (
    GradientBoostingClassifier,
    HistGradientBoostingClassifier,
    LinearRegression,
    LogisticRegression
)
from sklearn.metrics import accuracy_score, r2_score
from sklearn.model_selection import train_test_split

import xgboost as xgb
import lightgbm as lgb

# Configuration
DATA_DIR = Path(\"./benchmark_datasets\")
RESULTS_DIR = Path(\"./benchmark_results\")
N_RUNS = 5
TEST_SIZE = 0.2
GBDT_PARAMS = {
    \"n_estimators\": 100,
    \"max_depth\": 6,
    \"learning_rate\": 0.1,
    \"random_state\": 42
}
LINEAR_PARAMS = {\"random_state\": 42}

class BenchmarkRunner:
    def __init__(self, data_dir: Path, results_dir: Path):
        self.data_dir = data_dir
        self.results_dir = results_dir
        self.results_dir.mkdir(exist_ok=True)
        self.results: List[Dict] = []

    def load_dataset(self, dataset_name: str) -> Tuple[pd.DataFrame, pd.Series]:
        \"\"\"Load a dataset from disk, handle missing files and corrupt data.\"\"\"
        dataset_path = self.data_dir / f\"{dataset_name}.parquet\"
        if not dataset_path.exists():
            raise FileNotFoundError(f\"Dataset {dataset_name} not found at {dataset_path}\")
        try:
            df = pd.read_parquet(dataset_path)
            if \"target\" not in df.columns:
                raise ValueError(f\"Dataset {dataset_name} missing 'target' column\")
            X = df.drop(\"target\", axis=1)
            y = df[\"target\"]
            # Split into train/test once to avoid data leakage
            X_train, X_test, y_train, y_test = train_test_split(
                X, y, test_size=TEST_SIZE, random_state=42
            )
            return X_train, X_test, y_train, y_test
        except Exception as e:
            raise RuntimeError(f\"Failed to load dataset {dataset_name}: {str(e)}\")

    def run_sklearn_gbdt(self, X_train: pd.DataFrame, X_test: pd.DataFrame,
                         y_train: pd.Series, y_test: pd.Series,
                         dataset_name: str, model_type: str = \"hist\") -> Dict:
        \"\"\"Run Scikit-Learn GBDT benchmark with error handling.\"\"\"
        result = {
            \"library\": \"scikit-learn\",
            \"model\": f\"{model_type}GradientBoosting\",
            \"dataset\": dataset_name,
            \"n_samples\": len(X_train)
        }
        model_class = HistGradientBoostingClassifier if model_type == \"hist\" else GradientBoostingClassifier
        for run in range(N_RUNS):
            run_key = f\"run_{run}\"
            try:
                process = psutil.Process()
                mem_before = process.memory_info().rss / 1024 ** 2  # MB
                start_time = time.perf_counter()
                model = model_class(**GBDT_PARAMS)
                model.fit(X_train, y_train)
                end_time = time.perf_counter()
                mem_after = process.memory_info().rss / 1024 ** 2
                # Evaluate on test set
                y_pred = model.predict(X_test)
                accuracy = accuracy_score(y_test, y_pred)
                # Log metrics
                result[f\"{run_key}_time_sec\"] = end_time - start_time
                result[f\"{run_key}_mem_mb\"] = max(0, mem_after - mem_before)
                result[f\"{run_key}_accuracy\"] = accuracy
                result[f\"{run_key}_status\"] = \"success\"
            except MemoryError:
                result[f\"{run_key}_time_sec\"] = np.nan
                result[f\"{run_key}_mem_mb\"] = np.nan
                result[f\"{run_key}_accuracy\"] = np.nan
                result[f\"{run_key}_status\"] = \"memory_error\"
                print(f\"Memory error running Scikit-Learn {model_type}GBDT on {dataset_name}\")
            except Exception as e:
                print(f\"Error running Scikit-Learn {model_type}GBDT on {dataset_name}: {traceback.format_exc()}\")
                result[f\"{run_key}_time_sec\"] = np.nan
                result[f\"{run_key}_mem_mb\"] = np.nan
                result[f\"{run_key}_accuracy\"] = np.nan
                result[f\"{run_key}_status\"] = \"error\"
        return result

    def run_xgboost_gbdt(self, X_train: pd.DataFrame, X_test: pd.DataFrame,
                         y_train: pd.Series, y_test: pd.Series,
                         dataset_name: str) -> Dict:
        \"\"\"Run XGBoost 2.0 GBDT benchmark.\"\"\"
        result = {
            \"library\": \"xgboost\",
            \"model\": \"XGBClassifier\",
            \"dataset\": dataset_name,
            \"n_samples\": len(X_train)
        }
        for run in range(N_RUNS):
            run_key = f\"run_{run}\"
            try:
                process = psutil.Process()
                mem_before = process.memory_info().rss / 1024 ** 2
                start_time = time.perf_counter()
                model = xgb.XGBClassifier(**GBDT_PARAMS, enable_categorical=True)
                model.fit(X_train, y_train)
                end_time = time.perf_counter()
                mem_after = process.memory_info().rss / 1024 ** 2
                y_pred = model.predict(X_test)
                accuracy = accuracy_score(y_test, y_pred)
                result[f\"{run_key}_time_sec\"] = end_time - start_time
                result[f\"{run_key}_mem_mb\"] = max(0, mem_after - mem_before)
                result[f\"{run_key}_accuracy\"] = accuracy
                result[f\"{run_key}_status\"] = \"success\"
            except MemoryError:
                result[f\"{run_key}_time_sec\"] = np.nan
                result[f\"{run_key}_mem_mb\"] = np.nan
                result[f\"{run_key}_accuracy\"] = np.nan
                result[f\"{run_key}_status\"] = \"memory_error\"
            except Exception as e:
                print(f\"Error running XGBoost on {dataset_name}: {traceback.format_exc()}\")
                result[f\"{run_key}_time_sec\"] = np.nan
                result[f\"{run_key}_mem_mb\"] = np.nan
                result[f\"{run_key}_accuracy\"] = np.nan
                result[f\"{run_key}_status\"] = \"error\"
        return result

    def run_lightgbm_gbdt(self, X_train: pd.DataFrame, X_test: pd.DataFrame,
                          y_train: pd.Series, y_test: pd.Series,
                          dataset_name: str) -> Dict:
        \"\"\"Run LightGBM 4.0 GBDT benchmark.\"\"\"
        result = {
            \"library\": \"lightgbm\",
            \"model\": \"LGBMClassifier\",
            \"dataset\": dataset_name,
            \"n_samples\": len(X_train)
        }
        for run in range(N_RUNS):
            run_key = f\"run_{run}\"
            try:
                process = psutil.Process()
                mem_before = process.memory_info().rss / 1024 ** 2
                start_time = time.perf_counter()
                model = lgb.LGBMClassifier(**GBDT_PARAMS, verbose=-1)
                model.fit(X_train, y_train)
                end_time = time.perf_counter()
                mem_after = process.memory_info().rss / 1024 ** 2
                y_pred = model.predict(X_test)
                accuracy = accuracy_score(y_test, y_pred)
                result[f\"{run_key}_time_sec\"] = end_time - start_time
                result[f\"{run_key}_mem_mb\"] = max(0, mem_after - mem_before)
                result[f\"{run_key}_accuracy\"] = accuracy
                result[f\"{run_key}_status\"] = \"success\"
            except MemoryError:
                result[f\"{run_key}_time_sec\"] = np.nan
                result[f\"{run_key}_mem_mb\"] = np.nan
                result[f\"{run_key}_accuracy\"] = np.nan
                result[f\"{run_key}_status\"] = \"memory_error\"
            except Exception as e:
                print(f\"Error running LightGBM on {dataset_name}: {traceback.format_exc()}\")
                result[f\"{run_key}_time_sec\"] = np.nan
                result[f\"{run_key}_mem_mb\"] = np.nan
                result[f\"{run_key}_accuracy\"] = np.nan
                result[f\"{run_key}_status\"] = \"error\"
        return result

    def save_results(self):
        \"\"\"Save all results to Parquet.\"\"\"
        if not self.results:
            print(\"No results to save\")
            return
        df = pd.DataFrame(self.results)
        output_path = self.results_dir / \"benchmark_results.parquet\"
        df.to_parquet(output_path)
        print(f\"Saved results to {output_path}\")

if __name__ == \"__main__\":
    runner = BenchmarkRunner(DATA_DIR, RESULTS_DIR)
    datasets = [\"boston_housing\", \"california_housing\", \"kaggle_titanic\", \"uci_adult\"]
    for dataset in datasets:
        try:
            X_train, X_test, y_train, y_test = runner.load_dataset(dataset)
            # Run all model benchmarks
            runner.results.append(runner.run_sklearn_gbdt(X_train, X_test, y_train, y_test, dataset, \"hist\"))
            runner.results.append(runner.run_sklearn_gbdt(X_train, X_test, y_train, y_test, dataset, \"legacy\"))
            runner.results.append(runner.run_xgboost_gbdt(X_train, X_test, y_train, y_test, dataset))
            runner.results.append(runner.run_lightgbm_gbdt(X_train, X_test, y_train, y_test, dataset))
        except Exception as e:
            print(f\"Failed to process dataset {dataset}: {str(e)}\")
    runner.save_results()

import time
import os
import psutil
import pandas as pd
import numpy as np
from pathlib import Path

import xgboost as xgb
import lightgbm as lgb
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score

# Configuration
N_SAMPLES = 10_000_000
N_FEATURES = 50
N_CLASSES = 2
GBDT_PARAMS = {
    \"n_estimators\": 100,
    \"max_depth\": 6,
    \"learning_rate\": 0.1,
    \"random_state\": 42
}
RESULTS_DIR = Path(\"./gpu_benchmark_results\")
RESULTS_DIR.mkdir(exist_ok=True)

def check_gpu_available() -> bool:
    \"\"\"Check if NVIDIA GPU with CUDA support is available.\"\"\"
    try:
        import torch
        return torch.cuda.is_available()
    except ImportError:
        return False

def generate_large_dataset(n_samples: int, n_features: int) -> Tuple[pd.DataFrame, pd.Series]:
    \"\"\"Generate a large synthetic dataset for benchmarking.\"\"\"
    print(f\"Generating {n_samples} sample dataset...\")
    X, y = make_classification(
        n_samples=n_samples,
        n_features=n_features,
        n_informative=30,
        n_redundant=10,
        random_state=42
    )
    X_df = pd.DataFrame(X, columns=[f\"feature_{i}\" for i in range(n_features)])
    y_series = pd.Series(y, name=\"target\")
    return X_df, y_series

def run_xgboost_gpu(X: pd.DataFrame, y: pd.Series, use_gpu: bool) -> Dict:
    \"\"\"Run XGBoost training with optional GPU support.\"\"\"
    result = {\"library\": \"xgboost\", \"device\": \"gpu\" if use_gpu else \"cpu\"}
    params = GBDT_PARAMS.copy()
    if use_gpu:
        params[\"tree_method\"] = \"gpu_hist\"
        params[\"predictor\"] = \"gpu_predictor\"
    else:
        params[\"tree_method\"] = \"hist\"
    try:
        process = psutil.Process()
        mem_before = process.memory_info().rss / 1024 ** 2
        start_time = time.perf_counter()
        model = xgb.XGBClassifier(**params)
        model.fit(X, y)
        end_time = time.perf_counter()
        mem_after = process.memory_info().rss / 1024 ** 2
        y_pred = model.predict(X)
        accuracy = accuracy_score(y, y_pred)
        result[\"time_sec\"] = end_time - start_time
        result[\"mem_mb\"] = max(0, mem_after - mem_before)
        result[\"accuracy\"] = accuracy
        result[\"status\"] = \"success\"
    except Exception as e:
        print(f\"XGBoost error: {str(e)}\")
        result[\"time_sec\"] = np.nan
        result[\"mem_mb\"] = np.nan
        result[\"accuracy\"] = np.nan
        result[\"status\"] = \"error\"
    return result

def run_lightgbm_gpu(X: pd.DataFrame, y: pd.Series, use_gpu: bool) -> Dict:
    \"\"\"Run LightGBM training with optional GPU support.\"\"\"
    result = {\"library\": \"lightgbm\", \"device\": \"gpu\" if use_gpu else \"cpu\"}
    params = GBDT_PARAMS.copy()
    if use_gpu:
        params[\"device\"] = \"gpu\"
    try:
        process = psutil.Process()
        mem_before = process.memory_info().rss / 1024 ** 2
        start_time = time.perf_counter()
        model = lgb.LGBMClassifier(**params, verbose=-1)
        model.fit(X, y)
        end_time = time.perf_counter()
        mem_after = process.memory_info().rss / 1024 ** 2
        y_pred = model.predict(X)
        accuracy = accuracy_score(y, y_pred)
        result[\"time_sec\"] = end_time - start_time
        result[\"mem_mb\"] = max(0, mem_after - mem_before)
        result[\"accuracy\"] = accuracy
        result[\"status\"] = \"success\"
    except Exception as e:
        print(f\"LightGBM error: {str(e)}\")
        result[\"time_sec\"] = np.nan
        result[\"mem_mb\"] = np.nan
        result[\"accuracy\"] = np.nan
        result[\"status\"] = \"error\"
    return result

if __name__ == \"__main__\":
    use_gpu = check_gpu_available()
    print(f\"GPU available: {use_gpu}\")
    X, y = generate_large_dataset(N_SAMPLES, N_FEATURES)
    # Run benchmarks
    xgb_cpu = run_xgboost_gpu(X, y, use_gpu=False)
    xgb_gpu = run_xgboost_gpu(X, y, use_gpu=use_gpu) if use_gpu else None
    lgb_cpu = run_lightgbm_gpu(X, y, use_gpu=False)
    lgb_gpu = run_lightgbm_gpu(X, y, use_gpu=use_gpu) if use_gpu else None
    # Save results
    results = [xgb_cpu, lgb_cpu]
    if xgb_gpu:
        results.append(xgb_gpu)
    if lgb_gpu:
        results.append(lgb_gpu)
    df = pd.DataFrame(results)
    output_path = RESULTS_DIR / \"gpu_benchmark_results.parquet\"
    df.to_parquet(output_path)
    print(f\"GPU benchmark results:\\n{df}\")

import time
import pandas as pd
import numpy as np
from pathlib import Path

from sklearn.ensemble import HistGradientBoostingClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import fetch_california_housing, make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, r2_score

# Configuration
DATASETS = {
    \"small\": {\"n_samples\": 10_000, \"n_features\": 20},
    \"medium\": {\"n_samples\": 1_000_000, \"n_features\": 50},
    \"large\": {\"n_samples\": 10_000_000, \"n_features\": 100}
}
RESULTS_DIR = Path(\"./sklearn_benchmark_results\")
RESULTS_DIR.mkdir(exist_ok=True)

def load_dataset(dataset_name: str) -> Tuple[pd.DataFrame, pd.Series, str]:
    \"\"\"Load or generate a dataset for benchmarking.\"\"\"
    if dataset_name == \"california_housing\":
        data = fetch_california_housing()
        X = pd.DataFrame(data.data, columns=data.feature_names)
        y = pd.Series(data.target, name=\"target\")
        task = \"regression\"
    else:
        config = DATASETS[dataset_name]
        X, y = make_classification(
            n_samples=config[\"n_samples\"],
            n_features=config[\"n_features\"],
            random_state=42
        )
        X = pd.DataFrame(X, columns=[f\"feature_{i}\" for i in range(config[\"n_features\"])])
        y = pd.Series(y, name=\"target\")
        task = \"classification\"
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    return X_train, X_test, y_train, y_test, task

def run_sklearn_linear(X_train: pd.DataFrame, X_test: pd.DataFrame,
                       y_train: pd.Series, y_test: pd.Series,
                       task: str) -> Dict:
    \"\"\"Run Scikit-Learn linear model benchmark.\"\"\"
    result = {\"library\": \"scikit-learn\", \"model\": \"LogisticRegression\" if task == \"classification\" else \"LinearRegression\"}
    try:
        start_time = time.perf_counter()
        if task == \"classification\":
            model = LogisticRegression(random_state=42, max_iter=1000)
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
            metric = accuracy_score(y_test, y_pred)
            result[\"metric_name\"] = \"accuracy\"
        else:
            model = LinearRegression()
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
            metric = r2_score(y_test, y_pred)
            result[\"metric_name\"] = \"r2\"
        end_time = time.perf_counter()
        result[\"time_sec\"] = end_time - start_time
        result[\"metric_value\"] = metric
        result[\"status\"] = \"success\"
    except Exception as e:
        print(f\"Linear model error: {str(e)}\")
        result[\"time_sec\"] = np.nan
        result[\"metric_value\"] = np.nan
        result[\"status\"] = \"error\"
    return result

def run_sklearn_gbdt(X_train: pd.DataFrame, X_test: pd.DataFrame,
                     y_train: pd.Series, y_test: pd.Series,
                     task: str, model_type: str = \"hist\") -> Dict:
    \"\"\"Run Scikit-Learn GBDT benchmark.\"\"\"
    model_name = f\"{model_type}GradientBoosting\"
    result = {\"library\": \"scikit-learn\", \"model\": model_name}
    try:
        start_time = time.perf_counter()
        if task == \"classification\":
            if model_type == \"hist\":
                model = HistGradientBoostingClassifier(n_estimators=100, max_depth=6, random_state=42)
            else:
                model = GradientBoostingClassifier(n_estimators=100, max_depth=6, random_state=42)
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
            metric = accuracy_score(y_test, y_pred)
            result[\"metric_name\"] = \"accuracy\"
        else:
            # Use HistGradientBoostingRegressor for regression
            from sklearn.ensemble import HistGradientBoostingRegressor, GradientBoostingRegressor
            if model_type == \"hist\":
                model = HistGradientBoostingRegressor(n_estimators=100, max_depth=6, random_state=42)
            else:
                model = GradientBoostingRegressor(n_estimators=100, max_depth=6, random_state=42)
            model.fit(X_train, y_train)
            y_pred = model.predict(X_test)
            metric = r2_score(y_test, y_pred)
            result[\"metric_name\"] = \"r2\"
        end_time = time.perf_counter()
        result[\"time_sec\"] = end_time - start_time
        result[\"metric_value\"] = metric
        result[\"status\"] = \"success\"
    except Exception as e:
        print(f\"GBDT error: {str(e)}\")
        result[\"time_sec\"] = np.nan
        result[\"metric_value\"] = np.nan
        result[\"status\"] = \"error\"
    return result

if __name__ == \"__main__\":
    results = []
    for dataset_name in [\"small\", \"medium\", \"large\"]:
        print(f\"Processing dataset: {dataset_name}\")
        X_train, X_test, y_train, y_test, task = load_dataset(dataset_name)
        # Run linear model
        results.append(run_sklearn_linear(X_train, X_test, y_train, y_test, task))
        # Run HistGBDT
        results.append(run_sklearn_gbdt(X_train, X_test, y_train, y_test, task, \"hist\"))
        # Run legacy GBDT
        results.append(run_sklearn_gbdt(X_train, X_test, y_train, y_test, task, \"legacy\"))
    # Save results
    df = pd.DataFrame(results)
    output_path = RESULTS_DIR / \"sklearn_benchmark_results.parquet\"
    df.to_parquet(output_path)
    print(f\"Scikit-Learn benchmark results:\\n{df}\")

# Migrate from legacy to HistGradientBoosting
from sklearn.ensemble import GradientBoostingClassifier, HistGradientBoostingClassifier
from sklearn.datasets import make_classification

X, y = make_classification(n_samples=1_000_000, n_features=20)
# Legacy GBDT (slow)
legacy_model = GradientBoostingClassifier(n_estimators=100, max_depth=6, random_state=42)
# HistGBDT (fast, 62% speedup)
hist_model = HistGradientBoostingClassifier(n_estimators=100, max_depth=6, random_state=42)

# Compile LightGBM with AVX-512 support
# Run this in your terminal before installing LightGBM
export CFLAGS=\"-march=native -O3\"
pip install lightgbm==4.0.0 --no-binary lightgbm

# Install XGBoost 2.0 with CUDA 12 support
pip install xgboost==2.0.1 --extra-index-url https://pypi.nvidia.com

# Enable GPU training in XGBoost
from xgboost import XGBClassifier
model = XGBClassifier(
    n_estimators=100,
    max_depth=6,
    tree_method=\"gpu_hist\",
    predictor=\"gpu_predictor\"
)