import pandas as pd
import time
import sys
from typing import Optional

def load_and_clean_sales_data(
    file_path: str,
    use_legacy: bool = False,
    chunk_size: Optional[int] = None
) -> pd.DataFrame:
    """
    Load and clean retail sales data, compatible with both Pandas 1.x and 2.x.
    Args:
        file_path: Path to CSV file with columns: order_id, customer_id, order_date, product, quantity, unit_price, region
        use_legacy: If True, force legacy dtype behavior (Pandas 1.x style)
        chunk_size: Optional chunk size for large files (Pandas 2.x only optimizes chunked reads)
    Returns:
        Cleaned DataFrame with calculated total_price, parsed dates, and filtered invalid rows
    Raises:
        FileNotFoundError: If file_path does not exist
        ValueError: If required columns are missing
    """
    start_time = time.time()

    # Check Pandas version for compatibility
    pandas_version = tuple(int(x) for x in pd.__version__.split(".")[:2])
    if pandas_version[0] == 1 and use_legacy:
        print(f"Running legacy mode (Pandas {pd.__version__})")
        # Pandas 1.x default dtypes: object for strings, float for nullable ints
        dtype_map = {
            "order_id": "int64",
            "customer_id": "object",  # Legacy uses object for string-like IDs
            "product": "object",
            "region": "object"
        }
        parse_dates = ["order_date"]
    elif pandas_version[0] == 2:
        print(f"Running current mode (Pandas {pd.__version__})")
        # Pandas 2.x default: StringDtype for strings, nullable Int64 for integers
        dtype_map = {
            "order_id": "Int64",
            "customer_id": "string",
            "product": "string",
            "region": "string"
        }
        parse_dates = ["order_date"]
    else:
        raise RuntimeError(f"Unsupported Pandas version: {pd.__version__}")

    try:
        if chunk_size:
            # Chunked read for files >1GB (optimized in Pandas 2.x)
            chunks = []
            for chunk in pd.read_csv(
                file_path,
                dtype=dtype_map,
                parse_dates=parse_dates,
                chunksize=chunk_size,
                dtype_backend="numpy_nullable" if pandas_version[0] == 2 else None
            ):
                # Clean each chunk
                chunk = chunk.dropna(subset=["order_id", "quantity", "unit_price"])
                chunk["total_price"] = chunk["quantity"] * chunk["unit_price"]
                chunk = chunk[chunk["total_price"] > 0]  # Remove invalid negative totals
                chunks.append(chunk)
            df = pd.concat(chunks, ignore_index=True)
        else:
            df = pd.read_csv(
                file_path,
                dtype=dtype_map,
                parse_dates=parse_dates,
                dtype_backend="numpy_nullable" if pandas_version[0] == 2 else None
            )
            # Clean full dataset
            df = df.dropna(subset=["order_id", "quantity", "unit_price"])
            df["total_price"] = df["quantity"] * df["unit_price"]
            df = df[df["total_price"] > 0]

        # Validate required columns
        required_cols = {"order_id", "customer_id", "order_date", "product", "total_price", "region"}
        missing = required_cols - set(df.columns)
        if missing:
            raise ValueError(f"Missing required columns: {missing}")

        print(f"Loaded and cleaned {len(df)} rows in {time.time() - start_time:.2f}s")
        return df

    except FileNotFoundError:
        print(f"Error: File not found at {file_path}")
        raise
    except pd.errors.EmptyDataError:
        print(f"Error: No data in file {file_path}")
        raise
    except Exception as e:
        print(f"Unexpected error: {e}")
        raise

if __name__ == "__main__":
    # Example usage: Compare legacy vs current mode
    try:
        # Test with 1M row synthetic dataset (generated separately)
        df_legacy = load_and_clean_sales_data("sales_data_1m.csv", use_legacy=True)
        df_current = load_and_clean_sales_data("sales_data_1m.csv", use_legacy=False)
        print(f"Legacy memory usage: {df_legacy.memory_usage(deep=True).sum() / 1e6:.2f}MB")
        print(f"Current memory usage: {df_current.memory_usage(deep=True).sum() / 1e6:.2f}MB")
    except Exception as e:
        print(f"Script failed: {e}")
        sys.exit(1)

import pandas as pd
import time
import numpy as np
from typing import Tuple

def generate_synthetic_data(n_rows: int) -> Tuple[pd.DataFrame, pd.DataFrame]:
    """Generate synthetic order and customer DataFrames for benchmarking."""
    np.random.seed(42)  # Reproducible results

    # Orders DataFrame: 100k-100M rows
    orders = pd.DataFrame({
        "order_id": np.arange(n_rows),
        "customer_id": np.random.randint(0, n_rows // 10, size=n_rows),  # 10 customers per order on average
        "product_id": np.random.randint(0, 1000, size=n_rows),
        "quantity": np.random.randint(1, 10, size=n_rows),
        "order_date": pd.date_range("2020-01-01", periods=n_rows, freq="s")[:n_rows]
    })

    # Customers DataFrame: 1/10th the size of orders
    customers = pd.DataFrame({
        "customer_id": np.arange(n_rows // 10),
        "customer_name": [f"Customer_{i}" for i in range(n_rows // 10)],
        "region": np.random.choice(["North", "South", "East", "West"], size=n_rows // 10)
    })

    return orders, customers

def benchmark_join(
    orders: pd.DataFrame,
    customers: pd.DataFrame,
    join_type: str = "inner",
    pandas_version: str = "2.x"
) -> float:
    """
    Benchmark inner join of orders and customers DataFrames.
    Returns elapsed time in seconds.
    """
    start = time.perf_counter()

    if pandas_version.startswith("1."):
        # Pandas 1.x: no Arrow optimization, default object dtypes
        result = orders.merge(
            customers,
            on="customer_id",
            how=join_type
        )
    else:
        # Pandas 2.x: use Arrow backend if available
        result = orders.merge(
            customers,
            on="customer_id",
            how=join_type,
            dtype_backend="arrow" if hasattr(pd, "ArrowDtype") else None
        )

    elapsed = time.perf_counter() - start
    print(f"Join {join_type} ({len(orders)} orders, {len(customers)} customers): {elapsed:.2f}s, {len(result)} rows")
    return elapsed

def run_benchmarks():
    """Run join benchmarks across dataset sizes and Pandas versions."""
    benchmark_sizes = [1_000_000, 10_000_000, 100_000_000]
    pandas_ver = pd.__version__
    print(f"Running benchmarks with Pandas {pandas_ver}")

    results = []
    for size in benchmark_sizes:
        print(f"\nGenerating {size} row dataset...")
        orders, customers = generate_synthetic_data(size)

        # Benchmark inner join
        inner_time = benchmark_join(orders, customers, "inner", pandas_ver)
        # Benchmark left join
        left_time = benchmark_join(orders, customers, "left", pandas_ver)

        results.append({
            "size": size,
            "inner_join_time": inner_time,
            "left_join_time": left_time,
            "pandas_version": pandas_ver
        })

    # Save results to CSV for comparison
    results_df = pd.DataFrame(results)
    results_df.to_csv(f"join_benchmarks_pandas_{pandas_ver.replace('.', '_')}.csv", index=False)
    print(f"\nBenchmark results saved to join_benchmarks_pandas_{pandas_ver.replace('.', '_')}.csv")

if __name__ == "__main__":
    try:
        run_benchmarks()
    except Exception as e:
        print(f"Benchmark failed: {e}")
        raise

import pandas as pd
import numpy as np
from typing import List

def test_nullable_dtypes():
    """Compare nullable dtype handling between Pandas 1.x and 2.x."""
    pandas_ver = pd.__version__
    print(f"Testing nullable dtypes with Pandas {pandas_ver}")

    # Create test DataFrame with missing values
    test_data = {
        "int_column": [1, 2, None, 4, 5],
        "string_column": ["a", None, "c", "d", None],
        "float_column": [1.1, 2.2, 3.3, None, 5.5]
    }

    if pd.__version__.startswith("1."):
        print("Pandas 1.x: Using legacy dtype handling")
        # Pandas 1.x: None becomes NaN for floats, object for strings, error for int
        try:
            df_legacy = pd.DataFrame(test_data)
            print(f"Legacy int column dtype: {df_legacy['int_column'].dtype}")  # float64 (NaN forces float)
            print(f"Legacy string column dtype: {df_legacy['string_column'].dtype}")  # object
        except Exception as e:
            print(f"Legacy dtype error: {e}")

        # Explicit nullable dtypes in 1.x
        df_legacy_nullable = pd.DataFrame({
            "int_column": pd.array([1, 2, None, 4, 5], dtype="Int64"),
            "string_column": pd.array(["a", None, "c", "d", None], dtype="string"),
            "float_column": pd.array([1.1, 2.2, 3.3, None, 5.5], dtype="Float64")
        })
        print(f"Legacy nullable int dtype: {df_legacy_nullable['int_column'].dtype}")  # Int64
        print(f"Legacy nullable string dtype: {df_legacy_nullable['string_column'].dtype}")  # string
    else:
        print("Pandas 2.x: Using default nullable dtypes")
        # Pandas 2.x: read_csv defaults to nullable dtypes, StringDtype for strings
        df_current = pd.DataFrame(test_data)
        print(f"Current int column dtype: {df_current['int_column'].dtype}")  # Int64 (nullable)
        print(f"Current string column dtype: {df_current['string_column'].dtype}")  # string
        print(f"Current float column dtype: {df_current['float_column'].dtype}")  # Float64 (nullable)

    return df_legacy if pd.__version__.startswith("1.") else df_current

def benchmark_string_operations(df: pd.DataFrame, pandas_ver: str) -> dict:
    """Benchmark common string operations."""
    print(f"\nBenchmarking string operations with Pandas {pandas_ver}")
    results = {}

    # Benchmark str.contains
    start = time.perf_counter()
    df["string_column"].str.contains("a")
    results["str_contains"] = time.perf_counter() - start

    # Benchmark str.upper
    start = time.perf_counter()
    df["string_column"].str.upper()
    results["str_upper"] = time.perf_counter() - start

    # Benchmark str.replace
    start = time.perf_counter()
    df["string_column"].str.replace("a", "b")
    results["str_replace"] = time.perf_counter() - start

    for op, time_taken in results.items():
        print(f"{op}: {time_taken:.4f}s")

    return results

def run_string_benchmarks():
    """Run string benchmarks across Pandas versions."""
    # Generate 10M row string dataset
    np.random.seed(42)
    n_rows = 10_000_000
    print(f"Generating {n_rows} row string dataset...")

    if pd.__version__.startswith("1."):
        df = pd.DataFrame({
            "text": np.random.choice(["apple", "banana", "cherry", "date", None], size=n_rows)
        })
    else:
        df = pd.DataFrame({
            "text": pd.array(np.random.choice(["apple", "banana", "cherry", "date", None], size=n_rows), dtype="string")
        })

    benchmark_string_operations(df, pd.__version__)

if __name__ == "__main__":
    try:
        test_nullable_dtypes()
        run_string_benchmarks()
    except Exception as e:
        print(f"Test failed: {e}")
        raise

# Short snippet: Check for deprecated append() usage
import ast
import os

def find_deprecated_append(file_path: str) -> list:
    with open(file_path, "r") as f:
        tree = ast.parse(f.read())
    deprecated = []
    for node in ast.walk(tree):
        if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
            if node.func.attr == "append" and isinstance(node.func.value, ast.Name):
                if node.func.value.id == "DataFrame" or node.func.value.id == "df":
                    deprecated.append(f"Line {node.lineno}: {ast.unparse(node)}")
    return deprecated

# Short snippet: Enable Arrow backend for CSV read
import pandas as pd

df = pd.read_csv(
    "large_sales_data.csv",
    dtype_backend="arrow",  # Use Arrow for all columns
    parse_dates=["order_date"]
)
print(f"Arrow backend enabled: {type(df['product'].dtype)}")  # Should be ArrowDtype

# Short snippet: Check Pandas version in new project setup
import pandas as pd

if tuple(int(x) for x in pd.__version__.split(".")[:2]) < (2, 0):
    raise RuntimeError("New projects must use Pandas 2.x or higher")