Step-by-Step Guide to Deploying Local AI Assistants with Ollama 0.5 and Docker 26 on M3 Ultra Macs

import docker
import json
import os
import sys
import logging
from typing import Dict, Any, Optional

# Configure logging for audit trail
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

DOCKER_MIN_VERSION = "26.0.0"
REQUIRED_MACS_OS = "14.4"
OLLAMA_MIN_VERSION = "0.5.0"

def validate_docker_version(client: docker.DockerClient) -> bool:
    """Check if Docker version meets minimum requirements for M3 Ultra compatibility."""
    try:
        version_info = client.version()
        current_version = version_info["Version"]
        logger.info(f"Detected Docker version: {current_version}")

        # Split version into parts for comparison
        current_parts = list(map(int, current_version.split(".")))
        min_parts = list(map(int, DOCKER_MIN_VERSION.split(".")))

        # Pad versions to same length
        max_len = max(len(current_parts), len(min_parts))
        current_parts += [0] * (max_len - len(current_parts))
        min_parts += [0] * (max_len - len(min_parts))

        for curr, min_v in zip(current_parts, min_parts):
            if curr > min_v:
                return True
            elif curr < min_v:
                return False
        return True  # Versions equal
    except docker.errors.APIError as e:
        logger.error(f"Failed to fetch Docker version: {e}")
        return False
    except Exception as e:
        logger.error(f"Unexpected error validating Docker version: {e}")
        return False

def configure_docker_daemon() -> Dict[str, Any]:
    """Update Docker daemon.json with M3 Ultra optimized settings."""
    daemon_path = "/etc/docker/daemon.json" if os.name != "posix" else os.path.expanduser("~/.docker/daemon.json")
    default_config = {
        "features": {"buildkit": True},
        "insecure-registries": [],
        "registry-mirrors": [],
        "max-concurrent-downloads": 10,
        "max-concurrent-uploads": 5,
        "storage-driver": "overlay2",
        "log-driver": "json-file",
        "log-opts": {"max-size": "100m", "max-file": "3"},
        "live-restore": True,
        "ipv6": False,
        "fixed-cidr-v6": "",
        "default-ulimits": {
            "memlock": {"soft": -1, "hard": -1},
            "stack": {"soft": 65536, "hard": 65536}
        }
    }

    try:
        # Read existing config if present
        if os.path.exists(daemon_path):
            with open(daemon_path, "r") as f:
                existing_config = json.load(f)
            # Merge with defaults, preserving existing keys
            for key, value in default_config.items():
                if key not in existing_config:
                    existing_config[key] = value
            config = existing_config
        else:
            config = default_config
            os.makedirs(os.path.dirname(daemon_path), exist_ok=True)

        # Write updated config
        with open(daemon_path, "w") as f:
            json.dump(config, f, indent=2)
        logger.info(f"Updated Docker daemon config at {daemon_path}")
        return config
    except PermissionError:
        logger.error(f"Permission denied writing to {daemon_path}. Run with sudo.")
        sys.exit(1)
    except json.JSONDecodeError as e:
        logger.error(f"Invalid existing daemon.json: {e}")
        sys.exit(1)
    except Exception as e:
        logger.error(f"Failed to configure daemon: {e}")
        sys.exit(1)

if __name__ == "__main__":
    try:
        client = docker.from_env()
        logger.info("Initialized Docker client")
    except docker.errors.DockerException as e:
        logger.error(f"Failed to connect to Docker daemon: {e}")
        logger.error("Ensure Docker Desktop is running and Docker CLI is authenticated.")
        sys.exit(1)

    if not validate_docker_version(client):
        logger.error(f"Docker version must be >= {DOCKER_MIN_VERSION}. Please upgrade.")
        sys.exit(1)

    daemon_config = configure_docker_daemon()
    logger.info("Docker configuration complete. Restart Docker Desktop to apply changes.")

import docker
import time
import logging
import sys
import os
from typing import List, Optional

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

OLLAMA_IMAGE = "ollama/ollama:0.5.0"
MODEL_STORAGE_VOLUME = "ollama_models"
CONTAINER_NAME = "ollama_m3_ultra"
OLLAMA_PORT = 11434
M3_ULTRA_DEVICE = "/dev/dri/renderD128"  # MPS device path for M3 Ultra

def create_model_volume(client: docker.DockerClient) -> None:
    """Create persistent Docker volume for Ollama model storage."""
    try:
        existing_volumes = [v.name for v in client.volumes.list()]
        if MODEL_STORAGE_VOLUME not in existing_volumes:
            client.volumes.create(MODEL_STORAGE_VOLUME)
            logger.info(f"Created Docker volume: {MODEL_STORAGE_VOLUME}")
        else:
            logger.info(f"Using existing volume: {MODEL_STORAGE_VOLUME}")
    except docker.errors.APIError as e:
        logger.error(f"Failed to create volume: {e}")
        sys.exit(1)

def deploy_ollama_container(client: docker.DockerClient) -> docker.models.containers.Container:
    """Deploy Ollama 0.5 container with M3 Ultra optimized settings."""
    try:
        # Remove existing container if present
        try:
            existing = client.containers.get(CONTAINER_NAME)
            logger.warning(f"Stopping existing container: {CONTAINER_NAME}")
            existing.stop(timeout=10)
            existing.remove()
        except docker.errors.NotFound:
            pass

        # Check if MPS device exists (M3 Ultra specific)
        if not os.path.exists(M3_ULTRA_DEVICE):
            logger.error(f"MPS device not found at {M3_ULTRA_DEVICE}. Ensure Docker Desktop has GPU support enabled.")
            sys.exit(1)

        container = client.containers.run(
            image=OLLAMA_IMAGE,
            name=CONTAINER_NAME,
            detach=True,
            ports={f"{OLLAMA_PORT}/tcp": OLLAMA_PORT},
            volumes={
                MODEL_STORAGE_VOLUME: {"bind": "/root/.ollama", "mode": "rw"},
                M3_ULTRA_DEVICE: {"bind": M3_ULTRA_DEVICE, "mode": "rw"}
            },
            environment={
                "OLLAMA_HOST": "0.0.0.0:11434",
                "OLLAMA_NUM_GPU": "32",  # Use all 32 GPU cores on M3 Ultra
                "OLLAMA_METAL": "1",  # Enable Metal backend
                "OLLAMA_DEBUG": "0"
            },
            devices=[M3_ULTRA_DEVICE],  # Pass MPS device to container
            restart_policy={"Name": "always", "MaximumRetryCount": 5},
            mem_limit="100g",  # Limit to 100GB of 128GB unified memory
            cpu_count=24  # Use 24 of 32 CPU cores
        )
        logger.info(f"Started Ollama container: {container.id[:12]}")
        return container
    except docker.errors.ImageNotFound:
        logger.info(f"Pulling Ollama image: {OLLAMA_IMAGE}")
        client.images.pull(OLLAMA_IMAGE)
        return deploy_ollama_container(client)  # Retry after pull
    except docker.errors.APIError as e:
        logger.error(f"Failed to deploy container: {e}")
        sys.exit(1)

def pull_models(container: docker.models.containers.Container, models: List[str]) -> None:
    """Pull required LLM models into Ollama container."""
    for model in models:
        logger.info(f"Pulling model: {model}")
        try:
            exit_code, output = container.exec_run(
                cmd=f"ollama pull {model}",
                stream=True,
                demux=True
            )
            for chunk in output:
                if chunk[0] is not None:  # Stdout
                    logger.info(f"Pull progress: {chunk[0].decode().strip()}")
            if exit_code != 0:
                logger.error(f"Failed to pull model {model}: exit code {exit_code}")
                sys.exit(1)
            logger.info(f"Successfully pulled model: {model}")
        except docker.errors.APIError as e:
            logger.error(f"Error pulling model {model}: {e}")
            sys.exit(1)

if __name__ == "__main__":
    try:
        client = docker.from_env()
        logger.info("Connected to Docker daemon")
    except docker.errors.DockerException as e:
        logger.error(f"Docker connection failed: {e}")
        sys.exit(1)

    create_model_volume(client)
    ollama_container = deploy_ollama_container(client)

    # Wait for Ollama to start
    time.sleep(10)
    logger.info("Verifying Ollama health...")
    try:
        exit_code, output = ollama_container.exec_run("ollama list")
        if exit_code == 0:
            logger.info(f"Ollama health check passed: {output.decode().strip()}")
        else:
            logger.error("Ollama health check failed")
            sys.exit(1)
    except Exception as e:
        logger.error(f"Health check error: {e}")
        sys.exit(1)

    # Pull default models
    default_models = ["llama3:70b", "codellama:13b", "mistral:7b"]
    pull_models(ollama_container, default_models)
    logger.info("Ollama deployment complete. API available at http://localhost:11434")

from fastapi import FastAPI, HTTPException, Request, Depends
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
import httpx
import time
import logging
import sys
import os
from typing import Dict, Any, Optional
from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
from slowapi import Limiter, _rate_limit_exceeded_handler
from slowapi.util import get_remote_address
from slowapi.errors import RateLimitExceeded

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)

# Prometheus metrics
REQUEST_COUNT = Counter(
    "ai_assistant_requests_total",
    "Total AI assistant requests",
    ["model", "status"]
)
REQUEST_LATENCY = Histogram(
    "ai_assistant_request_latency_seconds",
    "Request latency in seconds",
    ["model"]
)
TOKEN_THROUGHPUT = Counter(
    "ai_assistant_tokens_total",
    "Total tokens generated",
    ["model"]
)

# Rate limiter
limiter = Limiter(key_func=get_remote_address)
app = FastAPI(title="M3 Ultra Local AI Assistant Middleware")
app.state.limiter = limiter
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)

# CORS config for local development
app.add_middleware(
    CORSMiddleware,
    allow_origins=["http://localhost:3000", "http://localhost:8080"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"]
)

OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
RATE_LIMIT = os.getenv("RATE_LIMIT", "10/minute")  # 10 requests per minute per IP

async def forward_request(model: str, prompt: str, max_tokens: int = 2048) -> Dict[str, Any]:
    """Forward request to Ollama API and track metrics."""
    start_time = time.time()
    try:
        async with httpx.AsyncClient(timeout=300.0) as client:
            response = await client.post(
                f"{OLLAMA_BASE_URL}/api/generate",
                json={
                    "model": model,
                    "prompt": prompt,
                    "max_tokens": max_tokens,
                    "stream": False
                }
            )
            response.raise_for_status()
            result = response.json()

            # Track metrics
            latency = time.time() - start_time
            REQUEST_COUNT.labels(model=model, status="success").inc()
            REQUEST_LATENCY.labels(model=model).observe(latency)
            token_count = result.get("eval_count", 0)
            TOKEN_THROUGHPUT.labels(model=model).inc(token_count)

            logger.info(f"Model: {model}, Latency: {latency:.2f}s, Tokens: {token_count}")
            return result
    except httpx.HTTPStatusError as e:
        REQUEST_COUNT.labels(model=model, status="error").inc()
        logger.error(f"Ollama request failed: {e}")
        raise HTTPException(status_code=e.response.status_code, detail=str(e))
    except Exception as e:
        REQUEST_COUNT.labels(model=model, status="error").inc()
        logger.error(f"Unexpected error: {e}")
        raise HTTPException(status_code=500, detail="Internal server error")

@app.post("/v1/generate")
@limiter.limit(RATE_LIMIT)
async def generate_text(request: Request, model: str, prompt: str, max_tokens: int = 2048):
    """Generate text using specified LLM model."""
    valid_models = ["llama3:70b", "codellama:13b", "mistral:7b"]
    if model not in valid_models:
        raise HTTPException(status_code=400, detail=f"Invalid model. Valid models: {valid_models}")
    return await forward_request(model, prompt, max_tokens)

@app.get("/metrics")
async def get_metrics():
    """Expose Prometheus metrics."""
    return generate_latest(), 200, {"Content-Type": CONTENT_TYPE_LATEST}

@app.get("/health")
async def health_check():
    """Health check endpoint."""
    try:
        async with httpx.AsyncClient(timeout=5.0) as client:
            response = await client.get(f"{OLLAMA_BASE_URL}/api/tags")
            response.raise_for_status()
            return {"status": "healthy", "ollama": "reachable"}
    except Exception as e:
        logger.error(f"Health check failed: {e}")
        raise HTTPException(status_code=503, detail="Ollama unreachable")

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)

# Docker environment variables for memory tuning
environment:
  OLLAMA_METAL_LAYERS: "80"  # All layers for 70B model
  OLLAMA_NUM_GPU: "32"       # Use all 32 GPU cores
  OLLAMA_MAX_LOADED_MODELS: "2"  # Limit to 2 models in memory at once

# Run Ollama with host network mode (Docker 26+)
docker run -d \
  --name ollama_m3_ultra \
  --network host \
  --add-host.docker.internal.host-gateway \
  -v ollama_models:/root/.ollama \
  -e OLLAMA_METAL=1 \
  ollama/ollama:0.5.0

# Add to crontab (crontab -e)
0 2 * * * /usr/bin/docker exec ollama_m3_ultra ollama pull llama3:70b >> /var/log/ollama-update.log 2>&1
0 2 * * * /usr/bin/docker exec ollama_m3_ultra ollama pull codellama:13b >> /var/log/ollama-update.log 2>&1

m3-ultra-local-ai/
├── docker/
│   ├── daemon.json          # Optimized Docker daemon config
│   ├── docker-compose.yml   # Full stack deployment config
│   └── ollama/
│       └── Dockerfile       # Custom Ollama image with M3 tweaks
├── middleware/
│   ├── main.py              # FastAPI middleware code
│   ├── requirements.txt     # Python dependencies
│   └── prometheus.yml       # Prometheus scrape config
├── scripts/
│   ├── deploy_ollama.py     # Step 2 deployment script
│   ├── configure_docker.py  # Step 1 Docker config script
│   └── update_models.sh     # Cron job model update script
├── benchmarks/
│   ├── results.json         # Token throughput/latency data
│   └── run_benchmarks.py    # Automated benchmark script
└── README.md                # Full setup instructions