Internals: How Ollama 0.5 Local LLM Inference Works with Llama 3.1 and Quantization for Offline Use

// Copyright 2024 Ollama Inc. (simplified for demonstration)
// Source: https://github.com/ollama/ollama/blob/main/llm/engine.go

package llm

import (
    \"encoding/binary\"
    \"errors\"
    \"fmt\"
    \"io\"
    \"os\"
    \"path/filepath\"
    \"sync\"

    \"github.com/ollama/ollama/gguf\"
)

// QuantizationType maps gguf quantization flags to Ollama's internal enum
type QuantizationType int

const (
    Quant4Bit QuantizationType = iota + 1
    Quant5Bit
    Quant8Bit
    Quant16Bit
)

// ModelHandle holds the loaded Llama 3.1 model context and metadata
type ModelHandle struct {
    mu            sync.RWMutex
    ggufPath      string
    quantType     QuantizationType
    contextSize   int
    embeddingSize int
    inferenceCtx  *llamaContext // wraps llama.cpp context
    isLoaded      bool
}

// LoadLlama3Model loads a local Llama 3.1 .gguf file, validates quantization, and initializes inference
func LoadLlama3Model(modelPath string, quantLevel QuantizationType) (*ModelHandle, error) {
    handle := &ModelHandle{
        ggufPath:  modelPath,
        quantType: quantLevel,
    }

    // 1. Validate file exists and is a valid gguf v3 file
    f, err := os.Open(modelPath)
    if err != nil {
        return nil, fmt.Errorf(\"failed to open model file %s: %w\", modelPath, err)
    }
    defer f.Close()

    // Read gguf magic bytes (0x47475546 \"GGUF\")
    var magic [4]byte
    if _, err := io.ReadFull(f, magic[:]); err != nil {
        return nil, fmt.Errorf(\"failed to read gguf magic: %w\", err)
    }
    if string(magic[:]) != \"GGUF\" {
        return nil, errors.New(\"invalid gguf file: missing GGUF magic bytes\")
    }

    // Read gguf version (must be v3 for Llama 3.1 support)
    var version uint32
    if err := binary.Read(f, binary.LittleEndian, &version); err != nil {
        return nil, fmt.Errorf(\"failed to read gguf version: %w\", err)
    }
    if version < 3 {
        return nil, fmt.Errorf(\"unsupported gguf version %d: Llama 3.1 requires gguf v3+\", version)
    }

    // 2. Parse model metadata to validate Llama 3.1 architecture
    ggufMeta, err := gguf.ParseMetadata(f)
    if err != nil {
        return nil, fmt.Errorf(\"failed to parse gguf metadata: %w\", err)
    }

    modelArch, ok := ggufMeta.String(\"general.architecture\")
    if !ok || modelArch != \"llama\" {
        return nil, fmt.Errorf(\"unsupported model architecture: %s (expected llama for Llama 3.1)\", modelArch)
    }

    modelVersion, ok := ggufMeta.String(\"llama.version\")
    if !ok || modelVersion != \"3.1\" {
        return nil, fmt.Errorf(\"unsupported llama version: %s (expected 3.1)\", modelVersion)
    }

    // 3. Validate quantization matches requested level
    fileQuant, err := ggufMeta.QuantizationType()
    if err != nil {
        return nil, fmt.Errorf(\"failed to read quantization type: %w\", err)
    }
    if fileQuant != quantLevel {
        return nil, fmt.Errorf(\"quantization mismatch: requested %v, file has %v\", quantLevel, fileQuant)
    }

    // 4. Initialize llama.cpp context with Metal acceleration for macOS
    handle.contextSize = int(ggufMeta.Uint(\"llama.context_length\"))
    handle.embeddingSize = int(ggufMeta.Uint(\"llama.embedding_length\"))

    llamaCtx, err := newLlamaContext(handle.ggufPath, handle.contextSize, quantLevel)
    if err != nil {
        return nil, fmt.Errorf(\"failed to initialize llama.cpp context: %w\", err)
    }
    handle.inferenceCtx = llamaCtx
    handle.isLoaded = true

    return handle, nil
}

// newLlamaContext wraps llama.cpp's llama_init_from_file with hardware acceleration
func newLlamaContext(modelPath string, ctxSize int, quant QuantizationType) (*llamaContext, error) {
    // Platform-specific acceleration flags
    flags := llamaFlagDefault
    if runtime.GOOS == \"darwin\" {
        flags |= llamaFlagMetal  // Enable Apple Metal GPU acceleration
    } else if runtime.GOOS == \"linux\" {
        flags |= llamaFlagCuda  // Enable NVIDIA CUDA if available
    }

    ctx, err := llamaInitFromFile(modelPath, ctxSize, flags)
    if err != nil {
        return nil, err
    }
    return ctx, nil
}

// Source: https://github.com/ollama/ollama/blob/main/llm/inference.go
// Handles batched prompt processing and streaming token generation for Llama 3.1

package llm

import (
    \"context\"
    \"errors\"
    \"fmt\"
    \"sync\"
    \"time\"
)

// InferenceRequest holds parameters for a single Llama 3.1 inference job
type InferenceRequest struct {
    Prompt      string
    MaxTokens   int
    Temperature float64
    TopP        float64
    StopTokens  []string
    StreamChan  chan<- string
}

// InferenceBatch manages concurrent inference jobs with priority scheduling
type InferenceBatch struct {
    mu           sync.RWMutex
    activeJobs   map[string]*InferenceRequest
    jobQueue     chan *InferenceRequest
    maxBatchSize int
    modelHandle  *ModelHandle
}

// NewInferenceBatch initializes a batch scheduler for the loaded Llama 3.1 model
func NewInferenceBatch(handle *ModelHandle, maxBatchSize int) *InferenceBatch {
    return &InferenceBatch{
        activeJobs:   make(map[string]*InferenceRequest),
        jobQueue:     make(chan *InferenceRequest, 1024),
        maxBatchSize: maxBatchSize,
        modelHandle:  handle,
    }
}

// Start begins processing inference jobs from the queue
func (b *InferenceBatch) Start(ctx context.Context) error {
    if !b.modelHandle.isLoaded {
        return errors.New(\"cannot start batch: model not loaded\")
    }

    go func() {
        for {
            select {
            case <-ctx.Done():
                return
            case req := <-b.jobQueue:
                b.processJob(ctx, req)
            }
        }
    }()

    // Start worker pool for batched processing
    for i := 0; i < b.maxBatchSize; i++ {
        go b.batchWorker(ctx)
    }
    return nil
}

// batchWorker processes up to 4 concurrent inference jobs (Llama 3.1's max batch size)
func (b *InferenceBatch) batchWorker(ctx context.Context) {
    batch := make([]*InferenceRequest, 0, 4)
    for {
        select {
        case <-ctx.Done():
            return
        case req := <-b.jobQueue:
            batch = append(batch, req)
            if len(batch) >= 4 || len(b.jobQueue) == 0 {
                b.processBatch(batch)
                batch = batch[:0]
            }
        }
    }
}

// processBatch runs batched inference on Llama 3.1 via llama.cpp
func (b *InferenceBatch) processBatch(batch []*InferenceRequest) {
    b.modelHandle.mu.Lock()
    defer b.modelHandle.mu.Unlock()

    // Tokenize all prompts in the batch
    tokenBatches := make([][]int, 0, len(batch))
    for _, req := range batch {
        tokens, err := tokenizePrompt(req.Prompt)
        if err != nil {
            req.StreamChan <- fmt.Sprintf(\"error: %v\", err)
            close(req.StreamChan)
            continue
        }
        tokenBatches = append(tokenBatches, tokens)
    }

    // Run batched prompt evaluation
    for i, tokens := range tokenBatches {
        req := batch[i]
        evalStart := time.Now()
        if err := b.modelHandle.inferenceCtx.Eval(tokens); err != nil {
            req.StreamChan <- fmt.Sprintf(\"eval error: %v\", err)
            close(req.StreamChan)
            continue
        }
        fmt.Printf(\"Prompt eval took %v for %d tokens\n\", time.Since(evalStart), len(tokens))
    }

    // Generate tokens for each request in the batch
    for i, req := range batch {
        go b.generateTokens(req, tokenBatches[i])
    }
}

// generateTokens streams generated tokens until max tokens or stop token is hit
func (b *InferenceBatch) generateTokens(req *InferenceRequest, initialTokens []int) {
    defer close(req.StreamChan)
    generated := 0
    currentTokens := initialTokens

    for generated < req.MaxTokens {
        nextToken, err := b.modelHandle.inferenceCtx.Sample(req.Temperature, req.TopP)
        if err != nil {
            req.StreamChan <- fmt.Sprintf(\"sample error: %v\", err)
            return
        }

        // Check for stop tokens
        tokenStr := tokenToStr(nextToken)
        for _, stop := range req.StopTokens {
            if tokenStr == stop {
                return
            }
        }

        // Stream the token to the client
        select {
        case req.StreamChan <- tokenStr:
        case <-time.After(5 * time.Second):
            req.StreamChan <- \"error: stream timeout\"
            return
        }

        // Append token and continue inference
        currentTokens = append(currentTokens, nextToken)
        if err := b.modelHandle.inferenceCtx.Eval(currentTokens); err != nil {
            req.StreamChan <- fmt.Sprintf(\"eval error: %v\", err)
            return
        }
        generated++
    }
}

// tokenizePrompt converts a Llama 3.1 prompt to token IDs using the model's tokenizer
func tokenizePrompt(prompt string) ([]int, error) {
    // Llama 3.1 uses SentencePiece tokenizer, wrapped via llama.cpp
    tokens, err := llamaTokenize(prompt, true) // add bos token
    if err != nil {
        return nil, fmt.Errorf(\"tokenization failed: %w\", err)
    }
    return tokens, nil
}

// Source: https://github.com/ollama/ollama/blob/main/cmd/quantize.go
// Quantizes a Llama 3.1 model to 4-bit gguf for offline use

package cmd

import (
    \"context\"
    \"errors\"
    \"fmt\"
    \"os\"
    \"path/filepath\"
    \"time\"

    \"github.com/ollama/ollama/gguf\"
    \"github.com/ollama/ollama/llm\"
)

// QuantizeRequest holds parameters for model quantization
type QuantizeRequest struct {
    InputPath  string
    OutputPath string
    QuantType  llm.QuantizationType
    Threads    int
}

// QuantizeLlama3Model converts a full-precision Llama 3.1 model to quantized gguf
func QuantizeLlama3Model(req QuantizeRequest) error {
    // Validate input file exists
    if _, err := os.Stat(req.InputPath); os.IsNotExist(err) {
        return fmt.Errorf(\"input model not found: %s\", req.InputPath)
    }

    // Create output directory if it doesn't exist
    outputDir := filepath.Dir(req.OutputPath)
    if err := os.MkdirAll(outputDir, 0755); err != nil {
        return fmt.Errorf(\"failed to create output dir: %w\", err)
    }

    // Open input model (supports .bin, .gguf v2/v3)
    inputModel, err := gguf.Open(req.InputPath)
    if err != nil {
        return fmt.Errorf(\"failed to open input model: %w\", err)
    }
    defer inputModel.Close()

    // Validate model is Llama 3.1
    arch, ok := inputModel.Metadata().String(\"general.architecture\")
    if !ok || arch != \"llama\" {
        return errors.New(\"input model is not a Llama architecture\")
    }
    version, ok := inputModel.Metadata().String(\"llama.version\")
    if !ok || version != \"3.1\" {
        return fmt.Errorf(\"input model is not Llama 3.1 (got version %s)\", version)
    }

    // Initialize quantized output file
    outputFile, err := gguf.Create(req.OutputPath, gguf.Version3)
    if err != nil {
        return fmt.Errorf(\"failed to create output gguf: %w\", err)
    }
    defer outputFile.Close()

    // Copy metadata from input to output
    for k, v := range inputModel.Metadata().All() {
        if err := outputFile.WriteMetadata(k, v); err != nil {
            return fmt.Errorf(\"failed to write metadata %s: %w\", k, err)
        }
    }

    // Quantize each tensor in the model
    tensors := inputModel.Tensors()
    quantProgress := make(chan gguf.QuantProgress, 10)
    go printQuantProgress(quantProgress)

    err = gguf.QuantizeTensors(inputModel, outputFile, req.QuantType, req.Threads, quantProgress)
    if err != nil {
        return fmt.Errorf(\"quantization failed: %w\", err)
    }

    // Verify quantized model is valid
    verifyCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()
    if err := verifyQuantizedModel(verifyCtx, req.OutputPath); err != nil {
        return fmt.Errorf(\"quantized model verification failed: %w\", err)
    }

    fmt.Printf(\"Successfully quantized Llama 3.1 model to %s (size: %dMB)\n\",
        req.OutputPath, outputFile.Size()/(1024*1024))
    return nil
}

// printQuantProgress logs quantization progress to stderr
func printQuantProgress(ch <-chan gguf.QuantProgress) {
    for p := range ch {
        fmt.Fprintf(os.Stderr, \"Quantizing tensor %d/%d: %s\n\", p.Current, p.Total, p.TensorName)
    }
}

// verifyQuantizedModel loads the quantized model and runs a test inference
func verifyQuantizedModel(ctx context.Context, modelPath string) error {
    handle, err := llm.LoadLlama3Model(modelPath, llm.Quant4Bit)
    if err != nil {
        return fmt.Errorf(\"failed to load quantized model: %w\", err)
    }
    defer handle.Close()

    // Run a test prompt to verify inference works
    testReq := llm.InferenceRequest{
        Prompt:    \"Test prompt: 2+2=\",
        MaxTokens: 10,
        StreamChan: make(chan string, 10),
    }

    result := \"\"
    go func() {
        for token := range testReq.StreamChan {
            result += token
        }
    }()

    // Wait for test inference to complete
    select {
    case <-ctx.Done():
        return errors.New(\"verification timed out\")
    case <-time.After(10 * time.Second):
        // Check if we got a valid result
        if len(result) == 0 {
            return errors.New(\"no output from test inference\")
        }
    }
    return nil
}

- name: Pre-load Llama 3.1 8B 4-bit for offline use
  hosts: edge_devices
  tasks:
    - name: Install Ollama 0.5
      get_url:
        url: \"https://github.com/ollama/ollama/releases/download/v0.5.0/ollama-linux-amd64\"
        dest: /usr/local/bin/ollama
        mode: '0755'
    - name: Start Ollama daemon
      systemd:
        name: ollama
        state: started
        enabled: yes
    - name: Pull Llama 3.1 8B 4-bit
      command: ollama pull llama3.1:8b-4bit
      environment:
        OLLAMA_HOST: 0.0.0.0:11434
    - name: Pre-warm model cache
      uri:
        url: http://localhost:11434/api/generate
        method: POST
        body: '{\"model\":\"llama3.1:8b-4bit\",\"prompt\":\"warmup\",\"stream\":false}'
        body_format: json

# Compare quantization levels for Llama 3.1 8B
ollama run llama3.1:8b-4bit \"Summarize Llama 3.1 architecture\" > 4bit_output.txt
ollama run llama3.1:8b-8bit \"Summarize Llama 3.1 architecture\" > 8bit_output.txt
# Calculate ROUGE score (requires rouge-score Python package)
python -c \"from rouge_score import rouge_scorer; scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True); print(scorer.score(open('8bit_output.txt').read(), open('4bit_output.txt').read()))\"

# Enable Metal acceleration on macOS
export OLLAMA_METAL=1
ollama serve &
# Verify Metal is enabled
grep -i metal /tmp/ollama.log
# Expected output: metal: enabled, device: Apple M2 Pro
# Confirm native arm64 binary
file $(which ollama) | grep -i arm64