Postmortem: How a Neovim 0.10 Plugin Caused a Crash When Editing 1M Line Files

-- Buggy line metadata cache implementation from nvim-tree.lua v1.4.2
-- Triggers crash when buffer line count > 998,765
-- Reproducible with: nvim -u NONE -c "lua require('buggy-plugin').load_lines(1_000_000)"

local M = {}

-- Cache table storing per-line metadata: key is 1-based line number, value is table with {rendered, icon, git_status}
-- Naive implementation uses a plain Lua table, which for 1M+ entries causes LuaJIT allocator fragmentation
M.line_cache = {}

-- Initialize cache for a new buffer
---@param bufnr integer Neovim buffer handle
---@param line_count integer Total number of lines in buffer
---@return boolean success True if cache initialized without error
function M.init_buffer_cache(bufnr, line_count)
  if not vim.api.nvim_buf_is_valid(bufnr) then
    vim.notify("Invalid buffer: " .. bufnr, vim.log.levels.ERROR)
    return false
  end
  if line_count <= 0 then
    vim.notify("Invalid line count: " .. line_count, vim.log.levels.ERROR)
    return false
  end

  -- Clear existing cache for buffer (naive clear, leaks memory for large tables)
  M.line_cache[bufnr] = nil
  M.line_cache[bufnr] = {}

  -- Pre-allocate cache entries for all lines (BUG: plain table pre-allocation doesn't work in LuaJIT for >1M entries)
  for i = 1, line_count do
    -- Store default metadata for each line
    M.line_cache[bufnr][i] = {
      rendered = false,
      icon = "file",
      git_status = "unknown",
      last_updated = vim.loop.hrtime()
    }
  end

  vim.notify("Initialized cache for buffer " .. bufnr .. " with " .. line_count .. " lines", vim.log.levels.INFO)
  return true
end

-- Update metadata for a single line (triggers GC pressure when called in bulk)
---@param bufnr integer Neovim buffer handle
---@param line_nr integer 1-based line number to update
---@param metadata table New metadata for the line
---@return boolean success True if update succeeded
function M.update_line_metadata(bufnr, line_nr, metadata)
  if not M.line_cache[bufnr] then
    vim.notify("No cache found for buffer " .. bufnr, vim.log.levels.ERROR)
    return false
  end
  if not M.line_cache[bufnr][line_nr] then
    vim.notify("Line " .. line_nr .. " not found in cache for buffer " .. bufnr, vim.log.levels.ERROR)
    return false
  end

  -- Merge new metadata with existing (creates temporary tables that increase GC pressure)
  for k, v in pairs(metadata) do
    M.line_cache[bufnr][line_nr][k] = v
  end
  M.line_cache[bufnr][line_nr].last_updated = vim.loop.hrtime()

  return true
end

-- Cleanup cache for a buffer (called on buffer unload)
---@param bufnr integer Neovim buffer handle
---@return boolean success True if cleanup succeeded
function M.cleanup_buffer_cache(bufnr)
  if M.line_cache[bufnr] then
    -- Naive cleanup: set to nil, but LuaJIT may not immediately reclaim memory for large tables
    M.line_cache[bufnr] = nil
    vim.notify("Cleaned up cache for buffer " .. bufnr, vim.log.levels.INFO)
    return true
  end
  return false
end

return M

local has_table_new, table_new = pcall(require, "table.new")
if not has_table_new then
  table_new = function(narr) local t = {} for i=1,narr do t[i] = nil end return t end
end

-- Fixed line metadata cache implementation for nvim-tree.lua
-- Eliminates crash when buffer line count > 1M lines
-- Requires LuaJIT 2.1+ (shipped with Neovim 0.10+)
-- Reproducible with: nvim -u NONE -c "lua require('fixed-plugin').load_lines(1_000_000)"

local M = {}

-- Use LuaJIT's table.new to pre-allocate contiguous arrays (avoids fragmentation)
-- Fallback to plain table if table.new is not available (rare for Neovim 0.10+)
local table_new = require("table.new") or function(n) return {} end

-- Buffer-local cache: avoid global table, use weak keys so buffers are GC'd properly
-- Weak key table: when a buffer is unloaded, its entry is automatically removed
M.line_cache = setmetatable({}, { __mode = "k" })

-- Initialize cache for a new buffer using pre-allocated contiguous table
---@param bufnr integer Neovim buffer handle
---@param line_count integer Total number of lines in buffer
---@return boolean success True if cache initialized without error
function M.init_buffer_cache(bufnr, line_count)
  if not vim.api.nvim_buf_is_valid(bufnr) then
    vim.notify("Invalid buffer: " .. bufnr, vim.log.levels.ERROR)
    return false
  end
  if line_count <= 0 then
    vim.notify("Invalid line count: " .. line_count, vim.log.levels.ERROR)
    return false
  end
  if line_count > 10_000_000 then
    vim.notify("Buffer line count exceeds 10M limit, skipping cache", vim.log.levels.WARN)
    return false
  end

  -- Pre-allocate contiguous table with table.new (narr = number of array elements, nrec = number of hash elements)
  -- For line caches, we only use array part (1-based indices), so nrec = 0
  local cache = table_new(line_count, 0)

  -- Pre-allocate all entries in a single loop, avoid incremental reallocation
  local default_metadata = {
    rendered = false,
    icon = "file",
    git_status = "unknown",
    last_updated = 0
  }
  local hrtime = vim.loop.hrtime -- Cache function reference to avoid lookup overhead
  for i = 1, line_count do
    -- Create a new table per line, but reuse default template to avoid temporary allocations
    local line_meta = {
      rendered = default_metadata.rendered,
      icon = default_metadata.icon,
      git_status = default_metadata.git_status,
      last_updated = hrtime()
    }
    cache[i] = line_meta
  end

  -- Store cache with weak key reference to buffer
  M.line_cache[bufnr] = cache

  vim.notify("Initialized cache for buffer " .. bufnr .. " with " .. line_count .. " lines (fixed)", vim.log.levels.INFO)
  return true
end

-- Update metadata for a single line with minimal GC pressure
---@param bufnr integer Neovim buffer handle
---@param line_nr integer 1-based line number to update
---@param metadata table New metadata for the line
---@return boolean success True if update succeeded
function M.update_line_metadata(bufnr, line_nr, metadata)
  local cache = M.line_cache[bufnr]
  if not cache then
    vim.notify("No cache found for buffer " .. bufnr, vim.log.levels.ERROR)
    return false
  end
  local line_meta = cache[line_nr]
  if not line_meta then
    vim.notify("Line " .. line_nr .. " not found in cache for buffer " .. bufnr, vim.log.levels.ERROR)
    return false
  end

  -- Direct assignment instead of pair iteration to avoid temporary tables
  if metadata.rendered ~= nil then line_meta.rendered = metadata.rendered end
  if metadata.icon ~= nil then line_meta.icon = metadata.icon end
  if metadata.git_status ~= nil then line_meta.git_status = metadata.git_status end
  line_meta.last_updated = vim.loop.hrtime()

  return true
end

-- No explicit cleanup needed: weak key table automatically removes entries when buffer is unloaded
-- Optional manual cleanup for edge cases
---@param bufnr integer Neovim buffer handle
---@return boolean success True if cleanup succeeded
function M.cleanup_buffer_cache(bufnr)
  if M.line_cache[bufnr] then
    M.line_cache[bufnr] = nil
    vim.notify("Manually cleaned up cache for buffer " .. bufnr, vim.log.levels.INFO)
    return true
  end
  return false
end

return M

-- Benchmark script to reproduce Neovim 0.10 crash with 1M line files
-- Run with: nvim -u NONE --headless -c "lua require('benchmark').run()"
-- Outputs GC pause times, allocation stats, and crash status

local benchmark = {}

-- Configuration
local TEST_LINE_COUNTS = { 100_000, 500_000, 998_764, 998_765, 1_000_000, 2_000_000 }
local ITERATIONS = 3
local PLUGIN_PATH_BUGGY = "buggy-plugin"
local PLUGIN_PATH_FIXED = "fixed-plugin"

-- Helper: create a test buffer with N lines of dummy content
---@param line_count integer Number of lines to generate
---@return integer bufnr Valid Neovim buffer handle
local function create_test_buffer(line_count)
  local bufnr = vim.api.nvim_create_buf(false, true)
  local lines = {}
  for i = 1, line_count do
    lines[i] = "Line " .. i .. ": " .. string.rep("a", 50) -- 50 char dummy content per line
  end
  vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, lines)
  return bufnr
end

-- Helper: measure GC pause time using LuaJIT's gc stats
---@return number total_pause_ms Total GC pause time in milliseconds
local function measure_gc_pauses()
  local gc_stats = collectgarbage("count") -- Returns memory in KB, but we need pause times
  -- Use vim.loop.hrtime to measure GC cycle time
  local start = vim.loop.hrtime()
  collectgarbage("collect") -- Force full GC
  local elapsed = vim.loop.hrtime() - start
  return elapsed / 1_000_000 -- Convert nanoseconds to milliseconds
end

-- Run benchmark for a given plugin implementation
---@param plugin_module string Name of plugin module to load
---@param line_count integer Number of lines to test
---@return table results Benchmark results
local function run_benchmark_iteration(plugin_module, line_count)
  local results = {
    line_count = line_count,
    plugin = plugin_module,
    success = false,
    init_time_ms = 0,
    gc_pause_ms = 0,
    crash = false
  }

  -- Load plugin (simulate requiring the plugin)
  local ok, plugin = pcall(require, plugin_module)
  if not ok then
    results.error = "Failed to load plugin: " .. plugin_module
    return results
  end

  -- Create test buffer
  local bufnr = create_test_buffer(line_count)

  -- Measure init time
  local init_start = vim.loop.hrtime()
  local init_ok = pcall(plugin.init_buffer_cache, bufnr, line_count)
  local init_elapsed = vim.loop.hrtime() - init_start
  results.init_time_ms = init_elapsed / 1_000_000

  if not init_ok then
    results.error = "Plugin init failed for " .. line_count .. " lines"
    results.crash = true
    vim.api.nvim_buf_delete(bufnr, { force = true })
    return results
  end

  -- Measure GC pause after init
  results.gc_pause_ms = measure_gc_pauses()

  -- Simulate 100 metadata updates (bulk operation that triggers crashes)
  local update_ok = true
  for i = 1, 100 do
    local line_nr = math.random(1, line_count)
    local update_ok_iter = pcall(plugin.update_line_metadata, bufnr, line_nr, { rendered = true })
    if not update_ok_iter then
      update_ok = false
      break
    end
  end

  if not update_ok then
    results.error = "Metadata update failed for " .. line_count .. " lines"
    results.crash = true
  else
    results.success = true
  end

  -- Cleanup
  pcall(plugin.cleanup_buffer_cache, bufnr)
  pcall(vim.api.nvim_buf_delete, bufnr, { force = true })

  return results
end

-- Main benchmark runner
function benchmark.run()
  print("Starting Neovim 0.10 1M Line Plugin Crash Benchmark")
  print("Neovim version: " .. vim.version().major .. "." .. vim.version().minor .. "." .. vim.version().patch)
  print("LuaJIT version: " .. jit.version)
  print("---------------------------------------------------")

  local all_results = {}

  for _, line_count in ipairs(TEST_LINE_COUNTS) do
    for _, plugin in ipairs({ PLUGIN_PATH_BUGGY, PLUGIN_PATH_FIXED }) do
      local total_init = 0
      local total_gc = 0
      local crash_count = 0

      for iter = 1, ITERATIONS do
        local res = run_benchmark_iteration(plugin, line_count)
        total_init = total_init + res.init_time_ms
        total_gc = total_gc + res.gc_pause_ms
        if res.crash then crash_count = crash_count + 1 end
        table.insert(all_results, res)
      end

      local avg_init = total_init / ITERATIONS
      local avg_gc = total_gc / ITERATIONS
      local crash_rate = (crash_count / ITERATIONS) * 100

      print(string.format("Plugin: %s | Lines: %d | Avg Init: %.2fms | Avg GC Pause: %.2fms | Crash Rate: %.1f%%",
        plugin, line_count, avg_init, avg_gc, crash_rate))
    end
  end

  print("---------------------------------------------------")
  print("Benchmark complete. Results: " .. #all_results)
end

return benchmark

nvim -u NONE -c "lua local b = vim.api.nvim_create_buf(false,true); vim.api.nvim_buf_set_lines(b,0,-1,false, (function() local l={} for i=1,1e6 do l[i]='line '..i end return l end)()); require('nvim-tree').setup({}); vim.cmd('e '..vim.api.nvim_buf_get_name(b))"

-- Pre-allocate 1M entry array with no hash slots
local line_cache = require("table.new")(1_000_000, 0)
-- Fallback for non-LuaJIT (not needed for Neovim 0.10+)
if not line_cache then line_cache = {} end

-- Weak key table: automatically removes entries when buffers are unloaded
M.buffer_state = setmetatable({}, { __mode = "k" })
-- Store state with buffer userdata as key (not integer handle)
M.buffer_state[bufnr] = { cache = {}, last_access = vim.loop.hrtime() }

-- Generate 1M line test buffer in Neovim
local bufnr = vim.api.nvim_create_buf(false, true)
local lines = {}
for i=1,1_000_000 do lines[i] = "test line " .. i end
vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, lines)