test: wasm tree-sitter #40304

Problem:
No CI coverage for wasm builds.

Solution:
Add a basic workflow that builds with ENABLE_WASMTIME
and runs wasm-specific tests.

.github/workflows/test_wasm.yml

@@ -0,0 +1,53 @@
+name: wasm
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
+  pull_request:
+    branches:
+      - 'master'
+      - 'release-[0-9]+.[0-9]+'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  ASAN_OPTIONS: detect_leaks=1:check_initialization_order=1:log_path=${{ github.workspace }}/build/log/asan:intercept_tls_get_addr=0
+  BIN_DIR: ${{ github.workspace }}/bin
+  BUILD_DIR: ${{ github.workspace }}/build
+  INSTALL_PREFIX: ${{ github.workspace }}/nvim-install
+  LOG_DIR: ${{ github.workspace }}/build/log
+  NVIM_LOG_FILE: ${{ github.workspace }}/build/nvim.log
+  TSAN_OPTIONS: log_path=${{ github.workspace }}/build/log/tsan
+  VALGRIND_LOG: ${{ github.workspace }}/build/log/valgrind-%p.log
+  # TEST_FILE: test/functional/core/startup_spec.lua
+  # TEST_FILTER: foo
+
+jobs:
+  wasmtime:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v6.0.3
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup
+        with:
+          install_flags: "--test"
+      - name: Build
+        run: |
+          cmake -S cmake.deps --preset ci -D ENABLE_WASMTIME=ON -D USE_BUNDLED_TS_PARSERS=ON
+          cmake --build .deps
+          cmake --preset ci -D ENABLE_WASMTIME=ON
+          cmake --build build
+      - name: functionaltest
+        timeout-minutes: 10
+        env:
+          TEST_FILE: test/functional/treesitter/wasm_spec.lua
+        run: cmake --build build --target functionaltest

cmake.deps/cmake/BuildTreesitterParsers.cmake

@@ -32,3 +32,22 @@ foreach(lang c lua vim vimdoc query)
   BuildTSParser(LANG ${lang})
 endforeach()
 BuildTSParser(LANG markdown CMAKE_FILE MarkdownParserCMakeLists.txt)
+
+if(USE_BUNDLED_TS_PARSERS AND ENABLE_WASMTIME)
+  if(DEPS_IGNORE_SHA)
+    set(_lua_wasm_hash "")
+  else()
+    set(_lua_wasm_hash "URL_HASH;SHA256=${TREESITTER_LUA_WASM_SHA256}")
+  endif()
+  ExternalProject_Add(treesitter_lua_wasm
+    DOWNLOAD_NO_PROGRESS TRUE
+    DOWNLOAD_NO_EXTRACT TRUE
+    URL ${TREESITTER_LUA_WASM_URL}
+    ${_lua_wasm_hash}
+    DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/treesitter_lua_wasm
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
+      ${DEPS_DOWNLOAD_DIR}/treesitter_lua_wasm/tree-sitter-lua.wasm
+      ${DEPS_INSTALL_DIR}/lib/nvim/parser/lua.wasm)
+endif()

cmake.deps/deps.txt

@@ -35,6 +35,8 @@ TREESITTER_C_URL https://github.com/tree-sitter/tree-sitter-c/archive/v0.24.2.ta
 TREESITTER_C_SHA256 2eeb4db31f8fa0865e45488503d13403923bcb485a1bdb637abff8c42dd97364
 TREESITTER_LUA_URL https://github.com/tree-sitter-grammars/tree-sitter-lua/archive/v0.5.0.tar.gz
 TREESITTER_LUA_SHA256 cf01b93f4b61b96a6d27942cf28eeda4cbce7d503c3bef773a8930b3d778a2d9
+TREESITTER_LUA_WASM_URL https://github.com/tree-sitter-grammars/tree-sitter-lua/releases/download/v0.5.0/tree-sitter-lua.wasm
+TREESITTER_LUA_WASM_SHA256 df08a1704e504c70b8dba4a3e6f8e0c99a4fb94e1b1693d2969f53141d09f0d4
 TREESITTER_VIM_URL https://github.com/tree-sitter-grammars/tree-sitter-vim/archive/v0.8.1.tar.gz
 TREESITTER_VIM_SHA256 93cafb9a0269420362454ace725a118ff1c3e08dcdfdc228aa86334b54d53c2a
 TREESITTER_VIMDOC_URL https://github.com/neovim/tree-sitter-vimdoc/archive/v4.1.0.tar.gz

scripts/bump_deps.lua

@@ -78,8 +78,9 @@ local function get_archive_info(repo, ref)
   local temp_dir = os.getenv('TMPDIR') or os.getenv('TEMP') or '/tmp'
 
   local archive_name = ref .. '.tar.gz'
-  local archive_path = temp_dir .. '/' .. archive_name
-  local archive_url = 'https://github.com/' .. repo .. '/archive/' .. archive_name
+  local archive_path = vim.fs.joinpath(temp_dir, archive_name)
+  local archive_url = ('https://github.com/%s/archive/%s'):format(repo, archive_name)
+  print(vim.inspect(archive_path), vim.inspect(archive_url))
 
   run_die(
     { 'curl', '-sfL', archive_url, '-o', archive_path },
@@ -90,10 +91,32 @@ local function get_archive_info(repo, ref)
     vim.fn.executable('sha256sum') == 1 and { 'sha256sum', archive_path }
     or { 'shasum', '-a', '256', archive_path }
   )
-  local archive_sha = run(shacmd):gmatch('%w+')()
+  local archive_sha = run_die(shacmd):gmatch('%w+')()
   return { url = archive_url, sha = archive_sha }
 end
 
+--- @param repo string
+--- @param ref string
+--- @param filename string
+local function get_release_artifact_info(repo, ref, filename)
+  local temp_dir = os.getenv('TMPDIR') or os.getenv('TEMP') or '/tmp'
+
+  local artifact_path = vim.fs.joinpath(temp_dir, filename)
+  local artifact_url = ('https://github.com/%s/releases/download/%s/%s'):format(repo, ref, filename)
+
+  run_die(
+    { 'curl', '-sfL', artifact_url, '-o', artifact_path },
+    'Failed to download release artifact from GitHub'
+  )
+
+  local shacmd = (
+    vim.fn.executable('sha256sum') == 1 and { 'sha256sum', artifact_path }
+    or { 'shasum', '-a', '256', artifact_path }
+  )
+  local artifact_sha = run_die(shacmd):gmatch('%w+')()
+  return { url = artifact_url, sha = artifact_sha }
+end
+
 local function get_gh_commit_sha(repo, ref)
   local full_repo = string.format('https://github.com/%s.git', repo)
   local tag_exists = run_die({ 'git', 'ls-remote', full_repo, 'refs/tags/' .. ref }) ~= ''
@@ -151,6 +174,15 @@ local function ref(name, _ref)
   print('Updating ' .. name .. ' to ' .. archive.url .. '\n')
   update_deps_file(symbol_upper, 'URL', archive.url:gsub('/', '\\/'))
   update_deps_file(symbol_upper, 'SHA256', archive.sha)
+
+  -- Keep wasm version in sync with native version
+  if name == 'tree-sitter-lua' then
+    local wasm = get_release_artifact_info(repo, _ref, 'tree-sitter-lua.wasm')
+    print('Updating ' .. name .. ' WASM to ' .. wasm.url .. '\n')
+    update_deps_file(symbol_upper .. '_WASM', 'URL', wasm.url:gsub('/', '\\/'))
+    update_deps_file(symbol_upper .. '_WASM', 'SHA256', wasm.sha)
+  end
+
   run_die({ 'git', 'add', deps_file })
 
   local zig = zig_mode[symbol]

test/functional/treesitter/wasm_spec.lua

@@ -0,0 +1,85 @@
+local t = require('test.testutil')
+local n = require('test.functional.testnvim')()
+
+local clear = n.clear
+local command = n.command
+local eq = t.eq
+local exec_lua = n.exec_lua
+local insert = n.insert
+local skip = t.skip
+
+local lua_text = [[
+local M = {}
+
+-- returns the sum of two numbers
+local function add(a, b)
+  return a + b
+end
+
+M.add = add
+return M
+]]
+
+describe('treesitter WASM parser', function()
+  before_each(clear)
+
+  it('lua WASM parser produces same highlight captures as native parser', function()
+    skip(
+      exec_lua('return vim._ts_add_language_from_wasm == nil'),
+      'N/A ENABLE_WASMTIME not enabled'
+    )
+
+    local wasm_files = exec_lua(function()
+      return vim.api.nvim_get_runtime_file('parser/lua.wasm', true)
+    end)
+    assert(
+      #wasm_files > 0,
+      'lua.wasm not found in runtimepath (rebuild with ENABLE_WASMTIME=ON USE_BUNDLED_TS_PARSERS=ON)'
+    )
+
+    local wasm_path = wasm_files[1]
+
+    local query_str = exec_lua(function()
+      local files = vim.api.nvim_get_runtime_file('queries/lua/highlights.scm', false)
+      assert(#files > 0, 'queries/lua/highlights.scm not found')
+      local f = assert(io.open(files[1]))
+      local s = f:read('*a')
+      f:close()
+      return s
+    end)
+
+    -- Buffer 1: parse with the native lua parser.
+    insert(lua_text)
+    local native_captures = exec_lua(function(qstr)
+      local query = vim.treesitter.query.parse('lua', qstr)
+      local parser = vim.treesitter.get_parser(0, 'lua')
+      local tree = parser:parse()[1]
+      local result = {}
+      for id, node in query:iter_captures(tree:root(), 0) do
+        local r = { node:range() }
+        table.insert(result, { query.captures[id], r[1], r[2], r[3], r[4] })
+      end
+      return result
+    end, query_str)
+
+    -- Buffer 2: same content, but the lua language loaded from WASM.
+    command('new')
+    insert(lua_text)
+    local wasm_captures = exec_lua(function(path, qstr)
+      -- Remove the native registration so the WASM one can take its place.
+      vim._ts_remove_language('lua')
+      vim._ts_add_language_from_wasm(path, 'lua')
+      local query = vim.treesitter.query.parse('lua', qstr)
+      local parser = vim.treesitter.get_parser(0, 'lua')
+      local tree = parser:parse()[1]
+      local result = {}
+      for id, node in query:iter_captures(tree:root(), 0) do
+        local r = { node:range() }
+        table.insert(result, { query.captures[id], r[1], r[2], r[3], r[4] })
+      end
+      return result
+    end, wasm_path, query_str)
+
+    eq(native_captures, wasm_captures)
+  end)
+end)