diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index 7f8a21bcc02..fa99cf48be4 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -296,10 +296,11 @@ jobs: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 sudo apt-get update && sudo apt-get install -y bubblewrap socat - # The semcode workflow builds semcode into the actions cache, under a key - # that ends in its commit. The newest entry for the prefix is restored. The - # path is part of the cache version, so it is the one semcode.yml saves. A - # review without semcode is still a review, so a miss only logs. + # The semcode workflow builds semcode and indexes the kernel with it, both + # into the actions cache, under a key that ends in the commit of each. The + # newest entry for the prefix is restored. The path is part of the cache + # version, so it is the one semcode.yml saves. A review without semcode is + # still a review, so a miss only logs. - name: Restore semcode uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: @@ -320,12 +321,29 @@ jobs: echo "Using semcode ${ref}." echo "ref=${ref}" >>"$GITHUB_OUTPUT" + # An index is only readable by the semcode that wrote it, so the kernel + # index is looked up under the commit of the restored binaries. + - name: Restore the kernel index + if: steps.semcode.outputs.ref != '' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/semcode-linux + key: semcode-linux-${{ runner.os }}-${{ runner.arch }}-${{ steps.semcode.outputs.ref }}- + restore-keys: semcode-linux-${{ runner.os }}-${{ runner.arch }}-${{ steps.semcode.outputs.ref }}- + # The model's knowledge of the kernel, glibc, musl, and UAPI specifications can # be wrong or out of date. The review agents therefore check details in the # cloned repositories. A review without a repository is still useful, so a # failed clone does not fail the job. git clones into a temporary directory # first. If the timeout kills git, the final directory does not exist, and # the agents are not told about it. + # + # The kernel index only answers for the commit it was built from, so with an + # index in the cache the kernel is fetched at that commit, which mainline + # keeps, instead of the latest. The fetch has depth two: at startup + # semcode-mcp compares HEAD with its parent to see that the index covers + # HEAD, and without the parent it checks every file instead. When the fetch + # fails, the latest mainline is cloned and the index stays unused. - name: Clone the Linux kernel, glibc, musl, and UAPI specifications continue-on-error: true timeout-minutes: 10 @@ -337,8 +355,24 @@ jobs: mv "$dir.tmp" "$dir" && git -C "$dir" log -1 --format="Cloned $name at %H (%s)" } + clone_linux() { + local url=https://github.com/torvalds/linux.git dir="$RUNNER_TEMP/claude-review/linux" commit + commit="$(cat "$RUNNER_TEMP/semcode-linux/commit" 2>/dev/null || :)" + if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then + if git init --quiet "$dir.tmp" && + git -C "$dir.tmp" fetch --quiet --depth 2 "$url" "$commit" && + git -C "$dir.tmp" checkout --quiet --detach FETCH_HEAD; then + mv "$dir.tmp" "$dir" && + git -C "$dir" log -1 --format="Cloned linux at %H (%s), the commit of the semcode index" + return + fi + rm -rf "$dir.tmp" + echo "Fetching the indexed kernel commit ${commit} failed, cloning the latest mainline instead." + fi + clone linux "$url" --depth 1 --single-branch + } pids=() - clone linux https://github.com/torvalds/linux.git --depth 1 --single-branch & + clone_linux & pids+=($!) clone glibc https://sourceware.org/git/glibc.git --depth 1 --single-branch & pids+=($!) @@ -462,7 +496,7 @@ jobs: cat > ~/.claude/settings.json << 'SETTINGS' { "permissions": { - "allow": ["Bash", "Read", "Grep", "Glob", "mcp__systemd__*"] + "allow": ["Bash", "Read", "Grep", "Glob", "mcp__systemd__*", "mcp__linux__*"] }, "hooks": { "PreToolUse": [ @@ -593,7 +627,7 @@ jobs: if [[ -d "$linux" ]]; then context+=" - A shallow clone of the latest mainline Linux kernel is at ${linux}. It has no history. When a + A shallow clone of a recent mainline Linux kernel is at ${linux}. It has no history. When a finding depends on kernel behavior, such as system call semantics, flags, error codes, or ioctl and netlink interfaces, check the kernel source there instead of relying on memory." fi @@ -651,7 +685,7 @@ jobs: sandbox=(bwrap --ro-bind / / --tmpfs /run --dev /dev --proc /proc --tmpfs /tmp --clearenv --setenv PATH /usr/bin:/bin --setenv HOME "$HOME" --unshare-all --die-with-parent --new-session) if ! "${sandbox[@]}" -- /bin/true; then echo "bubblewrap does not work, reviewing without semcode." - rm -rf "$RUNNER_TEMP/claude-review/semcode-systemd" + rm -rf "$RUNNER_TEMP/claude-review/semcode-systemd" "$RUNNER_TEMP/semcode-linux" fi add_server() { local name=$1 db=$2 repo=$3 wrapper="$RUNNER_TEMP/claude-review/semcode-mcp-$1" @@ -691,6 +725,20 @@ jobs: indexed; read those files directly." fi + linux_index="$RUNNER_TEMP/semcode-linux" + linux_indexed="" + if [[ -d "$linux_index/.semcode.db" && -d "$linux" ]] && + [[ "$(cat "$linux_index/commit" 2>/dev/null)" =~ ^[0-9a-f]{40}$ ]] && + [[ "$(git -C "$linux" rev-parse HEAD)" == "${BASH_REMATCH[0]}" ]] && + add_server linux "$linux_index/.semcode.db" "$linux"; then + linux_indexed="$linux" + context+=" + + The kernel tree is indexed with semcode; its tools are the mcp__linux__* tools. Use + find_function, find_type, find_callers, find_calls, find_callchain and grep_functions to read + and search kernel code instead of grep." + fi + if [[ "$servers" != "{}" ]]; then jq -n --argjson servers "$servers" '{mcpServers: $servers}' >"$RUNNER_TEMP/claude-review/mcp.json" mcp_args=(--mcp-config "$RUNNER_TEMP/claude-review/mcp.json" --strict-mcp-config) diff --git a/.github/workflows/semcode.yml b/.github/workflows/semcode.yml index e41fc3efd5e..37b46f4f56a 100644 --- a/.github/workflows/semcode.yml +++ b/.github/workflows/semcode.yml @@ -1,15 +1,21 @@ # SPDX-License-Identifier: LGPL-2.1-or-later -# Builds semcode for the Claude review. +# Builds semcode for the Claude review and indexes the Linux kernel with it. # # The review agents look up functions, types and call graphs with semcode # instead of grepping through the trees. A cold build of semcode takes about -# twenty minutes, far more than a review may spend on it, so the binaries go -# into the actions cache under +# twenty minutes and a kernel index longer than that, far more than a review +# may spend on them. Both go into the actions cache, from which the review job +# restores them: # -# semcode-bin--- +# semcode-bin--- the binaries +# semcode-linux---- the kernel index # -# from which the review job restores the newest entry for the prefix. A cache -# nobody restored for a week is evicted, so this also runs weekly. +# The review restores the newest entry for the prefix it knows. The kernel +# index only answers for the commit it was built from, which the review +# therefore clones instead of the latest mainline. An index is tied to the +# semcode that wrote it, so a new SEMCODE_REF rebuilds both. A cache nobody +# restored for a week is evicted, and the kernel moves on, so this also runs +# weekly. name: semcode @@ -78,3 +84,52 @@ jobs: install -D -s -t "$RUNNER_TEMP/semcode/bin" "$src/target/release/semcode-index" "$src/target/release/semcode-mcp" echo "$SEMCODE_REF" >"$RUNNER_TEMP/semcode/ref" ls -l "$RUNNER_TEMP/semcode/bin" + + linux: + runs-on: ubuntu-latest + needs: build + timeout-minutes: 120 + + steps: + - name: Restore semcode + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/semcode + key: ${{ needs.build.outputs.key }} + fail-on-cache-miss: true + + # The review runs semcode in a bubblewrap sandbox and so does this, for + # one less difference between the two. Ubuntu restricts the unprivileged + # user namespaces it needs. + - name: Install the sandbox + run: | + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + sudo apt-get update && sudo apt-get install -y bubblewrap + + - name: Clone the Linux kernel + id: linux + run: | + git clone --quiet --depth 1 --single-branch https://github.com/torvalds/linux.git "$RUNNER_TEMP/linux" + git -C "$RUNNER_TEMP/linux" log -1 --format="Cloned linux at %H (%s)" + echo "commit=$(git -C "$RUNNER_TEMP/linux" rev-parse HEAD)" >>"$GITHUB_OUTPUT" + + # Two analysis threads keep the indexer below nine gigabytes, which a + # runner has; with four it peaks above eleven. + - name: Index the Linux kernel + env: + LINUX_COMMIT: ${{ steps.linux.outputs.commit }} + run: | + index="$RUNNER_TEMP/semcode-linux" + mkdir -p "$index" + bwrap --ro-bind / / --tmpfs /run --dev /dev --proc /proc --tmpfs /tmp --bind "$index" "$index" \ + --clearenv --setenv PATH /usr/bin:/bin --setenv HOME "$HOME" \ + --unshare-all --die-with-parent --new-session -- \ + "$RUNNER_TEMP/semcode/bin/semcode-index" --jobs 2 --source "$RUNNER_TEMP/linux" --database "$index/.semcode.db" + echo "$LINUX_COMMIT" >"$index/commit" + du -sh "$index" + + - name: Save the kernel index + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/semcode-linux + key: semcode-linux-${{ runner.os }}-${{ runner.arch }}-${{ env.SEMCODE_REF }}-${{ steps.linux.outputs.commit }}