fix(skills-index): stop shipping skills.sh rows that cannot install

The builder shipped ~1.9k skills.sh rows per run with no resolved_github_id.
Most were rows whose repo 404s or whose fully-read tree has no matching
SKILL.md dir (renamed/removed upstream); every client install of one spent
~40 GitHub calls and failed as stale_index.

- fuzzy match ignores both '-' and '_' (pdb-database slug vs pdb_database dir)
- a single-skill repo (root or generic skills/ SKILL.md) resolves like the client does
- repo 404 or complete tree with no match: the row is dropped, counts logged
- truncated tree: bounded contents/<candidate>/SKILL.md probes, row kept if none hit
- transient errors keep the row unresolved, as before

scripts/build_skills_index.py

@@ -18,9 +18,10 @@ Output: website/static/api/skills-index.json
 
 import json
 import os
+import re
 import sys
 import time
-from collections import defaultdict
+from collections import Counter, defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone, UTC
 
@@ -42,6 +43,8 @@ import httpx
 OUTPUT_PATH = os.path.join(REPO_ROOT, "website", "static", "api", "skills-index.json")
 INDEX_VERSION = 1
 CLAWHUB_ENRICH_BUDGET_SECONDS = 480
+# Per-repo cap on contents/<candidate>/SKILL.md probes when the recursive tree is truncated.
+MAX_TRUNCATED_TREE_PROBES = 10
 
 
 def _meta_to_dict(meta: SkillMeta) -> dict:
@@ -103,8 +106,9 @@ def crawl_skills_sh(source: SkillsShSource) -> list:
     return list(all_skills.values())
 
 
-def _fetch_repo_tree(repo: str, auth: GitHubAuth) -> list:
-    """Fetch the recursive tree for a repo. Returns list of tree entries."""
+def _fetch_repo_tree(repo: str, auth: GitHubAuth) -> tuple[str, list]:
+    """Recursive tree for a repo as ``(status, entries)``: ``ok``, ``truncated`` (partial
+    entries), ``gone`` (the repo 404s) or ``error`` (anything transient — the row is kept)."""
     headers = auth.get_headers()
     try:
         resp = httpx.get(
@@ -112,7 +116,7 @@ def _fetch_repo_tree(repo: str, auth: GitHubAuth) -> list:
             headers=headers, timeout=15, follow_redirects=True,
         )
         if resp.status_code != 200:
-            return []
+            return ("gone" if resp.status_code == 404 else "error"), []
         branch = resp.json().get("default_branch", "main")
 
         resp = httpx.get(
@@ -121,13 +125,27 @@ def _fetch_repo_tree(repo: str, auth: GitHubAuth) -> list:
             headers=headers, timeout=30, follow_redirects=True,
         )
         if resp.status_code != 200:
-            return []
+            return "error", []
         data = resp.json()
-        if data.get("truncated"):
-            return []
-        return data.get("tree", [])
+        return ("truncated" if data.get("truncated") else "ok"), data.get("tree", [])
     except Exception:
-        return []
+        return "error", []
+
+
+def _probe_skill_md(repo: str, identifier: str, auth: GitHubAuth) -> str | None:
+    """First of the client's own candidate paths for ``owner/repo/slug`` (``SkillsShSource._candidate_identifiers``)
+    that holds a SKILL.md, for repos too big for one recursive tree (posthog/posthog)."""
+    for candidate in SkillsShSource._candidate_identifiers(identifier):
+        try:
+            resp = httpx.get(
+                f"https://api.github.com/repos/{repo}/contents/{candidate.split('/', 2)[2]}/SKILL.md",
+                headers=auth.get_headers(), timeout=15, follow_redirects=True,
+            )
+        except httpx.HTTPError:
+            continue
+        if resp.status_code == 200:
+            return candidate
+    return None
 
 
 def batch_resolve_paths(skills: list, auth: GitHubAuth) -> list:
@@ -158,31 +176,41 @@ def batch_resolve_paths(skills: list, auth: GitHubAuth) -> list:
     print(f"    {len(by_repo)} unique repos to scan", flush=True)
 
     resolved_count = 0
+    dropped: dict[str, str] = {}  # identifier -> "repo_gone" | "not_in_repo"
+
+    def _norm(name: str) -> str:
+        return re.sub(r"[-_]", "", name.lower())
 
     # Fetch trees in parallel (up to 6 concurrent)
     def _resolve_repo(repo: str, entries: list):
-        tree = _fetch_repo_tree(repo, auth)
-        if not tree:
+        status, tree = _fetch_repo_tree(repo, auth)
+        if status == "gone":
+            # A 404 repo can never install: shipping the row only made every client spend
+            # ~40 API calls on it and fail as stale_index.
+            dropped.update((e["identifier"], "repo_gone") for e in entries)
+            return 0
+        if status == "error":
             return 0
 
         # Find all SKILL.md paths in this repo
         skill_paths = {}  # skill_dir_name -> full_path
+        skill_dirs = []
         for item in tree:
             if item.get("type") != "blob":
                 continue
             path = item.get("path", "")
-            if path.endswith("/SKILL.md"):
-                skill_dir = path[: -len("/SKILL.md")]
-                dir_name = skill_dir.split("/")[-1]
-                skill_paths[dir_name.lower()] = f"{repo}/{skill_dir}"
-
-                # Also check SKILL.md frontmatter name if we can match by path
-                # For now, just index by directory name
-            elif path == "SKILL.md":
-                # Root-level SKILL.md
-                skill_paths["_root_"] = f"{repo}"
+            if path.endswith("/SKILL.md") or path == "SKILL.md":
+                skill_dir = path[: -len("SKILL.md")].rstrip("/")
+                skill_dirs.append(skill_dir)
+                if skill_dir:
+                    skill_paths[skill_dir.split("/")[-1].lower()] = f"{repo}/{skill_dir}"
+        # The client installs a single-skill repo whose only SKILL.md sits at the root or in a
+        # generic dir whatever the slug (GitHubSource._find_skill_in_repo_tree / _find_repo_root_skill).
+        lone_skill = (f"{repo}/{skill_dirs[0]}"
+                      if skill_dirs in ([""], ["skills"], [".agents/skills"], [".claude/skills"]) else None)
 
         count = 0
+        probes_left = MAX_TRUNCATED_TREE_PROBES
         for entry in entries:
             # Try to match the skill's name/path to a tree entry
             skill_name = entry.get("name", "").lower()
@@ -194,26 +222,25 @@ def batch_resolve_paths(skills: list, auth: GitHubAuth) -> list:
             parts = identifier.replace("skills-sh/", "").replace("skills.sh/", "")
             skill_token = parts.split("/")[-1].lower() if "/" in parts else ""
 
-            # Try matching in order of likelihood
-            for candidate in [skill_token, skill_name, skill_path]:
-                if not candidate:
-                    continue
-                matched = skill_paths.get(candidate)
-                if matched:
-                    entry["resolved_github_id"] = matched
-                    count += 1
-                    break
-            else:
-                # Try fuzzy: skill_token with common transformations
-                for tree_name, tree_path in skill_paths.items():
-                    if (skill_token and (
-                        tree_name.replace("-", "") == skill_token.replace("-", "")
-                        or skill_token in tree_name
-                        or tree_name in skill_token
-                    )):
-                        entry["resolved_github_id"] = tree_path
-                        count += 1
-                        break
+            # Try matching in order of likelihood, then fuzzy (``pdb-database`` slug vs
+            # ``pdb_database`` dir), then the lone-skill layout.
+            matched = next((skill_paths[c] for c in (skill_token, skill_name, skill_path)
+                            if c and c in skill_paths), None) or next(
+                (tree_path for tree_name, tree_path in skill_paths.items() if skill_token and (
+                    _norm(tree_name) == _norm(skill_token)
+                    or skill_token in tree_name
+                    or tree_name in skill_token
+                )), None) or lone_skill
+            if not matched and status == "truncated" and probes_left > 0:
+                probes_left -= 1
+                matched = _probe_skill_md(repo, parts, auth)
+            if matched:
+                entry["resolved_github_id"] = matched
+                count += 1
+            elif status == "ok":
+                # The whole tree was read and no SKILL.md dir is this skill: renamed or removed
+                # upstream while skills.sh still lists it.
+                dropped[identifier] = "not_in_repo"
 
         return count
 
@@ -230,9 +257,11 @@ def batch_resolve_paths(skills: list, auth: GitHubAuth) -> list:
                 print(f"    Warning: {repo}: {e}", file=sys.stderr)
 
     elapsed = time.time() - start
-    print(f"  Resolved {resolved_count}/{len(skills_sh)} paths ({elapsed:.1f}s)",
+    reasons = Counter(dropped.values())
+    print(f"  Resolved {resolved_count}/{len(skills_sh)} paths, dropped {len(dropped)} uninstallable "
+          f"({reasons['repo_gone']} repo gone, {reasons['not_in_repo']} not in repo) ({elapsed:.1f}s)",
           flush=True)
-    return skills
+    return [s for s in skills if s.get("identifier") not in dropped]
 
 
 def main():
@@ -359,7 +388,6 @@ def main():
                     "browse-sh": 5, "lobehub": 6}
     deduped.sort(key=lambda s: (source_order.get(s["source"], 99), s["name"]))
 
-    from collections import Counter
     by_source = Counter(s["source"] for s in deduped)
     print(f"\nCrawled {len(deduped)} skills in {time.time() - overall_start:.0f}s")
     for src, count in sorted(by_source.items(), key=lambda x: -x[1]):

tests/scripts/test_build_skills_index_resolve.py

@@ -0,0 +1,83 @@
+"""Invariants for scripts/build_skills_index.py's skills.sh path resolution.
+
+Every build shipped ~1.9k skills.sh rows with no ``resolved_github_id``; most were rows whose
+repo 404s or whose repo no longer has the skill, and each cost a client ~40 GitHub calls before it
+failed as ``stale_index``. Tree payloads below are trimmed copies of the real GitHub responses
+(google-deepmind/science-skills, expo/skills, PostHog/posthog, serpdownloaders/skills).
+"""
+
+import httpx
+
+import scripts.build_skills_index as build_mod
+
+_REPOS = {
+    "google-deepmind/science-skills": {"tree": [
+        {"path": "skills/pdb_database", "type": "tree"},
+        {"path": "skills/pdb_database/SKILL.md", "type": "blob"},
+        {"path": "skills/alphafold_database_fetch_and_analyze/SKILL.md", "type": "blob"},
+    ], "truncated": False},
+    "expo/skills": {"tree": [
+        {"path": "plugins/expo/skills/eas-hosting/SKILL.md", "type": "blob"},
+    ], "truncated": False},
+    # 58k entries live; GitHub caps the recursive listing and flags it truncated.
+    "posthog/posthog": {"tree": [{"path": "frontend/src/index.tsx", "type": "blob"}], "truncated": True},
+    "flaky/repo": None,  # 502 on the repo lookup: transient, never a reason to drop
+}
+_CONTENTS = {"posthog/posthog/.agents/skills/react-doctor/SKILL.md"}
+
+
+def _fake_get(url, **_kwargs):
+    req = httpx.Request("GET", url)
+    rest = url.removeprefix("https://api.github.com/repos/")
+    repo = "/".join(rest.split("/")[:2])
+    if repo not in _REPOS:
+        return httpx.Response(404, json={"message": "Not Found"}, request=req)
+    if _REPOS[repo] is None:
+        return httpx.Response(502, request=req)
+    tail = rest[len(repo):]
+    if not tail:
+        return httpx.Response(200, json={"default_branch": "main"}, request=req)
+    if tail.startswith("/git/trees/"):
+        return httpx.Response(200, json=_REPOS[repo], request=req)
+    path = tail.removeprefix("/contents/")
+    return httpx.Response(200 if f"{repo}/{path}" in _CONTENTS else 404, json={}, request=req)
+
+
+class _Auth:
+    def get_headers(self):
+        return {}
+
+
+def _row(identifier):
+    repo = "/".join(identifier.split("/")[1:3])
+    slug = identifier.rsplit("/", 1)[-1]
+    return {"name": slug, "source": "skills.sh", "identifier": identifier, "repo": repo, "path": slug}
+
+
+def _resolve(monkeypatch, *identifiers):
+    monkeypatch.setattr(build_mod.httpx, "get", _fake_get)
+    out = build_mod.batch_resolve_paths([_row(i) for i in identifiers], _Auth())
+    return {s["identifier"]: s.get("resolved_github_id") for s in out}
+
+
+def test_underscore_dirs_resolve_and_uninstallable_rows_are_not_shipped(monkeypatch):
+    shipped = _resolve(
+        monkeypatch,
+        "skills-sh/google-deepmind/science-skills/pdb-database",
+        "skills-sh/expo/skills/building-native-ui",      # tree read fine, skill gone upstream
+        "skills-sh/serpdownloaders/skills/hulu-downloader",  # repo 404s
+        "skills-sh/flaky/repo/some-skill",
+    )
+    assert shipped == {
+        "skills-sh/google-deepmind/science-skills/pdb-database": "google-deepmind/science-skills/skills/pdb_database",
+        "skills-sh/flaky/repo/some-skill": None,
+    }
+
+
+def test_truncated_tree_resolves_through_bounded_skill_md_probes(monkeypatch):
+    shipped = _resolve(monkeypatch, "skills-sh/posthog/posthog/react-doctor", "skills-sh/posthog/posthog/gone-skill")
+    # A partial tree proves nothing about absence: the unmatched row stays, unresolved.
+    assert shipped == {
+        "skills-sh/posthog/posthog/react-doctor": "posthog/posthog/.agents/skills/react-doctor",
+        "skills-sh/posthog/posthog/gone-skill": None,
+    }