diff --git a/scripts/docs-i18n/doc_chunked_raw.go b/scripts/docs-i18n/doc_chunked_raw.go index 93ad4f38e2b..417ab822e12 100644 --- a/scripts/docs-i18n/doc_chunked_raw.go +++ b/scripts/docs-i18n/doc_chunked_raw.go @@ -26,8 +26,9 @@ var docsProtocolTokens = []string{ } type docChunkStructure struct { - fenceCount int - tagCounts map[string]int + fenceCount int + tagCounts map[string]int + headingLevels []int } type docChunkSplitPlan struct { @@ -204,6 +205,9 @@ func validateDocChunkTranslation(source, translated string) error { if sourceStructure.fenceCount != translatedStructure.fenceCount { return fmt.Errorf("code fence mismatch: source=%d translated=%d", sourceStructure.fenceCount, translatedStructure.fenceCount) } + if !slices.Equal(sourceStructure.headingLevels, translatedStructure.headingLevels) { + return fmt.Errorf("heading structure mismatch: source=%v translated=%v", sourceStructure.headingLevels, translatedStructure.headingLevels) + } if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) { return fmt.Errorf("component tag set mismatch") } @@ -335,8 +339,9 @@ func summarizeDocChunkStructure(text string) docChunkStructure { } } return docChunkStructure{ - fenceCount: counts["__fence_toggle__"], - tagCounts: countsWithoutFence(counts), + fenceCount: counts["__fence_toggle__"], + tagCounts: countsWithoutFence(counts), + headingLevels: extractMarkdownHeadingLevels(text), } } diff --git a/scripts/docs-i18n/doc_mode_test.go b/scripts/docs-i18n/doc_mode_test.go index 05a49164200..ac268ff1b6a 100644 --- a/scripts/docs-i18n/doc_mode_test.go +++ b/scripts/docs-i18n/doc_mode_test.go @@ -475,6 +475,108 @@ func TestValidateDocChunkTranslationRejectsInventedI18NPlaceholder(t *testing.T) } } +func TestValidateDocChunkTranslationRejectsHeadingLoss(t *testing.T) { + t.Parallel() + + source := "## Detailed behavior and rationale\n\nExplanation.\n" + translated := "详细说明。\n" + + err := validateDocChunkTranslation(source, translated) + if err == nil { + t.Fatal("expected heading loss to be rejected") + } + if !strings.Contains(err.Error(), "heading structure mismatch: source=[2] translated=[]") { + t.Fatalf("expected heading structure error, got %v", err) + } +} + +func TestValidateDocChunkTranslationAcceptsTranslatedHeadingText(t *testing.T) { + t.Parallel() + + source := "## Detailed behavior and rationale\n\nExplanation.\n" + translated := "## 详细行为与设计理由\n\n说明。\n" + + if err := validateDocChunkTranslation(source, translated); err != nil { + t.Fatalf("expected translated heading text with the same level to pass, got %v", err) + } +} + +func TestValidateDocChunkTranslationRejectsSetextHeadingLoss(t *testing.T) { + t.Parallel() + + source := "Detailed behavior and rationale\n---------------------------------\n\nExplanation.\n" + translated := "详细说明。\n" + + err := validateDocChunkTranslation(source, translated) + if err == nil { + t.Fatal("expected Setext heading loss to be rejected") + } + if !strings.Contains(err.Error(), "heading structure mismatch: source=[2] translated=[]") { + t.Fatalf("expected Setext heading structure error, got %v", err) + } +} + +func TestValidateDocChunkTranslationAcceptsTranslatedSetextHeading(t *testing.T) { + t.Parallel() + + source := "Detailed behavior and rationale\n---------------------------------\n\nExplanation.\n" + translated := "详细行为与设计理由\n------------------\n\n说明。\n" + + if err := validateDocChunkTranslation(source, translated); err != nil { + t.Fatalf("expected translated Setext heading with the same level to pass, got %v", err) + } +} + +func TestValidateDocChunkTranslationDoesNotTreatThematicBreakAsSetextHeading(t *testing.T) { + t.Parallel() + + source := "- First item\n\n---\n\nParagraph.\n" + translated := "- 第一项\n\n---\n\n段落。\n" + + if err := validateDocChunkTranslation(source, translated); err != nil { + t.Fatalf("expected thematic break to remain distinct from a Setext heading, got %v", err) + } +} + +func TestValidateDocChunkTranslationRejectsNestedHeadingLoss(t *testing.T) { + t.Parallel() + + source := "> ## Warning\n>\n> Keep this setting enabled.\n\n- ### Step\n Run the command.\n" + translated := "> 警告\n>\n> 保持此设置启用。\n\n- 步骤\n 运行命令。\n" + + err := validateDocChunkTranslation(source, translated) + if err == nil { + t.Fatal("expected nested heading loss to be rejected") + } + if !strings.Contains(err.Error(), "heading structure mismatch: source=[2 3] translated=[]") { + t.Fatalf("expected nested heading structure error, got %v", err) + } +} + +func TestValidateDocChunkTranslationRejectsComponentNestedHeadingLoss(t *testing.T) { + t.Parallel() + + source := "\n## Important\n\nKeep this setting enabled.\n\n" + translated := "\n重要\n\n保持此设置启用。\n\n" + + err := validateDocChunkTranslation(source, translated) + if err == nil { + t.Fatal("expected component-nested heading loss to be rejected") + } + if !strings.Contains(err.Error(), "heading structure mismatch: source=[2] translated=[]") { + t.Fatalf("expected component-nested heading structure error, got %v", err) + } +} + +func TestHeadingExtractionIgnoresComponentExamplesInsideCodeFences(t *testing.T) { + t.Parallel() + + text := "```mdx\n\n## Example only\n\n```\n" + if levels := extractMarkdownHeadingLevels(text); len(levels) != 0 { + t.Fatalf("expected no headings from fenced component example, got %v", levels) + } +} + func TestValidateDocChunkTranslationRejectsTranscriptArtifact(t *testing.T) { t.Parallel() diff --git a/scripts/docs-i18n/markdown_segments.go b/scripts/docs-i18n/markdown_segments.go index 5f77c54beb9..c8d5e77fe57 100644 --- a/scripts/docs-i18n/markdown_segments.go +++ b/scripts/docs-i18n/markdown_segments.go @@ -94,6 +94,38 @@ func extractSegments(body, relPath string) ([]Segment, error) { return filtered, nil } +func extractMarkdownHeadingLevels(body string) []int { + source := []byte(stripDocComponentTagsForHeadingParse(body)) + doc := goldmark.New(goldmark.WithExtensions(extension.GFM)).Parser().Parse(text.NewReader(source)) + levels := []int{} + _ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + heading, ok := node.(*ast.Heading) + if ok { + levels = append(levels, heading.Level) + } + return ast.WalkContinue, nil + }) + return levels +} + +func stripDocComponentTagsForHeadingParse(body string) string { + lines := strings.Split(body, "\n") + fenceDelimiter := "" + for index, line := range lines { + wasInFence := fenceDelimiter != "" + var toggled bool + fenceDelimiter, toggled = updateFenceDelimiter(fenceDelimiter, line) + if wasInFence || toggled || fenceDelimiter != "" { + continue + } + lines[index] = docsComponentTagRE.ReplaceAllString(line, "") + } + return strings.Join(lines, "\n") +} + func blockParent(n ast.Node) ast.Node { for node := n.Parent(); node != nil; node = node.Parent() { if isTranslatableBlock(node) { diff --git a/scripts/docs-i18n/util.go b/scripts/docs-i18n/util.go index 6c1965dccd0..295c8a0c64d 100644 --- a/scripts/docs-i18n/util.go +++ b/scripts/docs-i18n/util.go @@ -12,7 +12,7 @@ import ( const ( workflowVersion = 16 - promptVersion = 7 + promptVersion = 8 docsI18nEngineName = "codex" envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER" envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL" diff --git a/scripts/docs-i18n/util_test.go b/scripts/docs-i18n/util_test.go index 44bb8a70ec4..0dd0e42c3f6 100644 --- a/scripts/docs-i18n/util_test.go +++ b/scripts/docs-i18n/util_test.go @@ -8,7 +8,7 @@ import ( func TestCacheNamespaceIncludesPromptVersion(t *testing.T) { t.Parallel() - if want := "prompt=7"; !strings.Contains(cacheNamespace(), want) { + if want := "prompt=8"; !strings.Contains(cacheNamespace(), want) { t.Fatalf("expected cache namespace to contain %q, got %q", want, cacheNamespace()) } }