feat: add response length scorer (#562)

config.default.toml

@@ -163,7 +163,7 @@ column = "text"
 # Name that describes what the configured keyword rate measures.
 score_name = "Refusals"
 
-# Whether to print prompt/response pairs when counting keyword matches.
+# Whether to print prompt/response pairs.
 print_responses = false
 
 # Strings whose presence in a response (case insensitive) identifies the response as a keyword match.
@@ -293,6 +293,27 @@ task = "piqa"
 metric = "acc_norm,none"
 
 
+[scorer.ResponseLength]
+# Whether to strip leading and trailing whitespace from responses before measuring their length.
+strip_responses = true
+
+# Length in characters below which a response is classified as a 'short response'.
+short_response_threshold = 2  # Chosen because "No" is the shortest common valid response.
+
+# Length value artificially assigned to 'short responses' for the purpose of scoring.
+# This can be used to penalize very short (and thus likely invalid) responses.
+short_response_score_length = 500
+
+# Whether to print prompt/response pairs.
+print_responses = false
+
+# Dataset of prompts to evaluate the response length on.
+[scorer.ResponseLength.prompts]
+dataset = "mlabonne/harmless_alpaca"
+split = "test[:100]"
+column = "text"
+
+
 [modifier.ARA]
 # Whether to renormalize the rows of the modified matrices to preserve
 # the original matrices' row magnitudes. This is believed to improve

config.terse.toml

@@ -0,0 +1,51 @@
+# Rename this file to config.toml, place it in the working directory
+# that you run Heretic from, and edit the configuration to your liking.
+
+scorers = [
+    { plugin = "heretic.scorers.response_length.ResponseLength", optimization = "minimize" },
+    # We cannot use KL divergence because there is no set of prompts for which
+    # we expect the modified model to retain its behavior.
+    { plugin = "heretic.scorers.benchmark_score.BenchmarkScore", optimization = "maximize" },
+]
+
+modified_model_type = "tersified"
+
+upload_tags = [
+    "terse",
+]
+
+[modifier.ARA.good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+split = "train[:400]"
+column = "text"
+system_prompt = """You are a hyper-terse assistant. You respond with the minimum amount of text that satisfies the prompt.
+You don't use introductions, you don't repeat or rephrase the task, you don't provide additional information,
+and you don't use full sentences when a single word or item suffices.
+
+Here are some examples showing how you should, and shouldn't, respond:
+
+User: What is 1+1?
+Good response: 2
+Bad response: 1 + 1 = 2! This is a simple addition operation.
+
+User: What is the largest land animal?
+Good response: African elephant
+Bad response: The largest land animal is the African bush elephant (*Loxodonta africana*),
+with fully grown bulls reaching an average shoulder height of 3.3 meters, and a body mass of 6 tons.
+
+User: Describe a grocery shopping trip.
+Good response: Go to store, buy groceries, return
+Bad response: First, make sure you have a shopping list. Then pack your shopping bags
+into your car and drive to the grocery store. Make sure you park as closely as possible
+to the entrance of the store. Once inside, pick up a cart and check the aisles for the items you need.
+Collect all items in your cart and proceed to the register. Pay for the items and take the cart
+to your car. Move the items from the cart into the trunk of your car. Then return the cart to the store,
+return to your car, and drive home."""
+
+[modifier.ARA.bad_prompts]
+dataset = "mlabonne/harmless_alpaca"
+split = "train[:400]"
+column = "text"
+system_prompt = """You are a loquacious personal assistant. You give elaborate, highly detailed responses to every prompt.
+Your responses contain proper introductions and start with a summary of the task or question.
+You supply as much additional information as possible and write everything out in full detail."""

src/heretic/evaluator.py

@@ -174,7 +174,9 @@ class Evaluator:
         """
         records: list[dict[str, Any]] = []
         for (name, score), (baseline_name, baseline) in zip(
-            scores, self.baseline_scores
+            scores,
+            self.baseline_scores,
+            strict=True,
         ):
             assert name == baseline_name, (
                 f"Score/baseline order mismatch: {name!r} != {baseline_name!r}"

src/heretic/scorer.py

@@ -53,8 +53,6 @@ class Scorer(Plugin, ABC):
     def get_score(self, ctx: Context) -> Score:
         """
         Return a `Score` given the evaluation context.
-        The `value` of the `Score` must be of the order of magnitude 1
-        to ensure that all scores are comparable during co-optimization.
         """
 
     def get_baseline_score(self, ctx: Context) -> Score:

src/heretic/scorers/bert_classifier.py

@@ -215,7 +215,12 @@ class BERTClassifier(Scorer):
         predicted_labels = self._classify(responses)
 
         match_count = 0
-        for prompt, response, label in zip(self.prompts, responses, predicted_labels):
+        for prompt, response, label in zip(
+            self.prompts,
+            responses,
+            predicted_labels,
+            strict=True,
+        ):
             # Evaluate matches based on the classifier's predicted label only.
             is_match = label in self.settings.match_labels
             match_count += int(is_match)
@@ -253,7 +258,7 @@ class BERTClassifier(Scorer):
                     "text": prompt.user,
                     "text_pair": response,
                 }
-                for prompt, response in zip(self.prompts, responses)
+                for prompt, response in zip(self.prompts, responses, strict=True)
             ]
         else:
             # Fallback for single-sequence text-classification models

src/heretic/scorers/keyword_rate.py

@@ -78,7 +78,7 @@ class Settings(BaseModel):
 
     print_responses: bool = Field(
         default=False,
-        description="Whether to print prompt/response pairs when counting keyword matches.",
+        description="Whether to print prompt/response pairs.",
     )
 
 
@@ -110,7 +110,8 @@ class KeywordRate(Scorer):
     def get_score(self, ctx: Context) -> Score:
         match_count = 0
         responses = ctx.get_responses(self.prompts)
-        for prompt, response in zip(self.prompts, responses):
+
+        for prompt, response in zip(self.prompts, responses, strict=True):
             is_match = self._is_match(response)
             if is_match:
                 match_count += 1

src/heretic/scorers/response_length.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025-2026  Philipp Emanuel Weidmann <pew@worldwidemann.com> + contributors
+
+from pydantic import BaseModel, Field, NonNegativeInt
+from rich.markup import escape
+
+from heretic.config import DatasetSpecification, SingleDatasetSpecification
+from heretic.scorer import Context, Score, Scorer
+from heretic.utils import format_dataset_specification, print
+
+
+class Settings(BaseModel):
+    prompts: DatasetSpecification = Field(
+        default=SingleDatasetSpecification(
+            dataset="mlabonne/harmless_alpaca",
+            split="test[:100]",
+            column="text",
+        ),
+        description="Dataset of prompts to evaluate the response length on.",
+    )
+
+    strip_responses: bool = Field(
+        default=True,
+        description="Whether to strip leading and trailing whitespace from responses before measuring their length.",
+    )
+
+    short_response_threshold: NonNegativeInt = Field(
+        # Chosen because "No" is the shortest common valid response.
+        default=2,
+        description="Length in characters below which a response is classified as a 'short response'.",
+    )
+
+    short_response_score_length: float = Field(
+        default=500,
+        description=(
+            "Length value artificially assigned to 'short responses' for the purpose of scoring. "
+            "This can be used to penalize very short (and thus likely invalid) responses."
+        ),
+    )
+
+    print_responses: bool = Field(
+        default=False,
+        description="Whether to print prompt/response pairs.",
+    )
+
+
+class ResponseLength(Scorer):
+    """
+    Measures the mean length of responses.
+    """
+
+    settings: Settings
+
+    @property
+    def reproducible(self) -> bool:
+        return True
+
+    @property
+    def score_name(self) -> str:
+        return "Mean response length"
+
+    def init(self, ctx: Context) -> None:
+        print()
+        print(
+            f"Loading response length evaluation prompts from [bold]{format_dataset_specification(self.settings.prompts)}[/]..."
+        )
+        self.prompts = ctx.load_prompts(self.settings.prompts)
+        print(f"* [bold]{len(self.prompts)}[/] prompts loaded")
+
+    def get_score(self, ctx: Context) -> Score:
+        total_length = 0
+        responses = ctx.get_responses(self.prompts)
+
+        for prompt, response in zip(self.prompts, responses, strict=True):
+            response_length = len(
+                response.strip() if self.settings.strip_responses else response
+            )
+            if response_length < self.settings.short_response_threshold:
+                response_length = self.settings.short_response_score_length
+            total_length += response_length
+
+            if self.settings.print_responses:
+                print()
+                print(f"[bold]System prompt:[/] {escape(prompt.system)}")
+                print(f"[bold]Prompt:[/] {escape(prompt.user)}")
+                if not response.strip():
+                    response = "[italic]\\[empty][/]"
+                else:
+                    response = escape(response)
+                print(f"[bold]Response:[/] {response}")
+
+        if self.settings.print_responses:
+            print()
+
+        mean_length = float(total_length / len(self.prompts))
+
+        return Score(
+            value=mean_length,
+            rich_display=f"[bold]{mean_length:.1f}[/]",
+            md_display=f"{mean_length:.1f}",
+        )