Coverage for transformer_lens/benchmarks/text_quality.py: 80%
327 statements
« prev ^ index » next coverage.py v7.10.1, created at 2026-09-01 16:23 +0000
1"""Text quality benchmark for TransformerBridge.
3Generates text the way a real user of the model would (its prompt profile:
4chat template, translation source, code, own-language continuation — see
5``text_quality_profiles``) and scores each output against a known-good
6reference completion with one pinned multilingual judge. The score derives
7from the perplexity ratio PPL_judge(generated)/PPL_judge(reference), which
8cancels the judge's per-language handicap; a repetition penalty catches
9degenerate loops (which the ratio alone rewards) and a length penalty
10catches truncated output.
12Generation is seeded per prompt for reproducibility, and the judge is loaded
13once (CPU/fp32 always, so scores do not depend on the verifying machine) and
14reused across all prompts.
15"""
17import gc
18import math
19from typing import Any, List, Optional, Tuple, Union
21import torch
22from transformers import (
23 AutoModelForCausalLM,
24 AutoTokenizer,
25 PreTrainedModel,
26 PreTrainedTokenizerBase,
27)
29from transformer_lens.benchmarks.text_quality_profiles import (
30 CAPTION_REFERENCES,
31 JUDGE_CONTEXT_KINDS,
32 JUDGE_R_FAIL,
33 LANG_ISO3,
34 LANG_NAMES,
35 MAX_NEW_TOKENS_BY_KIND,
36 NLLB_CODES,
37 PREPEND_BOS_BY_KIND,
38 T5_PREFIX_ARCHITECTURES,
39 TEMPERATURE_BY_KIND,
40 ProfilePrompt,
41 ProfileSpec,
42 p4_pass_threshold,
43 prompts_for,
44)
45from transformer_lens.benchmarks.utils import (
46 BenchmarkResult,
47 BenchmarkSeverity,
48 deterministic_rng,
49)
50from transformer_lens.model_bridge import TransformerBridge
52# The one judge every model is scored with, pinned by revision so a Hub update
53# can never silently move every score. Selection + measurements live in
54# scripts/text_quality_judge_bakeoff.py; separation is weakest in de/ru, so
55# scores there carry wider error bars.
56JUDGE_MODEL_ID = "Qwen/Qwen2.5-0.5B"
57JUDGE_REVISION = "060db6499f32faf8b98477b0a26969ef7d8b9987"
60def load_judge() -> Tuple[PreTrainedModel, PreTrainedTokenizerBase]:
61 """Load the pinned judge on CPU in fp32 (machine-independent scores)."""
62 tokenizer = AutoTokenizer.from_pretrained(JUDGE_MODEL_ID, revision=JUDGE_REVISION)
63 model = AutoModelForCausalLM.from_pretrained(
64 JUDGE_MODEL_ID, revision=JUDGE_REVISION, dtype=torch.float32
65 )
66 torch.nn.Module.to(model, "cpu")
67 model.eval()
68 return model, tokenizer
71def _judge_perplexity(
72 text: str,
73 context: str,
74 tokenizer: Any,
75 judge: Any,
76) -> Tuple[float, Optional[str]]:
77 """Judge perplexity of ``text``; ``context`` tokens are label-masked so only
78 ``text`` is scored. Returns (ppl, error)."""
79 try:
80 # Tokenize the pieces separately: tokenizing the concatenated string
81 # lets BPE merge across the boundary and shifts the label mask into
82 # the scored text.
83 text_ids = tokenizer(text, return_tensors="pt")["input_ids"]
84 context_len = 0
85 input_ids = text_ids
86 if context:
87 context_ids = tokenizer(context, return_tensors="pt")["input_ids"]
88 context_len = context_ids.shape[1]
89 input_ids = torch.cat([context_ids, text_ids], dim=1)
91 if text_ids.shape[1] < 2: 91 ↛ 92line 91 didn't jump to line 92 because the condition on line 91 was never true
92 return float("inf"), "Scored text too short (< 2 judge tokens)"
94 labels = input_ids.clone()
95 if context_len:
96 labels[0, :context_len] = -100
98 with torch.no_grad():
99 loss = judge(input_ids, labels=labels).loss.item()
100 return math.exp(loss), None
101 except Exception as e:
102 return float("inf"), f"Perplexity computation failed: {str(e)}"
105def _compute_repetition_penalty(text: str, ns: Tuple[int, ...] = (2, 3, 4)) -> float:
106 """Minimum unique-n-gram ratio in [0, 1]; low values mean looping output.
108 Load-bearing under ratio scoring: a degenerate loop has LOW judge
109 perplexity, so without this multiplier it would score 100.
110 """
111 words = text.lower().split()
112 # Scriptio continua (zh/ja): word n-grams are inert exactly where the
113 # judge rewards loops most, and a single stray space would restore the
114 # word path — so char n-grams whenever the text is CJK-dominated.
115 compact = "".join(text.split())
116 if compact: 116 ↛ 120line 116 didn't jump to line 120 because the condition on line 116 was always true
117 cjk = sum(1 for c in compact if 0x3040 <= ord(c) <= 0x30FF or 0x4E00 <= ord(c) <= 0x9FFF)
118 if cjk / len(compact) >= 0.3 and len(compact) >= 8:
119 words = list(compact)
120 if len(words) < 2: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true
121 return 1.0
123 min_ratio = 1.0
124 for n in ns:
125 if len(words) < n: 125 ↛ 126line 125 didn't jump to line 126 because the condition on line 125 was never true
126 continue
127 ngrams = [tuple(words[i : i + n]) for i in range(len(words) - n + 1)]
128 if len(ngrams) == 0: 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true
129 continue
130 unique_ratio = len(set(ngrams)) / len(ngrams)
131 min_ratio = min(min_ratio, unique_ratio)
133 return min_ratio
136def _ratio_to_score(ratio: float) -> float:
137 """Map generated/reference perplexity ratio to 0-100.
139 score = 100 - 100*ln(ratio)/ln(R_FAIL), clamped: ratio<=1 (as good as the
140 reference) scores 100, ratio=R_FAIL scores 0, and score 50 falls at
141 sqrt(R_FAIL) — the geometric midpoint between reference quality and
142 unambiguously broken output, which keeps the registry's phase-4 floor of
143 50 principled.
144 """
145 if ratio <= 0 or math.isinf(ratio) or math.isnan(ratio): 145 ↛ 146line 145 didn't jump to line 146 because the condition on line 145 was never true
146 return 0.0
147 if ratio <= 1.0:
148 return 100.0
149 return max(0.0, min(100.0, 100.0 - 100.0 * math.log(ratio) / math.log(JUDGE_R_FAIL)))
152_SCRIPT_RANGES: dict[str, Tuple[Tuple[int, int], ...]] = {
153 "zh": ((0x4E00, 0x9FFF),),
154 "ja": ((0x3040, 0x30FF), (0x4E00, 0x9FFF)),
155 "ar": ((0x0600, 0x06FF),),
156 "ru": ((0x0400, 0x04FF),),
157 "hi": ((0x0900, 0x097F),),
158}
160_LATIN_STOPWORDS: dict[str, frozenset] = {
161 "en": frozenset("the and of to is that with for was are it in on".split()),
162 "fr": frozenset(
163 "le la les des une est que je pas dans de et il elle un en du au pour sur ne ce se".split()
164 ),
165 "es": frozenset("el los una es que no por con para como de en la y se del las".split()),
166 "de": frozenset(
167 "der die das und ist nicht ich ein eine mit den von zu im auf f\u00fcr sich".split()
168 ),
169 "it": frozenset("il la di che non per una sono del gli e in un le si con".split()),
170 "nl": frozenset("de het een en is niet ik van dat met op voor aan zijn".split()),
171 "pt": frozenset("o os uma de e que n\u00e3o para com por em um as dos da".split()),
172}
175def _wrong_language(text: str, lang: str) -> bool:
176 """Conservatively true only when the text is clearly NOT in ``lang``.
178 Ratio scoring alone measures fluency, not language: fluent English output
179 beats a short German reference and clamps to 100, so an untranslated echo
180 would otherwise score perfectly. Non-Latin targets check script presence;
181 Latin targets require zero expected-language stopwords while another
182 covered language has several.
183 """
184 if lang in ("code", ""):
185 return False
186 ranges = _SCRIPT_RANGES.get(lang)
187 if ranges is not None:
188 letters = [c for c in text if c.isalpha()]
189 if not letters: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 return False
191 in_script = sum(1 for c in letters if any(lo <= ord(c) <= hi for lo, hi in ranges))
192 return in_script / len(letters) < 0.3
193 expected = _LATIN_STOPWORDS.get(lang)
194 if expected is None: 194 ↛ 195line 194 didn't jump to line 195 because the condition on line 194 was never true
195 return False
196 tokens = [w.strip(".,;:!?\"'()") for w in text.lower().split()]
197 hits = {code: sum(1 for w in tokens if w in stops) for code, stops in _LATIN_STOPWORDS.items()}
198 return hits[lang] == 0 and max(hits.values(), default=0) >= 3
201def _length_penalty(gen_tokens: int, ref_tokens: int) -> float:
202 """Penalize output far shorter OR far longer than its reference.
204 Neutral band [0.5x, 3x] of reference length. The old 25% floor never
205 fired once in four validation sweeps — a contentless 13-token chat stub
206 against a 41-token reference scored 93.6; at a 0.5x floor it drops below
207 the pass line. The 3x cap is the second net for rambling output the
208 repetition penalty misses."""
209 if ref_tokens <= 0:
210 return 1.0
211 under = gen_tokens / (0.5 * ref_tokens)
212 over = (3.0 * ref_tokens) / max(gen_tokens, 1)
213 return max(0.0, min(1.0, under, over))
216def _build_caption_test_images(n: int = 3) -> list:
217 """A few distinct synthetic images so caption scoring averages over samples
218 (content is unimportant — P4 only measures whether generation is grammatical)."""
219 from PIL import Image, ImageDraw
221 specs = [
222 (
223 "white",
224 [("rectangle", (30, 30, 110, 150), "blue"), ("ellipse", (120, 60, 200, 140), "green")],
225 ),
226 ("black", [("ellipse", (40, 40, 180, 180), "yellow")]),
227 (
228 "skyblue",
229 [
230 ("rectangle", (20, 120, 200, 200), "darkgreen"),
231 ("ellipse", (140, 20, 200, 80), "orange"),
232 ],
233 ),
234 ]
235 images = []
236 for bg, shapes in specs[:n]:
237 img = Image.new("RGB", (224, 224), color=bg)
238 draw = ImageDraw.Draw(img)
239 for kind, box, color in shapes:
240 getattr(draw, kind)(box, fill=color)
241 images.append(img)
242 return images
245def _generate_image_conditioned_captions(
246 bridge: TransformerBridge, max_new_tokens: int
247) -> List[Tuple[int, str]]:
248 """Caption synthetic images for image-conditioned seq2seq (Florence-2 emits
249 nothing text-only, so text-only P4 is uninformative); [] if no processor/PIL."""
250 processor = getattr(bridge, "processor", None)
251 if processor is None: 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true
252 return []
253 try:
254 images = _build_caption_test_images()
255 except Exception:
256 return []
258 # Task captioners (Florence-2) map a task token to an internal prompt.
259 # <DETAILED_CAPTION> yields a full grammatical sentence; plain <CAPTION> is
260 # only 2-3 words (too short to score) and <MORE_DETAILED_CAPTION> tends to
261 # loop on out-of-distribution synthetic images.
262 is_task_captioner = hasattr(processor, "post_process_generation")
263 task = "<DETAILED_CAPTION>" if is_task_captioner else "Describe this image in detail."
265 samples: List[Tuple[int, str]] = []
266 for i, image in enumerate(images):
267 try:
268 inputs = processor(text=task, images=image, return_tensors="pt")
269 input_ids = inputs["input_ids"].to(bridge.cfg.device)
270 extra = {
271 k: (v.to(bridge.cfg.device) if hasattr(v, "to") else v)
272 for k, v in inputs.items()
273 if k != "input_ids"
274 }
275 out = bridge.generate(
276 input_ids, max_new_tokens=max_new_tokens, return_type="tokens", **extra
277 )
278 if isinstance(out, torch.Tensor): 278 ↛ 266line 278 didn't jump to line 266 because the condition on line 278 was always true
279 is_encoder_decoder = bool(
280 getattr(getattr(bridge, "original_model", None), "config", None)
281 and getattr(bridge.original_model.config, "is_encoder_decoder", False)
282 )
283 # Decoder-only VLM output is prompt + continuation; scoring the
284 # fluent prompt as caption text would inflate every sample.
285 caption_ids = out[0] if is_encoder_decoder else out[0, input_ids.shape[-1] :]
286 text = bridge.tokenizer.decode(caption_ids, skip_special_tokens=True).strip()
287 if text: 287 ↛ 266line 287 didn't jump to line 266 because the condition on line 287 was always true
288 samples.append((i, text))
289 except Exception:
290 continue
291 return samples
294def _architecture_id(bridge: Any) -> str:
295 """First HF architecture name of the wrapped model, or ''."""
296 config = getattr(getattr(bridge, "original_model", None), "config", None)
297 architectures = getattr(config, "architectures", None) or []
298 return architectures[0] if architectures else ""
301def _resolve_lang_code(tokenizer, lang: str) -> Optional[str]:
302 """The tokenizer's own code string for ``lang`` ("de" / "de_DE" /
303 "deu_Latn"), or None. transformers 5.x NllbTokenizer exposes neither
304 get_lang_id nor lang_code_to_id, so candidates are probed through the
305 vocab as well."""
306 lang = lang.lower()
307 lang_code_to_id = getattr(tokenizer, "lang_code_to_id", None)
308 if isinstance(lang_code_to_id, dict): 308 ↛ 309line 308 didn't jump to line 309 because the condition on line 308 was never true
309 if lang in lang_code_to_id:
310 return lang
311 iso3 = LANG_ISO3.get(lang, "")
312 for code in lang_code_to_id:
313 code_lower = code.lower()
314 if code_lower.startswith(lang + "_") or (iso3 and code_lower.startswith(iso3 + "_")):
315 return code
316 # Vocab probing is only safe for DISTINCTIVE code forms ("deu_Latn"): a
317 # bare ISO code collides with ordinary subwords (T5's "de", Marian's "en")
318 # and would be injected as a forced decoder token.
319 nllb = NLLB_CODES.get(lang)
320 unk_id = getattr(tokenizer, "unk_token_id", None)
321 convert = getattr(tokenizer, "convert_tokens_to_ids", None)
322 if nllb and callable(convert):
323 try:
324 token_id = convert(nllb)
325 except Exception:
326 return None
327 if isinstance(token_id, int) and token_id >= 0 and token_id != unk_id:
328 return nllb
329 return None
332def _forced_bos_for_target(tokenizer, tgt_lang: str) -> Optional[int]:
333 """Target-language decoder token for multilingual translators, or None."""
334 get_lang_id = getattr(tokenizer, "get_lang_id", None)
335 if callable(get_lang_id):
336 try:
337 return int(get_lang_id(tgt_lang))
338 except Exception:
339 return None
340 code = _resolve_lang_code(tokenizer, tgt_lang)
341 if code is None:
342 return None
343 lang_code_to_id = getattr(tokenizer, "lang_code_to_id", None)
344 if isinstance(lang_code_to_id, dict) and code in lang_code_to_id: 344 ↛ 345line 344 didn't jump to line 345 because the condition on line 344 was never true
345 return int(lang_code_to_id[code])
346 try:
347 token_id = tokenizer.convert_tokens_to_ids(code)
348 except Exception:
349 return None
350 if ( 350 ↛ 356line 350 didn't jump to line 356 because the condition on line 350 was always true
351 isinstance(token_id, int)
352 and token_id >= 0
353 and token_id != getattr(tokenizer, "unk_token_id", None)
354 ):
355 return int(token_id)
356 return None
359def _build_model_input(
360 bridge: Any,
361 spec: ProfileSpec,
362 prompt: ProfilePrompt,
363 architecture_id: str,
364) -> str:
365 """Render one profile prompt into the text this model expects."""
366 if spec.kind == "chat":
367 return bridge.tokenizer.apply_chat_template(
368 [{"role": "user", "content": prompt.prompt}],
369 add_generation_prompt=True,
370 tokenize=False,
371 )
372 if spec.kind == "task:translation" and architecture_id in T5_PREFIX_ARCHITECTURES: 372 ↛ 373line 372 didn't jump to line 373 because the condition on line 372 was never true
373 src_name = LANG_NAMES.get(spec.src or "en", "English")
374 tgt_name = LANG_NAMES.get(spec.lang, "German")
375 return f"translate {src_name} to {tgt_name}: {prompt.prompt}"
376 if spec.kind == "task:summarization" and architecture_id in T5_PREFIX_ARCHITECTURES: 376 ↛ 377line 376 didn't jump to line 377 because the condition on line 376 was never true
377 return f"summarize: {prompt.prompt}"
378 return prompt.prompt
381def benchmark_text_quality(
382 bridge: Any,
383 profile: Union[str, ProfileSpec] = "continuation",
384 *,
385 max_new_tokens: Optional[int] = None,
386 judge_model: Optional[Any] = None,
387 judge_tokenizer: Optional[Any] = None,
388 model_name: Optional[str] = None,
389) -> BenchmarkResult:
390 """Benchmark text generation quality with profile prompts and reference-ratio scoring.
392 Generates from the model's prompt-profile prompts through the real user
393 path (``bridge.generate``), then scores each output against the prompt's
394 reference completion via the pinned judge's perplexity ratio, with
395 repetition and length penalties.
396 """
397 if model_name is not None and model_name.lower() == JUDGE_MODEL_ID.lower():
398 # Ratio scoring against the judge's own perplexity is self-grading.
399 return BenchmarkResult(
400 name="text_quality",
401 severity=BenchmarkSeverity.SKIPPED,
402 message=f"P4 skipped: {model_name} is the pinned judge — cannot self-score",
403 )
404 _loaded_locally = False
405 tokenizer = judge_tokenizer
406 try:
407 spec = ProfileSpec.parse(profile) if isinstance(profile, str) else profile
409 # Diffusion LMs produce text through their native sampler; scoring that
410 # text is as meaningful as scoring autoregressive output.
411 from transformer_lens.benchmarks.generation import resolve_text_generator
413 generator = resolve_text_generator(bridge)
414 if generator is None: 414 ↛ 415line 414 didn't jump to line 415 because the condition on line 414 was never true
415 return BenchmarkResult(
416 name="text_quality",
417 severity=BenchmarkSeverity.INFO,
418 message="Skipped: architecture supports no text generation",
419 )
421 is_encoder_decoder = bool(
422 getattr(getattr(bridge, "original_model", None), "config", None)
423 and getattr(bridge.original_model.config, "is_encoder_decoder", False)
424 )
425 is_multimodal = bool(getattr(getattr(bridge, "cfg", None), "is_multimodal", False))
427 # Effective-profile adjustments. Image-conditioned seq2seq (Florence-2)
428 # emits a bare EOS for text-only prompts — caption real images instead.
429 # A chat profile without a chat template downgrades to continuation;
430 # never the other direction (base models may ship templates).
431 adjustment = ""
432 if is_encoder_decoder and is_multimodal:
433 spec = ProfileSpec("caption")
434 elif spec.kind == "chat":
435 if getattr(bridge.tokenizer, "chat_template", None) is None:
436 spec = ProfileSpec("continuation", spec.lang)
437 adjustment = "chat profile downgraded: tokenizer has no chat template"
438 else:
439 try:
440 bridge.tokenizer.apply_chat_template(
441 [{"role": "user", "content": "probe"}],
442 add_generation_prompt=True,
443 tokenize=False,
444 )
445 except Exception as template_error:
446 spec = ProfileSpec("continuation", spec.lang)
447 adjustment = f"chat profile downgraded: template raised {template_error!r}"
449 denoise_style = "mask" if getattr(bridge.tokenizer, "mask_token", None) else "t5"
450 profile_prompts = prompts_for(spec, denoise_style=denoise_style)
451 if profile_prompts is None:
452 return BenchmarkResult(
453 name="text_quality",
454 severity=BenchmarkSeverity.SKIPPED,
455 message=(
456 f"P4 skipped: no prompts for profile '{spec}' — file a "
457 "TransformerLens issue to add coverage in "
458 "benchmarks/text_quality_profiles.py"
459 ),
460 )
462 if max_new_tokens is None: 462 ↛ 465line 462 didn't jump to line 465 because the condition on line 462 was always true
463 max_new_tokens = MAX_NEW_TOKENS_BY_KIND.get(spec.kind, 50)
465 architecture_id = _architecture_id(bridge)
466 forced_bos: Optional[int] = None
467 if spec.kind == "task:translation":
468 src_lang_attr = getattr(bridge.tokenizer, "src_lang", None)
469 if src_lang_attr is not None and spec.src:
470 src_code = _resolve_lang_code(bridge.tokenizer, spec.src)
471 if src_code is not None: 471 ↛ 472line 471 didn't jump to line 472 because the condition on line 471 was never true
472 try:
473 bridge.tokenizer.src_lang = src_code
474 except Exception:
475 pass
476 forced_bos = _forced_bos_for_target(bridge.tokenizer, spec.lang)
478 # Generate: (profile_prompt, generated_text) pairs. Token-level slicing —
479 # generate() decodes with skip_special_tokens, so the prompt string is
480 # not reliably a prefix of the output string (chat templates).
481 generations: List[Tuple[ProfilePrompt, str]] = []
482 primary_generated = ""
483 if spec.kind == "caption":
484 with deterministic_rng():
485 captions = _generate_image_conditioned_captions(bridge, max_new_tokens)
486 if not captions: 486 ↛ 487line 486 didn't jump to line 487 because the condition on line 486 was never true
487 return BenchmarkResult(
488 name="text_quality",
489 severity=BenchmarkSeverity.SKIPPED,
490 message="Skipped: image-conditioned model; image processor/PIL unavailable",
491 )
492 generations = [
493 (ProfilePrompt(prompt="", reference=CAPTION_REFERENCES[i]), text)
494 for i, text in captions
495 if i < len(CAPTION_REFERENCES)
496 ]
497 primary_generated = captions[0][1]
498 else:
499 prepend_bos = PREPEND_BOS_BY_KIND.get(spec.kind)
500 # Native diffusion samplers take neither return_type nor forced_bos;
501 # bound-method identity can't detect them (new object per access).
502 is_autoregressive = getattr(bridge.adapter, "supports_generation", True)
503 for prompt in profile_prompts:
504 model_input = _build_model_input(bridge, spec, prompt, architecture_id)
505 if is_encoder_decoder:
506 # Encoder input follows the tokenizer's own recipe (lang
507 # token + trailing </s>); to_tokens' BOS policy corrupts it
508 # (m2m100 loops on a stray <s>).
509 prompt_ids = bridge.tokenizer(model_input, return_tensors="pt")["input_ids"].to(
510 bridge.cfg.device
511 )
512 else:
513 prompt_ids = bridge.to_tokens(model_input, prepend_bos=prepend_bos)
514 gen_kwargs: dict = {
515 "max_new_tokens": max_new_tokens,
516 "temperature": TEMPERATURE_BY_KIND.get(spec.kind, 0.7),
517 }
518 if is_autoregressive: 518 ↛ 524line 518 didn't jump to line 524 because the condition on line 518 was always true
519 gen_kwargs["return_type"] = "tokens"
520 if forced_bos is not None:
521 gen_kwargs["forced_bos_token_id"] = forced_bos
522 # Seeded per prompt so each sample stream is independent of the
523 # previous prompt's length.
524 with deterministic_rng():
525 out = generator(prompt_ids, **gen_kwargs)
526 if not isinstance(out, torch.Tensor): 526 ↛ 527line 526 didn't jump to line 527 because the condition on line 526 was never true
527 continue
528 generated_ids = out[0, 1:] if is_encoder_decoder else out[0, prompt_ids.shape[-1] :]
529 generated = bridge.tokenizer.decode(generated_ids, skip_special_tokens=True)
530 if spec.kind == "task:denoise" and denoise_style == "t5":
531 # Splice the fill back so both ratio sides are full
532 # sentences (bare fragments judge in the thousands). An
533 # EMPTY fill must stay empty or a dead model inherits the
534 # near-reference sentence and a free 100.
535 if generated.strip():
536 generated = prompt.prompt.replace("<extra_id_0>", generated.strip())
537 # Empty output is a scored failure (0), not a dropped sample —
538 # dropping it would average only over the prompts that worked.
539 generations.append((prompt, generated))
540 if not primary_generated:
541 primary_generated = generated
543 if len(generations) == 0: 543 ↛ 544line 543 didn't jump to line 544 because the condition on line 543 was never true
544 return BenchmarkResult(
545 name="text_quality",
546 severity=BenchmarkSeverity.DANGER,
547 message="Generation produced no scoreable output for any prompt",
548 passed=False,
549 )
551 if judge_model is None or tokenizer is None: 551 ↛ 552line 551 didn't jump to line 552 because the condition on line 551 was never true
552 judge_model, tokenizer = load_judge()
553 _loaded_locally = True
555 # Judge context per kind is JUDGE_CONTEXT_KINDS' call. Translation is
556 # scored jointly: per-sentence judge perplexity on the short pivots is
557 # unstable (measured spread 4.8-3497), so the samples concatenate into
558 # one gen/ref pair.
559 # Captured pre-merge so translation keeps its per-sentence texts.
560 all_generated_texts = [text for _, text in generations]
562 if spec.kind == "task:translation" and len(generations) > 1:
563 joiner = "" if spec.lang in ("zh", "ja") else " "
564 joint = ProfilePrompt(
565 prompt="",
566 reference=joiner.join(g[0].reference for g in generations),
567 lang=spec.lang,
568 )
569 generations = [(joint, joiner.join(g[1] for g in generations))]
571 sample_lang = spec.lang if spec.kind != "caption" else "en"
572 per_prompt_scores = []
573 per_prompt_ratios = []
574 per_prompt_penalties = []
575 prompt_details_parts = []
577 for prompt, generated in generations:
578 context = prompt.prompt if spec.kind in JUDGE_CONTEXT_KINDS else ""
580 gen_token_count = len(tokenizer(generated)["input_ids"]) if generated.strip() else 0
581 if gen_token_count < 2:
582 # Empty or one-token output is a scored failure, not a dropped
583 # sample (Florence-style bare EOS, dead generation).
584 per_prompt_scores.append(0.0)
585 per_prompt_ratios.append(float("inf"))
586 per_prompt_penalties.append(0.0)
587 prompt_details_parts.append("score=0.0 (output < 2 tokens)")
588 continue
589 check_lang = prompt.lang if spec.kind != "task:translation" else sample_lang
590 if _wrong_language(generated, check_lang):
591 # Fluency-only ratio scoring would rate untranslated or
592 # wrong-language output above the reference; hard zero.
593 per_prompt_scores.append(0.0)
594 per_prompt_ratios.append(float("inf"))
595 per_prompt_penalties.append(0.0)
596 prompt_details_parts.append(f"score=0.0 (output not in '{check_lang}')")
597 continue
599 gen_ppl, gen_err = _judge_perplexity(generated, context, tokenizer, judge_model)
600 ref_ppl, ref_err = _judge_perplexity(prompt.reference, context, tokenizer, judge_model)
601 if gen_err is not None: 601 ↛ 603line 601 didn't jump to line 603 because the condition on line 601 was never true
602 # The model's own output was unjudgeable — scored failure.
603 per_prompt_scores.append(0.0)
604 per_prompt_ratios.append(float("inf"))
605 per_prompt_penalties.append(0.0)
606 prompt_details_parts.append(f"score=0.0 ({gen_err})")
607 continue
608 if ref_err is not None: 608 ↛ 611line 608 didn't jump to line 611 because the condition on line 608 was never true
609 # Our reference failed to judge — a data problem, not the
610 # model's; exclude the sample and say so.
611 prompt_details_parts.append(f"excluded (reference: {ref_err})")
612 continue
614 ratio = gen_ppl / ref_ppl if ref_ppl > 0 else float("inf")
615 rep_penalty = _compute_repetition_penalty(generated)
616 ref_token_count = len(tokenizer(prompt.reference)["input_ids"])
617 len_penalty = _length_penalty(gen_token_count, ref_token_count)
618 adjusted_score = _ratio_to_score(ratio) * rep_penalty * len_penalty
620 per_prompt_scores.append(adjusted_score)
621 per_prompt_ratios.append(ratio)
622 per_prompt_penalties.append(rep_penalty)
623 prompt_details_parts.append(
624 f"ratio={ratio:.2f} ppl={gen_ppl:.1f} ref_ppl={ref_ppl:.1f} "
625 f"rep={rep_penalty:.2f} len={len_penalty:.2f} score={adjusted_score:.1f}"
626 )
628 if len(per_prompt_scores) == 0: 628 ↛ 629line 628 didn't jump to line 629 because the condition on line 628 was never true
629 return BenchmarkResult(
630 name="text_quality",
631 severity=BenchmarkSeverity.ERROR,
632 message="Scoring failed for all prompts",
633 details={"generated_text": primary_generated},
634 passed=False,
635 )
637 avg_score = sum(per_prompt_scores) / len(per_prompt_scores)
638 finite_ratios = [r for r in per_prompt_ratios if math.isfinite(r)]
639 avg_ratio = sum(finite_ratios) / len(finite_ratios) if finite_ratios else float("inf")
640 avg_rep_penalty = sum(per_prompt_penalties) / len(per_prompt_penalties)
642 pass_threshold = p4_pass_threshold()
643 details = {
644 "score": round(avg_score, 1),
645 "prompt_profile": str(spec),
646 "judge_model": JUDGE_MODEL_ID,
647 "judge_revision": JUDGE_REVISION,
648 "avg_ratio": round(avg_ratio, 3) if math.isfinite(avg_ratio) else "inf",
649 "avg_repetition_penalty": round(avg_rep_penalty, 2),
650 "num_prompts": len(per_prompt_scores),
651 "per_prompt": " | ".join(prompt_details_parts),
652 "max_new_tokens": max_new_tokens,
653 "generated_text": primary_generated,
654 "generated_texts": all_generated_texts,
655 }
656 if adjustment:
657 details["profile_adjustment"] = adjustment
659 if avg_score >= pass_threshold:
660 return BenchmarkResult(
661 name="text_quality",
662 severity=BenchmarkSeverity.INFO,
663 message=(
664 f"Text quality score: {avg_score:.1f}/100 "
665 f"(profile {spec}, {len(per_prompt_scores)} prompts)"
666 ),
667 details=details,
668 )
669 elif avg_score >= pass_threshold / 2:
670 return BenchmarkResult(
671 name="text_quality",
672 severity=BenchmarkSeverity.WARNING,
673 message=(
674 f"Text quality score: {avg_score:.1f}/100 "
675 f"(below {pass_threshold:.0f}, profile {spec})"
676 ),
677 details=details,
678 passed=False,
679 )
680 else:
681 return BenchmarkResult(
682 name="text_quality",
683 severity=BenchmarkSeverity.DANGER,
684 message=(
685 f"Text quality score: {avg_score:.1f}/100 "
686 f"(profile {spec}) — generated text may be incoherent"
687 ),
688 details=details,
689 passed=False,
690 )
692 except Exception as e:
693 return BenchmarkResult(
694 name="text_quality",
695 severity=BenchmarkSeverity.ERROR,
696 message=f"Text quality benchmark failed: {str(e)}",
697 passed=False,
698 )
700 finally:
701 if _loaded_locally:
702 if judge_model is not None:
703 del judge_model
704 if tokenizer is not None:
705 del tokenizer
706 gc.collect()