Coverage for transformer_lens/benchmarks/text_quality_profiles.py: 86%

1"""Prompt profiles and reference data for the Phase-4 text-quality benchmark. 

2 

3Each verified model is scored on prompts a real user would feed it (its 

4``prompt_profile``): chat models get their chat template, translation models get 

5source sentences, code models get code, multilingual models get their own 

6language. Every prompt carries a known-good reference completion; scoring is the 

7ratio of judge perplexities PPL(generated)/PPL(reference), which cancels the 

8judge's per-language handicap. 

9 

10Profile resolution is curation-first because Hub metadata is unreliable 

11(observed live 2026-08-20): ``bigscience/mt0-base`` is mis-tagged 

12``text-generation``; ``facebook/m2m100_418M`` and ``google/long-t5-tglobal-base`` 

13have no ``pipeline_tag`` at all; the ``conversational`` tag is added by HF for 

14*any* repo shipping a chat template, including base models like 

15``Qwen/Qwen2.5-0.5B``; Helsinki-NLP language tags are unordered, so Marian 

16direction must come from the model id. Precedence: per-model override > 

17architecture rule > fetched HF signals > stored registry value > default. 

18 

19Pivot sentences are from Tatoeba (https://tatoeba.org, CC BY 2.0 FR); source 

20sentence ids are noted inline. Everything else is hand-authored. 

21 

22This module stays stdlib-only: the registry scraper imports it at scan time. 

23 

24Language x kind coverage (prompts exist where marked; uncovered combinations 

25SKIP with a file-an-issue message, they never score against wrong-language 

26data): 

27 

28 kind en fr es de zh ja ru ar hi it nl pt ro code 

29 continuation x x x x x x x x - - - - - x 

30 chat x x x x x x x x - - - - - - 

31 task:instruction x x - - x - - - - - - - - - 

32 task:summarization x x - - x - - - - - - - - - 

33 task:denoise x - - - - - - - - - - - - - 

34 PIVOT (translation) x x x x x x x x x x x x - - 

35 

36hi/it/nl/pt have pivot coverage only (translation targets); ro exists only in 

37NLLB_CODES. Filling continuation/chat for those plus it/nl/pt/hi bake-off 

38calibration is tracked as a follow-up. 

39""" 

40 

41from __future__ import annotations 

42 

43import math 

44from dataclasses import dataclass 

45from typing import Optional 

46 

47PROFILE_KINDS = ( 

48 "continuation", 

49 "chat", 

50 "task:instruction", 

51 "task:translation", 

52 "task:summarization", 

53 "task:denoise", 

54 "caption", 

55) 

56 

57 

58@dataclass(frozen=True) 

59class ProfileSpec: 

60 """A parsed prompt profile: what to feed the model and in which language.""" 

61 

62 kind: str 

63 lang: str = "en" 

64 src: Optional[str] = None # translation source language 

65 

66 @classmethod 

67 def parse(cls, spec: str) -> "ProfileSpec": 

68 """Parse ``kind[@lang]`` (translation: ``@src-tgt``); '@' because task kinds contain ':'.""" 

69 kind, _, lang = spec.partition("@") 

70 if kind not in PROFILE_KINDS: 

71 raise ValueError(f"Unknown profile kind {kind!r} in {spec!r}") 

72 if not lang: 

73 return cls(kind=kind) 

74 if kind == "task:translation": 

75 src, sep, tgt = lang.partition("-") 

76 if not sep or not src or not tgt: 

77 raise ValueError(f"Translation profile needs '@src-tgt', got {spec!r}") 

78 return cls(kind=kind, lang=tgt, src=src) 

79 return cls(kind=kind, lang=lang) 

80 

81 def __str__(self) -> str: 

82 if self.kind == "task:translation" and self.src: 

83 return f"{self.kind}@{self.src}-{self.lang}" 

84 if self.lang != "en": 

85 return f"{self.kind}@{self.lang}" 

86 return self.kind 

87 

88 

89@dataclass(frozen=True) 

90class ProfilePrompt: 

91 """One scored sample: model input and a known-good reference completion.""" 

92 

93 prompt: str 

94 reference: str 

95 lang: str = "en" 

96 

97 

98DEFAULT_PROFILE = ProfileSpec("continuation", "en") 

99 

100 

101def is_default_profile(profile) -> bool: 

102 """One sparse-encoding rule for every registry writer: the bare default 

103 continuation@en profile is never stored (a lang-tagged continuation is).""" 

104 if isinstance(profile, ProfileSpec): 104 ↛ 105line 104 didn't jump to line 105 because the condition on line 104 was never true

105 return profile == DEFAULT_PROFILE 

106 try: 

107 return ProfileSpec.parse(str(profile)) == DEFAULT_PROFILE 

108 except ValueError: 

109 return False 

110 

111 

112# --------------------------------------------------------------------------- 

113# Pivot sentences (Tatoeba, CC BY 2.0 FR) — index-aligned across languages. 

114# English #1277 / #1284 / #1315; per-language ids in row comments. 

115# Feed translation pairs and the judge bake-off's fluent corpus. 

116# --------------------------------------------------------------------------- 

117 

118PIVOT_SENTENCES: dict[str, tuple[str, str, str]] = { 

119 "en": ( # 1277, 1284, 1315 

120 "I have to go to sleep.", 

121 "I will be back soon.", 

122 "I can't live that kind of life.", 

123 ), 

124 "fr": ( # 373908, 3099, 3131 

125 "Je dois aller dormir.", 

126 "Je serai bientôt de retour.", 

127 "Je ne peux pas vivre comme ça.", 

128 ), 

129 "es": ( # 2482, 2489, 2521 

130 "Tengo que irme a dormir.", 

131 "Volveré pronto.", 

132 "No puedo vivir así.", 

133 ), 

134 "de": ( # 1195088, 85, 117 

135 "Ich muss schlafen.", 

136 "Ich werde bald zurück sein.", 

137 "Ich kann so ein Leben nicht leben.", 

138 ), 

139 "it": ( # 4369, 375118, 2733911 

140 "Devo andare a dormire.", 

141 "Torno subito.", 

142 "Non posso vivere quel tipo di vita.", 

143 ), 

144 "nl": ( # 5966, 5984, 378741 

145 "Ik moet gaan slapen.", 

146 "Ik ben zo terug.", 

147 "Ik kan zo niet leven.", 

148 ), 

149 "pt": ( # 182184, 331974, 405254 

150 "Preciso ir dormir.", 

151 "Voltarei em breve.", 

152 "Eu não posso viver esse tipo de vida.", 

153 ), 

154 "ru": ( # 5410, 374353, 5449 

155 "Мне пора идти спать.", 

156 "Я скоро вернусь.", 

157 "Я так жить не могу.", 

158 ), 

159 "zh": ( # 2, 9 (Hans transcription), 35 — one script; mixing traditional 

160 # into a simplified-dominant judge destroys that row's zero point. 

161 "我该去睡觉了。", 

162 "我很快就会回来。", 

163 "我不能这样活着。", 

164 ), 

165 "ja": ( # 4703, 4709, 4742 

166 "私は眠らなければなりません。", 

167 "すぐに戻ります。", 

168 "私はそんな風には生きられない。", 

169 ), 

170 "ar": ( # 372962, 400781, 549626 

171 "عليّ أن أنام.", 

172 "سأعود قريباً.", 

173 "لا أستطيع أن أعيش حياة كتلك.", 

174 ), 

175 "hi": ( # 3792910, 3793971, 11371181 

176 "मुझे सोना है।", 

177 "मैं जल्द लौटूंगी।", 

178 "मैं ऐसी जिंदगी नहीं जी सकता।", 

179 ), 

180} 

181 

182# --------------------------------------------------------------------------- 

183# Continuation prompts. English is seeded from the pre-rework default prompts so 

184# control-model scores stay comparable. "code" is a language here: code models 

185# continue code the way prose models continue prose. 

186# --------------------------------------------------------------------------- 

187 

188CONTINUATION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { 

189 "en": ( 

190 ProfilePrompt( 

191 "The theory of relativity explains that", 

192 " time and space are not absolute but depend on the observer's " 

193 "motion, so clocks moving at high speed tick more slowly than " 

194 "clocks at rest.", 

195 ), 

196 ProfilePrompt( 

197 "In the dense forests of the Amazon,", 

198 " thousands of plant and animal species live in a delicate " 

199 "balance, and scientists continue to discover new ones every year.", 

200 ), 

201 ProfilePrompt( 

202 "Modern computing relies heavily on", 

203 " fast processors and large amounts of memory, which allow " 

204 "software to handle enormous quantities of data in real time.", 

205 ), 

206 ProfilePrompt( 

207 "The city library opens early on weekdays, and", 

208 # Judge PPL 8.7 (en median 8.9). References must stay within 

209 # ~3.5x of the language median or this prompt's bar loosens 

210 # proportionally; the integration test pins the band. 

211 " many people stop by in the morning to read or borrow books before work.", 

212 ), 

213 ), 

214 "fr": ( 

215 ProfilePrompt( 

216 "La tour Eiffel est l'un des monuments", 

217 " les plus célèbres du monde, et des millions de visiteurs " 

218 "montent chaque année à son sommet pour admirer Paris.", 

219 lang="fr", 

220 ), 

221 ProfilePrompt( 

222 "Chaque matin, le boulanger du village", 

223 " prépare du pain frais et des croissants que les habitants " 

224 "viennent acheter dès l'ouverture de la boutique.", 

225 lang="fr", 

226 ), 

227 ProfilePrompt( 

228 "La science moderne repose sur", 

229 " l'observation, l'expérience et le raisonnement, qui permettent " 

230 "de comprendre les lois de la nature.", 

231 lang="fr", 

232 ), 

233 ProfilePrompt( 

234 "Pendant l'hiver, les montagnes", 

235 " se couvrent de neige et attirent de nombreux skieurs venus de " "toute l'Europe.", 

236 lang="fr", 

237 ), 

238 ), 

239 "es": ( 

240 ProfilePrompt( 

241 "El clima de la región mediterránea es", 

242 " templado, con veranos secos y calurosos e inviernos suaves y " 

243 "lluviosos, ideal para el cultivo de olivos.", 

244 lang="es", 

245 ), 

246 ProfilePrompt( 

247 "Cada domingo por la mañana, el mercado", 

248 " se llena de gente que compra fruta fresca, verduras y flores a " 

249 "los vendedores locales.", 

250 lang="es", 

251 ), 

252 ProfilePrompt( 

253 "La historia de América Latina está marcada por", 

254 " una gran diversidad cultural, fruto del encuentro entre pueblos " 

255 "indígenas, europeos y africanos.", 

256 lang="es", 

257 ), 

258 ProfilePrompt( 

259 "Los avances de la medicina moderna permiten", 

260 " tratar enfermedades que hace pocas décadas se consideraban " 

261 "incurables, y prolongar la vida de millones de personas.", 

262 lang="es", 

263 ), 

264 ), 

265 "de": ( 

266 ProfilePrompt( 

267 "Der Schwarzwald ist bekannt für", 

268 " seine dichten Wälder, tiefen Täler und traditionellen " 

269 "Bauernhäuser, die jedes Jahr viele Wanderer anziehen.", 

270 lang="de", 

271 ), 

272 ProfilePrompt( 

273 "Jeden Morgen fährt der Zug", 

274 " pünktlich um sieben Uhr vom Hauptbahnhof ab und bringt die " 

275 "Pendler in die umliegenden Städte zur Arbeit.", 

276 lang="de", 

277 ), 

278 ProfilePrompt( 

279 "Die deutsche Sprache hat", 

280 " viele lange zusammengesetzte Wörter, die Lernende oft " 

281 "überraschen, aber einer klaren Logik folgen.", 

282 lang="de", 

283 ), 

284 ProfilePrompt( 

285 "In der modernen Industrie spielen Roboter", 

286 " eine immer größere Rolle, weil sie schwere und gefährliche " 

287 "Arbeiten schneller und sicherer erledigen können.", 

288 lang="de", 

289 ), 

290 ), 

291 "zh": ( 

292 ProfilePrompt( 

293 "长城是中国古代", 

294 "伟大的防御工程，绵延数千公里，每年吸引大量游客前来参观。", 

295 lang="zh", 

296 ), 

297 ProfilePrompt( 

298 "每天早晨，公园里", 

299 "有许多老人打太极拳、散步和下棋，气氛十分热闹。", 

300 lang="zh", 

301 ), 

302 ProfilePrompt( 

303 "现代科技的发展使得", 

304 "人们的生活越来越方便，购物、学习和工作都可以在网上完成。", 

305 lang="zh", 

306 ), 

307 ProfilePrompt( 

308 "春天到了，山上的", 

309 "花都开了，许多家庭趁着周末去郊外踏青赏花。", 

310 lang="zh", 

311 ), 

312 ), 

313 "ja": ( 

314 ProfilePrompt( 

315 "日本の四季は", 

316 "それぞれ美しく、春には桜、秋には紅葉を楽しむために多くの人が旅行に出かけます。", 

317 lang="ja", 

318 ), 

319 ProfilePrompt( 

320 "毎朝、駅の周りには", 

321 "通勤や通学の人々が行き交い、店が次々と開き始めます。", 

322 lang="ja", 

323 ), 

324 ProfilePrompt( 

325 "現代の技術の進歩により、", 

326 "私たちの生活はますます便利になり、買い物も勉強も家にいながらできるようになりました。", 

327 lang="ja", 

328 ), 

329 ProfilePrompt( 

330 "図書館は静かな場所で、", 

331 "学生たちが本を読んだり、勉強したりするのに最適です。", 

332 lang="ja", 

333 ), 

334 ), 

335 "ru": ( 

336 ProfilePrompt( 

337 "Зимой в Сибири", 

338 " очень холодно, температура часто опускается ниже сорока " 

339 "градусов, но местные жители привыкли к таким морозам.", 

340 lang="ru", 

341 ), 

342 ProfilePrompt( 

343 "Каждое утро студенты", 

344 " спешат на занятия в университет, а вечером собираются в " 

345 "библиотеке, чтобы готовиться к экзаменам.", 

346 lang="ru", 

347 ), 

348 ProfilePrompt( 

349 "Современная наука позволяет", 

350 " лечить болезни, которые раньше считались неизлечимыми, и " 

351 "продлевать жизнь миллионам людей.", 

352 lang="ru", 

353 ), 

354 ProfilePrompt( 

355 "Русская литература известна", 

356 " во всём мире благодаря произведениям Толстого, Достоевского и " 

357 "Чехова, которые переведены на десятки языков.", 

358 lang="ru", 

359 ), 

360 ), 

361 "ar": ( 

362 ProfilePrompt( 

363 "تشتهر مدينة القاهرة", 

364 " بتاريخها العريق ومساجدها القديمة وأسواقها الشعبية التي يزورها " 

365 "السياح من جميع أنحاء العالم.", 

366 lang="ar", 

367 ), 

368 ProfilePrompt( 

369 "في كل صباح يذهب الطلاب", 

370 " إلى المدرسة مبكرين، ويقضون اليوم في تعلم القراءة والكتابة " "والعلوم.", 

371 lang="ar", 

372 ), 

373 ProfilePrompt( 

374 "يساعد التقدم العلمي الحديث", 

375 " الأطباء على علاج أمراض كانت تعتبر مستعصية قبل عقود قليلة.", 

376 lang="ar", 

377 ), 

378 ProfilePrompt( 

379 "تعتبر اللغة العربية", 

380 " من أقدم اللغات الحية في العالم، ويتحدث بها ملايين الناس في " "الوطن العربي وخارجه.", 

381 lang="ar", 

382 ), 

383 ), 

384 "code": ( 

385 ProfilePrompt( 

386 'def is_prime(n):\n """Return True if n is a prime number."""\n', 

387 " if n < 2:\n return False\n" 

388 " for i in range(2, int(n ** 0.5) + 1):\n" 

389 " if n % i == 0:\n return False\n" 

390 " return True\n", 

391 lang="code", 

392 ), 

393 ProfilePrompt( 

394 'def count_words(text):\n """Count occurrences of each word in text."""\n', 

395 " counts = {}\n for word in text.split():\n" 

396 " counts[word] = counts.get(word, 0) + 1\n" 

397 " return counts\n", 

398 lang="code", 

399 ), 

400 ProfilePrompt( 

401 "def fibonacci(n):\n" ' """Return the first n Fibonacci numbers as a list."""\n', 

402 " result = []\n a, b = 0, 1\n" 

403 " for _ in range(n):\n result.append(a)\n" 

404 " a, b = b, a + b\n return result\n", 

405 lang="code", 

406 ), 

407 ProfilePrompt( 

408 "// Return the largest number in the array.\n" "function findMax(numbers) {\n", 

409 " let max = numbers[0];\n" 

410 " for (const n of numbers) {\n" 

411 " if (n > max) max = n;\n }\n" 

412 " return max;\n}\n", 

413 lang="code", 

414 ), 

415 ), 

416} 

417 

418# --------------------------------------------------------------------------- 

419# Chat prompts: realistic user turns (rendered through the tokenizer's chat 

420# template at run time) with a good assistant reply as reference. 

421# --------------------------------------------------------------------------- 

422 

423CHAT_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { 

424 "en": ( 

425 ProfilePrompt( 

426 "How do I keep basil alive indoors?", 

427 "Keep basil in a warm spot with at least six hours of sunlight a " 

428 "day, water it when the top of the soil feels dry, and pinch off " 

429 "flower buds so the plant keeps producing leaves.", 

430 ), 

431 ProfilePrompt( 

432 "What's a good way to remember people's names?", 

433 "Repeat the name right after you hear it, use it once or twice in " 

434 "conversation, and link it to something memorable about the " 

435 "person, like their job or where you met.", 

436 ), 

437 ProfilePrompt( 

438 "Explain why the sky is blue in simple terms.", 

439 "Sunlight is made of many colors, and the air scatters blue light " 

440 "more than the other colors because blue travels in shorter " 

441 "waves. When you look up, that scattered blue light is what you " 

442 "see.", 

443 ), 

444 ), 

445 "fr": ( 

446 ProfilePrompt( 

447 "Comment préparer un bon café à la maison ?", 

448 "Utilisez du café fraîchement moulu, une eau à environ 90 degrés " 

449 "et un dosage d'une cuillère à soupe par tasse. Laissez infuser " 

450 "quelques minutes avant de servir.", 

451 lang="fr", 

452 ), 

453 ProfilePrompt( 

454 "Quels sont les avantages de la lecture quotidienne ?", 

455 "Lire chaque jour enrichit le vocabulaire, améliore la " 

456 "concentration et réduit le stress. C'est aussi un excellent " 

457 "moyen de découvrir de nouvelles idées.", 

458 lang="fr", 

459 ), 

460 ProfilePrompt( 

461 "Explique-moi simplement pourquoi les feuilles tombent en automne.", 

462 "En automne, les jours raccourcissent et les arbres reçoivent " 

463 "moins de lumière. Ils cessent de nourrir leurs feuilles, qui " 

464 "changent de couleur puis tombent pour économiser l'énergie " 

465 "pendant l'hiver.", 

466 lang="fr", 

467 ), 

468 ), 

469 "es": ( 

470 ProfilePrompt( 

471 "¿Cómo puedo mejorar mi memoria para estudiar?", 

472 "Estudia en sesiones cortas y regulares, repasa lo aprendido al " 

473 "día siguiente y explica el tema en voz alta con tus propias " 

474 "palabras. Dormir bien también ayuda mucho a fijar los " 

475 "recuerdos.", 

476 lang="es", 

477 ), 

478 ProfilePrompt( 

479 "¿Qué debo tener en cuenta al adoptar un gato?", 

480 "Prepara un espacio tranquilo con comida, agua y un arenero " 

481 "limpio. Dale tiempo para adaptarse, llévalo al veterinario para " 

482 "sus vacunas y juega con él todos los días.", 

483 lang="es", 

484 ), 

485 ProfilePrompt( 

486 "Explícame de forma sencilla cómo funciona un molino de viento.", 

487 "El viento empuja las aspas del molino y las hace girar. Ese giro " 

488 "mueve un eje conectado a una máquina o a un generador, que " 

489 "convierte el movimiento en trabajo útil o en electricidad.", 

490 lang="es", 

491 ), 

492 ), 

493 "de": ( 

494 ProfilePrompt( 

495 "Wie kann ich beim Einkaufen Geld sparen?", 

496 "Schreiben Sie vorher eine Einkaufsliste und halten Sie sich " 

497 "daran, vergleichen Sie Preise und kaufen Sie saisonale " 

498 "Produkte. Große Packungen lohnen sich nur, wenn Sie alles " 

499 "verbrauchen.", 

500 lang="de", 

501 ), 

502 ProfilePrompt( 

503 "Was ist ein guter Weg, eine neue Sprache zu lernen?", 

504 "Üben Sie jeden Tag ein wenig, hören Sie Podcasts oder Musik in " 

505 "der Sprache und sprechen Sie so früh wie möglich mit " 

506 "Muttersprachlern. Regelmäßigkeit ist wichtiger als lange " 

507 "Lerneinheiten.", 

508 lang="de", 

509 ), 

510 ProfilePrompt( 

511 "Erkläre mir einfach, warum es Ebbe und Flut gibt.", 

512 "Der Mond zieht mit seiner Schwerkraft am Wasser der Ozeane. Auf " 

513 "der dem Mond zugewandten Seite der Erde hebt sich das Wasser, " 

514 "und während sich die Erde dreht, wandert dieser Wasserberg — so " 

515 "entstehen Ebbe und Flut.", 

516 lang="de", 

517 ), 

518 ), 

519 "zh": ( 

520 ProfilePrompt( 

521 "怎样才能养成早起的习惯？", 

522 "每天固定同一时间睡觉和起床，睡前少看手机，把闹钟放在离床远一点的地方。坚持两三个星期，身体就会慢慢适应新的作息。", 

523 lang="zh", 

524 ), 

525 ProfilePrompt( 

526 "第一次做饭应该注意什么？", 

527 "先从简单的菜开始，提前准备好所有材料，注意用火安全，切菜时小心手指。做完后记得关闭燃气，慢慢积累经验就会越来越熟练。", 

528 lang="zh", 

529 ), 

530 ProfilePrompt( 

531 "请用简单的话解释为什么会下雨。", 

532 "太阳把地面上的水晒热，水变成水蒸气升到天上，遇冷凝结成小水滴，聚在一起形成云。当水滴越来越重，云托不住它们时，就落下来变成雨。", 

533 lang="zh", 

534 ), 

535 ), 

536 "ja": ( 

537 ProfilePrompt( 

538 "朝型の生活に変えるにはどうすればいいですか？", 

539 "毎日同じ時間に寝起きし、寝る前はスマートフォンを見ないようにしましょう。朝に日光を浴びると体内時計が整い、二、三週間続ければ自然に朝型になります。", 

540 lang="ja", 

541 ), 

542 ProfilePrompt( 

543 "初めての一人暮らしで気をつけることは何ですか?", 

544 "毎月の家賃や食費など生活費の計画を立て、無理のない範囲で貯金をしましょう。防犯のために戸締まりを忘れず、近所のスーパーや病院の場所も早めに確認しておくと安心です。", 

545 lang="ja", 

546 ), 

547 ProfilePrompt( 

548 "虹がどうしてできるのか、簡単に説明してください。", 

549 "雨上がりの空気中には小さな水滴がたくさん残っています。太陽の光がその水滴の中で曲がって反射すると、光が七つの色に分かれて見えます。これが虹です。", 

550 lang="ja", 

551 ), 

552 ), 

553 "ru": ( 

554 ProfilePrompt( 

555 "Как научиться рано вставать?", 

556 "Ложитесь и вставайте в одно и то же время каждый день, не " 

557 "смотрите в телефон перед сном и ставьте будильник подальше от " 

558 "кровати. Через пару недель организм привыкнет к новому режиму.", 

559 lang="ru", 

560 ), 

561 ProfilePrompt( 

562 "Что почитать, чтобы полюбить чтение?", 

563 "Начните с коротких книг на темы, которые вам действительно " 

564 "интересны, — детективы, приключения или научно-популярные " 

565 "рассказы. Главное — читать понемногу каждый день и не " 

566 "заставлять себя дочитывать скучное.", 

567 lang="ru", 

568 ), 

569 ProfilePrompt( 

570 "Объясни простыми словами, почему летом жарко, а зимой холодно.", 

571 "Земля вращается вокруг Солнца с наклонённой осью. Летом наше " 

572 "полушарие наклонено к Солнцу, лучи падают прямее и сильнее " 

573 "нагревают землю. Зимой оно отклонено от Солнца, лучи идут под " 

574 "углом и греют слабее.", 

575 lang="ru", 

576 ), 

577 ), 

578 "ar": ( 

579 ProfilePrompt( 

580 "كيف أنظم وقتي أثناء الدراسة؟", 

581 "قسّم يومك إلى فترات قصيرة للدراسة مع فترات راحة منتظمة، وابدأ " 

582 "بأصعب المواد عندما يكون ذهنك صافياً. اكتب قائمة بالمهام كل صباح " 

583 "والتزم بها قدر الإمكان.", 

584 lang="ar", 

585 ), 

586 ProfilePrompt( 

587 "ما هي فوائد المشي اليومي؟", 

588 "المشي كل يوم يقوي القلب والعضلات ويساعد على تخفيف التوتر " 

589 "وتحسين المزاج. كما أنه يساعد على النوم بشكل أفضل ولا يحتاج إلى " 

590 "أي معدات خاصة.", 

591 lang="ar", 

592 ), 

593 ProfilePrompt( 

594 "اشرح لي ببساطة كيف تصنع النحلة العسل.", 

595 "تجمع النحلة رحيق الأزهار وتخزنه في معدة خاصة، ثم تعود إلى " 

596 "الخلية وتسلمه لنحلات أخرى تضيف إليه مواد تحوله إلى عسل. بعد ذلك " 

597 "يوضع العسل في الأقراص الشمعية ويجفف بتحريك الأجنحة حتى ينضج.", 

598 lang="ar", 

599 ), 

600 ), 

601} 

602 

603# --------------------------------------------------------------------------- 

604# Task prompts. 

605# --------------------------------------------------------------------------- 

606 

607SUMMARIZATION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { 

608 "en": ( 

609 ProfilePrompt( 

610 "The city council voted on Tuesday to approve funding for a new " 

611 "public library in the downtown district. The project, which has " 

612 "been debated for over two years, will cost an estimated twelve " 

613 "million dollars and is expected to open in the spring of 2028. " 

614 "Supporters argued that the current library, built in 1962, is " 

615 "too small and lacks modern facilities. Opponents raised " 

616 "concerns about the cost and the loss of a parking lot at the " 

617 "proposed site. The mayor said the new building would include " 

618 "community meeting rooms, a children's wing, and free computer " 

619 "access for residents.", 

620 "The city council approved a twelve million dollar downtown " 

621 "library, expected to open in spring 2028, replacing the " 

622 "outdated 1962 building despite concerns over cost and parking.", 

623 ), 

624 ProfilePrompt( 

625 "Researchers at a European university have published a study " 

626 "showing that regular walking can significantly improve sleep " 

627 "quality in adults over sixty. The study followed four hundred " 

628 "participants for one year, half of whom walked for thirty " 

629 "minutes a day while the other half kept their usual habits. " 

630 "Those in the walking group fell asleep faster, woke less often " 

631 "during the night, and reported feeling more rested in the " 

632 "morning. The researchers noted that the benefits appeared " 

633 "within the first two months and lasted for the rest of the " 

634 "study.", 

635 "A year-long study of four hundred older adults found that " 

636 "walking thirty minutes daily improved sleep quality within two " 

637 "months, helping participants fall asleep faster and wake less " 

638 "often.", 

639 ), 

640 ProfilePrompt( 

641 "A severe storm swept through the coastal region on Friday " 

642 "night, leaving thousands of homes without electricity and " 

643 "forcing the closure of the main highway. Emergency crews worked " 

644 "through the weekend to clear fallen trees and restore power " 

645 "lines. Officials said no serious injuries were reported, though " 

646 "several boats were damaged in the harbor. Schools in the area " 

647 "remained closed on Monday while cleanup continued, and " 

648 "residents were advised to avoid the beachfront until inspectors " 

649 "declared it safe.", 

650 "A Friday night storm cut power to thousands of coastal homes " 

651 "and closed the main highway; crews restored services over the " 

652 "weekend with no serious injuries reported.", 

653 ), 

654 ), 

655} 

656 

657INSTRUCTION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { 

658 "en": ( 

659 ProfilePrompt( 

660 "List three things to pack for a day hike.", 

661 "Water, snacks, and a map of the trail.", 

662 ), 

663 ProfilePrompt( 

664 "Write one sentence describing what a lighthouse does.", 

665 "A lighthouse shines a bright light to guide ships safely along " "the coast at night.", 

666 ), 

667 ProfilePrompt( 

668 "Name the four seasons of the year.", 

669 "Spring, summer, autumn, and winter.", 

670 ), 

671 ), 

672} 

673 

674# Pretrained-only seq2seq models (T5, BART) were trained to fill masked spans, 

675# not to follow instructions; feed them their native denoising format. 

676DENOISE_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { 

677 # ONE sentinel per "t5" prompt: the whole special-stripped output is the 

678 # fill, spliced back by the runner so both ratio sides are full sentences 

679 # (bare span fragments judge in the thousands). 

680 "t5": ( 

681 ProfilePrompt( 

682 "The children <extra_id_0> in the park until the sun went down.", 

683 "The children played happily in the park until the sun went down.", 

684 ), 

685 ProfilePrompt( 

686 "Every morning she drinks a cup of <extra_id_0> and reads the newspaper.", 

687 "Every morning she drinks a cup of coffee and reads the newspaper.", 

688 ), 

689 ProfilePrompt( 

690 "The old bridge across the <extra_id_0> was built many years ago.", 

691 "The old bridge across the river was built many years ago.", 

692 ), 

693 ), 

694 "mask": ( 

695 ProfilePrompt( 

696 "The children played <mask> in the park until the sun went down.", 

697 "The children played happily in the park until the sun went down.", 

698 ), 

699 ProfilePrompt( 

700 "Every morning she drinks a cup of <mask> and reads the newspaper.", 

701 "Every morning she drinks a cup of coffee and reads the newspaper.", 

702 ), 

703 ProfilePrompt( 

704 "The old bridge across the <mask> was built many years ago.", 

705 "The old bridge across the river was built many years ago.", 

706 ), 

707 ), 

708} 

709 

710# References for the synthetic caption images built by the text-quality 

711# benchmark (index-aligned with _build_caption_test_images). 

712CAPTION_REFERENCES: tuple[str, ...] = ( 

713 "The image shows a blue rectangle and a green oval on a white background.", 

714 "The image shows a large yellow circle on a black background.", 

715 "The image shows a dark green rectangle and an orange oval on a light " "blue background.", 

716) 

717 

718# --------------------------------------------------------------------------- 

719# Per-kind generation and judging knobs. 

720# --------------------------------------------------------------------------- 

721 

722MAX_NEW_TOKENS_BY_KIND: dict[str, int] = { 

723 "continuation": 50, 

724 "chat": 64, 

725 "task:instruction": 48, 

726 "task:translation": 48, 

727 "task:summarization": 48, 

728 "task:denoise": 24, 

729 "caption": 50, 

730} 

731 

732# Chat prompts arrive pre-templated (the template supplies its own BOS); 

733# everything else follows the adapter default. 

734PREPEND_BOS_BY_KIND: dict[str, Optional[bool]] = { 

735 "chat": False, 

736} 

737 

738# Bake-off-measured scoring anchors; scripts/text_quality_judge_bakeoff.py 

739# regenerates them (it prints these names verbatim; last run 2026-08-20, full 

740# 13-domain corpus). R_FAIL = geo-mean of per-language MEDIAN corrupted/fluent 

741# ratios — a low percentile degenerates below 1 in weak-separation languages. 

742# R_GOOD = the paraphrase noise floor; score(R_GOOD) is the pass line. 

743JUDGE_R_FAIL = 18.1 

744JUDGE_R_GOOD = 3.74 

745 

746 

747# Known scale properties (measured during the 2026-08 output audit): 

748# - Saturation: any ratio <= 1 scores 100 — "at least reference-fluent" is the 

749# top of the scale, with no resolution above it. 

750# - Judge family-favoring: the pinned Qwen judge rates Qwen-family models a 

751# few points friendlier than others; watch Qwen entries in campaign reruns. 

752 

753 

754def p4_pass_threshold() -> float: 

755 """The P4 pass line, derived from the bake-off noise floor. The registry 

756 floor imports this so [floor, pass) can never silently diverge again.""" 

757 return round(100.0 - 100.0 * math.log(JUDGE_R_GOOD) / math.log(JUDGE_R_FAIL), 1) 

758 

759 

760# Registry scale marker for phase4_score. Absent = v1 (unpinned GPT-2, 

761# 135-10*ln(ppl), pass 85); 2 = pinned-judge reference-ratio scale (pass 56). 

762# The column mixes populations until the backlog is re-run, so every P4 write 

763# stamps the scale it was measured on. 

764P4_SCORING_VERSION = 2 

765 

766# Task output is generated greedily — that is how users run translators and 

767# summarizers, and it removes sampling variance from a single-sample score. 

768# Open-ended kinds keep sampling (greedy makes base models loop). 

769TEMPERATURE_BY_KIND: dict[str, float] = { 

770 "continuation": 0.7, 

771 "chat": 0.7, 

772 "task:instruction": 0.0, 

773 "task:translation": 0.0, 

774 "task:summarization": 0.0, 

775 "task:denoise": 0.0, 

776 "caption": 0.0, 

777} 

778 

779# Kinds whose judge PPL is conditioned on the prompt — the relevance signal: 

780# unconditioned, fluent-but-off-topic or hallucinated output judges as well as 

781# a real answer. Translation stays unconditioned (cross-lingual conditioning 

782# is noisy; the language check covers it); caption's source is an image the 

783# judge cannot read. 

784JUDGE_CONTEXT_KINDS = frozenset( 

785 {"continuation", "chat", "task:instruction", "task:summarization", "task:denoise"} 

786) 

787 

788# T5-family checkpoints expect a natural-language task prefix on the source. 

789T5_PREFIX_ARCHITECTURES = frozenset( 

790 { 

791 "T5ForConditionalGeneration", 

792 "T5WithLMHeadModel", 

793 "MT5ForConditionalGeneration", 

794 "LongT5ForConditionalGeneration", 

795 "SwitchTransformersForConditionalGeneration", 

796 "UMT5ForConditionalGeneration", 

797 } 

798) 

799 

800# Full NLLB (flores-200) codes for covered languages; transformers 5.x 

801# NllbTokenizer resolves them only via convert_tokens_to_ids. 

802NLLB_CODES: dict[str, str] = { 

803 "en": "eng_Latn", 

804 "fr": "fra_Latn", 

805 "es": "spa_Latn", 

806 "de": "deu_Latn", 

807 "it": "ita_Latn", 

808 "nl": "nld_Latn", 

809 "pt": "por_Latn", 

810 "ru": "rus_Cyrl", 

811 "zh": "zho_Hans", 

812 "ja": "jpn_Jpan", 

813 "ar": "arb_Arab", 

814 "hi": "hin_Deva", 

815 "ro": "ron_Latn", 

816} 

817 

818# ISO 639-3 equivalents for NLLB-style language codes ("deu_Latn"). 

819LANG_ISO3: dict[str, str] = { 

820 "en": "eng", 

821 "fr": "fra", 

822 "es": "spa", 

823 "de": "deu", 

824 "it": "ita", 

825 "nl": "nld", 

826 "pt": "por", 

827 "ru": "rus", 

828 "zh": "zho", 

829 "ja": "jpn", 

830 "ar": "ara", 

831 "hi": "hin", 

832 "ro": "ron", 

833} 

834 

835LANG_NAMES: dict[str, str] = { 

836 "en": "English", 

837 "fr": "French", 

838 "es": "Spanish", 

839 "de": "German", 

840 "it": "Italian", 

841 "nl": "Dutch", 

842 "pt": "Portuguese", 

843 "ru": "Russian", 

844 "zh": "Chinese", 

845 "ja": "Japanese", 

846 "ar": "Arabic", 

847 "hi": "Hindi", 

848 "ro": "Romanian", 

849} 

850 

851# --------------------------------------------------------------------------- 

852# Curation: architecture rules and per-model overrides. 

853# --------------------------------------------------------------------------- 

854 

855# Architectures whose task is unambiguous. T5/BART/Switch and Falcon/MPT are 

856# deliberately absent: their task depends on the checkpoint, so they resolve 

857# through overrides or fetched Hub signals. 

858ARCHITECTURE_PROFILE_KINDS: dict[str, str] = { 

859 "MarianMTModel": "task:translation", 

860 "M2M100ForConditionalGeneration": "task:translation", 

861 "PegasusForConditionalGeneration": "task:summarization", 

862 "LEDForConditionalGeneration": "task:summarization", 

863 "BlenderbotForConditionalGeneration": "chat", 

864 "BlenderbotSmallForConditionalGeneration": "chat", 

865 "GPTBigCodeForCausalLM": "continuation@code", 

866 "CodeGenForCausalLM": "continuation@code", 

867} 

868 

869# An unlabelled seq2seq model cannot continue text; its pretraining task is the 

870# only prompt it understands. 

871SEQ2SEQ_FALLBACK_KIND = "task:denoise" 

872 

873MODEL_PROFILE_OVERRIDES: dict[str, str] = { 

874 # T5 v1.0 checkpoints were multitask-trained with task prefixes; the WMT 

875 # en-de pair is their canonical supervised task. 

876 "google-t5/t5-small": "task:translation@en-de", 

877 "google-t5/t5-base": "task:translation@en-de", 

878 "google-t5/t5-large": "task:translation@en-de", 

879 "t5-small": "task:translation@en-de", 

880 "t5-base": "task:translation@en-de", 

881 "t5-large": "task:translation@en-de", 

882 # mt0 is instruction-tuned MT5 (Hub mis-tags it text-generation). 

883 "bigscience/mt0-small": "task:instruction", 

884 "bigscience/mt0-base": "task:instruction", 

885 "bigscience/mt0-large": "task:instruction", 

886 # Pretrained-only checkpoints: denoising is their only language. 

887 "google/long-t5-tglobal-base": "task:denoise", 

888 "google/long-t5-local-base": "task:denoise", 

889 # Base model that ships a chat template (Hub tags it conversational). 

890 "Qwen/Qwen2.5-0.5B": "continuation", 

891 # Task depends on the checkpoint for BART (arch rule deliberately absent); 

892 # without a scraped registry profile these canonical ones need curation. 

893 "facebook/bart-large-cnn": "task:summarization", 

894 "facebook/bart-large-xsum": "task:summarization", 

895 # MBart has no arch rule (base checkpoints are denoising pretrains, task 

896 # varies by fine-tune) — the canonical translators are curated instead. 

897 "facebook/mbart-large-50-many-to-many-mmt": "task:translation@en-de", 

898 "facebook/mbart-large-50-one-to-many-mmt": "task:translation@en-de", 

899 "facebook/mbart-large-50-many-to-one-mmt": "task:translation@de-en", 

900 # Indic-language denoiser; English denoise prompts measure the wrong 

901 # thing, so this skips until Indic coverage exists. 

902 "ai4bharat/IndicBART": "task:denoise@hi", 

903 # Code checkpoints on general-purpose architectures. 

904 "Salesforce/codegen-350M-mono": "continuation@code", 

905 "bigcode/starcoderbase-1b": "continuation@code", 

906 "replit/replit-code-v1-3b": "continuation@code", 

907} 

908 

909# --------------------------------------------------------------------------- 

910# Hub-signal distillation and profile resolution. 

911# --------------------------------------------------------------------------- 

912 

913# Full ISO 639-1 code set, used to pick language codes out of unstructured Hub 

914# tag lists. Complete on purpose: a dropped code silently reroutes a model to 

915# English prompts. 

916ISO_639_1 = frozenset( 

917 "aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce " 

918 "ch co cr cs cu cv cy da de dv dz ee el en eo es et eu fa ff fi fj fo fr " 

919 "fy ga gd gl gn gu gv ha he hi ho hr ht hu hy hz ia id ie ig ii ik io is " 

920 "it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln " 

921 "lo lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv " 

922 "ny oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk " 

923 "sl sm sn so sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw " 

924 "ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu".split() 

925) 

926 

927_PIPELINE_TAG_KINDS: dict[str, str] = { 

928 "translation": "task:translation", 

929 "summarization": "task:summarization", 

930 "text2text-generation": "task:denoise", 

931 "text-generation": "continuation", 

932 "image-text-to-text": "caption", 

933 "image-to-text": "caption", 

934} 

935 

936# Hub tags that mark code models (`conversational` is deliberately NOT mapped 

937# to chat: HF adds it for any repo shipping a chat template, base models 

938# included). 

939_CODE_TAGS = frozenset({"code", "code-generation", "coding"}) 

940 

941 

942@dataclass(frozen=True) 

943class HFSignals: 

944 """Distilled Hub metadata for one model, as fetched by the scraper.""" 

945 

946 pipeline_tag: Optional[str] = None 

947 languages: tuple[str, ...] = () 

948 tags: tuple[str, ...] = () 

949 

950 

951def extract_languages(card_data_language: object, tags: object) -> tuple[str, ...]: 

952 """Normalize cardData.language (str or list) plus tag-list ISO codes, noise dropped.""" 

953 langs: list[str] = [] 

954 if isinstance(card_data_language, str): 

955 langs.append(card_data_language.lower()) 

956 elif isinstance(card_data_language, (list, tuple)): 

957 langs.extend(str(item).lower() for item in card_data_language) 

958 if isinstance(tags, (list, tuple)): 958 ↛ 960line 958 didn't jump to line 960 because the condition on line 958 was always true

959 langs.extend(str(t).lower() for t in tags) 

960 seen: list[str] = [] 

961 for lang in langs: 

962 # The ISO gate alone filters framework/task tag noise ("pytorch", 

963 # "marian", "multilingual" are not ISO 639-1 codes). 

964 if lang in ISO_639_1 and lang not in seen: 

965 seen.append(lang) 

966 if len(seen) >= 8: 

967 break 

968 return tuple(seen) 

969 

970 

971def _marian_pair_from_model_id(model_id: str) -> Optional[tuple[str, str]]: 

972 """Parse opus-mt-{src}-{tgt} from the id; Helsinki-NLP language tags are unordered.""" 

973 name = model_id.rsplit("/", 1)[-1].lower() 

974 if not name.startswith("opus-mt-"): 

975 return None 

976 parts = name[len("opus-mt-") :].split("-") 

977 if len(parts) == 2 and all(len(p) in (2, 3) for p in parts): 

978 return parts[0], parts[1] 

979 return None 

980 

981 

982_CHAT_ID_MARKERS = ("instruct", "-chat", "_chat") 

983 

984 

985def _id_says_chat(model_id: str) -> bool: 

986 """Instruction-tuned checkpoints are used through their chat template; the 

987 id is the only reliable signal (HF's `conversational` tag also covers base 

988 models, and no architecture distinguishes tuned from base).""" 

989 name = model_id.rsplit("/", 1)[-1].lower() 

990 if name.endswith("-it") or "-it-" in name: 

991 return True 

992 return any(marker in name for marker in _CHAT_ID_MARKERS) 

993 

994 

995def _first_covered_language(languages: tuple[str, ...], table: dict) -> Optional[str]: 

996 for lang in languages: 996 ↛ 997line 996 didn't jump to line 997 because the loop on line 996 never started

997 if lang in table: 

998 return lang 

999 return None 

1000 

1001 

1002def profile_from_hf_signals( 

1003 model_id: str, 

1004 architecture_id: str, 

1005 signals: HFSignals, 

1006) -> Optional[ProfileSpec]: 

1007 """Distill fetched Hub metadata into a profile, or None when it says nothing.""" 

1008 tags_lower = {t.lower() for t in signals.tags} 

1009 if tags_lower & _CODE_TAGS: 

1010 return ProfileSpec("continuation", "code") 

1011 kind = _PIPELINE_TAG_KINDS.get((signals.pipeline_tag or "").lower()) 

1012 if kind is None: 

1013 return None 

1014 if kind == "task:translation": 

1015 pair = _marian_pair_from_model_id(model_id) 

1016 if pair is not None: 1016 ↛ 1017line 1016 didn't jump to line 1017 because the condition on line 1016 was never true

1017 return ProfileSpec(kind, lang=pair[1], src=pair[0]) 

1018 non_en = [lang for lang in signals.languages if lang != "en"] 

1019 if "en" in signals.languages and non_en: 

1020 return ProfileSpec(kind, lang=non_en[0], src="en") 

1021 # Direction unknowable from tags alone (tag lists are unordered); 

1022 # fall through rather than guess a reversed or identity pair. 

1023 return None 

1024 lang = _first_covered_language(signals.languages, CONTINUATION_PROMPTS) or "en" 

1025 if kind == "continuation": 1025 ↛ 1027line 1025 didn't jump to line 1027 because the condition on line 1025 was always true

1026 return ProfileSpec(kind, lang) 

1027 return ProfileSpec(kind) 

1028