Coverage for transformer_lens/model_bridge/sources/_hf_format.py: 88%
244 statements
« prev ^ index » next coverage.py v7.10.1, created at 2026-09-21 19:27 +0000
1"""Shared HF-format utilities used by every source whose backend produces an
2HF-shaped config or ``PreTrainedTokenizerBase`` tokenizer.
4The transformers source loads HF objects directly; the vLLM source extracts the
5same HF-shaped config via ``llm.llm_engine.model_config.hf_config`` because vLLM
6re-uses the ``transformers`` config and tokenizer libraries internally. This
7module is loader-agnostic — it speaks HF format, not HF loading.
8"""
9from __future__ import annotations
11import copy
12from typing import Any
14from transformers import PreTrainedTokenizerBase
16from transformer_lens.factories.architecture_adapter_factory import (
17 SUPPORTED_ARCHITECTURES,
18)
19from transformer_lens.utilities import get_tokenizer_with_bos
20from transformer_lens.utilities.heterogeneous_config import (
21 het_safe_view,
22 majority_value,
23 per_layer_attr_names,
24 per_layer_values,
25)
28def get_effective_text_config(hf_config):
29 """Return the config that owns the language-model forward path."""
30 if getattr(hf_config, "text_config", None) is not None:
31 return hf_config.text_config
32 decoder = getattr(hf_config, "decoder", None)
33 if decoder is not None and hasattr(decoder, "hidden_size"):
34 return decoder
35 return hf_config
38def map_default_transformer_lens_config(hf_config):
39 """Map HuggingFace config fields to TransformerLens config format.
41 Standardized mapping from various HuggingFace config field names to the
42 consistent TransformerLens naming convention. For multimodal models (LLaVA,
43 Gemma3ForConditionalGeneration), the language model dimensions are nested
44 under ``text_config``; we extract from there first.
46 Args:
47 hf_config: The HuggingFace config object
49 Returns:
50 A copy of hf_config with additional TransformerLens fields
51 """
52 # Extract language model config from text_config for multimodal models
53 raw_source_config = get_effective_text_config(hf_config)
55 tl_config = copy.deepcopy(hf_config)
57 # transformers>=5.15 het configs raise (not AttributeError, so hasattr does
58 # NOT suppress it) on global reads of per-layer attrs; the view resolves
59 # them to majority-layer values so every probe below stays safe.
60 het_attrs = per_layer_attr_names(raw_source_config)
61 source_config = het_safe_view(raw_source_config)
63 def legacy_per_layer(base_name: str, global_name: str) -> Any:
64 """Pre-5.15 Gemma 4 split geometry across <field> (sliding-attention layers)
65 and global_<field> (full-attention layers); rebuild the per-layer view."""
66 if base_name in het_attrs or "layer_types" in het_attrs: 66 ↛ 67line 66 didn't jump to line 67 because the condition on line 66 was never true
67 return None
68 base = getattr(source_config, base_name, None)
69 full = getattr(source_config, global_name, None)
70 layer_types = getattr(source_config, "layer_types", None)
71 if base is None or full is None or full == base or not layer_types:
72 return None
73 return [full if t == "full_attention" else base for t in layer_types]
75 if hasattr(source_config, "n_embd"):
76 tl_config.d_model = source_config.n_embd
77 elif hasattr(source_config, "hidden_size"):
78 tl_config.d_model = source_config.hidden_size
79 elif hasattr(source_config, "model_dim"): 79 ↛ 80line 79 didn't jump to line 80 because the condition on line 79 was never true
80 tl_config.d_model = source_config.model_dim
81 elif hasattr(source_config, "d_model"): 81 ↛ 83line 81 didn't jump to line 83 because the condition on line 81 was always true
82 tl_config.d_model = source_config.d_model
83 elif hasattr(source_config, "hidden_dim"):
84 tl_config.d_model = source_config.hidden_dim
85 if hasattr(source_config, "n_head"):
86 tl_config.n_heads = source_config.n_head
87 elif hasattr(source_config, "num_attention_heads"):
88 n_heads = source_config.num_attention_heads
89 if isinstance(n_heads, list): 89 ↛ 90line 89 didn't jump to line 90 because the condition on line 89 was never true
90 n_heads = max(n_heads)
91 tl_config.n_heads = n_heads
92 elif hasattr(source_config, "num_heads"):
93 tl_config.n_heads = source_config.num_heads
94 elif hasattr(source_config, "n_heads"):
95 tl_config.n_heads = source_config.n_heads
96 elif hasattr(source_config, "num_query_heads") and isinstance( 96 ↛ 99line 96 didn't jump to line 99 because the condition on line 96 was never true
97 source_config.num_query_heads, list
98 ):
99 tl_config.n_heads = max(source_config.num_query_heads)
100 if "num_key_value_heads" in het_attrs:
101 per_layer_kv = per_layer_values(source_config, "num_key_value_heads")
102 elif getattr(source_config, "attention_k_eq_v", True):
103 # HF applies num_global_key_value_heads only on attention_k_eq_v models;
104 # the 5.15 per-layer migration gates identically, defaulting True when absent.
105 per_layer_kv = legacy_per_layer("num_key_value_heads", "num_global_key_value_heads")
106 else:
107 per_layer_kv = None
108 kv_values = [v for v in per_layer_kv if v is not None] if per_layer_kv else []
109 if kv_values:
110 # Heterogeneous KV geometry (e.g. Gemma 4 31B: 16 KV heads on sliding layers,
111 # 4 on full-attention layers). The scalar keeps the majority-layer value —
112 # attention math is delegated to HF for these architectures — and the
113 # per-layer truth is preserved alongside it.
114 tl_config.per_layer_num_key_value_heads = per_layer_kv
115 try:
116 num_kv_heads = int(majority_value(kv_values))
117 num_heads = int(getattr(tl_config, "n_heads", 0))
118 if num_kv_heads != num_heads: 118 ↛ 165line 118 didn't jump to line 165 because the condition on line 118 was always true
119 tl_config.n_key_value_heads = num_kv_heads
120 except (TypeError, ValueError):
121 pass
122 elif (
123 hasattr(source_config, "num_key_value_heads")
124 and source_config.num_key_value_heads is not None
125 ):
126 try:
127 num_kv_heads = source_config.num_key_value_heads
128 # Per-layer lists (e.g., OpenELM) collapse to the max.
129 if isinstance(num_kv_heads, list): 129 ↛ 130line 129 didn't jump to line 130 because the condition on line 129 was never true
130 num_kv_heads = max(num_kv_heads)
131 if hasattr(num_kv_heads, "item"): 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true
132 num_kv_heads = num_kv_heads.item()
133 num_kv_heads = int(num_kv_heads)
134 num_heads = tl_config.n_heads
135 if hasattr(num_heads, "item"): 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 num_heads = num_heads.item()
137 num_heads = int(num_heads)
138 if num_kv_heads != num_heads:
139 tl_config.n_key_value_heads = num_kv_heads
140 except (TypeError, ValueError, AttributeError):
141 pass
142 elif hasattr(source_config, "num_kv_heads") and source_config.num_kv_heads is not None:
143 try:
144 num_kv_heads = source_config.num_kv_heads
145 if isinstance(num_kv_heads, list): 145 ↛ 146line 145 didn't jump to line 146 because the condition on line 145 was never true
146 num_kv_heads = max(num_kv_heads)
147 if hasattr(num_kv_heads, "item"): 147 ↛ 148line 147 didn't jump to line 148 because the condition on line 147 was never true
148 num_kv_heads = num_kv_heads.item()
149 num_kv_heads = int(num_kv_heads)
150 num_heads = tl_config.n_heads
151 if hasattr(num_heads, "item"): 151 ↛ 152line 151 didn't jump to line 152 because the condition on line 151 was never true
152 num_heads = num_heads.item()
153 num_heads = int(num_heads)
154 if num_kv_heads != num_heads:
155 tl_config.n_key_value_heads = num_kv_heads
156 except (TypeError, ValueError, AttributeError):
157 pass
158 elif hasattr(source_config, "n_kv_heads") and source_config.n_kv_heads is not None:
159 try:
160 num_kv_heads = int(source_config.n_kv_heads)
161 if num_kv_heads != getattr(tl_config, "n_heads", None):
162 tl_config.n_key_value_heads = num_kv_heads
163 except (TypeError, ValueError, AttributeError):
164 pass
165 if hasattr(source_config, "n_layer"):
166 tl_config.n_layers = source_config.n_layer
167 elif hasattr(source_config, "num_hidden_layers"):
168 tl_config.n_layers = source_config.num_hidden_layers
169 elif hasattr(source_config, "num_transformer_layers"): 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 tl_config.n_layers = source_config.num_transformer_layers
171 elif hasattr(source_config, "num_layers"): 171 ↛ 172line 171 didn't jump to line 172 because the condition on line 171 was never true
172 tl_config.n_layers = source_config.num_layers
173 elif hasattr(source_config, "n_layers"):
174 tl_config.n_layers = source_config.n_layers
175 elif hasattr(source_config, "n_blocks"): 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true
176 tl_config.n_layers = source_config.n_blocks
177 if hasattr(source_config, "vocab_size") and isinstance(source_config.vocab_size, int):
178 tl_config.d_vocab = source_config.vocab_size
179 if hasattr(source_config, "n_positions"):
180 tl_config.n_ctx = source_config.n_positions
181 elif hasattr(source_config, "max_position_embeddings"):
182 tl_config.n_ctx = source_config.max_position_embeddings
183 elif hasattr(source_config, "max_context_length"): 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true
184 tl_config.n_ctx = source_config.max_context_length
185 elif hasattr(source_config, "max_length"):
186 tl_config.n_ctx = source_config.max_length
187 elif hasattr(source_config, "seq_length"):
188 tl_config.n_ctx = source_config.seq_length
189 elif hasattr(source_config, "max_seq_len"):
190 # MPT-family field name.
191 tl_config.n_ctx = source_config.max_seq_len
192 elif hasattr(source_config, "image_size") and hasattr(source_config, "patch_size"):
193 # Vision Transformers calculate sequence length dynamically:
194 # (image_size / patch_size)^2 + 1 (for the CLS token)
195 image_size = source_config.image_size
196 patch_size = source_config.patch_size
198 # HF configs allow these to be integers or tuples/lists
199 img_h = image_size[0] if isinstance(image_size, (list, tuple)) else image_size
200 img_w = image_size[1] if isinstance(image_size, (list, tuple)) else image_size
201 patch_h = patch_size[0] if isinstance(patch_size, (list, tuple)) else patch_size
202 patch_w = patch_size[1] if isinstance(patch_size, (list, tuple)) else patch_size
204 tl_config.n_ctx = (img_h // patch_h) * (img_w // patch_w) + 1
205 elif hasattr(source_config, "max_sequence_length"):
206 tl_config.n_ctx = source_config.max_sequence_length
207 else:
208 # ALiBi models (Bloom) have no context length field; 2048 is a safe fallback.
209 tl_config.n_ctx = 2048
210 if hasattr(source_config, "n_inner"):
211 tl_config.d_mlp = source_config.n_inner
212 elif "intermediate_size" in het_attrs:
213 # Same max collapse as the per-layer-list case below.
214 mlp_values = [v for v in per_layer_values(source_config, "intermediate_size") if v]
215 tl_config.d_mlp = max(mlp_values) if mlp_values else None
216 elif hasattr(source_config, "intermediate_size"):
217 intermediate_size = source_config.intermediate_size
218 # Gemma 3n exposes a per-layer intermediate_size list (the MatFormer design permits
219 # variation). All released checkpoints (E2B/E4B) are uniform, and d_mlp is scalar
220 # metadata (the bridge defers MLP math to HF), so collapse to max — the shared value
221 # when uniform, an upper bound otherwise.
222 if isinstance(intermediate_size, (list, tuple)):
223 intermediate_size = max(intermediate_size) if intermediate_size else None
224 tl_config.d_mlp = intermediate_size
225 elif hasattr(source_config, "mlp_hidden_size"):
226 tl_config.d_mlp = source_config.mlp_hidden_size
227 elif hasattr(tl_config, "d_model"): 227 ↛ 229line 227 didn't jump to line 229 because the condition on line 227 was always true
228 tl_config.d_mlp = getattr(source_config, "n_inner", 4 * tl_config.d_model)
229 if "head_dim" in het_attrs:
230 per_layer_hd = per_layer_values(source_config, "head_dim")
231 else:
232 per_layer_hd = legacy_per_layer("head_dim", "global_head_dim")
233 hd_values = [v for v in per_layer_hd if v is not None] if per_layer_hd else []
234 if hd_values:
235 # Heterogeneous head_dim (e.g. Gemma 4: 256 on sliding layers, 512 on
236 # full-attention layers). Scalar d_head keeps the majority-layer value;
237 # the per-layer truth is preserved alongside it.
238 tl_config.per_layer_head_dim = per_layer_hd
239 tl_config.d_head = majority_value(hd_values)
240 elif hasattr(source_config, "head_dim") and source_config.head_dim is not None:
241 tl_config.d_head = source_config.head_dim
242 elif hasattr(tl_config, "d_model") and hasattr(tl_config, "n_heads"):
243 tl_config.d_head = tl_config.d_model // tl_config.n_heads
244 elif hasattr(tl_config, "d_model"): 244 ↛ 248line 244 didn't jump to line 248 because the condition on line 244 was always true
245 # Attention-less architectures (Mamba SSMs): set d_head = d_model so
246 # __post_init__ computes n_heads = 1. Values are nominal.
247 tl_config.d_head = tl_config.d_model
248 if hasattr(source_config, "activation_function"):
249 tl_config.act_fn = source_config.activation_function
250 # Gemma family: transformers 5.x exposes only hidden_activation (hidden_act
251 # was removed); it is authoritative over hidden_act when both exist.
252 elif getattr(source_config, "hidden_activation", None) is not None:
253 tl_config.act_fn = source_config.hidden_activation
254 elif hasattr(source_config, "hidden_act"):
255 tl_config.act_fn = source_config.hidden_act
256 elif hasattr(source_config, "activation_type"):
257 activation_type = source_config.activation_type
258 tl_config.act_fn = getattr(activation_type, "value", activation_type)
259 elif getattr(source_config, "activation_fn_name", None) is not None:
260 # OpenELM spells it activation_fn_name ("swish"); without this the
261 # cfg keeps the "relu" default and reconstructed FFNs silently
262 # diverge ~30% from HF.
263 tl_config.act_fn = source_config.activation_fn_name
264 if hasattr(source_config, "rope_theta"):
265 tl_config.rotary_base = source_config.rope_theta
266 if hasattr(source_config, "weight_tying"):
267 tl_config.tie_word_embeddings = bool(source_config.weight_tying)
268 # LayerNorm / RMSNorm epsilon — HF uses 3 different field names.
269 if hasattr(source_config, "rms_norm_eps"):
270 tl_config.eps = source_config.rms_norm_eps
271 elif hasattr(source_config, "layer_norm_eps"):
272 tl_config.eps = source_config.layer_norm_eps
273 elif hasattr(source_config, "layer_norm_epsilon"):
274 tl_config.eps = source_config.layer_norm_epsilon
275 elif hasattr(source_config, "norm_eps"):
276 tl_config.eps = source_config.norm_eps
277 if hasattr(source_config, "num_experts"):
278 tl_config.num_experts = source_config.num_experts
279 elif hasattr(source_config, "num_local_experts"):
280 tl_config.num_experts = source_config.num_local_experts
281 if hasattr(source_config, "num_experts_per_tok"):
282 tl_config.experts_per_token = source_config.num_experts_per_tok
283 if hasattr(source_config, "sliding_window") and source_config.sliding_window is not None:
284 tl_config.sliding_window = source_config.sliding_window
285 if getattr(hf_config, "use_parallel_residual", False):
286 tl_config.parallel_attn_mlp = True
287 # GPT-J and CodeGen run parallel attn+MLP but don't set use_parallel_residual.
288 arch_classes = getattr(hf_config, "architectures", []) or []
289 if any(a in ("GPTJForCausalLM", "CodeGenForCausalLM") for a in arch_classes):
290 tl_config.parallel_attn_mlp = True
291 tl_config.default_prepend_bos = True
292 return tl_config
295def determine_architecture_from_hf_config(hf_config):
296 """Determine the architecture name from HuggingFace config.
298 Returns:
299 str: The architecture name (e.g., "GPT2LMHeadModel", "LlamaForCausalLM")
301 Raises:
302 ValueError: If architecture cannot be determined
303 """
304 architectures = []
305 if hasattr(hf_config, "original_architecture"): 305 ↛ 306line 305 didn't jump to line 306 because the condition on line 305 was never true
306 architectures.append(hf_config.original_architecture)
307 if hasattr(hf_config, "architectures") and hf_config.architectures:
308 architectures.extend(hf_config.architectures)
309 if hasattr(hf_config, "model_type"):
310 model_type = hf_config.model_type
311 model_type_mappings = {
312 "afmoe": "AfmoeForCausalLM",
313 "apertus": "ApertusForCausalLM",
314 "gpt2": "GPT2LMHeadModel",
315 "openai-gpt": "OpenAIGPTLMHeadModel",
316 "hubert": "HubertModel",
317 "bamba": "BambaForCausalLM",
318 "bitnet": "BitNetForCausalLM",
319 "blenderbot": "BlenderbotForConditionalGeneration",
320 "bart": "BartForConditionalGeneration",
321 "ernie4_5": "Ernie4_5ForCausalLM",
322 "ernie4_5_moe": "Ernie4_5_MoeForCausalLM",
323 "exaone": "ExaoneForCausalLM",
324 "exaone4": "Exaone4ForCausalLM",
325 "falcon_mamba": "FalconMambaForCausalLM",
326 "florence2": "Florence2ForConditionalGeneration",
327 "longt5": "LongT5ForConditionalGeneration",
328 "m2m_100": "M2M100ForConditionalGeneration",
329 "marian": "MarianMTModel",
330 "mbart": "MBartForConditionalGeneration",
331 "pegasus": "PegasusForConditionalGeneration",
332 "seed_oss": "SeedOssForCausalLM",
333 "starcoder2": "Starcoder2ForCausalLM",
334 "nemotron": "NemotronForCausalLM",
335 "idefics3": "Idefics3ForConditionalGeneration",
336 "qwen2_audio": "Qwen2AudioForConditionalGeneration",
337 "audioflamingo3": "AudioFlamingo3ForConditionalGeneration",
338 "musicflamingo": "MusicFlamingoForConditionalGeneration",
339 "jetmoe": "JetMoeForCausalLM",
340 "minimax_m2": "MiniMaxM2ForCausalLM",
341 "led": "LEDForConditionalGeneration",
342 "llama": "LlamaForCausalLM",
343 "llama4_text": "Llama4ForCausalLM",
344 "llama4": "Llama4ForConditionalGeneration",
345 "llada": "LLaDAModelLM",
346 "mamba": "MambaForCausalLM",
347 "mamba2": "Mamba2ForCausalLM",
348 "mistral": "MistralForCausalLM",
349 "olmo_hybrid": "OlmoHybridForCausalLM",
350 "mistral3": "Mistral3ForConditionalGeneration",
351 "mixtral": "MixtralForCausalLM",
352 "mpt": "MptForCausalLM",
353 "gemma": "GemmaForCausalLM",
354 "gemma2": "Gemma2ForCausalLM",
355 "gemma3": "Gemma3ForCausalLM",
356 # gemma3n is tri-modal; the text path loads as the full ForConditionalGeneration
357 # (vision/audio referenced but unbridged in the text-only adapter).
358 "gemma3n": "Gemma3nForConditionalGeneration",
359 # gemma4 is multimodal-only; all released checkpoints load as the full
360 # ForConditionalGeneration (vision/audio referenced but unbridged).
361 "gemma4": "Gemma4ForConditionalGeneration",
362 "gemma4_unified": "Gemma4UnifiedForConditionalGeneration",
363 "gemma4_text": "Gemma4ForCausalLM",
364 "glm": "GlmForCausalLM",
365 "glm4": "Glm4ForCausalLM",
366 "glm4v": "Glm4vForConditionalGeneration",
367 "glmasr": "GlmAsrForConditionalGeneration",
368 "glm4_moe": "Glm4MoeForCausalLM",
369 "glm4_moe_lite": "Glm4MoeLiteForCausalLM",
370 "glm_moe_dsa": "GlmMoeDsaForCausalLM",
371 "t5gemma": "T5GemmaForConditionalGeneration",
372 "t5gemma2": "T5Gemma2ForConditionalGeneration",
373 "bert": "BertForMaskedLM",
374 "bloom": "BloomForCausalLM",
375 "codegen": "CodeGenForCausalLM",
376 "cohere2": "Cohere2ForCausalLM",
377 "gptj": "GPTJForCausalLM",
378 "gpt_neo": "GPTNeoForCausalLM",
379 "gpt_neox": "GPTNeoXForCausalLM",
380 "opt": "OPTForCausalLM",
381 "phi": "PhiForCausalLM",
382 "phi3": "Phi3ForCausalLM",
383 "qwen": "QwenForCausalLM",
384 "qwen2": "Qwen2ForCausalLM",
385 "qwen2_5_vl": "Qwen2_5_VLForConditionalGeneration",
386 "qwen3_vl": "Qwen3VLForConditionalGeneration",
387 "qwen3_vl_moe": "Qwen3VLMoeForConditionalGeneration",
388 "qwen2_moe": "Qwen2MoeForCausalLM",
389 "qwen3": "Qwen3ForCausalLM",
390 # qwen3_5 is the top-level multimodal config type; qwen3_5_text is
391 # the text-only sub-config. Both map to the text-only adapter so
392 # Qwen3.5 checkpoints (which report qwen3_5 even when loaded as
393 # text-only) are routed to Qwen3_5ForCausalLM.
394 "qwen3_5": "Qwen3_5ForCausalLM",
395 "qwen3_5_text": "Qwen3_5ForCausalLM",
396 # Same routing convention for the MoE variant.
397 "qwen3_5_moe": "Qwen3_5MoeForCausalLM",
398 "qwen3_5_moe_text": "Qwen3_5MoeForCausalLM",
399 "smollm3": "SmolLM3ForCausalLM",
400 "openelm": "OpenELMForCausalLM",
401 "ouro": "OuroForCausalLM",
402 "stablelm": "StableLmForCausalLM",
403 "t5": "T5ForConditionalGeneration",
404 "mt5": "MT5ForConditionalGeneration",
405 }
406 if model_type in model_type_mappings:
407 architectures.append(model_type_mappings[model_type])
409 for arch in architectures:
410 if arch in SUPPORTED_ARCHITECTURES:
411 return arch
412 raise ValueError(
413 f"Could not determine supported architecture from config. Available architectures: "
414 f"{list(SUPPORTED_ARCHITECTURES.keys())}, Config architectures: {architectures}, "
415 f"Model type: {getattr(hf_config, 'model_type', None)}"
416 )
419def setup_tokenizer(tokenizer, default_padding_side=None):
420 """Normalize a HuggingFace tokenizer for use with the bridge.
422 Args:
423 tokenizer: A ``PreTrainedTokenizer`` or ``PreTrainedTokenizerFast``.
424 default_padding_side: ``"right"`` or ``"left"``; sets ``tokenizer.padding_side``.
425 """
426 assert isinstance(
427 tokenizer, PreTrainedTokenizerBase
428 ), f"{type(tokenizer)} is not a supported tokenizer; use PreTrainedTokenizer or PreTrainedTokenizerFast"
429 assert default_padding_side in [
430 "right",
431 "left",
432 None,
433 ], f"padding_side must be 'right', 'left' or None, got {default_padding_side}"
434 tokenizer = get_tokenizer_with_bos(tokenizer)
435 assert tokenizer is not None
436 if default_padding_side is not None:
437 tokenizer.padding_side = default_padding_side
438 if tokenizer.padding_side is None: 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true
439 tokenizer.padding_side = "right"
440 if tokenizer.eos_token is None:
441 tokenizer.eos_token = "<|endoftext|>"
442 if tokenizer.pad_token is None:
443 tokenizer.pad_token = tokenizer.eos_token
444 if tokenizer.bos_token is None:
445 tokenizer.bos_token = tokenizer.eos_token
447 # Some vocabularies lack default IDs for these tokens; register them.
448 if tokenizer.pad_token is not None and tokenizer.pad_token_id is None:
449 tokenizer.add_special_tokens({"pad_token": tokenizer.pad_token})
450 if tokenizer.eos_token is not None and tokenizer.eos_token_id is None: 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true
451 tokenizer.add_special_tokens({"eos_token": tokenizer.eos_token})
452 if tokenizer.bos_token is not None and tokenizer.bos_token_id is None: 452 ↛ 453line 452 didn't jump to line 453 because the condition on line 452 was never true
453 tokenizer.add_special_tokens({"bos_token": tokenizer.bos_token})
455 return tokenizer