Coverage for transformer_lens/model_bridge/supported_architectures/qwen2_5_vl.py: 100%

1"""Qwen2.5-VL architecture adapter. 

2 

3Alibaba's Qwen2.5-VL (``Qwen2_5_VLForConditionalGeneration``): a windowed 

4ViT at ``model.visual`` (window attention with a few full-attention 

5blocks, RMS block norms, gated vision MLP, 2D rotary) whose patch merger 

6feeds a Qwen2-layout text decoder at ``model.language_model``. Text 

7attention uses mRoPE — three position streams (temporal/height/width) 

8split across rotary channels — so the generic RoPE reconstruction would 

9be text-only-correct but wrong for image runs; attention stays HF-native. 

10""" 

11 

12from typing import Any 

13 

14from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter 

15from transformer_lens.model_bridge.generalized_components import ( 

16 AttentionBridge, 

17 BlockBridge, 

18 EmbeddingBridge, 

19 LinearBridge, 

20 RMSNormalizationBridge, 

21 UnembeddingBridge, 

22 VisionProjectionBridge, 

23) 

24from transformer_lens.model_bridge.generalized_components.base import ( 

25 GeneralizedComponent, 

26) 

27from transformer_lens.model_bridge.generalized_components.qwen3_5_vision_encoder import ( 

28 Qwen3_5VisionBlockBridge, 

29 Qwen3_5VisionEncoderBridge, 

30) 

31 

32 

33class Qwen2_5_VLArchitectureAdapter(ArchitectureAdapter): 

34 """Architecture adapter for Qwen2_5_VLForConditionalGeneration models.""" 

35 

36 required_libraries: list[str] = ["torchvision"] 

37 required_libraries_group: str = "multimodal" 

38 

39 def __init__(self, cfg: Any) -> None: 

40 """Initialize the Qwen2.5-VL architecture adapter.""" 

41 super().__init__(cfg) 

42 

43 self.cfg.is_multimodal = True 

44 self._set_rms_rotary_defaults() 

45 self.cfg.attn_implementation = "eager" 

46 # Qwen tokenizers have no BOS; the prepend fallback would inject 

47 # <|im_end|>, which reads as an ended turn. 

48 self.cfg.default_prepend_bos = False 

49 

50 self._extract_vision_dims(cfg) 

51 

52 # Qwen2.5-VL's text decoder keeps Qwen2's hardcoded q/k/v biases. 

53 self.weight_processing_conversions = { 

54 **self._qkvo_weight_conversions(include_biases=True), 

55 } 

56 

57 self.component_mapping = { 

58 # Qwen2.5-VL's tower has a rotary embedding where Qwen3.5 has a 

59 # learned pos_embed, and a gated vision MLP instead of fc1/fc2. 

60 "vision_encoder": Qwen3_5VisionEncoderBridge( 

61 name="model.visual", 

62 config=self.cfg, 

63 submodules={ 

64 "pos_embed": GeneralizedComponent(name="rotary_pos_emb"), 

65 "blocks": Qwen3_5VisionBlockBridge( 

66 name="blocks", 

67 submodules={ 

68 "mlp": GeneralizedComponent( 

69 name="mlp", 

70 submodules={ 

71 "gate": LinearBridge(name="gate_proj"), 

72 "in": LinearBridge(name="up_proj"), 

73 "out": LinearBridge(name="down_proj"), 

74 }, 

75 ), 

76 }, 

77 ), 

78 }, 

79 ), 

80 "vision_projector": VisionProjectionBridge(name="model.visual.merger"), 

81 "embed": EmbeddingBridge(name="model.language_model.embed_tokens"), 

82 "blocks": BlockBridge( 

83 name="model.language_model.layers", 

84 submodules={ 

85 "ln1": RMSNormalizationBridge(name="input_layernorm", config=self.cfg), 

86 "ln2": RMSNormalizationBridge(name="post_attention_layernorm", config=self.cfg), 

87 # mRoPE (3-section multimodal rotary) lives in HF's forward. 

88 "attn": AttentionBridge( 

89 name="self_attn", 

90 config=self.cfg, 

91 submodules={ 

92 "q": LinearBridge(name="q_proj"), 

93 "k": LinearBridge(name="k_proj"), 

94 "v": LinearBridge(name="v_proj"), 

95 "o": LinearBridge(name="o_proj"), 

96 }, 

97 maintain_native_attention=True, 

98 requires_attention_mask=True, 

99 ), 

100 "mlp": self._gated_mlp(), 

101 }, 

102 ), 

103 "ln_final": RMSNormalizationBridge(name="model.language_model.norm", config=self.cfg), 

104 "unembed": UnembeddingBridge(name="lm_head", config=self.cfg), 

105 }