Coverage for transformer_lens/model_bridge/supported_architectures/qwen3_5_multimodal.py: 100%

1"""Qwen3.5 multimodal (vision-language) adapter for ``Qwen3_5ForConditionalGeneration``. 

2 

3Reuses the text-only Qwen3.5 hybrid backbone nested under ``model.language_model`` and adds 

4the vision tower (``model.visual``) + merger. The HF model runs the vision computation during 

5forward; this adapter only supplies the component mapping (hooks + weights). 

6""" 

7 

8from typing import Any 

9 

10from transformer_lens.model_bridge.generalized_components import VisionProjectionBridge 

11from transformer_lens.model_bridge.generalized_components.qwen3_5_vision_encoder import ( 

12 Qwen3_5VisionEncoderBridge, 

13) 

14from transformer_lens.model_bridge.supported_architectures.qwen3 import ( 

15 Qwen3ArchitectureAdapter, 

16) 

17 

18 

19class Qwen3_5MultimodalArchitectureAdapter(Qwen3ArchitectureAdapter): 

20 """Full vision-language adapter for Qwen3_5ForConditionalGeneration.""" 

21 

22 _testing_lm_attr = "model.language_model" 

23 _testing_hybrid = True 

24 

25 # Qwen3.5's image/video processor (Qwen3VLProcessor) requires torchvision. 

26 required_libraries: list[str] = ["torchvision"] 

27 required_libraries_group: str = "multimodal" 

28 

29 def __init__(self, cfg: Any) -> None: 

30 # q_proj stays 2x-wide through weight processing: HF and the attention bridge both 

31 # split [query|gate] per head at forward time; slicing the gate out changes outputs. 

32 setattr(cfg, "gated_q_proj", True) 

33 super().__init__(cfg, hybrid=True, lm_prefix="model.language_model") 

34 

35 self.cfg.is_multimodal = True 

36 

37 self._extract_vision_dims(cfg) 

38 self.components["vision_encoder"] = Qwen3_5VisionEncoderBridge( 

39 name="model.visual", config=self.cfg 

40 ) 

41 self.components["vision_projector"] = VisionProjectionBridge(name="model.visual.merger")