Coverage for transformer_lens/model_bridge/supported_architectures/deepseek_v3.py: 100%

1"""DeepSeek V3 architecture adapter. 

2 

3Supports DeepSeek V3 and DeepSeek-R1 models (both use DeepseekV3ForCausalLM). 

4""" 

5 

6from transformer_lens.model_bridge.generalized_components import RMSNormalizationBridge 

7from transformer_lens.model_bridge.generalized_components.base import ( 

8 GeneralizedComponent, 

9) 

10from transformer_lens.model_bridge.supported_architectures.deepseek_v2 import ( 

11 DeepSeekMLAFamilyArchitectureAdapter, 

12) 

13 

14 

15class DeepSeekV3ArchitectureAdapter(DeepSeekMLAFamilyArchitectureAdapter): 

16 """Architecture adapter for DeepSeek V3 / R1 models. 

17 

18 Uses RMSNorm, MLA with compressed Q/KV projections, partial RoPE, 

19 MoE on most layers (dense MLP on first few), and no biases. HF builds the 

20 two-stage LoRA Q path only when q_lora_rank is set and a single q_proj 

21 otherwise (ai-sage/GigaChat3-10B-A1.8B), so both Q paths are mapped optional. 

22 """ 

23 

24 # HF defaults to SDPA which handles MLA correctly; HF's eager attention 

25 # crashes on MLA's asymmetric Q/K dimensions. 

26 _testing_eager = None 

27 

28 q_lora_optional = True 

29 

30 def _build_q_a_layernorm(self, optional: bool) -> GeneralizedComponent: 

31 # Nearly every V3 checkpoint compresses Q: keep the norm's hooks and weight 

32 # processing instead of downgrading to V2-Lite's plain component. 

33 return RMSNormalizationBridge(name="q_a_layernorm", config=self.cfg, optional=optional)