Coverage for transformer_lens/model_bridge/supported_architectures/florence2.py: 100%

1"""Florence-2 architecture adapter. 

2 

3Microsoft's Florence-2 (``Florence2ForConditionalGeneration``): a DaViT 

4vision backbone at ``model.vision_tower`` whose projected features are 

5scattered into image-placeholder tokens, feeding a BART encoder-decoder at 

6``model.language_model``. The text stack reuses the BART mapping with 

7re-prefixed paths; the vision tower and projector are opaque delegated 

8components (window/channel-attention DaViT has no TL analogue). 

9""" 

10 

11from typing import Any 

12 

13from transformer_lens.model_bridge.generalized_components.base import ( 

14 GeneralizedComponent, 

15) 

16from transformer_lens.model_bridge.supported_architectures.bart import ( 

17 BartArchitectureAdapter, 

18) 

19 

20_TEXT_PREFIX = "model.language_model." 

21 

22 

23class Florence2ArchitectureAdapter(BartArchitectureAdapter): 

24 """Architecture adapter for Florence2ForConditionalGeneration models.""" 

25 

26 def __init__(self, cfg: Any) -> None: 

27 """Initialize the Florence-2 architecture adapter.""" 

28 super().__init__(cfg) 

29 

30 self.cfg.is_multimodal = True 

31 if hasattr(cfg, "vision_config"): 

32 self.cfg.vision_hidden_size = getattr(cfg.vision_config, "hidden_size", None) 

33 

34 self._reprefix_components("model.", _TEXT_PREFIX) 

35 

36 self.components["vision_encoder"] = GeneralizedComponent(name="model.vision_tower") 

37 self.components["vision_projector"] = GeneralizedComponent( 

38 name="model.multi_modal_projector" 

39 )