Coverage for transformer_lens/model_bridge/supported_architectures/whisper.py: 100%

54 statements  

« prev     ^ index     » next       coverage.py v7.10.1, created at 2026-11-10 15:50 +0000

1"""Whisper (``WhisperForConditionalGeneration``) adapter: a BART-family pre-norm 

2encoder-decoder whose encoder reads log-mel audio through two convolutions and a 

3fixed sinusoidal position table instead of token embeddings. The text decoder is 

4the standard BART-family stack with learned positions and a ``proj_out`` 

5unembedding tied to ``decoder_embed``; there is no ``final_logits_bias``.""" 

6 

7from typing import Any, Dict 

8 

9from transformer_lens.model_bridge.architecture_adapter import ResidualLayout 

10from transformer_lens.model_bridge.generalized_components import PosEmbedBridge 

11from transformer_lens.model_bridge.generalized_components.base import ( 

12 GeneralizedComponent, 

13) 

14from transformer_lens.model_bridge.supported_architectures.bart import ( 

15 BartFamilyArchitectureAdapter, 

16) 

17 

18 

19class WhisperArchitectureAdapter(BartFamilyArchitectureAdapter): 

20 """Architecture adapter for WhisperForConditionalGeneration models. 

21 

22 Inputs: ``bridge(input_features, decoder_input=decoder_ids)``, where 

23 ``input_features`` is the log-mel spectrogram ``[batch, n_mels, 2 * n_frames]`` 

24 from ``bridge.processor`` and ``decoder_ids`` are decoder token ids (e.g. the 

25 start-of-transcript prompt), one row per clip or one row shared by all clips. 

26 ``bridge.to_tokens(text)`` builds that prompt from the tokenizer's prefix tokens 

27 (set the language and task with ``bridge.tokenizer.set_prefix_tokens``). Returns 

28 decoder logits under teacher forcing. 

29 

30 Transcription: ``bridge.generate(input_features, return_type="str")`` decodes from 

31 the same prefix (or ``decoder_input``), applying the checkpoint's 

32 ``suppress_tokens`` and ``begin_suppress_tokens`` as HF's generate does. 

33 

34 Encoder front end (channel-first, so its hooks declare position on dim 2): 

35 

36 - ``conv1`` / ``conv2``: the two ``nn.Conv1d`` layers. HF applies GELU outside 

37 the modules, so ``hook_out`` is pre-activation; ``conv2`` halves the frames. 

38 - ``pos_embed``: the fixed sinusoidal table, read whole as ``[n_frames, d_model]`` 

39 (no batch dim) and added to the transposed ``conv2`` output. 

40 

41 ``encoder_blocks.{i}.hook_out`` is ``[batch, n_frames, d_model]``: 1500 frames, 

42 one per 20 ms of the 30 s window, on every released checkpoint. 

43 

44 Attention biases: ``q_proj``, ``v_proj`` and ``out_proj`` carry a bias but 

45 ``k_proj`` does not (HF builds it with ``bias=False``), so ``attn.k.bias`` is 

46 ``None`` on every self- and cross-attention. 

47 

48 Stacks may be asymmetric (large-v3-turbo: 32 encoder / 4 decoder layers; 

49 distil-whisper: 2 decoder layers); ``n_layers`` follows the decoder, as for 

50 Blenderbot. 

51 """ 

52 

53 require_symmetric_layers = False 

54 n_layers_from = "decoder" 

55 # Checkpoints ship scale_embedding=False, and HF's forward never applies it. 

56 force_scale_embedding = False 

57 has_final_stack_norm = True 

58 unembed_name = "proj_out" 

59 audio_input_name = "input_features" 

60 # P1 (component + forward parity vs HF) only: P2-P4 drive the model with text prompts. 

61 applicable_phases = [1] 

62 

63 def __init__(self, cfg: Any) -> None: 

64 """Set the audio seq2seq flags on top of the BART-family setup.""" 

65 super().__init__(cfg) 

66 self.cfg.is_audio_model = True 

67 self.cfg.is_audio_seq2seq = True 

68 # The tokenizer's own <|startoftranscript|> prefix starts the decoder prompt; a 

69 # prepended BOS (<|endoftext|>) would land inside it. 

70 self.cfg.default_prepend_bos = False 

71 # Pre-LN with every residual read through a LayerNorm, so folding and centering 

72 # are exact; residual_layout() tells weight processing where things are. 

73 self.supports_fold_ln = True 

74 self.supports_center_writing_weights = True 

75 

76 def residual_layout(self) -> ResidualLayout: 

77 """Norm readers, residual writers and value/output pairs of both stacks. 

78 

79 Each block's norms feed its own sublayers. The encoder's final norm feeds the 

80 cross-attention keys and values of every decoder block, and the decoder's 

81 final norm feeds the unembedding. Encoder writes are the attention and MLP 

82 outputs plus the sinusoidal positions; the GELU'd conv output is not linear 

83 in any weight, so it stays uncentered (centering is exact either way, since 

84 every read is through a LayerNorm). 

85 """ 

86 encoder_layers = getattr(self.cfg, "encoder_layers", None) or self.cfg.n_layers 

87 decoder_layers = self.cfg.n_layers 

88 norm_readers: Dict[str, list[str]] = {} 

89 residual_writers: list[str] = [] 

90 value_output_pairs: list[tuple[str, str]] = [] 

91 for i in range(encoder_layers): 

92 block = f"encoder_blocks.{i}" 

93 attn = f"{block}.self_attn" 

94 norm_readers[f"{block}.self_attn_layer_norm"] = [f"{attn}.{p}" for p in "qkv"] 

95 norm_readers[f"{block}.final_layer_norm"] = [f"{block}.mlp.in"] 

96 residual_writers += [f"{attn}.o", f"{block}.mlp.out"] 

97 value_output_pairs.append((f"{attn}.v", f"{attn}.o")) 

98 cross_kv: list[str] = [] 

99 for i in range(decoder_layers): 

100 block = f"decoder_blocks.{i}" 

101 self_attn, cross_attn = f"{block}.self_attn", f"{block}.encoder_attn" 

102 norm_readers[f"{block}.self_attn_layer_norm"] = [f"{self_attn}.{p}" for p in "qkv"] 

103 norm_readers[f"{block}.encoder_attn_layer_norm"] = [f"{cross_attn}.q"] 

104 norm_readers[f"{block}.final_layer_norm"] = [f"{block}.mlp.in"] 

105 cross_kv += [f"{cross_attn}.k", f"{cross_attn}.v"] 

106 residual_writers += [f"{self_attn}.o", f"{cross_attn}.o", f"{block}.mlp.out"] 

107 value_output_pairs += [ 

108 (f"{self_attn}.v", f"{self_attn}.o"), 

109 (f"{cross_attn}.v", f"{cross_attn}.o"), 

110 ] 

111 norm_readers["encoder_ln_final"] = cross_kv 

112 norm_readers["decoder_ln_final"] = ["unembed"] 

113 return ResidualLayout( 

114 norm_readers=norm_readers, 

115 residual_writers=residual_writers, 

116 embedding_writers=["pos_embed", "decoder_embed", "decoder_pos_embed"], 

117 value_output_pairs=value_output_pairs, 

118 ) 

119 

120 def _encoder_embeddings(self) -> Dict[str, GeneralizedComponent]: 

121 conv1 = GeneralizedComponent(name="model.encoder.conv1") 

122 conv2 = GeneralizedComponent(name="model.encoder.conv2") 

123 pos_embed = PosEmbedBridge(name="model.encoder.embed_positions") 

124 conv1.mark_pos_dim(2) 

125 conv2.mark_pos_dim(2) 

126 pos_embed.mark_pos_dim(0) 

127 return {"conv1": conv1, "conv2": conv2, "pos_embed": pos_embed}