Coverage for transformer_lens/model_bridge/supported_architectures/whisper.py: 100%
54 statements
« prev ^ index » next coverage.py v7.10.1, created at 2026-11-10 15:50 +0000
« prev ^ index » next coverage.py v7.10.1, created at 2026-11-10 15:50 +0000
1"""Whisper (``WhisperForConditionalGeneration``) adapter: a BART-family pre-norm
2encoder-decoder whose encoder reads log-mel audio through two convolutions and a
3fixed sinusoidal position table instead of token embeddings. The text decoder is
4the standard BART-family stack with learned positions and a ``proj_out``
5unembedding tied to ``decoder_embed``; there is no ``final_logits_bias``."""
7from typing import Any, Dict
9from transformer_lens.model_bridge.architecture_adapter import ResidualLayout
10from transformer_lens.model_bridge.generalized_components import PosEmbedBridge
11from transformer_lens.model_bridge.generalized_components.base import (
12 GeneralizedComponent,
13)
14from transformer_lens.model_bridge.supported_architectures.bart import (
15 BartFamilyArchitectureAdapter,
16)
19class WhisperArchitectureAdapter(BartFamilyArchitectureAdapter):
20 """Architecture adapter for WhisperForConditionalGeneration models.
22 Inputs: ``bridge(input_features, decoder_input=decoder_ids)``, where
23 ``input_features`` is the log-mel spectrogram ``[batch, n_mels, 2 * n_frames]``
24 from ``bridge.processor`` and ``decoder_ids`` are decoder token ids (e.g. the
25 start-of-transcript prompt), one row per clip or one row shared by all clips.
26 ``bridge.to_tokens(text)`` builds that prompt from the tokenizer's prefix tokens
27 (set the language and task with ``bridge.tokenizer.set_prefix_tokens``). Returns
28 decoder logits under teacher forcing.
30 Transcription: ``bridge.generate(input_features, return_type="str")`` decodes from
31 the same prefix (or ``decoder_input``), applying the checkpoint's
32 ``suppress_tokens`` and ``begin_suppress_tokens`` as HF's generate does.
34 Encoder front end (channel-first, so its hooks declare position on dim 2):
36 - ``conv1`` / ``conv2``: the two ``nn.Conv1d`` layers. HF applies GELU outside
37 the modules, so ``hook_out`` is pre-activation; ``conv2`` halves the frames.
38 - ``pos_embed``: the fixed sinusoidal table, read whole as ``[n_frames, d_model]``
39 (no batch dim) and added to the transposed ``conv2`` output.
41 ``encoder_blocks.{i}.hook_out`` is ``[batch, n_frames, d_model]``: 1500 frames,
42 one per 20 ms of the 30 s window, on every released checkpoint.
44 Attention biases: ``q_proj``, ``v_proj`` and ``out_proj`` carry a bias but
45 ``k_proj`` does not (HF builds it with ``bias=False``), so ``attn.k.bias`` is
46 ``None`` on every self- and cross-attention.
48 Stacks may be asymmetric (large-v3-turbo: 32 encoder / 4 decoder layers;
49 distil-whisper: 2 decoder layers); ``n_layers`` follows the decoder, as for
50 Blenderbot.
51 """
53 require_symmetric_layers = False
54 n_layers_from = "decoder"
55 # Checkpoints ship scale_embedding=False, and HF's forward never applies it.
56 force_scale_embedding = False
57 has_final_stack_norm = True
58 unembed_name = "proj_out"
59 audio_input_name = "input_features"
60 # P1 (component + forward parity vs HF) only: P2-P4 drive the model with text prompts.
61 applicable_phases = [1]
63 def __init__(self, cfg: Any) -> None:
64 """Set the audio seq2seq flags on top of the BART-family setup."""
65 super().__init__(cfg)
66 self.cfg.is_audio_model = True
67 self.cfg.is_audio_seq2seq = True
68 # The tokenizer's own <|startoftranscript|> prefix starts the decoder prompt; a
69 # prepended BOS (<|endoftext|>) would land inside it.
70 self.cfg.default_prepend_bos = False
71 # Pre-LN with every residual read through a LayerNorm, so folding and centering
72 # are exact; residual_layout() tells weight processing where things are.
73 self.supports_fold_ln = True
74 self.supports_center_writing_weights = True
76 def residual_layout(self) -> ResidualLayout:
77 """Norm readers, residual writers and value/output pairs of both stacks.
79 Each block's norms feed its own sublayers. The encoder's final norm feeds the
80 cross-attention keys and values of every decoder block, and the decoder's
81 final norm feeds the unembedding. Encoder writes are the attention and MLP
82 outputs plus the sinusoidal positions; the GELU'd conv output is not linear
83 in any weight, so it stays uncentered (centering is exact either way, since
84 every read is through a LayerNorm).
85 """
86 encoder_layers = getattr(self.cfg, "encoder_layers", None) or self.cfg.n_layers
87 decoder_layers = self.cfg.n_layers
88 norm_readers: Dict[str, list[str]] = {}
89 residual_writers: list[str] = []
90 value_output_pairs: list[tuple[str, str]] = []
91 for i in range(encoder_layers):
92 block = f"encoder_blocks.{i}"
93 attn = f"{block}.self_attn"
94 norm_readers[f"{block}.self_attn_layer_norm"] = [f"{attn}.{p}" for p in "qkv"]
95 norm_readers[f"{block}.final_layer_norm"] = [f"{block}.mlp.in"]
96 residual_writers += [f"{attn}.o", f"{block}.mlp.out"]
97 value_output_pairs.append((f"{attn}.v", f"{attn}.o"))
98 cross_kv: list[str] = []
99 for i in range(decoder_layers):
100 block = f"decoder_blocks.{i}"
101 self_attn, cross_attn = f"{block}.self_attn", f"{block}.encoder_attn"
102 norm_readers[f"{block}.self_attn_layer_norm"] = [f"{self_attn}.{p}" for p in "qkv"]
103 norm_readers[f"{block}.encoder_attn_layer_norm"] = [f"{cross_attn}.q"]
104 norm_readers[f"{block}.final_layer_norm"] = [f"{block}.mlp.in"]
105 cross_kv += [f"{cross_attn}.k", f"{cross_attn}.v"]
106 residual_writers += [f"{self_attn}.o", f"{cross_attn}.o", f"{block}.mlp.out"]
107 value_output_pairs += [
108 (f"{self_attn}.v", f"{self_attn}.o"),
109 (f"{cross_attn}.v", f"{cross_attn}.o"),
110 ]
111 norm_readers["encoder_ln_final"] = cross_kv
112 norm_readers["decoder_ln_final"] = ["unembed"]
113 return ResidualLayout(
114 norm_readers=norm_readers,
115 residual_writers=residual_writers,
116 embedding_writers=["pos_embed", "decoder_embed", "decoder_pos_embed"],
117 value_output_pairs=value_output_pairs,
118 )
120 def _encoder_embeddings(self) -> Dict[str, GeneralizedComponent]:
121 conv1 = GeneralizedComponent(name="model.encoder.conv1")
122 conv2 = GeneralizedComponent(name="model.encoder.conv2")
123 pos_embed = PosEmbedBridge(name="model.encoder.embed_positions")
124 conv1.mark_pos_dim(2)
125 conv2.mark_pos_dim(2)
126 pos_embed.mark_pos_dim(0)
127 return {"conv1": conv1, "conv2": conv2, "pos_embed": pos_embed}