diff --git a/docs/source/content/model_structure.md b/docs/source/content/model_structure.md index 338a1603a..0a42db856 100644 --- a/docs/source/content/model_structure.md +++ b/docs/source/content/model_structure.md @@ -42,6 +42,8 @@ Each `blocks.{i}` is a `BlockBridge` with subcomponents: - `ln2`: normalization before MLP - `mlp`: MLP module +Normalizations applied to a sublayer's output, before the residual add, use `ln1_post` (after attention) and `ln2_post` (after the MLP). Keys follow the norm's position, not its Hugging Face name: Gemma 2/3 have all four, while OLMo 2-style post-norm blocks have only `ln1_post` and `ln2_post`. + ## Canonical Hook Names Use these canonical names when adding hooks or reading from the cache. Every name below exists on every `TransformerBridge` from boot, and the legacy `HookedTransformer` names in the [alias table](#legacy-alias-table) resolve to the same `HookPoint` objects whether or not compatibility mode is enabled. @@ -108,7 +110,7 @@ Each row is checked against a tiny GPT-2 bridge and a tiny Gemma 3 bridge by `te | `hook_pos_embed` | `pos_embed.hook_out` | `rotary_emb.hook_out` on rotary models, where it never fires | | `hook_unembed` | `unembed.hook_out` | | | `blocks.{i}.hook_resid_pre` | `blocks.{i}.hook_in` | | -| `blocks.{i}.hook_resid_mid` | `blocks.{i}.ln2.hook_in` | the residual sum, not `blocks.{i}.attn.hook_out` | +| `blocks.{i}.hook_resid_mid` | `blocks.{i}.ln2.hook_in` | the residual sum, not `blocks.{i}.attn.hook_out`; `blocks.{i}.mlp.hook_in` when there is no `ln2` | | `blocks.{i}.hook_resid_post` | `blocks.{i}.hook_out` | | | `blocks.{i}.hook_attn_out` | `blocks.{i}.attn.hook_out` | `blocks.{i}.ln1_post.hook_out` when a post-attention norm exists | | `blocks.{i}.hook_mlp_out` | `blocks.{i}.mlp.hook_out` | `blocks.{i}.ln2_post.hook_out` when a post-MLP norm exists | diff --git a/tests/integration/model_bridge/test_exaone4_adapter.py b/tests/integration/model_bridge/test_exaone4_adapter.py index a69c6b7d9..0adaa136e 100644 --- a/tests/integration/model_bridge/test_exaone4_adapter.py +++ b/tests/integration/model_bridge/test_exaone4_adapter.py @@ -61,9 +61,9 @@ def grab(tensor, hook): hooks = [ "blocks.0.attn.hook_out", - "blocks.0.ln1.hook_out", + "blocks.0.ln1_post.hook_out", "blocks.0.mlp.hook_out", - "blocks.0.ln2.hook_out", + "blocks.0.ln2_post.hook_out", ] with torch.no_grad(): bridge.run_with_hooks(sample_tokens, fwd_hooks=[(name, grab) for name in hooks]) diff --git a/tests/integration/model_bridge/test_flex_olmo_adapter.py b/tests/integration/model_bridge/test_flex_olmo_adapter.py index eb84535d1..cc21c6030 100644 --- a/tests/integration/model_bridge/test_flex_olmo_adapter.py +++ b/tests/integration/model_bridge/test_flex_olmo_adapter.py @@ -57,7 +57,7 @@ def test_bridge_structure(self) -> None: assert len(bridge.blocks) == 3 assert isinstance(bridge.blocks[0].mlp, MoEBridge) # OLMo-2 post-norm layout carried through the subclass - assert bridge.blocks[0].ln1.original_component is not None + assert bridge.blocks[0].ln1_post.original_component is not None def test_forward_matches_hf(self) -> None: bridge, hf_reference = tiny_flex_olmo_bridge() diff --git a/tests/integration/model_bridge/test_gemma3n_bridge.py b/tests/integration/model_bridge/test_gemma3n_bridge.py index 3afe1f802..ae3318386 100644 --- a/tests/integration/model_bridge/test_gemma3n_bridge.py +++ b/tests/integration/model_bridge/test_gemma3n_bridge.py @@ -64,3 +64,38 @@ def test_run_with_cache_text_only(bridge): logits, cache = bridge.run_with_cache(IDS) assert torch.isfinite(logits).all() assert len(cache) > 0 + + +def test_contribution_hooks_are_post_norm_outputs(bridge): + """hook_attn_out / hook_mlp_out carry the post-sublayer norm outputs, the tensors the + layer actually adds (Gemma 2/3 contract), checked against an independent HF run.""" + from transformers import AutoModelForCausalLM + + hf = AutoModelForCausalLM.from_pretrained( + MODEL_NAME, torch_dtype=torch.float32, attn_implementation="eager" + ).eval() + modules = dict(hf.named_modules()) + reference = {} + handles = [] + for layer in range(bridge.cfg.n_layers): + for hf_norm, alias in ( + ("post_attention_layernorm", "hook_attn_out"), + ("post_feedforward_layernorm", "hook_mlp_out"), + ): + path = next(n for n in modules if n.endswith(f"layers.{layer}.{hf_norm}")) + key = f"blocks.{layer}.{alias}" + handles.append( + modules[path].register_forward_hook( + lambda _m, _i, out, key=key: reference.__setitem__(key, out.detach()) + ) + ) + try: + with torch.no_grad(): + hf(IDS) + _, cache = bridge.run_with_cache(IDS) + finally: + for handle in handles: + handle.remove() + + for key, expected in reference.items(): + torch.testing.assert_close(cache[key], expected, atol=1e-5, rtol=1e-5) diff --git a/tests/integration/model_bridge/test_olmo2_hook_semantics.py b/tests/integration/model_bridge/test_olmo2_hook_semantics.py index eb768f3f0..65a2340a2 100644 --- a/tests/integration/model_bridge/test_olmo2_hook_semantics.py +++ b/tests/integration/model_bridge/test_olmo2_hook_semantics.py @@ -191,3 +191,14 @@ def test_compatibility_mode_preserves_residual_semantics() -> None: cache[f"blocks.{layer}.hook_resid_post"], cache[f"blocks.{layer}.hook_resid_mid"] + cache[f"blocks.{layer}.hook_mlp_out"], ) + + +def test_mid_layer_cached_ln_explains_missing_pre_norm( + olmo2_bridge: TransformerBridge, sample_tokens: torch.Tensor +) -> None: + """Post-norm-only blocks have no ln1 scale to cache; the error must say so rather + than point the user at a hook that cannot exist.""" + with torch.no_grad(): + _, cache = olmo2_bridge.run_with_cache(sample_tokens) + with pytest.raises(KeyError, match=r"no ln1 \(pre-attention norm\)"): + cache.decompose_resid(layer=1, apply_ln=True) diff --git a/tests/integration/model_bridge/test_olmo_hybrid_adapter.py b/tests/integration/model_bridge/test_olmo_hybrid_adapter.py index d7652cdba..200b0aaa5 100644 --- a/tests/integration/model_bridge/test_olmo_hybrid_adapter.py +++ b/tests/integration/model_bridge/test_olmo_hybrid_adapter.py @@ -123,6 +123,21 @@ def test_residual_contributions_decompose_stream(self, bridge, sample_tokens): + cache[f"blocks.{layer}.hook_mlp_out"], ) + def test_resid_mid_splits_the_stream_on_both_layer_types(self, bridge, sample_tokens): + with torch.no_grad(): + _, cache = bridge.run_with_cache(sample_tokens) + + for layer in range(bridge.cfg.n_layers): + resid_mid = cache[f"blocks.{layer}.hook_resid_mid"] + torch.testing.assert_close( + resid_mid, + cache[f"blocks.{layer}.hook_resid_pre"] + cache[f"blocks.{layer}.hook_attn_out"], + ) + torch.testing.assert_close( + cache[f"blocks.{layer}.hook_resid_post"], + resid_mid + cache[f"blocks.{layer}.hook_mlp_out"], + ) + def test_full_attention_contributions_are_post_norm(self, bridge, sample_tokens): """Layers 1/3 are full attention: aliases must differ from raw module outputs.""" with torch.no_grad(): diff --git a/tests/unit/model_bridge/generalized_components/test_block_post_norm_layout.py b/tests/unit/model_bridge/generalized_components/test_block_post_norm_layout.py new file mode 100644 index 000000000..3bab8b9b2 --- /dev/null +++ b/tests/unit/model_bridge/generalized_components/test_block_post_norm_layout.py @@ -0,0 +1,35 @@ +"""BlockBridge derives post-norm wiring from the canonical norm keys.""" + +import pytest + +from transformer_lens.model_bridge.generalized_components import BlockBridge +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + + +def _block(*norm_keys: str) -> BlockBridge: + submodules = {key: GeneralizedComponent(name=key) for key in norm_keys} + submodules["attn"] = GeneralizedComponent(name="self_attn") + submodules["mlp"] = GeneralizedComponent(name="mlp") + return BlockBridge(name="model.layers", submodules=submodules) + + +def test_post_norm_only_block_reads_resid_mid_at_the_mlp() -> None: + block = _block("ln1_post", "ln2_post") + assert block.hook_aliases["hook_resid_mid"] == "mlp.hook_in" + assert block.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" + assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" + assert block.mlp_reads_resid_directly is True + + +def test_sandwich_block_keeps_the_pre_mlp_norm_as_resid_mid() -> None: + block = _block("ln1", "ln1_post", "ln2", "ln2_post") + assert block.hook_aliases["hook_resid_mid"] == "ln2.hook_in" + assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" + assert block.mlp_reads_resid_directly is False + + +def test_sequential_block_without_an_mlp_side_norm_is_rejected() -> None: + with pytest.raises(ValueError, match="neither 'ln2' nor 'ln2_post'"): + _block("ln1") diff --git a/tests/unit/model_bridge/supported_architectures/test_exaone4_adapter.py b/tests/unit/model_bridge/supported_architectures/test_exaone4_adapter.py index f7f67d3be..51cd65865 100644 --- a/tests/unit/model_bridge/supported_architectures/test_exaone4_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_exaone4_adapter.py @@ -63,8 +63,9 @@ def test_no_bos_prepending(self, adapter): class TestExaone4ComponentMapping: def test_post_norm_positions(self, adapter): submodules = adapter.component_mapping["blocks"].submodules - assert submodules["ln1"].name == "post_attention_layernorm" - assert submodules["ln2"].name == "post_feedforward_layernorm" + assert submodules["ln1_post"].name == "post_attention_layernorm" + assert submodules["ln2_post"].name == "post_feedforward_layernorm" + assert "ln1" not in submodules and "ln2" not in submodules def test_attention_has_per_head_qk_norms(self, adapter): attn = adapter.component_mapping["blocks"].submodules["attn"] @@ -88,8 +89,8 @@ def test_model_type_detection(self): class TestExaone4PostNormResidMid: - """Post-norm layout: ln2.hook_in is the MLP-branch output, so the adapter - must override hook_resid_mid to mlp.hook_in (olmo2 precedent).""" + """Post-norm-only layout: no pre-MLP norm, so hook_resid_mid is mlp.hook_in + (olmo2 precedent).""" def test_hook_resid_mid_points_at_mlp_hook_in(self, adapter) -> None: block = adapter.component_mapping["blocks"] @@ -98,8 +99,8 @@ def test_hook_resid_mid_points_at_mlp_hook_in(self, adapter) -> None: def test_contribution_aliases_point_at_post_norm_outputs(self, adapter) -> None: """Inherited from olmo2: contributions are the post-norm outputs (#1648).""" block = adapter.component_mapping["blocks"] - assert block.hook_aliases["hook_attn_out"] == "ln1.hook_out" - assert block.hook_aliases["hook_mlp_out"] == "ln2.hook_out" + assert block.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" + assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" class TestExaone4NoPEGating: diff --git a/tests/unit/model_bridge/supported_architectures/test_flex_olmo_adapter.py b/tests/unit/model_bridge/supported_architectures/test_flex_olmo_adapter.py index abfe4737f..c6e0158d1 100644 --- a/tests/unit/model_bridge/supported_architectures/test_flex_olmo_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_flex_olmo_adapter.py @@ -48,9 +48,9 @@ def test_moe_replaces_dense_mlp(self, adapter): def test_inherits_olmo2_post_norm_layout(self, adapter): assert isinstance(adapter, Olmo2ArchitectureAdapter) blocks = adapter.component_mapping["blocks"] - assert isinstance(blocks.submodules["ln1"], RMSNormalizationBridge) - assert blocks.submodules["ln1"].name == "post_attention_layernorm" - assert blocks.submodules["ln2"].name == "post_feedforward_layernorm" + assert isinstance(blocks.submodules["ln1_post"], RMSNormalizationBridge) + assert blocks.submodules["ln1_post"].name == "post_attention_layernorm" + assert blocks.submodules["ln2_post"].name == "post_feedforward_layernorm" attn = blocks.submodules["attn"] assert attn.submodules["q_norm"].name == "q_norm" @@ -58,8 +58,8 @@ def test_inherits_post_norm_contribution_aliases(self, adapter): """hook_attn_out / hook_mlp_out must stay on the post-norm outputs (#1648); a future de-inheritance from olmo2 must not silently regress this.""" block = adapter.component_mapping["blocks"] - assert block.hook_aliases["hook_attn_out"] == "ln1.hook_out" - assert block.hook_aliases["hook_mlp_out"] == "ln2.hook_out" + assert block.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" + assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" assert block.hook_aliases["hook_resid_mid"] == "mlp.hook_in" diff --git a/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py b/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py index f5fb101b8..1a68b9028 100644 --- a/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py @@ -100,14 +100,21 @@ def test_altup_block_decomposition(): "mlp", ): assert name in blocks.submodules - for norm in ( - "input_layernorm", - "post_attention_layernorm", - "pre_feedforward_layernorm", - "post_feedforward_layernorm", - "post_per_layer_input_norm", + for key, hf_name in ( + ("ln1", "input_layernorm"), + ("ln1_post", "post_attention_layernorm"), + ("ln2", "pre_feedforward_layernorm"), + ("ln2_post", "post_feedforward_layernorm"), + ("post_per_layer_input_norm", "post_per_layer_input_norm"), ): - assert norm in blocks.submodules + assert blocks.submodules[key].name == hf_name + + +def test_contribution_aliases_point_at_post_norm_outputs(): + """The residual contribution is the post-sublayer norm output (Gemma 2/3 contract).""" + blocks = _adapter().component_mapping["blocks"] + assert blocks.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" + assert blocks.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" def test_kv_shared_submodules_are_optional(): diff --git a/tests/unit/model_bridge/supported_architectures/test_olmo2_adapter.py b/tests/unit/model_bridge/supported_architectures/test_olmo2_adapter.py index ad29cf79d..a246f6d4f 100644 --- a/tests/unit/model_bridge/supported_architectures/test_olmo2_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_olmo2_adapter.py @@ -6,11 +6,12 @@ - Weight conversions: the adapter uses the base QKVO helper, so only the exact key set is asserted at the adapter layer. - Component-mapping structure, bridge types, and HF module paths. -- Post-norm block wiring: ln1 maps to post_attention_layernorm and ln2 maps to - post_feedforward_layernorm. This is the central arch-specific decision. +- Post-norm block wiring: ln1_post maps to post_attention_layernorm and ln2_post + to post_feedforward_layernorm, with no ln1/ln2. This is the central + arch-specific decision. - Q/K-norm submodules under attention with the correct HF names. -- hook_alias_overrides: hook_resid_mid points at mlp.hook_in (the true post-attn - pre-mlp residual under post-norm), overriding BlockBridge's default ln2.hook_in. +- Derived hook aliases: hook_resid_mid points at mlp.hook_in (the true post-attn + pre-mlp residual under post-norm), not BlockBridge's default ln2.hook_in. - GQA forward hook shapes for Q and K/V with Q/K-norm wired into the fake attention. - setup_component_testing: rotary wiring on template and bridge-model attentions, plus forcing eager attention on the HF model and per-layer self_attn configs. @@ -205,8 +206,8 @@ def test_block_mlp_is_gated_mlp_bridge(self, adapter: Olmo2ArchitectureAdapter) def test_block_norms_are_rms(self, adapter: Olmo2ArchitectureAdapter) -> None: block = _mapping(adapter)["blocks"] - assert isinstance(block.submodules["ln1"], RMSNormalizationBridge) - assert isinstance(block.submodules["ln2"], RMSNormalizationBridge) + assert isinstance(block.submodules["ln1_post"], RMSNormalizationBridge) + assert isinstance(block.submodules["ln2_post"], RMSNormalizationBridge) def test_embed_and_unembed_are_correct_bridge_types( self, adapter: Olmo2ArchitectureAdapter @@ -227,26 +228,28 @@ def test_mlp_gate_in_out_are_linear_bridges(self, adapter: Olmo2ArchitectureAdap class TestOlmo2PostNormBlockWiring: - """OLMo 2 is post-norm: RMSNorm applies AFTER attention (ln1) and AFTER MLP (ln2). - The HF module names diverge from the pre-norm Llama family default, where ln1 - would be `input_layernorm` and ln2 would be `post_attention_layernorm`. This is - the central arch-specific decision and the single test most likely to catch a - porting regression.""" + """OLMo 2 is post-norm only: RMSNorm applies AFTER attention and AFTER MLP, so + the norms sit in the ln1_post/ln2_post slots and ln1/ln2 (pre-sublayer norms) + are absent. Tools that read ln1/ln2 as the norm feeding a sublayer must not + find these. The HF name `post_attention_layernorm` is the pre-MLP norm in the + Llama family, which is the porting trap this guards.""" - def test_ln1_maps_to_post_attention_layernorm(self, adapter: Olmo2ArchitectureAdapter) -> None: + def test_ln1_post_maps_to_post_attention_layernorm( + self, adapter: Olmo2ArchitectureAdapter + ) -> None: block = _mapping(adapter)["blocks"] - assert block.submodules["ln1"].name == "post_attention_layernorm" + assert block.submodules["ln1_post"].name == "post_attention_layernorm" - def test_ln2_maps_to_post_feedforward_layernorm( + def test_ln2_post_maps_to_post_feedforward_layernorm( self, adapter: Olmo2ArchitectureAdapter ) -> None: block = _mapping(adapter)["blocks"] - assert block.submodules["ln2"].name == "post_feedforward_layernorm" + assert block.submodules["ln2_post"].name == "post_feedforward_layernorm" def test_block_submodule_set(self, adapter: Olmo2ArchitectureAdapter) -> None: - """Exact block submodule set: sequential transformer with both ln1 and ln2.""" + """Exact block submodule set: post-norms only, no pre-sublayer ln1/ln2.""" block = _mapping(adapter)["blocks"] - assert set(block.submodules.keys()) == {"ln1", "ln2", "attn", "mlp"} + assert set(block.submodules.keys()) == {"ln1_post", "ln2_post", "attn", "mlp"} class TestOlmo2QKNormStructure: @@ -276,11 +279,9 @@ def test_k_norm_uses_k_norm_hf_name(self, adapter: Olmo2ArchitectureAdapter) -> class TestOlmo2HookAliasOverrides: - """Under post-norm, `ln2.hook_in` no longer captures the residual between - attention and MLP (ln2 applies AFTER MLP, so ln2.hook_in is the MLP output). - The true post-attn pre-mlp residual is at `mlp.hook_in`. The adapter overrides - `hook_resid_mid` accordingly. BlockBridge's default would point at the wrong - tensor without this override.""" + """With no pre-MLP norm, the post-attn pre-mlp residual is at `mlp.hook_in`. + BlockBridge derives that from the post-norm-only layout; its default + `ln2.hook_in` would not resolve.""" def test_hook_resid_mid_points_at_mlp_hook_in(self, adapter: Olmo2ArchitectureAdapter) -> None: block = _mapping(adapter)["blocks"] @@ -292,8 +293,8 @@ def test_contribution_aliases_point_at_post_norm_outputs( """hook_attn_out / hook_mlp_out must expose the tensor added to the residual stream, which under post-norm is the norm output (#1648).""" block = _mapping(adapter)["blocks"] - assert block.hook_aliases["hook_attn_out"] == "ln1.hook_out" - assert block.hook_aliases["hook_mlp_out"] == "ln2.hook_out" + assert block.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" + assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" def test_hook_mlp_in_captures_on_mlp(self, adapter: Olmo2ArchitectureAdapter) -> None: """No pre-MLP norm exists, so the hook_mlp_in capture sits on the MLP.""" diff --git a/tests/unit/model_bridge/supported_architectures/test_olmo3_adapter.py b/tests/unit/model_bridge/supported_architectures/test_olmo3_adapter.py index 39730d75a..4c19490a8 100644 --- a/tests/unit/model_bridge/supported_architectures/test_olmo3_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_olmo3_adapter.py @@ -50,6 +50,6 @@ def test_inherits_post_norm_contribution_aliases() -> None: """hook_attn_out / hook_mlp_out must stay on the post-norm outputs (#1648); a future de-inheritance from olmo2 must not silently regress this.""" block = Olmo3ArchitectureAdapter(_cfg("Olmo3ForCausalLM")).component_mapping["blocks"] - assert block.hook_aliases["hook_attn_out"] == "ln1.hook_out" - assert block.hook_aliases["hook_mlp_out"] == "ln2.hook_out" + assert block.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" + assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" assert block.hook_aliases["hook_resid_mid"] == "mlp.hook_in" diff --git a/tests/unit/model_bridge/supported_architectures/test_olmo_hybrid_adapter.py b/tests/unit/model_bridge/supported_architectures/test_olmo_hybrid_adapter.py index a7a29d5c9..d22b7b7ae 100644 --- a/tests/unit/model_bridge/supported_architectures/test_olmo_hybrid_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_olmo_hybrid_adapter.py @@ -67,18 +67,9 @@ def test_model_type_detection(self): assert determine_architecture_from_hf_config(cfg) == "OlmoHybridForCausalLM" -class TestOlmoHybridResidMidDropped: - """No single hook_resid_mid target is correct for both layer types, so the - block drops the alias type-visibly (ParallelBlockBridge precedent).""" - - def test_hook_resid_mid_alias_absent(self, adapter) -> None: - block = adapter.component_mapping["blocks"] - assert "hook_resid_mid" not in block.hook_aliases - - class TestOlmoHybridPerLayerContributionAliases: - """set_original_component selects hook_attn_out / hook_mlp_out per layer - type: full-attention layers are post-norm (contribution = norm output), + """set_original_component selects norm keys and aliases per layer type: + full-attention layers are post-norm (contribution = norm output), linear-attention layers are pre-norm (contribution = raw output). #1648.""" def _bind(self, adapter, hf_layer): @@ -94,8 +85,20 @@ def test_full_attention_layer_uses_post_norm_outputs(self, adapter) -> None: hf_layer = nn.Module() hf_layer.post_feedforward_layernorm = nn.LayerNorm(8) block = self._bind(adapter, hf_layer) - assert block.hook_aliases["hook_attn_out"] == "ln2.hook_out" + assert block.hook_aliases["hook_attn_out"] == "ln1_post.hook_out" assert block.hook_aliases["hook_mlp_out"] == "ln2_post.hook_out" + assert block.hook_aliases["hook_resid_mid"] == "mlp.hook_in" + + def test_full_attention_layer_rekeys_post_attention_norm(self, adapter) -> None: + """post_attention_layernorm follows attention here, so it must not sit in + the ln2 slot that tools read as the norm feeding the MLP.""" + import torch.nn as nn + + hf_layer = nn.Module() + hf_layer.post_feedforward_layernorm = nn.LayerNorm(8) + block = self._bind(adapter, hf_layer) + assert "ln2" not in block.submodules + assert block.submodules["ln1_post"].name == "post_attention_layernorm" def test_linear_attention_layer_uses_raw_outputs(self, adapter) -> None: import torch.nn as nn @@ -103,3 +106,5 @@ def test_linear_attention_layer_uses_raw_outputs(self, adapter) -> None: block = self._bind(adapter, nn.Module()) assert block.hook_aliases["hook_attn_out"] == "linear_attn.hook_out" assert block.hook_aliases["hook_mlp_out"] == "mlp.hook_out" + assert block.hook_aliases["hook_resid_mid"] == "ln2.hook_in" + assert block.submodules["ln2"].name == "post_attention_layernorm" diff --git a/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py b/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py index 4e9d80008..459d32bdd 100644 --- a/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py +++ b/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py @@ -9,12 +9,10 @@ behavior correct for these three documented edge cases, without needing to build a working forward-capable module tree per adapter: -- Gemma 3n: every per-block norm (including the per-head q/k/v norms) is a plain - ``GeneralizedComponent`` placeholder, and none of them is keyed "ln1"/"ln2". -- Gemma 4: ln1/ln2 exist (sandwich norms) but are plain ``GeneralizedComponent`` - placeholders, not ``NormalizationBridge`` -- rule-incapable even though the mount - name matches. The per-head q/k/v norms are, like Gemma 3n, keyed under "self_attn", - never "ln1"/"ln2". +- Gemma 3n and Gemma 4: ln1/ln2 exist (sandwich norms) but are plain + ``GeneralizedComponent`` placeholders, not ``NormalizationBridge`` -- + rule-incapable even though the mount name matches. Gemma 3n's per-head q/k/v + norms are keyed under "self_attn", never "ln1"/"ln2". - StableLM: ln1/ln2 ARE real ``NormalizationBridge`` instances (a genuine rule target), but the per-head norms are keyed "q_norm"/"k_norm" under "attn" even though the wrapped HF module is named "q_layernorm"/"k_layernorm" -- the mount @@ -53,10 +51,13 @@ def _adapter(self) -> Gemma3nArchitectureAdapter: cfg = make_bridge_cfg("Gemma3nForConditionalGeneration", n_key_value_heads=2, d_head=8) return Gemma3nArchitectureAdapter(cfg) - def test_no_ln1_or_ln2_key_at_block_level(self): + def test_sandwich_norms_are_plain_placeholders_not_normalization_bridge(self): block_submodules = _block_submodules(self._adapter()) - assert "ln1" not in block_submodules - assert "ln2" not in block_submodules + for key in ("ln1", "ln1_post", "ln2", "ln2_post"): + component = block_submodules[key] + assert type(component) is GeneralizedComponent + assert not isinstance(component, NormalizationBridge) + assert not isinstance(component, _RelevanceRuleCapable) def test_qkv_norm_placeholders_are_plain_and_rule_incapable(self): block_submodules = _block_submodules(self._adapter()) diff --git a/tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py b/tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py index c8cbd867a..f36d9932c 100644 --- a/tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py +++ b/tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py @@ -144,14 +144,10 @@ def test_attn_in_cross_run_patch_matches_legacy( @pytest.mark.slow @pytest.mark.skipif( os.getenv("RUN_OLMO2_GAP_TEST", "") != "1", - reason="Set RUN_OLMO2_GAP_TEST=1 to exercise the OLMo-2 post-norm gap (1B-param download).", + reason="Set RUN_OLMO2_GAP_TEST=1 to exercise OLMo-2 post-norm split-QKV (1B-param download).", ) -@pytest.mark.xfail( - strict=True, - reason="OLMo 2 post-norm: ln1 maps to post_attention_layernorm, so pre-ln1 capture " - "reads post-attention residual. Flip to passing when the carve-out is fixed.", -) -def test_olmo2_pre_ln_capture_known_gap() -> None: +def test_olmo2_post_norm_split_qkv_cross_run_patching() -> None: + """OLMo 2 has no pre-attention norm, so the fork must not re-normalize.""" bridge = TransformerBridge.boot_transformers("allenai/OLMo-2-0425-1B", device="cpu") bridge.enable_compatibility_mode(no_processing=True) bridge.set_use_split_qkv_input(True) diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index d247a7838..ad097942a 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -1191,6 +1191,20 @@ def _get_cached_ln_scale( try: scale = self[key] except KeyError as e: + norm = f"ln{2 if mlp_input else 1}" + blocks = getattr(self.model, "blocks", None) + block = blocks[layer] if blocks is not None and layer is not None else None + if ( + block is not None + and getattr(block, norm, None) is None + and getattr(block, f"{norm}_post", None) is not None + ): + sublayer = "MLP" if mlp_input else "attention" + raise KeyError( + f"Layer {layer} has no {norm} (pre-{sublayer} norm): this architecture " + f"normalizes sublayer outputs ({norm}_post) instead, so a mid-layer cached-scale " + f"LN is undefined. Use apply_ln=False, or apply LN at the final layer." + ) from e raise KeyError( f"Cached LN scale not found at '{key}'. apply_ln operations require the model " f"to have cached this hook (some non-decoder-only architectures expose LN scale " diff --git a/transformer_lens/model_bridge/generalized_components/altup_block.py b/transformer_lens/model_bridge/generalized_components/altup_block.py index c9fa50355..28c1ccd68 100644 --- a/transformer_lens/model_bridge/generalized_components/altup_block.py +++ b/transformer_lens/model_bridge/generalized_components/altup_block.py @@ -36,11 +36,20 @@ def __init__( submodules: Optional[Dict[str, GeneralizedComponent]] = None, hook_alias_overrides: Optional[Dict[str, str]] = None, ): + # Same post-norm redirect as BlockBridge: the residual contribution is the + # post-sublayer norm's output, not the raw sublayer output. + auto_overrides = {} + if submodules is not None: + if "ln1_post" in submodules: + auto_overrides["hook_attn_out"] = "ln1_post.hook_out" + if "ln2_post" in submodules: + auto_overrides["hook_mlp_out"] = "ln2_post.hook_out" + merged_overrides = {**auto_overrides, **(hook_alias_overrides or {})} super().__init__( name, config, submodules=submodules if submodules is not None else {}, - hook_alias_overrides=hook_alias_overrides, + hook_alias_overrides=merged_overrides if merged_overrides else None, ) self.altup_active_idx = int(getattr(config, "altup_active_idx", 0) or 0) # Active AltUp stream as a conventional residual (patchable). diff --git a/transformer_lens/model_bridge/generalized_components/block.py b/transformer_lens/model_bridge/generalized_components/block.py index a3a0728bf..f3bba6f66 100644 --- a/transformer_lens/model_bridge/generalized_components/block.py +++ b/transformer_lens/model_bridge/generalized_components/block.py @@ -70,11 +70,14 @@ def __init__( hook_alias_overrides: Optional dictionary to override default hook aliases. For example, {"hook_attn_out": "ln1_post.hook_out"} will make hook_attn_out point to ln1_post.hook_out instead of the default attn.hook_out. - mlp_reads_resid_directly: True for post-norm blocks where the MLP consumes - the mid-residual with no pre-MLP norm (OLMo 2 layout). Moves the - hook_mlp_in capture from ln2 (whose input there is the raw MLP output) - to the MLP itself. + mlp_reads_resid_directly: True when the MLP consumes the mid-residual with + no pre-MLP norm. Moves the hook_mlp_in capture from ln2 to the MLP + itself. Implied for post-norm-only blocks (ln2_post without ln2). """ + has_ln2 = submodules is not None and "ln2" in submodules + # Post-norm-only blocks (OLMo 2 layout) declare ln2_post and no ln2: the + # MLP reads the mid-residual directly. + post_norm_only_mlp = submodules is not None and "ln2_post" in submodules and not has_ln2 # ln1_post/ln2_post redirect attn_out/mlp_out to match HookedTransformer's # placement (hook fires after the post-norm, not before). auto_overrides = {} @@ -83,22 +86,29 @@ def __init__( auto_overrides["hook_attn_out"] = "ln1_post.hook_out" if "ln2_post" in submodules: auto_overrides["hook_mlp_out"] = "ln2_post.hook_out" + if post_norm_only_mlp: + auto_overrides["hook_resid_mid"] = "mlp.hook_in" merged_overrides = {**auto_overrides, **(hook_alias_overrides or {})} - # Guard against the bug where a sequential block (attn + mlp) with no ln2 - # silently points hook_resid_mid at the wrong tensor. Use + # Guard against the bug where a sequential block (attn + mlp) with no MLP-side + # norm silently points hook_resid_mid at the wrong tensor. Use # ParallelBlockBridge for parallel-residual architectures. # Skip the check on generic-container / attn-only uses (no mlp). has_attn_like = submodules is not None and any( k in submodules for k in _VARIANT_SUBMODULE_SET ) has_mlp = submodules is not None and "mlp" in submodules - has_ln2 = submodules is not None and "ln2" in submodules - if has_attn_like and has_mlp and not has_ln2 and type(self) is BlockBridge: + if ( + has_attn_like + and has_mlp + and not has_ln2 + and not post_norm_only_mlp + and type(self) is BlockBridge + ): raise ValueError( - f"BlockBridge at '{name}': 'ln2' submodule not declared. " - f"Either declare ln2, or use ParallelBlockBridge for a " - f"parallel-residual architecture." + f"BlockBridge at '{name}': neither 'ln2' nor 'ln2_post' declared. " + f"Declare ln2 (pre-MLP norm) or ln2_post (post-MLP norm), or use " + f"ParallelBlockBridge for a parallel-residual architecture." ) super().__init__( @@ -113,7 +123,7 @@ def __init__( self._capture_hook_handles: list[torch.utils.hooks.RemovableHandle] = [] # Fallback for _read_use_hook_mlp_in when block.config is None. self._use_hook_mlp_in: bool = False - self.mlp_reads_resid_directly = mlp_reads_resid_directly + self.mlp_reads_resid_directly = mlp_reads_resid_directly or post_norm_only_mlp # Fires on the MLP-branch entry (pre-ln2, or the MLP input on post-norm # blocks) when use_hook_mlp_in is set. See #1317. self.hook_mlp_in = HookPoint() diff --git a/transformer_lens/model_bridge/supported_architectures/AGENTS.md b/transformer_lens/model_bridge/supported_architectures/AGENTS.md index a926b53b8..82590074d 100644 --- a/transformer_lens/model_bridge/supported_architectures/AGENTS.md +++ b/transformer_lens/model_bridge/supported_architectures/AGENTS.md @@ -60,6 +60,8 @@ class MyArchitectureAdapter(ArchitectureAdapter): } ``` +Norm keys name the norm's position, not the HF attribute: `ln1`/`ln2` feed attention/MLP, `ln1_post`/`ln2_post` sit on their outputs before the residual add. A post-norm-only block (OLMo 2) declares only `ln1_post`/`ln2_post`; `BlockBridge` then derives `hook_attn_out`, `hook_mlp_out`, and `hook_resid_mid`. Never put a post-norm in the `ln1`/`ln2` slot — tools read those as the norm feeding the sublayer. + Optional overrides on the class: - `default_cfg` — class-level defaults merged into the runtime config diff --git a/transformer_lens/model_bridge/supported_architectures/gemma3n.py b/transformer_lens/model_bridge/supported_architectures/gemma3n.py index b6543ce4a..e151d9189 100644 --- a/transformer_lens/model_bridge/supported_architectures/gemma3n.py +++ b/transformer_lens/model_bridge/supported_architectures/gemma3n.py @@ -78,16 +78,12 @@ def __init__(self, cfg: Any) -> None: name="model.language_model.layers", config=self.cfg, submodules={ - "input_layernorm": GeneralizedComponent(name="input_layernorm"), - "post_attention_layernorm": GeneralizedComponent( - name="post_attention_layernorm" - ), - "pre_feedforward_layernorm": GeneralizedComponent( - name="pre_feedforward_layernorm" - ), - "post_feedforward_layernorm": GeneralizedComponent( - name="post_feedforward_layernorm" - ), + # Sandwich norms under the canonical keys (Gemma 2/3 layout), so + # hook_attn_out / hook_mlp_out resolve to the post-norm outputs. + "ln1": GeneralizedComponent(name="input_layernorm"), + "ln1_post": GeneralizedComponent(name="post_attention_layernorm"), + "ln2": GeneralizedComponent(name="pre_feedforward_layernorm"), + "ln2_post": GeneralizedComponent(name="post_feedforward_layernorm"), "post_per_layer_input_norm": GeneralizedComponent( name="post_per_layer_input_norm" ), diff --git a/transformer_lens/model_bridge/supported_architectures/olmo2.py b/transformer_lens/model_bridge/supported_architectures/olmo2.py index ff604e717..04ce424eb 100644 --- a/transformer_lens/model_bridge/supported_architectures/olmo2.py +++ b/transformer_lens/model_bridge/supported_architectures/olmo2.py @@ -22,7 +22,8 @@ class Olmo2ArchitectureAdapter(ArchitectureAdapter): pre-norm models like Llama: - Post-norm: RMSNorm is applied AFTER attention and AFTER MLP, not before. - ln1 maps to post_attention_layernorm, ln2 maps to post_feedforward_layernorm. + ln1_post maps to post_attention_layernorm, ln2_post to post_feedforward_layernorm; + there is no ln1/ln2 (no pre-sublayer norm). - Q/K normalization: Per-head RMSNorm applied to queries and keys after projection. - No biases on any projections. @@ -35,8 +36,8 @@ class Olmo2ArchitectureAdapter(ArchitectureAdapter): - blocks.{i}.mlp.b_in - No bias on MLP up_proj - blocks.{i}.mlp.b_gate - No bias on MLP gate_proj - blocks.{i}.mlp.b_out - No bias on MLP down_proj - - blocks.{i}.ln1.b - RMSNorm has no bias - - blocks.{i}.ln2.b - RMSNorm has no bias + - blocks.{i}.ln1_post.b - RMSNorm has no bias + - blocks.{i}.ln2_post.b - RMSNorm has no bias - ln_final.b - RMSNorm has no bias """ @@ -61,17 +62,18 @@ def __init__(self, cfg: Any) -> None: **self._qkvo_weight_conversions(), } - # Component mapping — POST-NORM architecture: - # ln1 = post_attention_layernorm (applied AFTER attention) - # ln2 = post_feedforward_layernorm (applied AFTER MLP) self.component_mapping = { "embed": EmbeddingBridge(name="model.embed_tokens"), "rotary_emb": RotaryEmbeddingBridge(name="model.rotary_emb", config=self.cfg), "blocks": BlockBridge( name="model.layers", submodules={ - "ln1": RMSNormalizationBridge(name="post_attention_layernorm", config=self.cfg), - "ln2": RMSNormalizationBridge( + # Post-norm only: BlockBridge derives hook_attn_out / hook_mlp_out + # (norm outputs) and hook_resid_mid (mlp.hook_in) from these keys. + "ln1_post": RMSNormalizationBridge( + name="post_attention_layernorm", config=self.cfg + ), + "ln2_post": RMSNormalizationBridge( name="post_feedforward_layernorm", config=self.cfg ), "attn": self._attention_bridge_cls( @@ -90,18 +92,6 @@ def __init__(self, cfg: Any) -> None: ), "mlp": self._build_mlp_bridge(), }, - # Post-norm overrides: ln1/ln2 are applied AFTER attention/MLP and - # BEFORE the residual add, so the residual mid-point is mlp.hook_in - # and the additive contributions are the norm outputs, not the raw - # module outputs (attn.hook_out / mlp.hook_out stay raw). - hook_alias_overrides={ - "hook_resid_mid": "mlp.hook_in", - "hook_attn_out": "ln1.hook_out", - "hook_mlp_out": "ln2.hook_out", - }, - # No pre-MLP norm: the MLP consumes the mid-residual directly, so - # the hook_mlp_in capture must sit on the MLP, not on ln2. - mlp_reads_resid_directly=True, ), "ln_final": RMSNormalizationBridge(name="model.norm", config=self.cfg), "unembed": UnembeddingBridge(name="lm_head", config=self.cfg), diff --git a/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py b/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py index 79a3910da..f7726951a 100644 --- a/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py +++ b/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py @@ -30,46 +30,39 @@ class _OlmoHybridBlockBridge(BlockBridge): - """BlockBridge with per-layer-type hook aliases and no hook_resid_mid. - - hook_resid_mid: no single target fits both layer types (ln2.hook_in is the - mid-point on linear-attention layers but the raw attn-branch output on - full-attention layers); dropped type-visibly, as on ParallelBlockBridge. - - hook_attn_out / hook_mlp_out must expose the tensor added to the residual - stream, which also differs by layer type: full-attention layers are OLMo2 - post-norm (contribution = norm output), linear-attention layers are pre-norm - (contribution = raw sublayer output). Both candidate targets exist on both - layer types, so alias fallback lists cannot discriminate; instead the - aliases are selected per layer at bind time in set_original_component. + """BlockBridge whose norm keys and hook aliases are chosen per layer type. + + One HF attribute plays two roles: ``post_attention_layernorm`` is the pre-MLP + norm on GatedDeltaNet (linear-attention) layers but the post-attention norm on + OLMo2-style full-attention layers. The template declares it as ``ln2``; + full-attention layers re-key it to ``ln1_post`` at bind time so ``ln2`` always + means the norm feeding the MLP. Both layer types then expose the tensor added + to the residual stream as hook_attn_out / hook_mlp_out, and a hook_resid_mid + (``ln2.hook_in`` or ``mlp.hook_in``). """ def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) if self.hook_aliases is BlockBridge.hook_aliases: self.hook_aliases = dict(self.hook_aliases) - self.hook_aliases.pop("hook_resid_mid", None) - # Full-attention (OLMo2 post-norm) layers route the MLP output through - # post_feedforward_layernorm, so the residual-facing MLP output is - # ln2_post.hook_out there; linear-attention layers have no ln2_post and - # fall back to the raw mlp.hook_out. + # Per-layer targets are set at bind time; this fallback covers the + # template, which has both ln2 and ln2_post declared. self.hook_aliases["hook_mlp_out"] = ["ln2_post.hook_out", "mlp.hook_out"] def set_original_component(self, original_component: nn.Module) -> None: super().set_original_component(original_component) if self.hook_aliases is BlockBridge.hook_aliases: self.hook_aliases = dict(self.hook_aliases) + # Runs before setup_submodules binds the norms, so re-keying is safe. if getattr(original_component, "post_feedforward_layernorm", None) is not None: - # Full-attention (post-norm) layer: ln2 = post_attention_layernorm - # applied after attention, ln2_post = post_feedforward_layernorm. - # The MLP consumes the mid-residual directly, so the hook_mlp_in - # capture must sit on the MLP, not on ln2 (whose input here is the - # raw attention output). - self.hook_aliases["hook_attn_out"] = "ln2.hook_out" + if "ln2" in self.submodules: + self.submodules["ln1_post"] = self.submodules.pop("ln2") + self.hook_aliases["hook_resid_mid"] = "mlp.hook_in" + self.hook_aliases["hook_attn_out"] = "ln1_post.hook_out" self.hook_aliases["hook_mlp_out"] = "ln2_post.hook_out" self.mlp_reads_resid_directly = True else: - # Linear-attention (pre-norm) layer. + self.hook_aliases["hook_resid_mid"] = "ln2.hook_in" self.hook_aliases["hook_attn_out"] = "linear_attn.hook_out" self.hook_aliases["hook_mlp_out"] = "mlp.hook_out" self.mlp_reads_resid_directly = False