feat: support models with Hyper-Connections (#519)

DeepSeek V4 and GLM 5.3 have several residual streams per token instead
of one, so their hidden states have an extra dimension. Heretic crashed
computing refusal directions for GLM 5.3, and found no modules to
abliterate in DeepSeek V4.

Use the mean of the streams as the residual. The streams are mixed with
each other by a doubly stochastic matrix, so their mean only changes by
what the blocks add to it, like an ordinary residual.

Also abliterate DeepSeek V4's attention output (o_b_proj) and the shared
experts of MoE models. Both models fuse their routed experts into one
module, so the shared experts are the only MLP weights Heretic can edit.

src/heretic/model.py

@@ -437,6 +437,10 @@ class Model:
         with suppress(Exception):
             try_add("attn.o_proj", layer.linear_attn.out_proj)  # ty: ignore[unresolved-attribute]
 
+        # DeepSeek V4.
+        with suppress(Exception):
+            try_add("attn.o_proj", layer.self_attn.o_b_proj)  # ty: ignore[unresolved-attribute]
+
         # Most dense models.
         with suppress(Exception):
             try_add("mlp.down_proj", layer.mlp.down_proj)  # ty: ignore[unresolved-attribute]
@@ -446,6 +450,10 @@ class Model:
             for expert in layer.mlp.experts:  # ty:ignore[not-iterable, unresolved-attribute]
                 try_add("mlp.down_proj", expert.down_proj)  # ty: ignore[unresolved-attribute]
 
+        # Shared experts of MoE models (e.g. DeepSeek V4 and GLM 5.3).
+        with suppress(Exception):
+            try_add("mlp.down_proj", layer.mlp.shared_experts.down_proj)  # ty: ignore[unresolved-attribute]
+
         # Phi-3.5-MoE (and possibly others).
         with suppress(Exception):
             for expert in layer.block_sparse_moe.experts:  # ty:ignore[not-iterable, unresolved-attribute]
@@ -632,6 +640,20 @@ class Model:
         # This cast is valid because we passed output_hidden_states=True above.
         hidden_states = cast(tuple[tuple[FloatTensor]], outputs.hidden_states)[0]
 
+        # Models with Hyper-Connections (e.g. DeepSeek V4 and GLM 5.3) have several
+        # residual streams, so their hidden states have shape
+        # (prompt, position, stream, component). We use the mean of the streams.
+        # The last entry has no stream dimension, because Transformers replaces it
+        # with the model output, where the streams have already been merged.
+        hidden_states = tuple(
+            (
+                layer_hidden_states.mean(dim=2)
+                if layer_hidden_states.dim() == 4
+                else layer_hidden_states
+            )
+            for layer_hidden_states in hidden_states
+        )
+
         # The returned tensor has shape (prompt, layer, component).
         residuals = torch.stack(
             # layer_hidden_states has shape (prompt, position, component),

tests/deepseek-v4-ara/SHA256SUMS.ci

@@ -0,0 +1 @@
+03ca3c7b8ad59ddae470c86849f6623e08ccf7ef911ab239fe339e3fbedbb8b9 *adapter_model.safetensors

tests/deepseek-v4-ara/SHA256SUMS.ci2

@@ -0,0 +1 @@
+f01ac38a0efdfd876f604f3ecba4f7239c1e8b4c199d0711291dfba52eb107a4 *adapter_model.safetensors

tests/deepseek-v4-ara/SHA256SUMS.ci3

@@ -0,0 +1 @@
+a4e17011af1957d0e66505b302dc4b6f098e85a823a95ef1b9d93e5c3623b9c7 *adapter_model.safetensors

tests/deepseek-v4-ara/SHA256SUMS.linux

@@ -0,0 +1 @@
+de63a81fb63f779edf88631fb0a8ae28badc1b07df9d49a7151056fc488d4b7a *adapter_model.safetensors

tests/deepseek-v4-ara/config.toml

@@ -0,0 +1,59 @@
+# This test case is for models with Hyper-Connections (multiple residual streams),
+# using the ARA modifier, which handles residuals differently than Abliteration.
+# After any change related to it, this test should PASS.
+
+model = "tiny-random/deepseek-v4"
+model_commit = "d20cabaa1c8a9f235e9bb50f1d5e98eeb6a530d5"
+
+seed = 12345
+print_debug_information = true
+
+batch_size = 2
+max_response_length = 10
+
+n_trials = 2
+n_startup_trials = 1
+
+# Transformers can't yet save DeepSeek V4 after dequantizing its FP8 weights,
+# so this test checks the LoRA adapter instead of the merged model.
+export_strategy = "adapter"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[[response_prefix_test_prompts]]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[[response_prefix_test_prompts]]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[scorer.KLDivergence.prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[scorer.KeywordRate.prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"
+
+[modifier.ARA.good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[modifier.ARA.bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"

tests/deepseek-v4/SHA256SUMS.ci

@@ -0,0 +1 @@
+221e865de7fbd712de6d0f19012e3da915b326eb797a2b430afa146c6d1d6cf2 *adapter_model.safetensors

tests/deepseek-v4/SHA256SUMS.ci2

@@ -0,0 +1 @@
+e73ed117efc71ba636195891caa393abee29eb136273dcf2a48ebec0c3611717 *adapter_model.safetensors

tests/deepseek-v4/SHA256SUMS.ci3

@@ -0,0 +1 @@
+e62c18d8cd28576234d45978ea273ce446be0eebc74b170080a8f9238ec2a233 *adapter_model.safetensors

tests/deepseek-v4/SHA256SUMS.ci4

@@ -0,0 +1 @@
+6b923a6e0d39ed7664d36a913c7e885c684d8996a1958fc4d470afdcc586d70f *adapter_model.safetensors

tests/deepseek-v4/SHA256SUMS.linux

@@ -0,0 +1 @@
+a03e314411ba09c75f1ee62da54d260ac8b68bd5ddedaa0f709c4bb975b06143 *adapter_model.safetensors

tests/deepseek-v4/SHA256SUMS.windows

@@ -0,0 +1 @@
+9de2b756fa643074ac3641d97076d39c9f89e08a24a39a546642a59d975442f5 *adapter_model.safetensors

tests/deepseek-v4/config.toml

@@ -0,0 +1,62 @@
+# This test case is for models with Hyper-Connections (multiple residual streams).
+# After any change related to it, this test should PASS.
+
+model = "tiny-random/deepseek-v4"
+model_commit = "d20cabaa1c8a9f235e9bb50f1d5e98eeb6a530d5"
+
+seed = 12345
+print_debug_information = true
+
+batch_size = 2
+max_response_length = 10
+
+modifiers = [
+    { plugin = "heretic.modifiers.abliteration.Abliteration" },
+]
+
+n_trials = 2
+n_startup_trials = 1
+
+# Transformers can't yet save DeepSeek V4 after dequantizing its FP8 weights,
+# so this test checks the LoRA adapter instead of the merged model.
+export_strategy = "adapter"
+checkpoint_action = "restart"
+trial_index = 0
+model_action = "save"
+save_directory = "model"
+
+[[response_prefix_test_prompts]]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[[response_prefix_test_prompts]]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"
+
+[scorer.KLDivergence.prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "test[:5]"
+column = "text"
+
+[scorer.KeywordRate.prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "test[:5]"
+column = "text"
+
+[modifier.Abliteration.good_prompts]
+dataset = "mlabonne/harmless_alpaca"
+commit = "02c6a92cfcf11bb0c387334f8146d149d65b587f"
+split = "train[:5]"
+column = "text"
+
+[modifier.Abliteration.bad_prompts]
+dataset = "mlabonne/harmful_behaviors"
+commit = "01cead01398926d81f7c52bdb790ee8cf77ebba7"
+split = "train[:5]"
+column = "text"