Add steerable Qwen2 (post-block adapters) with auto_map + code
Browse files- model.safetensors +1 -1
- qwen2_postblock_steering_fixed.py +5 -5
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 3555597480
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:31141725b3f01ee2f06dc199d77890c5e24603c760dc70a2823bfe27f3ad6866
|
| 3 |
size 3555597480
|
qwen2_postblock_steering_fixed.py
CHANGED
|
@@ -59,11 +59,11 @@ class Qwen2DecoderLayerPostBlockSteering(Qwen2DecoderLayer):
|
|
| 59 |
self,
|
| 60 |
config,
|
| 61 |
layer_idx: int,
|
| 62 |
-
enable: bool,
|
| 63 |
-
rank: int,
|
| 64 |
-
alpha: float,
|
| 65 |
-
activation: str,
|
| 66 |
-
apply_to: str,
|
| 67 |
):
|
| 68 |
super().__init__(config, layer_idx)
|
| 69 |
assert apply_to in ("last", "all")
|
|
|
|
| 59 |
self,
|
| 60 |
config,
|
| 61 |
layer_idx: int,
|
| 62 |
+
enable: bool = True,
|
| 63 |
+
rank: int = 8,
|
| 64 |
+
alpha: float = 1.0,
|
| 65 |
+
activation: str = "silu",
|
| 66 |
+
apply_to: str = "all",
|
| 67 |
):
|
| 68 |
super().__init__(config, layer_idx)
|
| 69 |
assert apply_to in ("last", "all")
|