Initial model conversion and upload.

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54566aee0d66e67f57658f178073695301dc769b173089d2355509be0e317be9
-size 1520485422
+oid sha256:f2fa66d389820b24dbcb4571ae71e26710ec4dc541b4386d2136c5797be2226c
+size 1520501998

model.safetensors.index.json

@@ -1,13 +1,15 @@
 {
   "metadata": {
-    "total_parameters": 8206434307,
-    "total_size": 16412868614
+    "total_parameters": 8206442499,
+    "total_size": 16412884998
   },
   "weight_map": {
     "hallucination_detector.bias": "model-00004-of-00004.safetensors",
     "hallucination_detector.weight": "model-00004-of-00004.safetensors",
     "hallucination_down_proj.weight": "model-00004-of-00004.safetensors",
     "hallucination_gate_proj.weight": "model-00004-of-00004.safetensors",
+    "hallucination_norm.bias": "model-00004-of-00004.safetensors",
+    "hallucination_norm.weight": "model-00004-of-00004.safetensors",
     "hallucination_up_proj.weight": "model-00004-of-00004.safetensors",
     "lm_head.weight": "model-00004-of-00004.safetensors",
     "model.embed_tokens.weight": "model-00001-of-00004.safetensors",

modeling.py

@@ -22,6 +22,7 @@ class SelfCorrectiveLlama(LlamaForCausalLM):
         self.hallucination_up_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
         self.hallucination_down_proj = nn.Linear(intermediate_size, config.hidden_size, bias=False)
         self.hallucination_detector = nn.Linear(config.hidden_size, self.num_new_tokens + 1)
+        self.hallucination_norm = nn.LayerNorm(config.hidden_size)
     
     def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
         # Get the full sequence of input IDs from the past, if available