Update README.md

README.md

@@ -180,8 +180,8 @@ def chat_loop(model, tokenizer):
                     input_ids=input_ids,
                     max_new_tokens=max_new_tokens,
                     do_sample=True,
-                    temperature=0.9,
-                    top_p=0.9,
+                    temperature=1.0,
+                    top_p=0.95,
                     top_k=40,
                     use_cache=False,
                     pad_token_id=tokenizer.pad_token_id,
@@ -222,15 +222,15 @@ layer_config = {}
 for i in range(48):  # 48 layers
     prefix = f"model.layers.{i}"
 
-    # Attention layers -> 8-bit
+    # Attention layers -> 16-bit
     if i in [3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47]:  # self_attn layers
         for proj in ["q_proj", "k_proj", "v_proj", "o_proj"]:
             layer_config[f"{prefix}.self_attn.{proj}"] = {"bits": 16}
-    else:  # linear_attn layers
+    else:  # linear_attn layers -> 16-bit
         for proj in ["in_proj_qkvz", "in_proj_ba", "out_proj"]:
             layer_config[f"{prefix}.linear_attn.{proj}"] = {"bits": 16}
 
-    # MLP gate -> 8-bit
+    # MLP gate -> 16-bit
     layer_config[f"{prefix}.mlp.gate"] = {"bits": 16}
 
     # shared_expert_gate -> 16-bit (skipped)