safetensor chat

chat_HF.py → chat.py

@@ -47,7 +47,7 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 # Default model repository
-DEFAULT_MODEL_REPO = "MistyozAI/CosmicFish-120M"
+DEFAULT_MODEL_REPO = "MistyozAI/CosmicFish-90M"
 
 # Default prompt template
 DEFAULT_PROMPT_TEMPLATE = "Below is a conversation between a helpful AI assistant and a human. The assistant is knowledgeable, friendly, and provides detailed and accurate responses.\n\n"
@@ -59,9 +59,9 @@ class CosmicConfig:
     def __init__(self,
                  vocab_size=50257,
                  block_size=512,
-                 n_layer=12,
+                 n_layer=10,
                  n_head=16,
-                 n_embd=704,
+                 n_embd=640,
                  bias=True,
                  dropout=0.0,
                  n_query_groups=4,
@@ -998,9 +998,9 @@ def main():
                         help="Device to use (cuda or cpu)")
 
     # Generation parameters
-    parser.add_argument("--temperature", type=float, default=0.7,
+    parser.add_argument("--temperature", type=float, default=0.5,
                         help="Temperature for sampling (default: 0.7)")
-    parser.add_argument("--max_tokens", type=int, default=1024,
+    parser.add_argument("--max_tokens", type=int, default=512,
                         help="Maximum number of tokens to generate per response")
     parser.add_argument("--min_tokens", type=int, default=10,
                         help="Minimum number of tokens to generate per response")
@@ -1019,7 +1019,7 @@ def main():
     parser.add_argument("--instruction", type=str,
                         default=DEFAULT_PROMPT_TEMPLATE,
                         help="Instruction prompt to prepend to the conversation")
-    parser.add_argument("--max_history", type=int, default=1024,
+    parser.add_argument("--max_history", type=int, default=512,
                         help="Maximum number of tokens to keep in history")
 
     # UI parameters