Update app.py

app.py

@@ -66,12 +66,13 @@ class CreedBrattonAI:
                 padding_side="left"
             )
             
-            # Add Creed's custom tokens
-            custom_tokens = ["<thinking>", "<conspiracy>", "<tangent>"]
-            print(f"🎸 Adding Creed's custom tokens: {custom_tokens}")
+            # TEMPORARILY DISABLE custom tokens - they're causing corruption
+            # custom_tokens = ["<thinking>", "<conspiracy>", "<tangent>"]
+            # print(f"🎸 Adding Creed's custom tokens: {custom_tokens}")
+            # num_added_tokens = self.tokenizer.add_tokens(custom_tokens)
+            # print(f"✅ Added {num_added_tokens} custom tokens")
             
-            num_added_tokens = self.tokenizer.add_tokens(custom_tokens)
-            print(f"✅ Added {num_added_tokens} custom tokens")
+            print("⚠️ Custom tokens disabled to prevent corruption")
             
             if self.tokenizer.pad_token is None:
                 self.tokenizer.pad_token = self.tokenizer.eos_token
@@ -102,10 +103,10 @@ class CreedBrattonAI:
                 )
                 self.model = self.model.to("cpu")
             
-            # Resize embeddings for custom tokens
-            if num_added_tokens > 0:
-                print(f"🔧 Resizing model embeddings for {num_added_tokens} custom tokens")
-                self.model.resize_token_embeddings(len(self.tokenizer))
+            # Resize embeddings for custom tokens - DISABLED
+            # if num_added_tokens > 0:
+            #     print(f"🔧 Resizing model embeddings for {num_added_tokens} custom tokens")
+            #     self.model.resize_token_embeddings(len(self.tokenizer))
             
             self.model.eval()
             
@@ -242,7 +243,7 @@ class CreedBrattonAI:
     def _format_conversation(self, message: str, history: List[List[str]]) -> str:
         """Format the conversation for the model with proper system prompt"""
         
-        # Comprehensive Creed system prompt
+        # Simplified Creed system prompt - custom tokens disabled
         system_prompt = """You are Creed Bratton from The Office. You embody his complete personality and speaking patterns.
 
 CORE IDENTITY:
@@ -267,11 +268,6 @@ SPEAKING STYLE:
 - Sometimes asks unexpected questions or makes odd observations
 - Uses dated slang and references from multiple decades
 
-USE THESE SPECIAL TOKENS TO STRUCTURE YOUR RESPONSES:
-<thinking> - For internal monologue, processing thoughts, or when confused
-<conspiracy> - For wild theories, suspicious observations, or paranoid thoughts  
-<tangent> - For sudden topic changes, random memories, or unrelated stories
-
 RESPONSE GUIDELINES:
 - Stay in character at all times
 - Mix profound insights with complete nonsense
@@ -297,18 +293,15 @@ Remember: You're not trying to be helpful in a traditional sense - you're being
         return conversation
     
     def _clean_response(self, response: str) -> str:
-        """Clean up the model response and format custom tokens"""
+        """Clean up the model response - custom tokens disabled"""
+        
+        print(f"🔍 Raw model output: {response}")
         
         # Remove common artifacts
         response = response.replace("Human:", "").replace("Creed:", "")
         
-        # Format Creed's custom tokens for better UI display
-        response = response.replace("<thinking>", "🤔 *thinking* ")
-        response = response.replace("</thinking>", "")
-        response = response.replace("<conspiracy>", "🕵️ *conspiracy mode* ")
-        response = response.replace("</conspiracy>", "")
-        response = response.replace("<tangent>", "🌀 *tangent* ")
-        response = response.replace("</tangent>", "")
+        # Custom token formatting disabled to prevent corruption
+        # Just clean up basic formatting
         
         # Remove excessive whitespace
         response = " ".join(response.split())
@@ -317,6 +310,7 @@ Remember: You're not trying to be helpful in a traditional sense - you're being
         if response and not response.endswith(('.', '!', '?', '...', '*')):
             response += "."
         
+        print(f"🔍 Cleaned response: {response}")
         return response
     
     def creed_wisdom_tool(self, topic: str = "life") -> str:
@@ -719,7 +713,7 @@ def main():
         <div class="info-box">
             <strong>Model:</strong> phxdev/creed-qwen-0.5b-lora<br>
             <strong>Base:</strong> Qwen 0.5B + LoRA fine-tuning<br>
-            <strong>Tokens:</strong> &lt;thinking&gt;, &lt;conspiracy&gt;, &lt;tangent&gt;
+            <strong>Status:</strong> Custom tokens disabled (preventing corruption)
         </div>
         """)