Upload app.py

app.py

@@ -274,6 +274,23 @@ def generate_text(prompt, max_new_tokens=100, temperature=0.8, top_k=50):
         
         # Decode
         generated_text = enc.decode(tokens[0].tolist())
+        
+        # Post-process to fix spacing issues (common with BPE tokenizers)
+        import re
+        # Fix 1: lowercase followed by uppercase (e.g., "perpetualWith" -> "perpetual With")
+        generated_text = re.sub(r'([a-z])([A-Z])', r'\1 \2', generated_text)
+        
+        # Fix 2: Common word boundaries that got merged (e.g., "perpetualwith" -> "perpetual with")
+        # Add space before common words that might have been merged
+        common_words = ['with', 'the', 'and', 'that', 'this', 'have', 'from', 'not', 'but', 'for', 'are', 'was', 'were', 'been', 'will', 'shall', 'would', 'could', 'should']
+        for word in common_words:
+            # Only add space if it's not already separated and follows a lowercase letter
+            pattern = r'([a-z])(' + word + r'\b)'
+            generated_text = re.sub(pattern, r'\1 \2', generated_text, flags=re.IGNORECASE)
+        
+        # Fix 3: Add space before character names (all caps words)
+        generated_text = re.sub(r'([a-z])([A-Z]{2,})', r'\1 \2', generated_text)
+        
         return generated_text
     except Exception as e:
         import traceback