special token removed

app.py

@@ -32,8 +32,6 @@ model.train(False)
 
 def generate_text(prompt, max_length=100, num_samples=1, temperature=0.8):
     enc = tiktoken.get_encoding('gpt2')
-    # Modify encoding behavior to allow special tokens
-    enc._special_tokens.add('<|endoftext|>')
     tokens = enc.encode(prompt)
     tokens = torch.tensor(tokens, dtype=torch.long)
     tokens = tokens.unsqueeze(0).repeat(num_samples, 1)
@@ -55,10 +53,8 @@ def generate_text(prompt, max_length=100, num_samples=1, temperature=0.8):
             
             tokens = torch.cat((tokens, next_token), dim=1)
             
-            # Check for end of text token
-            endoftext_token = enc.encode('<|endoftext|>')[0]
-            if next_token.item() == endoftext_token:
-                break
+            # Remove special token check entirely
+            # Just generate for the specified length or until context limit
     
     generated_texts = []
     for i in range(num_samples):