Start with semicolon, debug logits

app.py

@@ -375,7 +375,15 @@ def generate(prompt: str, temperature: float, max_tokens: int, num_steps: int, g
         # Latent -> Gcode via trained decoder (with debug)
         with torch.no_grad():
             batch_size = latent.shape[0]
-            input_ids = torch.full((batch_size, 1), gcode_tokenizer.pad_token_id, dtype=torch.long, device=device)
+            # Start with semicolon (gcode comment start) instead of pad
+            # Gcode files start with "; Source: ..." 
+            start_tokens = gcode_tokenizer.encode(";", add_special_tokens=False)
+            print(f"Start tokens for ';': {start_tokens}")
+            if start_tokens:
+                start_id = start_tokens[0]
+            else:
+                start_id = gcode_tokenizer.pad_token_id
+            input_ids = torch.tensor([[start_id]], dtype=torch.long, device=device)
             
             generated_tokens = []
             for step in range(min(max_tokens, 1024) - 1):
@@ -402,7 +410,10 @@ def generate(prompt: str, temperature: float, max_tokens: int, num_steps: int, g
                 # Debug first few tokens
                 if step < 5:
                     token_str = gcode_tokenizer.decode([token_id])
-                    print(f"Step {step}: token_id={token_id}, token='{token_str}'")
+                    # Check logits distribution
+                    top5_vals, top5_ids = torch.topk(logits[0, -1, :], 5)
+                    top5_tokens = [gcode_tokenizer.decode([i.item()]) for i in top5_ids]
+                    print(f"Step {step}: token_id={token_id}, token='{token_str}', top5={list(zip(top5_tokens, top5_vals.tolist()))}")
                 
                 if token_id == gcode_tokenizer.eos_token_id:
                     print(f"Hit EOS at step {step}")