Upload folder using huggingface_hub

conversion_metadata.json

@@ -1,7 +1,7 @@
 {
   "original_checkpoint": "checkpoints/base_20251213_164813/best.pt",
   "original_config": "embedded_in_checkpoint",
-  "converted_at": "2025-12-15 08:28:11.703039",
+  "converted_at": "2025-12-15 12:38:28.627410",
   "model_type": "base",
   "vocab_size": 43,
   "epoch": 38,

processing_swipe.py

@@ -111,9 +111,11 @@ class SwipeProcessor(ProcessorMixin):
                 path_coords = torch.cat([path_coords, torch.zeros(batch_size, pad_len, 3)], dim=1)
 
             # Create path mask (1 = real data, 0 = padding)
+            # Detect padding by checking for all-zero coordinates
             path_mask = torch.ones(batch_size, self.max_path_len, dtype=torch.long)
-            if padding and current_path_len < self.max_path_len:
-                path_mask[:, current_path_len:] = 0
+            # A point is padding if all its coordinates (x, y, t) are zero
+            is_padding = (path_coords == 0).all(dim=-1)  # [batch, path_len]
+            path_mask[is_padding] = 0
 
             result["path_coords"] = path_coords
             # Store path_mask internally for attention_mask construction

tokenization_swipe.py

@@ -141,7 +141,7 @@ class SwipeTokenizer(PreTrainedTokenizer):
         Returns:
             str: Concatenated string
         """
-        # Filter out special tokens
+        # Filter out special tokens (must include [PUNC] which represents punctuation)
         special_tokens = {
             self.pad_token,
             self.cls_token,
@@ -149,6 +149,7 @@ class SwipeTokenizer(PreTrainedTokenizer):
             self.mask_token,
             self.unk_token,
             self.eos_token,
+            "[PUNC]",  # Punctuation token from CharacterTokenizer
         }
         filtered = [t for t in tokens if t not in special_tokens]
         return "".join(filtered)