Upload folder using huggingface_hub

README.md

@@ -44,20 +44,19 @@ model.eval()
 
 # Example: Predict word from swipe path
 from datasets import load_dataset
-from swipealot.data.dataset import normalize_coordinates, sample_path_points
 
 # Load sample
 dataset = load_dataset("futo-org/swipe.futo.org", split="test[:1]")
 item = dataset[0]
 
-# Preprocess path
+# Preprocess swipe path using processor methods
 # 1. Normalize timestamps (x,y already normalized in futo dataset)
-normalized = normalize_coordinates(item["data"], item["canvas_width"], item["canvas_height"])
+normalized = processor.normalize_coordinates(item["data"], item["canvas_width"], item["canvas_height"])
 
 # 2. Resample to fixed length (max_path_len=128)
 #    - Pads with zeros if path < 128 points
 #    - Interpolates if path > 128 points
-path_coords, _ = sample_path_points(normalized, processor.max_path_len)
+path_coords, _ = processor.sample_path_points(normalized, processor.max_path_len)
 path = torch.tensor([path_coords], dtype=torch.float32)
 
 # Get predictions
@@ -196,3 +195,4 @@ outputs.last_hidden_state # [batch, seq_len, d_model] - Hidden representations
 ## License
 
 Apache 2.0
+

conversion_metadata.json

@@ -1,7 +1,7 @@
 {
   "original_checkpoint": "checkpoints/base_20251213_164813/best.pt",
   "original_config": "embedded_in_checkpoint",
-  "converted_at": "2025-12-15 08:03:26.074041",
+  "converted_at": "2025-12-15 08:28:11.703039",
   "model_type": "base",
   "vocab_size": 43,
   "epoch": 38,

preprocessing.py

@@ -0,0 +1,108 @@
+"""Standalone preprocessing utilities for swipe path data.
+
+This module provides preprocessing functions for the SwipeALot model that are
+completely standalone and don't require the full swipealot training package.
+"""
+
+import numpy as np
+
+
+def normalize_coordinates(
+    data_points: list[dict], canvas_width: float, canvas_height: float
+) -> list[dict]:
+    """
+    Normalize swipe coordinates and timestamps.
+
+    Args:
+        data_points: List of dicts with 'x', 'y', 't' keys
+        canvas_width: Canvas width (not used - kept for compatibility)
+        canvas_height: Canvas height (not used - kept for compatibility)
+
+    Returns:
+        List of normalized coordinate dicts with x, y in [0,1] and t in [0,1]
+
+    Note:
+        For futo-org/swipe.futo.org dataset, x and y are already normalized to [0,1].
+        This function clamps them to ensure they stay in bounds and normalizes timestamps.
+    """
+    if not data_points:
+        return []
+
+    # Extract timestamps for normalization
+    timestamps = [p["t"] for p in data_points]
+    t_min = min(timestamps)
+    t_max = max(timestamps)
+    t_range = t_max - t_min if t_max > t_min else 1.0
+
+    normalized = []
+    for point in data_points:
+        # x and y are already normalized to [0,1] in the dataset
+        # But sometimes they go slightly outside bounds, so clamp them
+        x_norm = max(0.0, min(1.0, point["x"]))
+        y_norm = max(0.0, min(1.0, point["y"]))
+
+        # Normalize timestamp to [0, 1]
+        t_norm = (point["t"] - t_min) / t_range
+
+        normalized.append({"x": x_norm, "y": y_norm, "t": t_norm})
+
+    return normalized
+
+
+def sample_path_points(data_points: list[dict], max_len: int) -> tuple:
+    """
+    Sample or pad path points to fixed length using linear interpolation.
+
+    Args:
+        data_points: List of coordinate dicts with 'x', 'y', 't' keys
+        max_len: Target length (typically 128 for SwipeALot models)
+
+    Returns:
+        Tuple of (sampled_points, mask) where:
+        - sampled_points: numpy array of shape [max_len, 3] with (x, y, t) coordinates
+        - mask: numpy array of shape [max_len] indicating valid (1) vs padding (0) points
+
+    Note:
+        - If path has fewer points than max_len, it's zero-padded
+        - If path has more points than max_len, it's downsampled using linear interpolation
+        - If path has exactly max_len points, it's returned as-is
+    """
+    num_points = len(data_points)
+
+    if num_points == max_len:
+        points = data_points
+        mask = [1] * max_len
+    elif num_points < max_len:
+        # Pad with zeros
+        points = data_points + [{"x": 0.0, "y": 0.0, "t": 0.0}] * (max_len - num_points)
+        mask = [1] * num_points + [0] * (max_len - num_points)
+    else:
+        # Downsample using linear interpolation
+        # Extract coordinates as arrays
+        x_coords = np.array([p["x"] for p in data_points])
+        y_coords = np.array([p["y"] for p in data_points])
+        t_coords = np.array([p["t"] for p in data_points])
+
+        # Original indices (parameter for interpolation)
+        original_indices = np.arange(num_points)
+
+        # Target indices for interpolation (evenly spaced)
+        target_indices = np.linspace(0, num_points - 1, max_len)
+
+        # Interpolate each coordinate independently
+        x_interp = np.interp(target_indices, original_indices, x_coords)
+        y_interp = np.interp(target_indices, original_indices, y_coords)
+        t_interp = np.interp(target_indices, original_indices, t_coords)
+
+        # Reconstruct points
+        points = [
+            {"x": float(x), "y": float(y), "t": float(t)}
+            for x, y, t in zip(x_interp, y_interp, t_interp, strict=True)
+        ]
+        mask = [1] * max_len
+
+    # Convert to numpy arrays
+    coords = np.array([[p["x"], p["y"], p["t"]] for p in points], dtype=np.float32)
+    mask = np.array(mask, dtype=np.int64)
+
+    return coords, mask