refactor(patch-utils): improve patch application check for tokenlearn

src/distiller/patch_utils.py

@@ -176,6 +176,18 @@ def is_patch_already_applied(patch_file: Path, target_dir: Path) -> bool:
 								return True
 							break
 
+		# For tokenlearn.patch, check if the indexing fix is already present
+		if "tokenlearn.patch" in patch_file.name:
+			pretrain_file = target_dir / "tokenlearn" / "pretrain.py"
+			if pretrain_file.exists():
+				pretrain_content = pretrain_file.read_text()
+				# Check for the specific fix we're adding
+				if (
+					"Fix for index out of bounds issue" in pretrain_content
+					and "torch.clamp(input_ids, 0, self.w.shape[0] - 1)" in pretrain_content
+				):
+					return True
+
 		return False
 
 	except Exception as e: