XiangpengYang commited on
Commit
a0288c0
·
1 Parent(s): b2cec57

feat: resolve pi05 checkpoint artifacts

Browse files
Files changed (2) hide show
  1. artifacts.py +64 -0
  2. tests/test_artifacts.py +38 -0
artifacts.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Resolve and validate Hugging Face π₀.₅ UR checkpoint artifacts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ import os
7
+ from pathlib import Path, PurePosixPath
8
+
9
+
10
+ DEFAULT_CHECKPOINT_PATH = "checkpoint"
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class ArtifactPaths:
15
+ checkpoint: Path
16
+
17
+
18
+ def snapshot_download(**kwargs) -> str:
19
+ """Import Hugging Face Hub lazily so validation tests stay lightweight."""
20
+ from huggingface_hub import snapshot_download as download
21
+
22
+ return download(**kwargs)
23
+
24
+
25
+ def normalize_model_id(value: str) -> str:
26
+ if not isinstance(value, str) or not value.strip():
27
+ raise ValueError("Hugging Face model ID is required")
28
+ result = value.strip()
29
+ if any(character.isspace() for character in result):
30
+ raise ValueError("Hugging Face model ID cannot contain whitespace")
31
+ return result
32
+
33
+
34
+ def normalize_checkpoint_path(value: str) -> str:
35
+ if not isinstance(value, str) or not value.strip():
36
+ raise ValueError("checkpoint path is required")
37
+ candidate = value.strip().replace("\\", "/")
38
+ path = PurePosixPath(candidate)
39
+ if path.is_absolute() or ".." in path.parts or path == PurePosixPath("."):
40
+ raise ValueError("checkpoint path must be a relative path without parent traversal")
41
+ return path.as_posix()
42
+
43
+
44
+ def resolve_model_id() -> str:
45
+ return os.getenv("PI05_MODEL_ID", "")
46
+
47
+
48
+ def resolve_checkpoint_path() -> str:
49
+ return os.getenv("PI05_CHECKPOINT_PATH", DEFAULT_CHECKPOINT_PATH)
50
+
51
+
52
+ def download_checkpoint(model_id: str, checkpoint_path: str) -> ArtifactPaths:
53
+ model_id = normalize_model_id(model_id)
54
+ relative = normalize_checkpoint_path(checkpoint_path)
55
+ root = Path(snapshot_download(repo_id=model_id))
56
+ checkpoint = root.joinpath(*PurePosixPath(relative).parts)
57
+ if not (checkpoint / "params").is_dir() and not (checkpoint / "model.safetensors").is_file():
58
+ raise FileNotFoundError(
59
+ f"checkpoint has neither params/ nor model.safetensors: {checkpoint}"
60
+ )
61
+ statistics = checkpoint / "assets/ur_demo/norm_stats.json"
62
+ if not statistics.is_file():
63
+ raise FileNotFoundError(f"UR normalization statistics not found: {statistics}")
64
+ return ArtifactPaths(checkpoint=checkpoint)
tests/test_artifacts.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ import tempfile
3
+ import unittest
4
+ from unittest import mock
5
+
6
+
7
+ class ArtifactTests(unittest.TestCase):
8
+ def test_checkpoint_path_rejects_absolute_parent_and_empty_paths(self):
9
+ import artifacts
10
+
11
+ for value in ("/tmp/model", "../model", "a/../../model", ""):
12
+ with self.subTest(value=value), self.assertRaises(ValueError):
13
+ artifacts.normalize_checkpoint_path(value)
14
+
15
+ def test_download_checkpoint_validates_required_layout(self):
16
+ import artifacts
17
+
18
+ with tempfile.TemporaryDirectory() as directory:
19
+ root = Path(directory)
20
+ checkpoint = root / "checkpoints/30000"
21
+ (checkpoint / "params").mkdir(parents=True)
22
+ (checkpoint / "assets/ur_demo").mkdir(parents=True)
23
+ (checkpoint / "assets/ur_demo/norm_stats.json").write_text("{}")
24
+ with mock.patch.object(artifacts, "snapshot_download", return_value=str(root)):
25
+ paths = artifacts.download_checkpoint("owner/model", "checkpoints/30000")
26
+ self.assertEqual(paths.checkpoint, checkpoint)
27
+
28
+ def test_download_checkpoint_reports_missing_weights(self):
29
+ import artifacts
30
+
31
+ with tempfile.TemporaryDirectory() as directory:
32
+ with mock.patch.object(artifacts, "snapshot_download", return_value=directory):
33
+ with self.assertRaisesRegex(FileNotFoundError, "params/|model.safetensors"):
34
+ artifacts.download_checkpoint("owner/model", "checkpoint")
35
+
36
+
37
+ if __name__ == "__main__":
38
+ unittest.main()