File size: 7,075 Bytes
0185029 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
RELEASE_ROOT = Path(__file__).resolve().parents[1]
CHECKER_PATH = RELEASE_ROOT / "scripts" / "check_release.py"
SPEC = importlib.util.spec_from_file_location("orarl_release_check", CHECKER_PATH)
assert SPEC is not None and SPEC.loader is not None
CHECKER = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = CHECKER
SPEC.loader.exec_module(CHECKER)
def _reasons(root: Path, max_bytes: int = CHECKER.DEFAULT_MAX_BYTES) -> list[str]:
return [finding.reason for finding in CHECKER.check_release(root, max_bytes)]
def test_clean_text_tree_passes(tmp_path: Path) -> None:
(tmp_path / "README.md").write_text("portable source\n", encoding="utf-8")
assert CHECKER.check_release(tmp_path) == []
def test_portable_evaluation_jsonl_is_release_source(tmp_path: Path) -> None:
annotation = tmp_path / "data" / "eval" / "annotations" / "task.jsonl"
annotation.parent.mkdir(parents=True)
natural_text = "".join(("fl", "uffy"))
annotation.write_text(
f'{{"problem":"{natural_text}","videos":["media/task/videos/clip.mp4"]}}\n',
encoding="utf-8",
)
assert CHECKER.check_release(tmp_path) == []
def test_canonical_evaluation_jsonl_can_exceed_the_generic_source_limit(
tmp_path: Path,
) -> None:
annotation = tmp_path / "data" / "eval" / "annotations" / "task.jsonl"
annotation.parent.mkdir(parents=True)
annotation.write_text('{"problem":"' + ("x" * 64) + '"}\n', encoding="utf-8")
assert CHECKER.check_release(tmp_path, max_bytes=16) == []
def test_evaluation_jsonl_still_rejects_private_paths(tmp_path: Path) -> None:
annotation = tmp_path / "data" / "eval" / "annotations" / "task.jsonl"
annotation.parent.mkdir(parents=True)
private_path = "/" + "mnt" + "/person/video.mp4"
annotation.write_text(
f'{{"videos":["{private_path}"]}}\n',
encoding="utf-8",
)
assert "private absolute path at line 1" in _reasons(tmp_path)
def test_non_evaluation_data_tree_is_rejected(tmp_path: Path) -> None:
generated = tmp_path / "data" / "training"
generated.mkdir(parents=True)
(generated / "rows.txt").write_text("private input\n", encoding="utf-8")
assert "generated root data directory" in _reasons(tmp_path)
def test_excluded_method_terms_are_detected_without_literals_in_test_source(
tmp_path: Path,
) -> None:
term_one = "".join(("c", "p", "p", "o"))
term_two = "".join(("lu", "ff", "y"))
(tmp_path / "source.py").write_text(
f"first = '{term_one}'\nsecond = '{term_two}'\n",
encoding="utf-8",
)
reasons = _reasons(tmp_path)
assert sum("excluded method term" in reason for reason in reasons) == 2
def test_legacy_oracle_names_are_detected(tmp_path: Path) -> None:
legacy_name = "".join(("build_", "g", "t", "_response"))
(tmp_path / "source.py").write_text(
f"def {legacy_name}():\n pass\n",
encoding="utf-8",
)
assert any("excluded method term" in reason for reason in _reasons(tmp_path))
def test_evaluation_runtime_may_name_benchmark_ground_truth(tmp_path: Path) -> None:
evaluator = tmp_path / "eval" / "task" / "tracking" / "eval_tracking.py"
evaluator.parent.mkdir(parents=True)
annotation_field = "".join(("g", "t", "_", "bbox"))
evaluator.write_text(
f'box = record["{annotation_field}"]\n',
encoding="utf-8",
)
assert CHECKER.check_release(tmp_path) == []
def test_evaluation_runtime_still_rejects_excluded_method_terms(tmp_path: Path) -> None:
evaluator = tmp_path / "eval" / "task" / "eval_vllm.py"
evaluator.parent.mkdir(parents=True)
legacy_name = "".join(("build_", "g", "t", "_response"))
evaluator.write_text(f"def {legacy_name}():\n pass\n", encoding="utf-8")
assert any("excluded method term" in reason for reason in _reasons(tmp_path))
def test_generated_test_caches_fail_the_source_gate(tmp_path: Path) -> None:
cache = tmp_path / ".pytest_cache"
cache.mkdir()
(cache / "state").write_text("generated\n", encoding="utf-8")
(tmp_path / "README.md").write_text("portable source\n", encoding="utf-8")
findings = CHECKER.check_release(tmp_path)
assert len(findings) == 1
assert findings[0].path == Path(".pytest_cache")
assert findings[0].reason == "generated directory"
def test_gitignored_generated_state_is_not_part_of_release(tmp_path: Path) -> None:
cache = tmp_path / ".pytest_cache"
cache.mkdir()
(cache / "state").write_text("generated\n", encoding="utf-8")
(tmp_path / ".gitignore").write_text(".pytest_cache/\n", encoding="utf-8")
assert CHECKER.check_release(tmp_path) == []
def test_gitignored_training_runs_are_not_part_of_release(tmp_path: Path) -> None:
run = tmp_path / "runs" / "smoke-training" / "grpo"
run.mkdir(parents=True)
(run / "smoke.log").write_text("/private/training/path\n", encoding="utf-8")
(tmp_path / ".gitignore").write_text("/runs/\n", encoding="utf-8")
assert CHECKER.check_release(tmp_path) == []
def test_local_checkpoint_tree_is_outside_release_scan(tmp_path: Path) -> None:
checkpoint = tmp_path / "checkpoint" / "Video-ORA-9B"
checkpoint.mkdir(parents=True)
(checkpoint / "model.safetensors").write_bytes(b"local weights")
(tmp_path / "README.md").write_text("portable source\n", encoding="utf-8")
assert CHECKER.check_release(tmp_path) == []
def test_documented_release_media_is_allowlisted(tmp_path: Path) -> None:
assets = tmp_path / "assets"
assets.mkdir()
(assets / "orarl-data-scaling.gif").write_bytes(b"documented data animation")
(assets / "orarl-hero.gif").write_bytes(b"documented preview")
(assets / "orarl-method.gif").write_bytes(b"documented method animation")
(assets / "orarl-model-scaling.gif").write_bytes(b"documented model animation")
(assets / "paper-results.png").write_bytes(b"documented figure")
(tmp_path / "orarl.pdf").write_bytes(b"documented paper")
assert CHECKER.check_release(tmp_path) == []
def test_private_paths_and_secret_values_are_detected(tmp_path: Path) -> None:
private_path = "/" + "mnt" + "/person/project/model"
access_value = "AKIA" + ("A" * 16)
(tmp_path / "settings.txt").write_text(
f"model={private_path}\ncloud={access_value}\n",
encoding="utf-8",
)
reasons = _reasons(tmp_path)
assert any("private absolute path" in reason for reason in reasons)
assert any("credential-like value" in reason for reason in reasons)
def test_generated_large_and_broken_entries_are_detected(tmp_path: Path) -> None:
(tmp_path / "weights.pt").write_bytes(b"payload")
(tmp_path / "large.txt").write_text("x" * 17, encoding="utf-8")
(tmp_path / "missing-link").symlink_to(tmp_path / "not-there")
reasons = _reasons(tmp_path, max_bytes=16)
assert "generated or binary artifact" in reasons
assert any("file is too large" in reason for reason in reasons)
assert "broken symbolic link" in reasons
|