| """Fail CI on accidental secrets or non-code payloads in the Git index.""" |
|
|
| from __future__ import annotations |
|
|
| import re |
| import io |
| import subprocess |
| import tarfile |
| import zipfile |
| from pathlib import Path, PurePosixPath |
|
|
| RELEASE_FILES = frozenset({ |
| ".github/workflows/ci.yml", |
| ".gitignore", |
| "CITATION.cff", |
| "CONTRIBUTING.md", |
| "LICENSE", |
| "MANIFEST.in", |
| "NOTICE", |
| "README.md", |
| "README_zh-CN.md", |
| "SECURITY.md", |
| "assets/architecture.webp", |
| "assets/badge-arxiv.svg", |
| "assets/badge-hf-paper.svg", |
| "assets/badge-models-data.svg", |
| "assets/badge-website.svg", |
| "docs/data.md", |
| "eviscene/__init__.py", |
| "eviscene/__main__.py", |
| "eviscene/environment.py", |
| "eviscene/eval/__init__.py", |
| "eviscene/eval/agent_private_gold.py", |
| "eviscene/eval/archive_adapter.py", |
| "eviscene/eval/archive_baseline.py", |
| "eviscene/eval/case_selection.py", |
| "eviscene/eval/evaluator_private_gold.py", |
| "eviscene/eval/gemini_run_guard.py", |
| "eviscene/eval/postprocess_run.py", |
| "eviscene/eval/private_gold_judge_contract.py", |
| "eviscene/eval/private_gold_metrics.py", |
| "eviscene/eval/public_release.py", |
| "eviscene/eval/release_adapter.py", |
| "eviscene/eval/result_records.py", |
| "eviscene/eval/rollout.py", |
| "eviscene/eval/run_artifacts.py", |
| "eviscene/eval/run_cases.py", |
| "eviscene/eval/run_eval.py", |
| "eviscene/eval/score_reference_chain.py", |
| "eviscene/eval/score_semantic_reward.py", |
| "eviscene/eval/score_sft_eligibility.py", |
| "eviscene/eval/scoring_release_adapter.py", |
| "eviscene/integrations/__init__.py", |
| "eviscene/integrations/async_runtime.py", |
| "eviscene/integrations/browse/__init__.py", |
| "eviscene/integrations/browse/jina_reader.py", |
| "eviscene/integrations/browse/passage_selection.py", |
| "eviscene/integrations/clock/__init__.py", |
| "eviscene/integrations/clock/system_clock.py", |
| "eviscene/integrations/gemini/__init__.py", |
| "eviscene/integrations/gemini/interactions.py", |
| "eviscene/integrations/gemini/runtime_metrics.py", |
| "eviscene/integrations/gemini/schema.py", |
| "eviscene/integrations/http_sessions.py", |
| "eviscene/integrations/http_transport.py", |
| "eviscene/integrations/llm/__init__.py", |
| "eviscene/integrations/llm/openai_compatible.py", |
| "eviscene/integrations/llm/prefix_cache.py", |
| "eviscene/integrations/ocr/__init__.py", |
| "eviscene/integrations/ocr/baidu.py", |
| "eviscene/integrations/search/__init__.py", |
| "eviscene/integrations/search/jina_reranker.py", |
| "eviscene/integrations/search/serper.py", |
| "eviscene/integrations/search/visual_search.py", |
| "eviscene/integrations/vlm/__init__.py", |
| "eviscene/integrations/vlm/factory.py", |
| "eviscene/integrations/vlm/openai_vlm.py", |
| "eviscene/integrations/vlm/qwen_vl.py", |
| "eviscene/orchestrator/__init__.py", |
| "eviscene/orchestrator/evidence_policy.py", |
| "eviscene/orchestrator/evidence_semantics.py", |
| "eviscene/orchestrator/investigation_models.py", |
| "eviscene/orchestrator/llm_backend.py", |
| "eviscene/orchestrator/pipeline.py", |
| "eviscene/orchestrator/react_runtime.py", |
| "eviscene/orchestrator/route_policy.py", |
| "eviscene/orchestrator/runtime_case.py", |
| "eviscene/orchestrator/runtime_events.py", |
| "eviscene/orchestrator/source_access.py", |
| "eviscene/orchestrator/source_provenance.py", |
| "eviscene/orchestrator/stage_native.py", |
| "eviscene/orchestrator/stage_runner.py", |
| "eviscene/orchestrator/stage_schemas.py", |
| "eviscene/orchestrator/stage_tools.py", |
| "eviscene/orchestrator/stage_types.py", |
| "eviscene/orchestrator/state.py", |
| "eviscene/orchestrator/tool_ablation.py", |
| "eviscene/orchestrator/tool_cache.py", |
| "eviscene/orchestrator/tool_execution.py", |
| "eviscene/orchestrator/tool_health.py", |
| "eviscene/orchestrator/tool_registry.py", |
| "eviscene/orchestrator/tool_result.py", |
| "eviscene/orchestrator/unified_prompts.py", |
| "eviscene/provider_profiles.py", |
| "eviscene/redaction.py", |
| "eviscene/render_trace_html.py", |
| "eviscene/storage.py", |
| "eviscene/tools/__init__.py", |
| "eviscene/tools/base.py", |
| "eviscene/tools/check_consistency.py", |
| "eviscene/tools/compare_reference.py", |
| "eviscene/tools/context_memory.py", |
| "eviscene/tools/count_objects.py", |
| "eviscene/tools/crop_and_inspect.py", |
| "eviscene/tools/crop_and_search.py", |
| "eviscene/tools/current_time.py", |
| "eviscene/tools/focused_visual_inspection.py", |
| "eviscene/tools/ocr_with_position.py", |
| "eviscene/tools/perceive_scene.py", |
| "eviscene/tools/reverse_image_search.py", |
| "eviscene/tools/text_image_search.py", |
| "eviscene/tools/text_search.py", |
| "eviscene/tools/vision_utils.py", |
| "eviscene/tools/visit.py", |
| "eviscene/tools/visual_anomaly.py", |
| "eviscene/trace_viewer.py", |
| "eviscene/trajectory/__init__.py", |
| "eviscene/trajectory/exporter.py", |
| "eviscene/trajectory/media_projection.py", |
| "eviscene/trajectory/perception_exporter.py", |
| "eviscene/trajectory/reference_chain.py", |
| "eviscene/trajectory/schema.py", |
| "eviscene/trajectory/scoring.py", |
| "eviscene/trajectory/semantic_reward.py", |
| "eviscene/trajectory/sft_eligibility.py", |
| "eviscene/workflow.py", |
| "pyproject.toml", |
| "scripts/__init__.py", |
| "scripts/audit_agent_private_gold.py", |
| "scripts/audit_direct_qa_baseline.py", |
| "scripts/audit_real_trace.py", |
| "scripts/check_release.py", |
| "scripts/collect_psd_rollouts.py", |
| "scripts/prepare_psd_paired_eval.py", |
| "scripts/prepare_runtime_input.py", |
| "scripts/run_agent_test_rollout.py", |
| "scripts/run_direct_qa_baseline.py", |
| "scripts/run_psd_judge.py", |
| "scripts/run_psd_repair_driver.py", |
| "scripts/server/retry_stream_agent_judge_failures.py", |
| "scripts/server/stream_agent_judges.py", |
| "scripts/summarize_agent_test_evaluation.py", |
| "scripts/trajectory/__init__.py", |
| "scripts/trajectory/build_sft_case_split.py", |
| "scripts/trajectory/build_sft_training_package.py", |
| "scripts/trajectory/export_dataset.py", |
| "scripts/trajectory/run_teacher_rollout_autopilot.py", |
| "scripts/trajectory/stage_accepted_teacher_release.py", |
| "training/LICENSE", |
| "training/MANIFEST.in", |
| "training/NOTICE", |
| "training/README.md", |
| "training/eviscene_training/__init__.py", |
| "training/eviscene_training/__main__.py", |
| "training/eviscene_training/_repo_import.py", |
| "training/eviscene_training/artifact_receipts.py", |
| "training/eviscene_training/audit.py", |
| "training/eviscene_training/checkpoint_io.py", |
| "training/eviscene_training/checkpoint_selection.py", |
| "training/eviscene_training/checkpoints.py", |
| "training/eviscene_training/cli.py", |
| "training/eviscene_training/contracts.py", |
| "training/eviscene_training/encode_cache.py", |
| "training/eviscene_training/io.py", |
| "training/eviscene_training/long_context.py", |
| "training/eviscene_training/manifests.py", |
| "training/eviscene_training/perception.py", |
| "training/eviscene_training/policy.py", |
| "training/eviscene_training/policy_contract.py", |
| "training/eviscene_training/profile.py", |
| "training/eviscene_training/psd.py", |
| "training/eviscene_training/psd_candidate_binding.py", |
| "training/eviscene_training/psd_candidates.py", |
| "training/eviscene_training/psd_capture_semantics.py", |
| "training/eviscene_training/psd_case_pool.py", |
| "training/eviscene_training/psd_collection.py", |
| "training/eviscene_training/psd_collection_recovery.py", |
| "training/eviscene_training/psd_datums.py", |
| "training/eviscene_training/psd_deterministic_sampler.py", |
| "training/eviscene_training/psd_fast_length.py", |
| "training/eviscene_training/psd_fsdp_resume.py", |
| "training/eviscene_training/psd_gemini_judge.py", |
| "training/eviscene_training/psd_hf_teacher.py", |
| "training/eviscene_training/psd_infrastructure_retry.py", |
| "training/eviscene_training/psd_initialization.py", |
| "training/eviscene_training/psd_judge_media.py", |
| "training/eviscene_training/psd_materialization.py", |
| "training/eviscene_training/psd_media.py", |
| "training/eviscene_training/psd_modality.py", |
| "training/eviscene_training/psd_model_route.py", |
| "training/eviscene_training/psd_ms_swift.py", |
| "training/eviscene_training/psd_pipeline_plan.py", |
| "training/eviscene_training/psd_preflight.py", |
| "training/eviscene_training/psd_repair.py", |
| "training/eviscene_training/psd_repair_finalize.py", |
| "training/eviscene_training/psd_repair_runtime.py", |
| "training/eviscene_training/psd_repair_search.py", |
| "training/eviscene_training/psd_repair_storage.py", |
| "training/eviscene_training/psd_repair_verifier.py", |
| "training/eviscene_training/psd_repairs.py", |
| "training/eviscene_training/psd_resume.py", |
| "training/eviscene_training/psd_round.py", |
| "training/eviscene_training/psd_slate.py", |
| "training/eviscene_training/psd_slate_feedback.py", |
| "training/eviscene_training/psd_slate_search.py", |
| "training/eviscene_training/psd_source_completion.py", |
| "training/eviscene_training/psd_source_review.py", |
| "training/eviscene_training/psd_storage_admission.py", |
| "training/eviscene_training/psd_target_balance.py", |
| "training/eviscene_training/psd_topk.py", |
| "training/eviscene_training/resource_monitor.py", |
| "training/eviscene_training/rewards.py", |
| "training/eviscene_training/sft_loss_scale.py", |
| "training/eviscene_training/watchdog.py", |
| "training/pyproject.toml", |
| }) |
|
|
| ARTIFACT_SUFFIXES = { |
| ".jsonl", |
| ".jsonl.gz", |
| ".csv", |
| ".tsv", |
| ".parquet", |
| ".arrow", |
| ".safetensors", |
| ".pt", |
| ".pth", |
| ".ckpt", |
| ".bin", |
| ".zip", |
| ".tar", |
| ".tar.gz", |
| ".png", |
| ".jpg", |
| ".jpeg", |
| ".gif", |
| ".webp", |
| ".pdf", |
| ".log", |
| ".whl", |
| ".pyc", |
| ".pyo", |
| } |
| PATTERNS = { |
| "access token": re.compile( |
| r"(?:hf_[A-Za-z0-9]{25,}|gh[pousr]_[A-Za-z0-9]{30,}" |
| r"|github_pat_[A-Za-z0-9_]{40,}|AIza[\w-]{30,}" |
| r"|sk-(?:proj-)?[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16})" |
| ), |
| "private key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), |
| "local user path": re.compile( |
| r"[A-Za-z]:[\\/]+Users[\\/]+(?!<|YOUR_)[\w.-]+[\\/]", re.I |
| ), |
| } |
| ALLOWED_ASSETS = {"assets/architecture.webp"} |
| FORBIDDEN_PARTS = { |
| "tests", |
| "operations", |
| "private", |
| "evaluation", |
| "runs", |
| "checkpoints", |
| "datasets", |
| "weights", |
| "models", |
| "data", |
| "outputs", |
| "cache", |
| ".cache", |
| "logs", |
| "traces", |
| "artifacts", |
| "exports", |
| "__pycache__", |
| ".pytest_cache", |
| ".ruff_cache", |
| ".venv", |
| ".git", |
| } |
|
|
|
|
| def _allowed_path(path: str, *, distribution: bool) -> bool: |
| if path in RELEASE_FILES: |
| return True |
| if not distribution: |
| return False |
| if path.startswith("eviscene_training/"): |
| return "training/" + path in RELEASE_FILES |
| if path in {"PKG-INFO", "setup.cfg"}: |
| return True |
| return bool(re.fullmatch( |
| r"(?:eviscene|eviscene_training)(?:-[0-9][\w.]*)?\.dist-info/" |
| r"(?:METADATA|WHEEL|RECORD|entry_points\.txt|top_level\.txt|licenses/(?:LICENSE|NOTICE))" |
| r"|(?:eviscene|eviscene_training)\.egg-info/" |
| r"(?:PKG-INFO|SOURCES\.txt|dependency_links\.txt|entry_points\.txt|requires\.txt|top_level\.txt)", |
| path, |
| )) |
|
|
|
|
| def inspect_file(path: str, content: bytes, *, distribution: bool = False) -> list[str]: |
| reasons = [] |
| normalized = path.replace("\\", "/") |
| relative = PurePosixPath(normalized) |
| if relative.is_absolute() or ".." in relative.parts or ":" in normalized: |
| reasons.append("invalid release path") |
| if not _allowed_path(normalized, distribution=distribution): |
| reasons.append("not in release file allowlist") |
| parts = tuple(part.lower() for part in relative.parts) |
| if ( |
| any(part in FORBIDDEN_PARTS for part in parts) |
| or relative.name.lower().startswith(".env") |
| or path.lower().endswith((".pem", ".key")) |
| ): |
| reasons.append("private/local file") |
| if ( |
| any(path.lower().endswith(suffix) for suffix in ARTIFACT_SUFFIXES) |
| and path not in ALLOWED_ASSETS |
| ): |
| reasons.append("non-code payload") |
| if path not in ALLOWED_ASSETS: |
| text = content.decode("utf-8", errors="replace") |
| reasons.extend( |
| name for name, pattern in PATTERNS.items() if pattern.search(text) |
| ) |
| return reasons |
|
|
|
|
| def inspect_distributions(root: Path) -> None: |
| failures = [] |
| archives = sorted(root.glob("dist/*.whl")) + sorted(root.glob("dist/*.tar.gz")) |
| archives += sorted(root.glob("training/dist/*.whl")) |
| archives += sorted(root.glob("training/dist/*.tar.gz")) |
| if not archives: |
| raise SystemExit("No built distributions to inspect") |
| for archive in archives: |
| if archive.suffix == ".whl": |
| with zipfile.ZipFile(archive) as package: |
| entries = [(name, package.read(name)) for name in package.namelist() |
| if not name.endswith("/")] |
| else: |
| with tarfile.open(archive, "r:gz") as package: |
| entries = [(str(PurePosixPath(member.name).relative_to( |
| PurePosixPath(member.name).parts[0])), package.extractfile(member).read()) |
| for member in package.getmembers() if member.isfile()] |
| for name, content in entries: |
| failures.extend(f"{archive.name}: {name}: {reason}" |
| for reason in inspect_file(name, content, distribution=True)) |
| if failures: |
| raise SystemExit("\n".join(failures)) |
| print(f"Distribution hygiene passed ({len(archives)} archives)") |
|
|
|
|
| def main() -> None: |
| import argparse |
|
|
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--distributions", action="store_true", |
| help="Inspect built wheels and source archives instead of the Git index") |
| args = parser.parse_args() |
| root = Path(__file__).resolve().parents[1] |
| if args.distributions: |
| inspect_distributions(root) |
| return |
| names = list( |
| filter( |
| None, |
| subprocess.check_output(["git", "ls-files", "-z", "--cached"], cwd=root) |
| .decode("utf-8") |
| .split("\0"), |
| ) |
| ) |
| |
| queries = "".join(":" + name + "\n" for name in names).encode("utf-8") |
| batch = subprocess.run( |
| ["git", "cat-file", "--batch"], |
| cwd=root, |
| input=queries, |
| stdout=subprocess.PIPE, |
| check=True, |
| ) |
| stream = io.BytesIO(batch.stdout) |
| failures = [] |
| for name in names: |
| header = stream.readline().split() |
| if len(header) != 3 or header[1] != b"blob": |
| raise SystemExit(f"Cannot inspect staged blob: {name}") |
| content = stream.read(int(header[2])) |
| if stream.read(1) != b"\n": |
| raise SystemExit("Malformed Git blob stream") |
| failures.extend(f"{name}: {reason}" for reason in inspect_file(name, content)) |
| if failures: |
| raise SystemExit("\n".join(failures)) |
| print(f"Release hygiene passed ({len(names)} tracked files)") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
