Upload Base/Datasets/rag_mcp_sft/push_to_hf.py with huggingface_hub

Base/Datasets/rag_mcp_sft/push_to_hf.py

@@ -0,0 +1,31 @@
+import argparse
+import os
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Upload the RAG+MCP SFT dataset folder to Hugging Face")
+    parser.add_argument("--repo-id", default="ASTERIZER/LUNA-RAG-MCP-SFT-10M")
+    parser.add_argument("--folder", default=str(Path(__file__).resolve().parent))
+    parser.add_argument("--private", action="store_true")
+    args = parser.parse_args()
+
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        raise RuntimeError("HF_TOKEN is required in the environment to push the dataset")
+
+    api = HfApi(token=token)
+    api.create_repo(repo_id=args.repo_id, repo_type="dataset", private=args.private, exist_ok=True)
+    api.upload_folder(
+        repo_id=args.repo_id,
+        repo_type="dataset",
+        folder_path=args.folder,
+        path_in_repo=".",
+    )
+    print(f"uploaded_dataset repo_id={args.repo_id} folder={args.folder}")
+
+
+if __name__ == "__main__":
+    main()