File size: 7,231 Bytes
48ecd01 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 | #!/usr/bin/env python3
"""Upload FRANKENSTALLM: model, eval reports, source code, and data scripts to Hugging Face.
Usage:
huggingface-cli login
# 모델 + README + 평가 결과 + 보고서
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --create-pr
# 위 + 소스 코드 + 데이터 스크립트 (모델/데이터/소스 전부)
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --with-source --with-data --create-pr
# 평가·보고서만
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --readme-only --create-pr
"""
import argparse
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
HF_CHECKPOINT = PROJECT_ROOT / "outputs" / "hf_checkpoint-best-fixed"
REPORTS_DIR = PROJECT_ROOT / "reports"
EVAL_RESULTS_DIR = PROJECT_ROOT / "eval" / "results" / "frankenstallm-3b-v2"
DATA_DIR = PROJECT_ROOT / "data"
SOURCE_DIRS = ["train", "model", "configs", "scripts", "tokenizer", "eval"]
def main():
parser = argparse.ArgumentParser(description="Upload model, eval reports, source, and data scripts to Hugging Face")
parser.add_argument("--repo-id", type=str, required=True, help="e.g. pathcosmos/frankenstallm")
parser.add_argument("--readme-only", action="store_true", help="Only push README + eval results + reports (no model)")
parser.add_argument("--create-pr", action="store_true", help="Create a Pull Request instead of pushing to main")
parser.add_argument("--with-source", action="store_true", help="Upload full source code (train, eval, model, configs, scripts, tokenizer)")
parser.add_argument("--with-data", action="store_true", help="Upload data scripts and DATA_README (no .bin files)")
args = parser.parse_args()
create_pr = getattr(args, "create_pr", False)
try:
from huggingface_hub import HfApi, create_repo
except ImportError:
print("Install: pip install huggingface_hub")
raise SystemExit(1)
api = HfApi()
# 레포 없으면 생성
# 레포가 없으면 생성 (본인 계정일 때만 성공)
try:
create_repo(args.repo_id, repo_type="model", exist_ok=True)
except Exception as e:
print(f"Note: create_repo skipped (use Hugging Face website to create repo if needed): {e}")
if not args.readme_only:
if not HF_CHECKPOINT.exists():
print(f"Checkpoint not found: {HF_CHECKPOINT}")
raise SystemExit(1)
print(f"Uploading model from {HF_CHECKPOINT} ...")
api.upload_folder(
folder_path=str(HF_CHECKPOINT),
repo_id=args.repo_id,
repo_type="model",
create_pr=create_pr,
)
print("Model upload done.")
# README는 체크포인트 폴더 것 사용 (이미 평가 요약 포함)
readme_src = HF_CHECKPOINT / "README.md"
if readme_src.exists():
print("Pushing README (model card) ...")
api.upload_file(
path_or_fileobj=str(readme_src),
path_in_repo="README.md",
repo_id=args.repo_id,
repo_type="model",
create_pr=create_pr,
)
print("README upload done.")
else:
print("No README.md in checkpoint dir; skipping README push.")
# 평가 결과 JSON
results_json = EVAL_RESULTS_DIR / "ollama_benchmark_results.json"
if results_json.exists():
print("Pushing ollama_benchmark_results.json ...")
api.upload_file(
path_or_fileobj=str(results_json),
path_in_repo="eval/ollama_benchmark_results.json",
repo_id=args.repo_id,
repo_type="model",
create_pr=create_pr,
)
print("Eval results upload done.")
# 배포·평가 보고서 (상세 기록)
for name, src in [
("2026-03-09_GGUF_DEPLOYMENT_AND_EVAL_REPORT.md", REPORTS_DIR / "2026-03-09_GGUF_DEPLOYMENT_AND_EVAL_REPORT.md"),
("2026-03-09_ORPO_EVALUATION_REPORT.md", REPORTS_DIR / "2026-03-09_ORPO_EVALUATION_REPORT.md"),
]:
if src.exists():
print(f"Pushing {name} ...")
api.upload_file(
path_or_fileobj=str(src),
path_in_repo=f"eval_reports/{name}",
repo_id=args.repo_id,
repo_type="model",
create_pr=create_pr,
)
print("Reports upload done.")
# ---------- 소스 코드 (--with-source) ----------
if getattr(args, "with_source", False):
print("Uploading source code ...")
ignore_common = ["**/__pycache__/**", "**/*.pyc", "**/.DS_Store"]
for dirname in ["train", "model", "configs", "scripts", "tokenizer"]:
src_dir = PROJECT_ROOT / dirname
if src_dir.exists():
api.upload_folder(
folder_path=str(src_dir),
path_in_repo=f"source/{dirname}",
repo_id=args.repo_id,
repo_type="model",
ignore_patterns=ignore_common,
create_pr=create_pr,
)
print(f" source/{dirname}/ done.")
# eval: outputs, results 제외 (대용량)
eval_dir = PROJECT_ROOT / "eval"
if eval_dir.exists():
api.upload_folder(
folder_path=str(eval_dir),
path_in_repo="source/eval",
repo_id=args.repo_id,
repo_type="model",
ignore_patterns=ignore_common + ["**/outputs/**", "**/results/**", "**/.compile_cache/**"],
create_pr=create_pr,
)
print(" source/eval/ done.")
# 루트 문서
for name in ["README.md", "CLAUDE.md", "requirements.txt", "PROGRESS.md"]:
src_file = PROJECT_ROOT / name
if src_file.exists():
api.upload_file(
path_or_fileobj=str(src_file),
path_in_repo=f"source/{name}",
repo_id=args.repo_id,
repo_type="model",
create_pr=create_pr,
)
for p in PROJECT_ROOT.glob("PLAN_*.md"):
api.upload_file(
path_or_fileobj=str(p),
path_in_repo=f"source/{p.name}",
repo_id=args.repo_id,
repo_type="model",
create_pr=create_pr,
)
print("Source upload done.")
# ---------- 데이터 스크립트 (--with-data, .bin 제외) ----------
if getattr(args, "with_data", False) and DATA_DIR.exists():
print("Uploading data scripts (no .bin) ...")
api.upload_folder(
folder_path=str(DATA_DIR),
path_in_repo="data",
repo_id=args.repo_id,
repo_type="model",
ignore_patterns=[
"**/*.bin",
"**/*.chunk*",
"**/__pycache__/**",
"**/code/**",
"**/*.pyc",
],
create_pr=create_pr,
)
print("Data scripts upload done.")
print(f"Done. https://huggingface.co/{args.repo_id}")
if __name__ == "__main__":
main()
|