Upload folder using huggingface_hub

README.md

@@ -594,6 +594,7 @@ upload_model_to_huggingface.py
 
 ```python
 
+#!/usr/bin/env python3
 import os
 from pathlib import Path
 
@@ -603,18 +604,19 @@ from huggingface_hub import HfApi, create_repo, upload_folder
 # 실제로 만들 Hugging Face 모델 repo ID (예시)
 REPO_ID = "AeiROBOT/SenseVoice-Small-ko"  # <-- 원하는 이름으로 수정
 
-# 업로드할 로컬 폴더 
-MODEL_DIR = Path("/home/khw/workspace/STTProject/SenseVoice-Small-ko")
+# 업로드할 로컬 폴더 (학습 결과)
+MODEL_DIR = Path("/home/khw/Workspace/SenseVoice/outputs")
 
 # 로컬에 있는 model.py를 함께 올리고 싶으면 (FunASR/SenseVoice용)
 # outputs 안에 이미 복사해 두었으면 생략 가능
-
+EXTRA_FILES = [
+    Path("/home/khw/Workspace/SenseVoice/model.py"),  # 없으면 주석 처리
+]
 
 
 def main():
     # 1) 토큰 가져오기 (환경변수 사용 권장)
     # 미리 export HUGGINGFACE_HUB_TOKEN=hf_xxx 하기
-    # 미리 export HF_HUB_DOWNLOAD_TIMEOUT=3600 하기 (업로드/다운로드 시간 늘리기)
     token = os.environ.get("HUGGINGFACE_HUB_TOKEN")
     if token is None:
         raise RuntimeError(
@@ -634,6 +636,26 @@ def main():
         repo_type="model",
     )
 
+    # 3) 추가로 올릴 파일(model.py 등)을 outputs 안으로 복사 (선택)
+    #    -> HF 리포 root에 README.md, model.pt, config.yaml, configuration.json, model.py 등이 같이 있도록 추천
+    for extra in EXTRA_FILES:
+        if extra.is_file():
+            target = MODEL_DIR / extra.name
+            if not target.exists():
+                print(f"[info] copy {extra} -> {target}")
+                target.write_bytes(extra.read_bytes())
+        else:
+            print(f"[warn] extra file not found: {extra}")
+
+    # 3-1) 모델 카드(README) 업로드: 실행 위치(CWD)의 README_huggingface.md를 outputs/README.md로 복사
+    #      - HF 모델 허브는 repo 루트의 README.md를 모델 카드로 인식합니다.
+    readme_src = Path.cwd() / "README_huggingface.md"
+    readme_dst = MODEL_DIR / "README.md"
+    if readme_src.is_file():
+        print(f"[info] copy {readme_src} -> {readme_dst}")
+        readme_dst.write_text(readme_src.read_text(encoding="utf-8"), encoding="utf-8")
+    else:
+        print(f"[warn] README_huggingface.md not found in CWD: {Path.cwd()}")
 
     # 4) 폴더 통째로 업로드
     print(f"[info] uploading folder: {MODEL_DIR} -> {REPO_ID}")
@@ -643,10 +665,10 @@ def main():
         path_in_repo=".",         # 리포 루트에 그대로 올리기
         token=token,
         repo_type="model",
-        # ignore_patterns=[
-        #     "model.pt.ep*",   # 체크포인트들 제외
-        #     "*.pt.ep*",       # 혹시 다른 파일명도 비슷하게 찍히면 같이 제외
-        # ],
+        ignore_patterns=[
+            "model.pt.ep*",   # 체크포인트들 제외
+            "*.pt.ep*",       # 혹시 다른 파일명도 비슷하게 찍히면 같이 제외
+        ],
     )
 
     print("[done] uploaded to:", f"https://huggingface.co/{REPO_ID}")
@@ -655,4 +677,5 @@ def main():
 if __name__ == "__main__":
     main()
 
+
 ```

config.yaml

@@ -41,7 +41,7 @@ dataset_conf:
   batch_sampler: BatchSampler
   data_split_num: 1
   batch_type: token
-  batch_size: 6000
+  batch_size: 32000
   max_token_length: 2000
   min_token_length: 60
   max_source_length: 2000
@@ -49,16 +49,16 @@ dataset_conf:
   max_target_length: 200
   min_target_length: 0
   shuffle: true
-  num_workers: 4
+  num_workers: 16
   sos: 1
   eos: 2
   IndexDSJsonl: IndexDSJsonl
   retry: 20
-  sort_size: 1024
+  sort_size: 4096
 train_conf:
   accum_grad: 1
   grad_clip: 5
-  max_epoch: 300
+  max_epoch: 200
   keep_nbest_models: 20
   avg_nbest_model: 10
   log_interval: 1
@@ -72,7 +72,7 @@ optim_conf:
   lr: 0.0002
 scheduler: warmuplr
 scheduler_conf:
-  warmup_steps: 25000
+  warmup_steps: 2000
 specaug: SpecAugLFR
 specaug_conf:
   apply_time_warp: false

model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f534febbb6bf61d7b81027e45ba2ab8e6b83706e20368514c95a2146e4bf999
-size 2809141646
+oid sha256:9ca020b5521f1faa9ac1e72b8031df29e087112bfcb1ff3fdfc73059e71341b4
+size 2809141902

model.pt.best

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f534febbb6bf61d7b81027e45ba2ab8e6b83706e20368514c95a2146e4bf999
-size 2809141646
+oid sha256:9ca020b5521f1faa9ac1e72b8031df29e087112bfcb1ff3fdfc73059e71341b4
+size 2809141902

tensorboard/events.out.tfevents.1768816105.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f6426ae5902fb7b1fbf457b08303a1d6f2f941d8cfa93db3209bda8838456a9
+size 42788368

tensorboard/events.out.tfevents.1768835645.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:469222f234780ca278d7fa6996574352d0ed9594bf8fade19cdf2ee7878ff5a6
+size 681556

tensorboard/events.out.tfevents.1768836232.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be8bd5e533a77be3e5b947e76621d6834a4e03a8712bd40f4473bc54ab2bf3a9
+size 3418988

tensorboard/events.out.tfevents.1768840269.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f35048e894ad9f722ca053a375d8e4fed82abbb2347c9300adc17e28051de09
+size 10314708

tensorboard/events.out.tfevents.1768862816.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02bbd925c0230b51aa640347b7a955cba9a90e773e9e0011389e5e95910436cb
+size 20926816

tensorboard/events.out.tfevents.1768880772.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2764a32313282d273e16752e2a707bdc8ec6a9c0828ab39daaccac293ab4b54
+size 26618567

tensorboard/events.out.tfevents.1768902675.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c5f4a781a543ddd0ae3df0b0b0b0d8f009a2c2bd8d473d43f7cde353d184ccd
+size 32222568

tensorboard/events.out.tfevents.1768951072.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1008384f288989af2bbbff0aa62559ac509e4f7713184e7d3e417d2eefb8e02e
+size 13351898

tensorboard/events.out.tfevents.1768969665.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7cbd9ba6318c94802ac3f4cd938009087dccaab54b7b815baf23ba9e613f90
+size 46889935

tensorboard/events.out.tfevents.1769042911.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc90dbe9f6b7bd03573d416ff01f8c04c7df2792b50785de2fb4caf36697589e
+size 2903564

tensorboard/events.out.tfevents.1769046565.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a70a3fee95da653e201d6bb66baba1756d4a4023a9ea8dd0b9f635bcf07b40
+size 374955

tensorboard/events.out.tfevents.1769047086.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f73323090bfb7096deda345036e1ed61ea4ece9f2bc542a1e2c3f6e239cfde91
+size 520192

tensorboard/events.out.tfevents.1769053292.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abba67941d21c81bded507c103ed5536b4d9572d7aa46a51b5f9565fe9008ba3
+size 8484400

tensorboard/events.out.tfevents.1769093025.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd9533ddd822dd36755969c4892ecf926eab3a001a69c92fe4a7c6b4ed00720d
+size 5609090

tensorboard/events.out.tfevents.1769179482.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb73e1615710038860ddb519099a82b7bc2e8ad21f42ea3bb1c955e1b79ae6e3
+size 5603370

tensorboard/events.out.tfevents.1769209818.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1c0196c8390fc83b418f07388603f3bbae8b544701b50faea07e7c386d50ac6
+size 5753520

tensorboard/events.out.tfevents.1769273521.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbcee1bdeb4d6ba65775e80523aeeb64d8b3d3d50c40482fad4baafe544972f5
+size 5601225

tensorboard/events.out.tfevents.1769341722.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04fd7231ff464e037d276dcafe55870d6e15d8cf1b23bad75422110e911ede74
+size 5606945

tensorboard/events.out.tfevents.1769380986.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c7cc3bfb7e81c11d0d0b9004583ad576a7444dd814af48da42d06bee4fb07f8
+size 5603370

tensorboard/events.out.tfevents.1769437862.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d8decfbbd2db6a404a581996a335cf5ad6fe5da30664af05a4aa14e442f5110
+size 2876780

tensorboard/events.out.tfevents.1769468740.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312c2bbfcf1c02bf032eeaf5fb09c0a2b6b1e56043c6357a45c8a827f79c15cb
+size 1438410

tensorboard/events.out.tfevents.1769529595.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a082fc822386bce1402871c201d679d31e0e849a3f038ad006f7d6ace70460
+size 14091305

tensorboard/events.out.tfevents.1769619507.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f883235bbbb0ac7e0f36019665dcf34c196a673efede6eae262f19943a560d59
+size 5601940

tensorboard/events.out.tfevents.1769699488.arobot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f1efe242ccf10bafb48b310c2f18d61d06c2bdcdb79dd113eea6f973cc5fac
+size 5753520