Fix: disable progress bars and memory copy during base checkpoint download

scripts/train.py

@@ -183,6 +183,13 @@ def run_live_eval_previews(model, tokenizer, device):
 # ------------------------------------------------------------------------------
 def ensure_dataset_and_base_ckpt(data_dir: str, base_ckpt_path: str, token: str = None):
     os.makedirs(data_dir, exist_ok=True)
+    os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+    try:
+        from huggingface_hub.utils import disable_progress_bars
+        disable_progress_bars()
+    except Exception:
+        pass
+
     shards = [
         "train_tokens.npy", "train_labels.npy", "train_offsets.npy", "train_domains.npy",
         "val_tokens.npy", "val_labels.npy", "val_offsets.npy", "val_domains.npy",
@@ -196,14 +203,16 @@ def ensure_dataset_and_base_ckpt(data_dir: str, base_ckpt_path: str, token: str
             if not os.path.exists(target):
                 print(f"   • Fetching {s}...")
                 dl = hf_hub_download(repo_id="ViuAI/ViuTranslate-Data", filename=s, repo_type="dataset", token=token)
-                shutil.copy(dl, target)
+                if dl != target and not os.path.exists(target):
+                    shutil.copy(dl, target)
         print("   ✅ All dataset shards downloaded.")
 
     if not os.path.exists(base_ckpt_path):
-        print(f"\n🌐 Base checkpoint not found. Downloading base weights from ViuAI/ViuAI-500M...")
-        os.makedirs(os.path.dirname(base_ckpt_path), exist_ok=True)
+        print(f"\n🌐 Base checkpoint not found. Downloading base weights (~5.9GB) from ViuAI/ViuAI-500M...")
+        os.makedirs(os.path.dirname(base_ckpt_path) if os.path.dirname(base_ckpt_path) else ".", exist_ok=True)
         dl_b = hf_hub_download(repo_id="ViuAI/ViuAI-500M", filename="checkpoints/ckpt_latest.pt", token=token)
-        shutil.copy(dl_b, base_ckpt_path)
+        if dl_b != base_ckpt_path and not os.path.exists(base_ckpt_path):
+            shutil.copy(dl_b, base_ckpt_path)
         print("   ✅ Base checkpoint ready.")
 
 # ------------------------------------------------------------------------------