Upload folder using huggingface_hub

src/debug_paths.py

@@ -0,0 +1,20 @@
+import os
+import glob
+from config import Config
+
+print(f"Current Working Directory: {os.getcwd()}")
+print(f"Config.RESULTS_DIR: {Config.RESULTS_DIR}")
+
+# Debug Finding Checkpoints
+candidates = glob.glob(os.path.join(Config.RESULTS_DIR, "checkpoint-*"))
+print(f"Found {len(candidates)} candidates:")
+for c in candidates:
+    print(f" - {c}")
+
+if not candidates:
+    # Try relative path manual
+    print("Trying relative path './results/checkpoint-*'...")
+    candidates = glob.glob("./results/checkpoint-*")
+    print(f"Found {len(candidates)} candidates via relative:")
+    for c in candidates:
+        print(f" - {c}")

src/upload_to_hf.py

@@ -1,85 +1,94 @@
 import os
 import sys
 import glob
+import shutil
 from huggingface_hub import HfApi, create_repo, upload_folder
 from config import Config
 
 def main():
-    print("🚀 开始上传全套项目 (代码 + 模型 + 数据) 到 Hugging Face...")
+    print("🚀 开始重新上传 (Code + Model Combined)...")
     
-    # 1. 检测登录
     api = HfApi()
     try:
         user_info = api.whoami()
         username = user_info['name']
-        print(f"✅ 当前登录用户: {username}")
-    except Exception as e:
-        print("❌ 未检测到登录状态！请先运行 'huggingface-cli login'")
+        print(f"✅ User: {username}")
+    except:
+        print("❌ Please login first.")
         return
 
-    # 定义仓库名称
     model_repo_id = f"{username}/sentiment-analysis-bert-finetuned"
-    dataset_repo_id = f"{username}/sentiment-analysis-dataset-processed"
-
-    # ========================================================
-    # 2. 上传模型与代码 (合并到一个 Model Repo)
-    # ========================================================
-    print(f"\n📦 正在准备模型仓库: {model_repo_id}")
-    create_repo(repo_id=model_repo_id, repo_type="model", exist_ok=True)
     
-    # A. 上传最新模型权重 (到根目录)
+    # 1. 准备临时上传目录 (Merge Strategy)
+    # create a temp dir to combine everything before uploading to ensure structure is perfect
+    upload_dir = "hf_upload_staging"
+    if os.path.exists(upload_dir):
+        shutil.rmtree(upload_dir)
+    os.makedirs(upload_dir)
+    
+    print(f"📦 Staging files to {upload_dir}...")
+    
+    # A. Copy Project Code (src, notebook, etc)
+    # We want these at the root
+    items_to_copy = ["src", "notebooks", "docs", "demo", "README.md", "requirements.txt", "*.pptx"]
+    for pattern in items_to_copy:
+        for item in glob.glob(pattern):
+            dest = os.path.join(upload_dir, item)
+            if os.path.isdir(item):
+                shutil.copytree(item, dest, dirs_exist_ok=True)
+            else:
+                shutil.copy2(item, dest)
+                
+    # B. Copy Model Weights (Flattened to root)
+    # Find latest checkpoint
     candidates = glob.glob(os.path.join(Config.RESULTS_DIR, "checkpoint-*"))
+    # Filter out zip files if any
+    candidates = [c for c in candidates if os.path.isdir(c)]
+    
     if candidates:
         candidates.sort(key=os.path.getmtime)
         latest_ckpt = candidates[-1]
-        print(f"   ➡️  发现最新模型: {latest_ckpt}")
-        print("   ⬆️  正在上传模型权重 (model.safetensors 等)... 这可能需要几分钟")
+        print(f"✅ Found latest checkpoint: {latest_ckpt}")
+        
+        # Files to copy from checkpoint to root
+        model_files = ["config.json", "model.safetensors", "pytorch_model.bin", "tokenizer.json", "vocab.txt", "tokenizer_config.json", "special_tokens_map.json"]
+        
+        found_weights = False
+        for fname in os.listdir(latest_ckpt):
+            if fname in model_files or fname.endswith(".safetensors") or fname.endswith(".bin"):
+                 # Copy to root of staging
+                 shutil.copy2(os.path.join(latest_ckpt, fname), os.path.join(upload_dir, fname))
+                 if "model" in fname or "pytorch" in fname:
+                     found_weights = True
         
-        upload_folder(
-            folder_path=latest_ckpt,
-            repo_id=model_repo_id,
-            repo_type="model",
-            ignore_patterns=["optimizer.pt", "scheduler.pt", "rng_state.pth"] # 剔除大文件
-        )
+        if not found_weights:
+            print("⚠️ WARNING: No model weights (.bin or .safetensors) found in checkpoint!")
     else:
-        print("   ⚠️  未找到 checkpoint，跳过模型权重上传。")
+        print("❌ No checkpoints found in results/!")
 
-    # B. 上传项目代码 (到根目录)
-    print("   ⬆️  正在上传项目代码 (src, notebook, docs...)...")
-    # 我们上传当前目录 '.'，但要排除 data, results, venv 等杂物
+    # 2. Upload the Staged Directory
+    print(f"\n⬆️ Uploading entire {upload_dir} to https://huggingface.co/{model_repo_id}")
+    create_repo(repo_id=model_repo_id, repo_type="model", exist_ok=True)
+    
     upload_folder(
-        folder_path=".",
+        folder_path=upload_dir,
         repo_id=model_repo_id,
-        repo_type="model",
-        ignore_patterns=[
-            "results/*", "data/*", "__pycache__", "*.pyc", ".git", ".DS_Store", 
-            "env", "venv", ".venv", ".ipynb_checkpoints", "**/*.pt"
-        ]
+        repo_type="model"
     )
-    print(f"✅ 代码与模型已同步: https://huggingface.co/{model_repo_id}")
-
-    # ========================================================
-    # 3. 上传数据集
-    # ========================================================
+    
+    # Cleanup
+    shutil.rmtree(upload_dir)
+    print("🎉 Done! Model and Code are now together in the repo root.")
+    
+    # Check dataset
+    dataset_repo_id = f"{username}/sentiment-analysis-dataset-processed"
     data_path = os.path.join(Config.DATA_DIR, "processed_dataset")
     if os.path.exists(data_path):
-        print(f"\n📚 正在准备数据集仓库: {dataset_repo_id}")
+        print(f"\n⬆️ Uploading dataset to https://huggingface.co/datasets/{dataset_repo_id}")
         create_repo(repo_id=dataset_repo_id, repo_type="dataset", exist_ok=True)
-        
-        print("   ⬆️  正在上传数据集...")
-        upload_folder(
-            folder_path=data_path,
-            repo_id=dataset_repo_id,
-            repo_type="dataset"
-        )
-        print(f"✅ 数据集已同步: https://huggingface.co/datasets/{dataset_repo_id}")
-    else:
-        print("⚠️ 未找到 data/processed_dataset，跳过数据集上传。")
-
-    print("\n🎉 全部上传任务完成！")
-
+        upload_folder(folder_path=data_path, repo_id=dataset_repo_id, repo_type="dataset")
+    
 if __name__ == "__main__":
-    # 解决相对导入
     current_dir = os.path.dirname(os.path.abspath(__file__))
     parent_dir = os.path.dirname(current_dir)
     sys.path.append(parent_dir)