Update backend/utils/analyze_prod_models.py

backend/utils/analyze_prod_models.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from pathlib import Path
 from huggingface_hub import HfApi
 from dotenv import load_dotenv
-from app.config.hf_config import HF_ORGANIZATION
+from app.config.hf_config import HF_TOKEN, HF_ORGANIZATION, HF_AGGREGATED
 
 # Get the backend directory path
 BACKEND_DIR = Path(__file__).parent.parent
@@ -22,22 +22,22 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 # Initialize Hugging Face API
-HF_TOKEN = os.getenv("HF_TOKEN")
-if not HF_TOKEN:
-    raise ValueError("HF_TOKEN not found in environment variables")
+# HF_TOKEN = os.getenv("HF_TOKEN")
+# if not HF_TOKEN:
+    # raise ValueError("HF_TOKEN not found in environment variables")
 api = HfApi(token=HF_TOKEN)
 
 def count_evaluated_models():
     """Count the number of evaluated models"""
     try:
         # Get dataset info
-        dataset_info = api.dataset_info(repo_id=f"{HF_ORGANIZATION}/greek-contents", repo_type="dataset")
+        dataset_info = api.dataset_info(repo_id=f"{HF_ORGANIZATION}/{HF_AGGREGATED}", repo_type="dataset")
         
         # Get file list
-        files = api.list_repo_files(f"{HF_ORGANIZATION}/greek-contents", repo_type="dataset")
+        files = api.list_repo_files(f"{HF_ORGANIZATION}/{HF_AGGREGATED}", repo_type="dataset")
         
         # Get last commit info
-        commits = api.list_repo_commits(f"{HF_ORGANIZATION}/greek-contents", repo_type="dataset")
+        commits = api.list_repo_commits(f"{HF_ORGANIZATION}/{HF_AGGREGATED}", repo_type="dataset")
         last_commit = next(commits, None)
         
         # Count lines in jsonl files
@@ -47,7 +47,7 @@ def count_evaluated_models():
                 try:
                     # Download file content
                     content = api.hf_hub_download(
-                        repo_id=f"{HF_ORGANIZATION}/greek-contents",
+                        repo_id=f"{HF_ORGANIZATION}/{HF_AGGREGATED}",
                         filename=file,
                         repo_type="dataset"
                     )