Upload src/download_from_hf.py with huggingface_hub

src/download_from_hf.py

@@ -12,7 +12,6 @@ def download_from_huggingface(repo_name, token):
     # Create necessary directories
     os.makedirs('cbow/checkpoints', exist_ok=True)
     os.makedirs('checkpoints', exist_ok=True)
-    os.makedirs('vocabulary', exist_ok=True)
     os.makedirs('data', exist_ok=True)
     os.makedirs('config', exist_ok=True)
     os.makedirs('src', exist_ok=True)
@@ -65,18 +64,18 @@ def download_from_huggingface(repo_name, token):
         except Exception as e:
             print(f"Error downloading {data_file}: {e}")
 
-    # Download vocabulary files
+    # Download all tokenizer files from cbow directory
     try:
-        vocab_files = snapshot_download(
+        cbow_files = snapshot_download(
             repo_id=repo_name,
             repo_type="model",
             token=token,
             local_dir="cbow",
-            allow_patterns="vocabulary/*.pkl"
+            allow_patterns="cbow/*.pkl"
         )
-        print("Downloaded vocabulary files")
+        print("Downloaded CBOW tokenizer files")
     except Exception as e:
-        print(f"Error downloading vocabulary files: {e}")
+        print(f"Error downloading CBOW tokenizer files: {e}")
 
     # Download configuration files
     config_files = ['sweep.yaml', 'requirements.txt']