File size: 2,507 Bytes
c3e06eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | import sys
from pathlib import Path
# Add parent directory to path to locate 'config' module
sys.path.append(str(Path(__file__).parent.parent))
from config.settings import Settings
import os
from huggingface_hub import HfApi
def verify_space_access():
"""Verify access to the Space and its settings"""
api = HfApi(token=Settings.HF_TOKEN)
try:
# Check if we have write access
space_info = api.space_info(Settings.HF_SPACE)
print(f"Space runtime: {space_info.runtime}")
print(f"Space sdk: {space_info.sdk}")
# Try to list the contents to verify write access
contents = api.list_repo_files(
repo_id=Settings.HF_SPACE,
repo_type="space"
)
print(f"Space contents: {len(contents)} files")
return True
except Exception as e:
print(f"Error verifying space access: {str(e)}")
return False
def main():
"""Upload a specific large embedding file to HuggingFace Space using HfApi.upload_file"""
if not Settings.HF_TOKEN:
raise ValueError("HF_TOKEN not found in environment variables")
print("Verifying Space access...")
if not verify_space_access():
raise RuntimeError("Failed to verify Space access")
print("Starting upload process...")
# Define the file to upload
local_file_path = Path(
"./data/processed/embeddings/chroma/fade0013-ed4b-4928-b81b-7435145156dc/data_level0.bin"
)
if not local_file_path.exists():
raise FileNotFoundError(f"File {local_file_path} does not exist")
# Define the path in the repository
repo_file_path = "data/processed/embeddings/chroma/fade0013-ed4b-4928-b81b-7435145156dc/data_level0.bin"
api = HfApi(token=Settings.HF_TOKEN)
print(f"Uploading {local_file_path} to {Settings.HF_SPACE}/{repo_file_path}...")
try:
api.upload_file(
path_or_fileobj=str(local_file_path),
path_in_repo=repo_file_path,
repo_id=Settings.HF_SPACE,
repo_type="space",
token=Settings.HF_TOKEN,
)
print("Upload complete!")
except Exception as e:
print(f"Error during upload: {str(e)}")
if "storage" in str(e).lower():
print("\nNOTE: This might be a Git LFS limitation.")
print("Consider splitting the file into smaller chunks or contacting Hugging Face support.")
raise
if __name__ == "__main__":
main()
|