import sys from pathlib import Path # Add parent directory to path to locate 'config' module sys.path.append(str(Path(__file__).parent.parent)) from config.settings import Settings import os from huggingface_hub import HfApi def verify_space_access(): """Verify access to the Space and its settings""" api = HfApi(token=Settings.HF_TOKEN) try: # Check if we have write access space_info = api.space_info(Settings.HF_SPACE) print(f"Space runtime: {space_info.runtime}") print(f"Space sdk: {space_info.sdk}") # Try to list the contents to verify write access contents = api.list_repo_files( repo_id=Settings.HF_SPACE, repo_type="space" ) print(f"Space contents: {len(contents)} files") return True except Exception as e: print(f"Error verifying space access: {str(e)}") return False def main(): """Upload a specific large embedding file to HuggingFace Space using HfApi.upload_file""" if not Settings.HF_TOKEN: raise ValueError("HF_TOKEN not found in environment variables") print("Verifying Space access...") if not verify_space_access(): raise RuntimeError("Failed to verify Space access") print("Starting upload process...") # Define the file to upload local_file_path = Path( "./data/processed/embeddings/chroma/fade0013-ed4b-4928-b81b-7435145156dc/data_level0.bin" ) if not local_file_path.exists(): raise FileNotFoundError(f"File {local_file_path} does not exist") # Define the path in the repository repo_file_path = "data/processed/embeddings/chroma/fade0013-ed4b-4928-b81b-7435145156dc/data_level0.bin" api = HfApi(token=Settings.HF_TOKEN) print(f"Uploading {local_file_path} to {Settings.HF_SPACE}/{repo_file_path}...") try: api.upload_file( path_or_fileobj=str(local_file_path), path_in_repo=repo_file_path, repo_id=Settings.HF_SPACE, repo_type="space", token=Settings.HF_TOKEN, ) print("Upload complete!") except Exception as e: print(f"Error during upload: {str(e)}") if "storage" in str(e).lower(): print("\nNOTE: This might be a Git LFS limitation.") print("Consider splitting the file into smaller chunks or contacting Hugging Face support.") raise if __name__ == "__main__": main()