from huggingface_hub import login, upload_file, hf_hub_download, delete_file import os from dotenv import load_dotenv def manage_model(action="push"): load_dotenv() hf_key = os.getenv('HUGGING_FACE_API_KEY') login(hf_key) assert action in ["push", "pull", "delete"] model_name = "hallucination" user_name = "ToiTenBao" repo_id = f"{user_name}/{model_name}" folder_path = "./cc3m_checkpoints" file_name = "topk_32.0_32_vision_model.encoder.layers.{layer}.hook_resid_post_0.001_256_0.03125_42.ckpt" # file_name = "topk_32.0_32_text_decoder.bert.encoder.layer.{layer}.crossattention.self.hook_resid_pre_0.001_256_0.03125_42.ckpt" # folder_path = "./vis_file" # file_name = "sae_graph_75719_nonhal.html" # Removed leading '/' max_layers = 12 os.makedirs(folder_path, exist_ok=True) for layer in range(9, max_layers): base_name = file_name.format(layer=layer) remote_path = f"cc3m_checkpoints/{base_name}" local_path = os.path.join(folder_path, base_name) if action.lower() == "push": upload_file( path_or_fileobj=local_path, path_in_repo=remote_path, repo_id=repo_id, ) elif action.lower() == "pull": local_dir = os.path.dirname(folder_path) if folder_path != "." else "." hf_hub_download( repo_id=repo_id, filename=remote_path, local_dir=local_dir, ) elif action.lower() == "delete": try: delete_file( path_in_repo=remote_path, repo_id=repo_id, ) except Exception as e: print(f"Error deleting {remote_path}: {e}") continue else: raise ValueError("Action must be 'push', 'pull', or 'delete'") def delete_single_file(file_layer): load_dotenv() hf_key = os.getenv('HUGGING_FACE_API_KEY') login(hf_key) model_name = "hallucination" user_name = "ToiTenBao" repo_id = f"{user_name}/{model_name}" file_name = "topk_16.0_32_text_decoder.bert.encoder.layer.{layer}.attention.self.hook_resid_pre_0.001_256_0.0_42.ckpt" # Removed leading '/' base_name = file_name.format(layer=file_layer) remote_path = f"checkpoints/{base_name}" try: delete_file( path_in_repo=remote_path, repo_id=repo_id, ) print(f"Deleted {remote_path}") except Exception as e: print(f"Error deleting {remote_path}: {e}") if __name__ == "__main__": manage_model(action="pull")