Add tool to validate datasets on HF Hub

app.py

@@ -79,34 +79,74 @@ def Dataset_Creator_Function(dataset_name: str, conversation_data: str) -> str:
         
         print(f"Creating dataset: {repo_id}")
         
-        # Create a simple dataset from a dictionary
+        # Create a simple dataset from a dictionary with a train split
         data = {
             "text": [conversation_data],
             "timestamp": [datetime.datetime.now().isoformat()],
             "id": [str(uuid.uuid4())]
         }
         
-        # Create the dataset directly
+        # Explicitly ensure the repository exists
+        try:
+            hf_api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True)
+            print(f"Repository {repo_id} created or already exists")
+        except Exception as repo_error:
+            print(f"Note about repository creation: {str(repo_error)}")
+        
+        # Create the dataset and push to hub
         dataset = Dataset.from_dict(data)
         
-        # Push to Hugging Face Hub using the simpler method from documentation
+        # Push to Hugging Face Hub using the standard method
         dataset.push_to_hub(
-            repo_id=repo_id,  # Include username in repo_id
-            token=api_key,    # Pass token explicitly
-            private=False     # Make it public
+            repo_id=repo_id,
+            token=api_key,
+            private=False
         )
         
         # Generate the URL for the dataset
         dataset_url = f"https://huggingface.co/datasets/{repo_id}"
         print(f"Dataset successfully pushed to: {dataset_url}")
         
-        return f"Successfully created dataset at {dataset_url}"
+        # Verify dataset exists by checking the API
+        verify_result = verify_dataset_exists(repo_id)
+        if verify_result["exists"]:
+            return f"Successfully created dataset at {dataset_url}"
+        else:
+            return f"Dataset was uploaded, but it may take a few minutes to appear at {dataset_url}. Error: {verify_result['message']}"
+            
     except Exception as e:
         import traceback
         error_trace = traceback.format_exc()
         print(f"Dataset creation error: {str(e)}\n{error_trace}")
         return f"Error creating dataset: {str(e)}\n\nTroubleshooting tips:\n1. Verify your HF_API_KEY is valid\n2. Try a simpler dataset name with only letters and underscores\n3. Check your permissions for the Misfits-and-Machines organization"
 
+def verify_dataset_exists(repo_id: str) -> dict:
+    """Verify that a dataset exists and is valid on the Hugging Face Hub.
+    
+    Args:
+        repo_id: Full repository ID in format "username/dataset_name"
+    
+    Returns:
+        Dict with "exists" boolean and "message" string
+    """
+    try:
+        # Check if dataset exists using the datasets-server API
+        api_url = f"https://datasets-server.huggingface.co/is-valid?dataset={repo_id}"
+        response = requests.get(api_url)
+        
+        # Parse the response
+        if response.status_code == 200:
+            data = response.json()
+            # If any of these are True, the dataset exists in some form
+            if data.get("viewer", False) or data.get("preview", False):
+                return {"exists": True, "message": "Dataset is valid and accessible"}
+            else:
+                return {"exists": False, "message": "Dataset exists but may not be fully processed yet"}
+        else:
+            return {"exists": False, "message": f"API returned status code {response.status_code}"}
+    except Exception as e:
+        return {"exists": False, "message": f"Error verifying dataset: {str(e)}"}
+
 @tool
 def Dataset_Creator_Tool(dataset_name: str, conversation_data: str) -> str:
     """A tool that posts a new dataset of the current conversation to Hugging Face.
@@ -129,6 +169,30 @@ def Dataset_Creator_Tool(dataset_name: str, conversation_data: str) -> str:
         error_trace = traceback.format_exc()
         return f"Error using Dataset Creator tool: {str(e)}\n{error_trace}"
 
+@tool
+def Check_Dataset_Validity(dataset_name: str) -> str:
+    """A tool that checks if a dataset exists and is valid on Hugging Face.
+    
+    Args:
+        dataset_name: Name of the dataset to check (with or without organization prefix)
+    
+    Returns:
+        Status message about the dataset validity
+    """
+    try:
+        # Ensure the dataset name has the organization prefix
+        if "/" not in dataset_name:
+            dataset_name = f"Misfits-and-Machines/{dataset_name.replace(' ', '_').lower()}"
+        
+        # Check dataset validity
+        result = verify_dataset_exists(dataset_name)
+        
+        if result["exists"]:
+            return f"Dataset '{dataset_name}' exists and is valid. You can access it at https://huggingface.co/datasets/{dataset_name}"
+        else:
+            return f"Dataset '{dataset_name}' could not be verified: {result['message']}. It may still be processing or may not exist."
+    except Exception as e:
+        return f"Error checking dataset validity: {str(e)}"
 
 @tool
 def get_current_time_in_timezone(timezone: str) -> str:
@@ -169,7 +233,8 @@ agent = CodeAgent(
         Sonar_Web_Search_Tool, 
         get_current_time_in_timezone, 
         image_generation_tool,
-        Dataset_Creator_Tool
+        Dataset_Creator_Tool,
+        Check_Dataset_Validity
     ],  
     max_steps=6,
     verbosity_level=1,