Upload folder using huggingface_hub

evaluate_atomic_matching.py

@@ -9,6 +9,8 @@ from litellm import completion
 import json
 import getpass
 import shutil
+import tempfile
+import subprocess
 
 # Initialize instructor client
 client = instructor.from_litellm(completion)
@@ -97,24 +99,34 @@ def process_questions(llm_model: str, openai_api_Key: str, deploy_key: str, metr
     
     # Push to git repo and verify
     try:
-        # Write deploy key to temporary file
+        # Write deploy key to temporary file with proper line endings
         key_file = os.path.join(script_dir, "deploy_key")
-        with open(key_file, 'w') as f:
-            f.write(deploy_key)
+        deploy_key_cleaned = deploy_key.strip().replace('\r\n', '\n')
+        with open(key_file, 'w', newline='\n') as f:
+            f.write(deploy_key_cleaned)
+        
+        # Ensure correct permissions for SSH key
         os.chmod(key_file, 0o600)
         
-        # Set up Git SSH command
-        os.environ['GIT_SSH_COMMAND'] = f'ssh -i {key_file} -o StrictHostKeyChecking=no'
+        # Set up Git SSH command with more verbose output for debugging
+        ssh_command = f'ssh -v -i {key_file} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
+        os.environ['GIT_SSH_COMMAND'] = ssh_command
         
-        # Initialize repo with remote URL if it doesn't exist
+        # Test SSH connection before attempting clone
+        try:
+            subprocess.run(['ssh', '-T', '-i', key_file, '-o', 'StrictHostKeyChecking=no', 
+                          '-o', 'UserKnownHostsFile=/dev/null', 'git@github.com'], 
+                         capture_output=True, timeout=10)
+        except subprocess.CalledProcessError as e:
+            raise Exception(f"SSH connection test failed: {e.stderr.decode() if e.stderr else str(e)}")
+        except Exception as e:
+            raise Exception(f"SSH connection test failed: {str(e)}")
+
+        # Rest of the git operations
         repo_url = "git@github.com:spotonix-inc/eval_results.git"
-        
-        # Create a temporary directory for cloning
-        import tempfile
         temp_dir = tempfile.mkdtemp()
         
         try:
-            # Clone into temporary directory
             repo = Repo.clone_from(repo_url, temp_dir)
             
             # Copy the new file to the cloned repo under atomic_concepts_gradio directory
@@ -137,19 +149,27 @@ def process_questions(llm_model: str, openai_api_Key: str, deploy_key: str, metr
             github_web_url = github_web_url.replace(".git", "")
             file_url = f"{github_web_url}/blob/main/atomic_concepts_gradio/{filename}"
             
-        finally:
-            # Clean up
-            shutil.rmtree(temp_dir)
-            os.remove(key_file)
+            final_message = (
+                f"Successfully processed {len(question_list)} questions.\n"
+                f"Results saved to {filename} and pushed to repository.\n"
+                f"View results at: {file_url}"
+            )
+            if error_messages:
+                final_message += "\n\nErrors encountered:\n" + "\n".join(error_messages)
+            return final_message, filepath
         
-        final_message = (
-            f"Successfully processed {len(question_list)} questions.\n"
-            f"Results saved to {filename} and pushed to repository.\n"
-            f"View results at: {file_url}"
-        )
-        if error_messages:
-            final_message += "\n\nErrors encountered:\n" + "\n".join(error_messages)
-        return final_message, filepath
+        except Exception as e:
+            raise Exception(f"Git operation failed: {str(e)}")
+        finally:
+            # Ensure cleanup happens even if there's an error
+            try:
+                shutil.rmtree(temp_dir)
+            except:
+                pass
+            try:
+                os.remove(key_file)
+            except:
+                pass
         
     except Exception as e:
         return f"Error pushing to repository: {str(e)}", None