update evaluator to start if pending models were found

src/evaluator/evaluate.py

@@ -295,9 +295,124 @@ def process_evaluation_queue():
         print(f"Evaluation requests path does not exist: {EVAL_REQUESTS_PATH}")
         return
     
+    # Find all model directories (each model has its own directory)
+    model_dirs = [d for d in os.listdir(EVAL_REQUESTS_PATH) if os.path.isdir(os.path.join(EVAL_REQUESTS_PATH, d))]
+    print(f"Found {len(model_dirs)} model directories")
+    
     pending_files = []
-    for file in os.listdir(EVAL_REQUESTS_PATH):
-        if file.endswith('.json'):
+    for model_dir in model_dirs:
+        model_dir_path = os.path.join(EVAL_REQUESTS_PATH, model_dir)
+        print(f"\nChecking model directory: {model_dir_path}")
+        
+        # Find all JSON files in the model directory
+        json_files = [f for f in os.listdir(model_dir_path) if f.endswith('.json')]
+        print(f"Found {len(json_files)} JSON files in {model_dir}")
+        
+        for file in json_files:
+            file_path = os.path.join(model_dir_path, file)
+            try:
+                with open(file_path, 'r') as f:
+                    eval_entry = json.load(f)
+                
+                # Check if this is a pending evaluation
+                if eval_entry.get('status') == EvaluationStatus.PENDING.value:
+                    print(f"\n=== Found pending evaluation ===")
+                    print(f"Model: {eval_entry['model']}")
+                    print(f"Revision: {eval_entry['revision']}")
+                    print(f"Precision: {eval_entry['precision']}")
+                    print(f"Weight type: {eval_entry['weight_type']}")
+                    
+                    # Update status to RUNNING
+                    eval_entry['status'] = EvaluationStatus.RUNNING.value
+                    with open(file_path, 'w') as f:
+                        json.dump(eval_entry, f, indent=2)
+                    
+                    # Run evaluation
+                    try:
+                        print("\n=== Starting evaluation ===")
+                        eval_result = evaluate_model(
+                            model_name=eval_entry['model'],
+                            revision=eval_entry['revision'],
+                            precision=eval_entry['precision'],
+                            weight_type=eval_entry['weight_type']
+                        )
+                        
+                        print("\n=== Evaluation completed ===")
+                        print(f"Results: {eval_result.results}")
+                        
+                        # Update status to FINISHED and add results
+                        eval_entry['status'] = EvaluationStatus.FINISHED.value
+                        eval_entry['results'] = eval_result.results
+                        
+                        if eval_result.error:
+                            eval_entry['error'] = eval_result.error
+                        
+                        # Save updated entry
+                        with open(file_path, 'w') as f:
+                            json.dump(eval_entry, f, indent=2)
+                        
+                        # Move file to results directory
+                        if not os.path.exists(EVAL_RESULTS_PATH):
+                            os.makedirs(EVAL_RESULTS_PATH)
+                        
+                        result_filename = os.path.basename(file_path)
+                        result_path = os.path.join(EVAL_RESULTS_PATH, result_filename)
+                        
+                        os.rename(file_path, result_path)
+                        print(f"\nMoved evaluation result to: {result_path}")
+                        
+                        # Upload to Hugging Face
+                        try:
+                            API.upload_file(
+                                path_or_fileobj=result_path,
+                                path_in_repo=result_filename,
+                                repo_id=RESULTS_REPO,
+                                repo_type="dataset",
+                                commit_message=f"Add evaluation results for {eval_entry['model']}"
+                            )
+                            print("\nResults uploaded to Hugging Face")
+                        except Exception as upload_error:
+                            print(f"Error uploading results: {str(upload_error)}")
+                            eval_entry['error'] = f"Evaluation completed but failed to upload results: {str(upload_error)}"
+                            with open(file_path, 'w') as f:
+                                json.dump(eval_entry, f, indent=2)
+                    except Exception as eval_error:
+                        print(f"\n=== Error during evaluation ===")
+                        print(f"Error: {str(eval_error)}")
+                        print(f"Full traceback: {traceback.format_exc()}")
+                        
+                        # Update status to FAILED and add error
+                        eval_entry['status'] = EvaluationStatus.FAILED.value
+                        eval_entry['error'] = str(eval_error)
+                        
+                        with open(file_path, 'w') as f:
+                            json.dump(eval_entry, f, indent=2)
+                        
+                        # Move failed evaluation to results directory
+                        if not os.path.exists(EVAL_RESULTS_PATH):
+                            os.makedirs(EVAL_RESULTS_PATH)
+                        
+                        result_filename = os.path.basename(file_path)
+                        result_path = os.path.join(EVAL_RESULTS_PATH, result_filename)
+                        
+                        os.rename(file_path, result_path)
+                        print(f"\nMoved failed evaluation to: {result_path}")
+                        
+                        # Upload error file
+                        try:
+                            API.upload_file(
+                                path_or_fileobj=result_path,
+                                path_in_repo=result_filename,
+                                repo_id=RESULTS_REPO,
+                                repo_type="dataset",
+                                commit_message=f"Add evaluation error for {eval_entry['model']}"
+                            )
+                            print("\nError file uploaded to Hugging Face")
+                        except Exception as upload_error:
+                            print(f"Error uploading error file: {str(upload_error)}")
+            except Exception as e:
+                print(f"Error processing file {file}: {str(e)}")
+                print(f"Full traceback: {traceback.format_exc()}")
             pending_files.append(os.path.join(EVAL_REQUESTS_PATH, file))
     
     print(f"Found {len(pending_files)} pending evaluation requests")