Spaces:
Sleeping
Sleeping
Sahil Garg
commited on
Commit
·
a7cd9e6
1
Parent(s):
ccbc6cd
unnecessary validation check removed.
Browse files- agents/generator_validator.py +29 -64
agents/generator_validator.py
CHANGED
|
@@ -622,11 +622,6 @@ class LLMNotesGenerator(BaseGenerator):
|
|
| 622 |
"""Refine LLM notes generation based on feedback"""
|
| 623 |
logger.info(f"Refining LLM notes generation with feedback: {feedback}")
|
| 624 |
|
| 625 |
-
# For LLM generation, we can try different approaches:
|
| 626 |
-
# 1. Switch to RLHF if not already using it
|
| 627 |
-
# 2. Retry with different parameters
|
| 628 |
-
# 3. Use fallback models
|
| 629 |
-
|
| 630 |
if not self.use_rlhf and "quality" in str(feedback).lower():
|
| 631 |
# If quality issues and not using RLHF, try RLHF
|
| 632 |
logger.info("Switching to RLHF for better quality")
|
|
@@ -640,80 +635,50 @@ class LLMNotesGenerator(BaseGenerator):
|
|
| 640 |
return self.generate(previous_result.data.get("file_path") if previous_result.data else None)
|
| 641 |
|
| 642 |
class NotesValidator(BaseValidator):
|
| 643 |
-
"""
|
| 644 |
|
| 645 |
def validate(self, generation_result: GenerationResult) -> ValidationResult:
|
| 646 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 647 |
if not generation_result.success or not generation_result.output_path:
|
| 648 |
return ValidationResult(
|
| 649 |
is_valid=False,
|
| 650 |
score=0.0,
|
| 651 |
feedback=["Generation failed - no output produced"],
|
| 652 |
suggestions=["Retry generation process"],
|
| 653 |
-
metadata={"validation_type": "
|
| 654 |
)
|
| 655 |
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
score = 1.0 # Start with perfect score
|
| 659 |
-
|
| 660 |
-
try:
|
| 661 |
-
# Check if output file exists
|
| 662 |
-
if not os.path.exists(generation_result.output_path):
|
| 663 |
-
feedback.append("Output file does not exist")
|
| 664 |
-
suggestions.append("Check file generation process")
|
| 665 |
-
score -= 0.5
|
| 666 |
-
|
| 667 |
-
# Check file size (reasonable minimum)
|
| 668 |
-
if os.path.exists(generation_result.output_path):
|
| 669 |
-
file_size = os.path.getsize(generation_result.output_path)
|
| 670 |
-
if file_size < 1000: # Less than 1KB
|
| 671 |
-
feedback.append("Output file is unusually small")
|
| 672 |
-
suggestions.append("Verify data processing and Excel generation")
|
| 673 |
-
score -= 0.3
|
| 674 |
-
|
| 675 |
-
# Check for common financial notes elements
|
| 676 |
-
# This would require reading the Excel file, but for now we'll do basic checks
|
| 677 |
-
|
| 678 |
-
# Check metadata completeness
|
| 679 |
-
metadata = generation_result.metadata
|
| 680 |
-
if not metadata.get("execution_id"):
|
| 681 |
-
feedback.append("Missing execution ID in metadata")
|
| 682 |
-
score -= 0.1
|
| 683 |
-
|
| 684 |
-
if generation_result.metadata.get("generation_method") == "llm":
|
| 685 |
-
# Additional checks for LLM-generated content
|
| 686 |
-
rlhf_metadata = metadata.get("rlhf_metadata", {})
|
| 687 |
-
if rlhf_metadata and rlhf_metadata.get("predicted_quality", 0) < 0.5:
|
| 688 |
-
feedback.append("Low quality score from RLHF validation")
|
| 689 |
-
suggestions.append("Consider regenerating with RLHF enabled")
|
| 690 |
-
score -= 0.4
|
| 691 |
-
|
| 692 |
-
# Ensure score doesn't go below 0
|
| 693 |
-
score = max(0.0, score)
|
| 694 |
-
|
| 695 |
-
return ValidationResult(
|
| 696 |
-
is_valid=score >= 0.6, # 60% threshold for validity
|
| 697 |
-
score=score,
|
| 698 |
-
feedback=feedback,
|
| 699 |
-
suggestions=suggestions,
|
| 700 |
-
metadata={
|
| 701 |
-
"validation_type": "comprehensive",
|
| 702 |
-
"file_size": os.path.getsize(generation_result.output_path) if os.path.exists(generation_result.output_path) else 0,
|
| 703 |
-
"has_rlhf": bool(generation_result.metadata.get("rlhf_metadata"))
|
| 704 |
-
}
|
| 705 |
-
)
|
| 706 |
-
|
| 707 |
-
except Exception as e:
|
| 708 |
-
logger.error(f"Validation failed: {e}")
|
| 709 |
return ValidationResult(
|
| 710 |
is_valid=False,
|
| 711 |
score=0.0,
|
| 712 |
-
feedback=[
|
| 713 |
-
suggestions=["Check
|
| 714 |
-
metadata={"validation_type": "
|
| 715 |
)
|
| 716 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 717 |
def get_validation_criteria(self) -> List[str]:
|
| 718 |
"""Return list of validation criteria"""
|
| 719 |
return [
|
|
|
|
| 622 |
"""Refine LLM notes generation based on feedback"""
|
| 623 |
logger.info(f"Refining LLM notes generation with feedback: {feedback}")
|
| 624 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
if not self.use_rlhf and "quality" in str(feedback).lower():
|
| 626 |
# If quality issues and not using RLHF, try RLHF
|
| 627 |
logger.info("Switching to RLHF for better quality")
|
|
|
|
| 635 |
return self.generate(previous_result.data.get("file_path") if previous_result.data else None)
|
| 636 |
|
| 637 |
class NotesValidator(BaseValidator):
|
| 638 |
+
"""Simplified validator for financial notes - passes user validation to interactive feedback system"""
|
| 639 |
|
| 640 |
def validate(self, generation_result: GenerationResult) -> ValidationResult:
|
| 641 |
+
"""
|
| 642 |
+
Simplified validation - since users provide direct feedback through interactive sessions,
|
| 643 |
+
this validator just does basic file existence checks and always passes validation
|
| 644 |
+
to let users provide their own feedback.
|
| 645 |
+
"""
|
| 646 |
if not generation_result.success or not generation_result.output_path:
|
| 647 |
return ValidationResult(
|
| 648 |
is_valid=False,
|
| 649 |
score=0.0,
|
| 650 |
feedback=["Generation failed - no output produced"],
|
| 651 |
suggestions=["Retry generation process"],
|
| 652 |
+
metadata={"validation_type": "basic_file_check"}
|
| 653 |
)
|
| 654 |
|
| 655 |
+
# Basic file existence check
|
| 656 |
+
if not os.path.exists(generation_result.output_path):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 657 |
return ValidationResult(
|
| 658 |
is_valid=False,
|
| 659 |
score=0.0,
|
| 660 |
+
feedback=["Output file does not exist"],
|
| 661 |
+
suggestions=["Check file generation process"],
|
| 662 |
+
metadata={"validation_type": "basic_file_check"}
|
| 663 |
)
|
| 664 |
|
| 665 |
+
# Get file size for metadata
|
| 666 |
+
file_size = os.path.getsize(generation_result.output_path) if os.path.exists(generation_result.output_path) else 0
|
| 667 |
+
|
| 668 |
+
# Always pass validation - let users provide feedback through interactive system
|
| 669 |
+
return ValidationResult(
|
| 670 |
+
is_valid=True, # Always pass - user feedback takes precedence
|
| 671 |
+
score=1.0, # Perfect score - user will validate
|
| 672 |
+
feedback=[], # No automatic feedback - user provides feedback
|
| 673 |
+
suggestions=[], # No automatic suggestions - user provides direction
|
| 674 |
+
metadata={
|
| 675 |
+
"validation_type": "user_feedback_based",
|
| 676 |
+
"file_size": file_size,
|
| 677 |
+
"automatic_validation_disabled": True,
|
| 678 |
+
"reason": "User feedback through interactive sessions replaces automatic validation"
|
| 679 |
+
}
|
| 680 |
+
)
|
| 681 |
+
|
| 682 |
def get_validation_criteria(self) -> List[str]:
|
| 683 |
"""Return list of validation criteria"""
|
| 684 |
return [
|