Sahil Garg commited on
Commit
a7cd9e6
·
1 Parent(s): ccbc6cd

unnecessary validation check removed.

Browse files
Files changed (1) hide show
  1. agents/generator_validator.py +29 -64
agents/generator_validator.py CHANGED
@@ -622,11 +622,6 @@ class LLMNotesGenerator(BaseGenerator):
622
  """Refine LLM notes generation based on feedback"""
623
  logger.info(f"Refining LLM notes generation with feedback: {feedback}")
624
 
625
- # For LLM generation, we can try different approaches:
626
- # 1. Switch to RLHF if not already using it
627
- # 2. Retry with different parameters
628
- # 3. Use fallback models
629
-
630
  if not self.use_rlhf and "quality" in str(feedback).lower():
631
  # If quality issues and not using RLHF, try RLHF
632
  logger.info("Switching to RLHF for better quality")
@@ -640,80 +635,50 @@ class LLMNotesGenerator(BaseGenerator):
640
  return self.generate(previous_result.data.get("file_path") if previous_result.data else None)
641
 
642
  class NotesValidator(BaseValidator):
643
- """Validator for financial notes quality"""
644
 
645
  def validate(self, generation_result: GenerationResult) -> ValidationResult:
646
- """Validate generated financial notes"""
 
 
 
 
647
  if not generation_result.success or not generation_result.output_path:
648
  return ValidationResult(
649
  is_valid=False,
650
  score=0.0,
651
  feedback=["Generation failed - no output produced"],
652
  suggestions=["Retry generation process"],
653
- metadata={"validation_type": "basic"}
654
  )
655
 
656
- feedback = []
657
- suggestions = []
658
- score = 1.0 # Start with perfect score
659
-
660
- try:
661
- # Check if output file exists
662
- if not os.path.exists(generation_result.output_path):
663
- feedback.append("Output file does not exist")
664
- suggestions.append("Check file generation process")
665
- score -= 0.5
666
-
667
- # Check file size (reasonable minimum)
668
- if os.path.exists(generation_result.output_path):
669
- file_size = os.path.getsize(generation_result.output_path)
670
- if file_size < 1000: # Less than 1KB
671
- feedback.append("Output file is unusually small")
672
- suggestions.append("Verify data processing and Excel generation")
673
- score -= 0.3
674
-
675
- # Check for common financial notes elements
676
- # This would require reading the Excel file, but for now we'll do basic checks
677
-
678
- # Check metadata completeness
679
- metadata = generation_result.metadata
680
- if not metadata.get("execution_id"):
681
- feedback.append("Missing execution ID in metadata")
682
- score -= 0.1
683
-
684
- if generation_result.metadata.get("generation_method") == "llm":
685
- # Additional checks for LLM-generated content
686
- rlhf_metadata = metadata.get("rlhf_metadata", {})
687
- if rlhf_metadata and rlhf_metadata.get("predicted_quality", 0) < 0.5:
688
- feedback.append("Low quality score from RLHF validation")
689
- suggestions.append("Consider regenerating with RLHF enabled")
690
- score -= 0.4
691
-
692
- # Ensure score doesn't go below 0
693
- score = max(0.0, score)
694
-
695
- return ValidationResult(
696
- is_valid=score >= 0.6, # 60% threshold for validity
697
- score=score,
698
- feedback=feedback,
699
- suggestions=suggestions,
700
- metadata={
701
- "validation_type": "comprehensive",
702
- "file_size": os.path.getsize(generation_result.output_path) if os.path.exists(generation_result.output_path) else 0,
703
- "has_rlhf": bool(generation_result.metadata.get("rlhf_metadata"))
704
- }
705
- )
706
-
707
- except Exception as e:
708
- logger.error(f"Validation failed: {e}")
709
  return ValidationResult(
710
  is_valid=False,
711
  score=0.0,
712
- feedback=[f"Validation error: {str(e)}"],
713
- suggestions=["Check system logs for detailed error information"],
714
- metadata={"validation_type": "error"}
715
  )
716
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
717
  def get_validation_criteria(self) -> List[str]:
718
  """Return list of validation criteria"""
719
  return [
 
622
  """Refine LLM notes generation based on feedback"""
623
  logger.info(f"Refining LLM notes generation with feedback: {feedback}")
624
 
 
 
 
 
 
625
  if not self.use_rlhf and "quality" in str(feedback).lower():
626
  # If quality issues and not using RLHF, try RLHF
627
  logger.info("Switching to RLHF for better quality")
 
635
  return self.generate(previous_result.data.get("file_path") if previous_result.data else None)
636
 
637
  class NotesValidator(BaseValidator):
638
+ """Simplified validator for financial notes - passes user validation to interactive feedback system"""
639
 
640
  def validate(self, generation_result: GenerationResult) -> ValidationResult:
641
+ """
642
+ Simplified validation - since users provide direct feedback through interactive sessions,
643
+ this validator just does basic file existence checks and always passes validation
644
+ to let users provide their own feedback.
645
+ """
646
  if not generation_result.success or not generation_result.output_path:
647
  return ValidationResult(
648
  is_valid=False,
649
  score=0.0,
650
  feedback=["Generation failed - no output produced"],
651
  suggestions=["Retry generation process"],
652
+ metadata={"validation_type": "basic_file_check"}
653
  )
654
 
655
+ # Basic file existence check
656
+ if not os.path.exists(generation_result.output_path):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
657
  return ValidationResult(
658
  is_valid=False,
659
  score=0.0,
660
+ feedback=["Output file does not exist"],
661
+ suggestions=["Check file generation process"],
662
+ metadata={"validation_type": "basic_file_check"}
663
  )
664
 
665
+ # Get file size for metadata
666
+ file_size = os.path.getsize(generation_result.output_path) if os.path.exists(generation_result.output_path) else 0
667
+
668
+ # Always pass validation - let users provide feedback through interactive system
669
+ return ValidationResult(
670
+ is_valid=True, # Always pass - user feedback takes precedence
671
+ score=1.0, # Perfect score - user will validate
672
+ feedback=[], # No automatic feedback - user provides feedback
673
+ suggestions=[], # No automatic suggestions - user provides direction
674
+ metadata={
675
+ "validation_type": "user_feedback_based",
676
+ "file_size": file_size,
677
+ "automatic_validation_disabled": True,
678
+ "reason": "User feedback through interactive sessions replaces automatic validation"
679
+ }
680
+ )
681
+
682
  def get_validation_criteria(self) -> List[str]:
683
  """Return list of validation criteria"""
684
  return [