udf added to /notes-llm alongwith RLHF

.gitignore

@@ -23,6 +23,16 @@ docker-compose.dev.yml
 file_cleanup.py
 agents/langgraph_routes.py
 
+# Test and debug files
+test_*.py
+debug_*.py
+check_*.py
+*_test.py
+*_debug.py
+*_check.py
+file_cleanup.py
+restart_server.py
+
 # RLHF related data
 data/feedback/
 data/models/

agents/feedback_manager.py

@@ -53,22 +53,6 @@ class FeedbackManager:
             "timestamp": time.time(),
             "reviewer_id": feedback.get("reviewer_id", "anonymous"),
             
-            # Technical accuracy metrics
-            "calculation_accuracy": feedback.get("calculation_accuracy"),
-            "account_classification": feedback.get("account_classification"), 
-            "statement_balance": feedback.get("statement_balance"),
-            
-            # Compliance metrics
-            "accounting_standards": feedback.get("accounting_standards"),
-            "regulatory_compliance": feedback.get("regulatory_compliance"),
-            
-            # Quality metrics
-            "completeness": feedback.get("completeness"),
-            "professional_presentation": feedback.get("professional_presentation"),
-            
-            # Overall quality score (computed)
-            "overall_score": self._compute_overall_score(feedback),
-            
             # Qualitative feedback
             "specific_errors": feedback.get("specific_errors", ""),
             "missing_items": feedback.get("missing_items", ""),
@@ -102,29 +86,17 @@ class FeedbackManager:
         # Filter and prepare training data
         training_data = []
         for feedback in feedback_data:
-            if feedback.get("overall_score") is not None:
-                training_sample = {
-                    "statement_id": feedback["statement_id"],
-                    "statement_type": feedback["statement_type"],
-                    "reward_score": feedback["overall_score"],
-                    "binary_approval": feedback["would_accept_for_audit"],
-                    "technical_metrics": {
-                        "calculation_accuracy": feedback.get("calculation_accuracy"),
-                        "account_classification": feedback.get("account_classification"),
-                        "statement_balance": feedback.get("statement_balance")
-                    },
-                    "quality_metrics": {
-                        "completeness": feedback.get("completeness"),
-                        "professional_presentation": feedback.get("professional_presentation"),
-                        "accounting_standards": feedback.get("accounting_standards")
-                    },
-                    "feedback_text": {
-                        "errors": feedback.get("specific_errors", ""),
-                        "missing": feedback.get("missing_items", ""),
-                        "suggestions": feedback.get("improvement_suggestions", "")
-                    }
+            training_sample = {
+                "statement_id": feedback["statement_id"],
+                "statement_type": feedback["statement_type"],
+                "binary_approval": feedback["would_accept_for_audit"],
+                "feedback_text": {
+                    "errors": feedback.get("specific_errors", ""),
+                    "missing": feedback.get("missing_items", ""),
+                    "suggestions": feedback.get("improvement_suggestions", "")
                 }
-                training_data.append(training_sample)
+            }
+            training_data.append(training_sample)
                 
         return training_data
         
@@ -157,23 +129,20 @@ class FeedbackManager:
             return {"total_feedback": 0, "total_statements": len(statements)}
             
         # Calculate statistics
-        scores = [fb["overall_score"] for fb in feedback_data if fb.get("overall_score")]
         audit_approvals = [fb["would_accept_for_audit"] for fb in feedback_data]
         
         stats = {
             "total_feedback": len(feedback_data),
             "total_statements": len(statements),
-            "avg_overall_score": sum(scores) / len(scores) if scores else 0,
             "audit_approval_rate": sum(audit_approvals) / len(audit_approvals) if audit_approvals else 0,
-            "feedback_by_type": {},
-            "recent_trend": self._calculate_trend()
+            "feedback_by_type": {}
         }
         
         # Group by statement type
         for fb in feedback_data:
             stmt_type = fb.get("statement_type", "unknown")
             if stmt_type not in stats["feedback_by_type"]:
-                stats["feedback_by_type"][stmt_type] = {"count": 0, "avg_score": 0}
+                stats["feedback_by_type"][stmt_type] = {"count": 0}
             stats["feedback_by_type"][stmt_type]["count"] += 1
             
         return stats
@@ -197,52 +166,3 @@ class FeedbackManager:
             except (json.JSONDecodeError, FileNotFoundError):
                 logger.warning("Could not load statements database, starting fresh")
         return []
-        
-    def _compute_overall_score(self, feedback: Dict[str, Any]) -> float:
-        """Compute overall quality score from individual metrics"""
-        metrics = [
-            feedback.get("calculation_accuracy"),
-            feedback.get("account_classification"),
-            feedback.get("statement_balance"),
-            feedback.get("accounting_standards"),
-            feedback.get("regulatory_compliance"),
-            feedback.get("completeness"),
-            feedback.get("professional_presentation")
-        ]
-        
-        # Filter out None values
-        valid_metrics = [m for m in metrics if m is not None]
-        
-        if not valid_metrics:
-            return 0.0
-            
-        return sum(valid_metrics) / len(valid_metrics)
-        
-    def _calculate_trend(self) -> Dict[str, float]:
-        """Calculate recent feedback trend"""
-        feedback_data = self._load_feedback()
-        
-        if len(feedback_data) < 5:
-            return {"trend": "insufficient_data"}
-            
-        # Sort by timestamp
-        sorted_feedback = sorted(feedback_data, key=lambda x: x.get("timestamp", 0))
-        
-        # Compare recent vs older feedback
-        mid_point = len(sorted_feedback) // 2
-        older_scores = [fb["overall_score"] for fb in sorted_feedback[:mid_point] if fb.get("overall_score")]
-        recent_scores = [fb["overall_score"] for fb in sorted_feedback[mid_point:] if fb.get("overall_score")]
-        
-        if older_scores and recent_scores:
-            older_avg = sum(older_scores) / len(older_scores)
-            recent_avg = sum(recent_scores) / len(recent_scores)
-            improvement = recent_avg - older_avg
-            
-            return {
-                "older_average": older_avg,
-                "recent_average": recent_avg,
-                "improvement": improvement,
-                "trend": "improving" if improvement > 0.1 else "stable" if abs(improvement) <= 0.1 else "declining"
-            }
-            
-        return {"trend": "insufficient_data"}

agents/generator_validator.py

@@ -9,8 +9,6 @@ from abc import ABC, abstractmethod
 from typing import Dict, Any, List, Optional, Tuple
 from dataclasses import dataclass
 from datetime import datetime
-import subprocess
-import shutil
 import uuid
 
 logger = logging.getLogger(__name__)
@@ -76,9 +74,14 @@ class InteractiveFeedbackManager:
                         # Convert datetime strings back to datetime objects
                         session_data['created_at'] = datetime.fromisoformat(session_data['created_at'])
                         session_data['last_updated'] = datetime.fromisoformat(session_data['last_updated'])
-                        # Convert feedback history timestamps
-                        for feedback in session_data['feedback_history']:
-                            feedback['timestamp'] = datetime.fromisoformat(feedback['timestamp'])
+
+                        # Convert feedback history dictionaries back to FeedbackData objects
+                        feedback_objects = []
+                        for feedback_dict in session_data['feedback_history']:
+                            feedback_dict['timestamp'] = datetime.fromisoformat(feedback_dict['timestamp'])
+                            feedback_objects.append(FeedbackData(**feedback_dict))
+                        session_data['feedback_history'] = feedback_objects
+
                         self.sessions[session_id] = InteractiveSession(**session_data)
             except Exception as e:
                 logger.error(f"Failed to load sessions: {e}")
@@ -90,28 +93,32 @@ class InteractiveFeedbackManager:
         try:
             data = {}
             for session_id, session in self.sessions.items():
-                session_dict = {
-                    'session_id': session.session_id,
-                    'original_file_path': session.original_file_path,
-                    'current_iteration': session.current_iteration,
-                    'feedback_history': [
-                        {
-                            'session_id': f.session_id,
-                            'feedback_text': f.feedback_text,
-                            'feedback_type': f.feedback_type,
-                            'iteration_number': f.iteration_number,
-                            'timestamp': f.timestamp.isoformat(),
-                            'changes_description': f.changes_description,
-                            'udf_function': f.udf_function,
-                            'udf_version': f.udf_version
-                        } for f in session.feedback_history
-                    ],
-                    'archived_udfs': session.archived_udfs,
-                    'final_udf': session.final_udf,
-                    'status': session.status,
-                    'created_at': session.created_at.isoformat(),
-                    'last_updated': session.last_updated.isoformat()
-                }
+                # Handle case where session might be a dict instead of InteractiveSession object
+                if isinstance(session, dict):
+                    session_dict = session
+                else:
+                    session_dict = {
+                        'session_id': session.session_id,
+                        'original_file_path': session.original_file_path,
+                        'current_iteration': session.current_iteration,
+                        'feedback_history': [
+                            {
+                                'session_id': f.session_id,
+                                'feedback_text': f.feedback_text,
+                                'feedback_type': f.feedback_type,
+                                'iteration_number': f.iteration_number,
+                                'timestamp': f.timestamp.isoformat(),
+                                'changes_description': f.changes_description,
+                                'udf_function': f.udf_function,
+                                'udf_version': f.udf_version
+                            } for f in session.feedback_history
+                        ],
+                        'archived_udfs': session.archived_udfs,
+                        'final_udf': session.final_udf,
+                        'status': session.status,
+                        'created_at': session.created_at.isoformat(),
+                        'last_updated': session.last_updated.isoformat()
+                    }
                 data[session_id] = session_dict
 
             with open(self.sessions_file, 'w') as f:
@@ -186,34 +193,186 @@ class InteractiveFeedbackManager:
         return self.sessions.get(session_id)
 
     def _generate_udf_from_feedback(self, feedback_text: str, feedback_type: str, iteration: int) -> str:
-        """Generate UDF function based on user feedback"""
-        # This is a simplified UDF generation - in practice, this would use LLM to create proper functions
-        udf_template = f"""
-def apply_user_feedback_v{iteration}(notes_data, feedback_type='{feedback_type}'):
-    '''
+        """Generate UDF function based on user feedback with actual analysis"""
+        # Analyze feedback content and create meaningful modifications
+        feedback_lower = feedback_text.lower()
+
+        # Determine what modifications to apply based on feedback
+        apply_detailed_depreciation = 'depreciation' in feedback_lower and 'asset' in feedback_lower
+        apply_increase_detail = 'detail' in feedback_lower
+
+        # Handle formula feedback specifically
+        if feedback_type == 'formula':
+            return self._generate_formula_udf(feedback_text, iteration)
+
+        # Create properly formatted UDF code
+        udf_code = f'''def apply_user_feedback_v{iteration}(notes_data, feedback_type='{feedback_type}'):
+    """
     UDF generated from user feedback iteration {iteration}
-    Feedback: {feedback_text}
+    Original Feedback: {feedback_text}
     Type: {feedback_type}
     Generated: {datetime.now().isoformat()}
-    '''
+    """
+    import pandas as pd
+    import re
+
     # Apply feedback-based modifications
-    if '{feedback_type}' == 'numeric':
-        # Handle numeric feedback (e.g., adjust calculations)
-        pass
-    elif '{feedback_type}' == 'formula':
-        # Handle formula suggestions
-        pass
-    elif '{feedback_type}' == 'text':
-        # Handle text-based suggestions
-        pass
-    elif '{feedback_type}' == 'suggestion':
-        # Handle general suggestions
-        pass
+    if notes_data and isinstance(notes_data, dict):
+        # Modify notes content based on feedback analysis
+        for sheet_name, df in notes_data.items():
+            if isinstance(df, pd.DataFrame):
+                df_copy = df.copy()
+
+                # Add detailed depreciation notes with asset categories
+                if {apply_detailed_depreciation}:
+                    if 'depreciation' in sheet_name.lower() or 'fixed asset' in sheet_name.lower():
+                        if len(df.columns) >= 1:
+                            # Add detailed descriptions to the first column
+                            if df_copy.columns[0] in df_copy.columns:
+                                mask = df_copy.iloc[:, 0].astype(str).str.contains('depreciation|asset', case=False, na=False)
+                                df_copy.loc[mask, df_copy.columns[0]] = df_copy.loc[mask, df_copy.columns[0]].astype(str) + \\
+                                    ' - Detailed breakdown by asset category including buildings, equipment, furniture, and motor vehicles'
+
+                # Increase detail level for all notes
+                if {apply_increase_detail}:
+                    if len(df.columns) >= 1 and df_copy.columns[0] in df_copy.columns:
+                        for idx in df_copy.index:
+                            if pd.notna(df_copy.iloc[idx, 0]):
+                                current_value = str(df_copy.iloc[idx, 0])
+                                if 'depreciation' in current_value.lower():
+                                    df_copy.iloc[idx, 0] = current_value + ' (Systematic allocation of asset cost over useful life)'
+                                elif 'inventory' in current_value.lower():
+                                    df_copy.iloc[idx, 0] = current_value + ' (Valued at lower of cost or net realizable value)'
+                                elif 'loans' in current_value.lower() or 'advances' in current_value.lower():
+                                    df_copy.iloc[idx, 0] = current_value + ' (Long-term financial assets with repayment terms)'
+
+                # Update the notes data with modified dataframe
+                notes_data[sheet_name] = df_copy
 
     return notes_data
-"""
+'''
+
+        return udf_code
+
+    def _generate_formula_udf(self, feedback_text: str, iteration: int) -> str:
+        """Generate UDF specifically for formula feedback"""
+        import re
+
+        # Parse the formula from feedback text
+        # First try the flexible pattern that captures full operand names
+        formula_match = re.search(r'=\s*([^-\n]+)\s*-\s*([^\n]+)', feedback_text, re.IGNORECASE)
+        if formula_match:
+            operand1 = formula_match.group(1).strip()
+            operand2 = formula_match.group(2).strip()
+        else:
+            # Fallback to other patterns
+            formula_match = re.search(r'total\s*=\s*(.+?)\s*-\s*(.+?)(?:\s|$)', feedback_text, re.IGNORECASE)
+            if formula_match:
+                operand1 = formula_match.group(1).strip()
+                operand2 = formula_match.group(2).strip()
+            else:
+                formula_match = re.search(r'(.+?)\s*-\s*(.+?)\s*=\s*total', feedback_text, re.IGNORECASE)
+                if formula_match:
+                    operand1 = formula_match.group(1).strip()
+                    operand2 = formula_match.group(2).strip()
+
+        if formula_match:
+            operand1 = formula_match.group(1).strip()
+            operand2 = formula_match.group(2).strip()
+
+            udf_code = f'''def apply_user_feedback_v{iteration}(notes_data, feedback_type='formula'):
+    """
+    UDF generated from formula feedback iteration {iteration}
+    Original Feedback: {feedback_text}
+    Formula: Total = {operand1} - {operand2}
+    Generated: {datetime.now().isoformat()}
+    """
+    import pandas as pd
+
+    # Apply formula modifications
+    if notes_data and isinstance(notes_data, dict):
+        for sheet_name, df in notes_data.items():
+            if isinstance(df, pd.DataFrame) and len(df.columns) >= 2:
+                df_copy = df.copy()
+
+                # Look for the operands in the dataframe
+                operand1_col = None
+                operand2_col = None
+                total_col = None
+
+                # Find columns containing the operands
+                for col in df_copy.columns:
+                    col_str = str(col).lower()
+                    if operand1.lower() in col_str:
+                        operand1_col = col
+                    if operand2.lower() in col_str:
+                        operand2_col = col
+                    if 'total' in col_str:
+                        total_col = col
+
+                # If we found the operand columns, create or update total
+                if operand1_col is not None and operand2_col is not None:
+                    # Calculate the formula: operand1 - operand2
+                    try:
+                        # Convert to numeric, handling any non-numeric values
+                        op1_values = pd.to_numeric(df_copy[operand1_col], errors='coerce')
+                        op2_values = pd.to_numeric(df_copy[operand2_col], errors='coerce')
+
+                        # Calculate total = operand1 - operand2
+                        calculated_total = op1_values - op2_values
+
+                        # Add or update total column
+                        if total_col is None:
+                            # Find a good position for total column (usually after the operands)
+                            cols = list(df_copy.columns)
+                            max_idx = max(cols.index(operand1_col), cols.index(operand2_col))
+                            cols.insert(max_idx + 1, 'Total')
+                            df_copy['Total'] = calculated_total
+                            df_copy = df_copy[cols]
+                        else:
+                            df_copy[total_col] = calculated_total
+
+                        print(f"Applied formula: Total = {operand1} - {operand2}")
+                        print(f"Sample calculation: {{op1_values.iloc[0] if len(op1_values) > 0 else 'N/A'}} - {{op2_values.iloc[0] if len(op2_values) > 0 else 'N/A'}} = {{calculated_total.iloc[0] if len(calculated_total) > 0 else 'N/A'}}")
+
+                    except Exception as e:
+                        print(f"Error applying formula: {{e}}")
+
+                notes_data[sheet_name] = df_copy
 
-        return udf_template.strip()
+    return notes_data
+'''
+        else:
+            # Fallback for unrecognized formula patterns
+            udf_code = f'''def apply_user_feedback_v{iteration}(notes_data, feedback_type='formula'):
+    """
+    UDF generated from formula feedback iteration {iteration}
+    Original Feedback: {feedback_text}
+    Generated: {datetime.now().isoformat()}
+    Note: Could not parse formula pattern, applying general enhancement
+    """
+    import pandas as pd
+
+    # Apply general formula-related enhancements
+    if notes_data and isinstance(notes_data, dict):
+        for sheet_name, df in notes_data.items():
+            if isinstance(df, pd.DataFrame):
+                df_copy = df.copy()
+
+                # Add formula indicators to relevant cells
+                if len(df.columns) >= 1:
+                    for idx in df_copy.index:
+                        if pd.notna(df_copy.iloc[idx, 0]):
+                            cell_value = str(df_copy.iloc[idx, 0])
+                            if 'total' in cell_value.lower():
+                                df_copy.iloc[idx, 0] = cell_value + ' (Calculated field)'
+
+                notes_data[sheet_name] = df_copy
+
+    return notes_data
+'''
+
+        return udf_code
 
 class BaseGenerator(ABC):
     """Abstract base class for financial statement generators"""
@@ -253,11 +412,16 @@ class LLMNotesGenerator(BaseGenerator):
         self.use_rlhf = use_rlhf
 
     def generate(self, file_path: str, **kwargs) -> GenerationResult:
-        """Generate notes using AI/LLM approach"""
+        """Generate notes using AI/LLM approach with feedback integration"""
         try:
             self.attempts_made += 1
             execution_id = f"notes_llm_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{self.attempts_made}"
 
+            # Check for feedback context
+            feedback_context = kwargs.get('feedback_context', {})
+            session_id = feedback_context.get('session_id')
+            udfs_to_apply = feedback_context.get('udfs', [])
+
             # Choose workflow based on RLHF preference
             if self.use_rlhf:
                 from agents.rlhf_workflows import run_rlhf_workflow
@@ -267,6 +431,10 @@ class LLMNotesGenerator(BaseGenerator):
                 result = run_workflow(file_path, "notes-llm")
 
             if result["status"] == "success":
+                # Apply UDFs to the result if available
+                if udfs_to_apply:
+                    result = self._apply_udfs_to_result(result, udfs_to_apply, feedback_context)
+
                 return GenerationResult(
                     success=True,
                     output_path=result["result"]["output_xlsx_path"],
@@ -277,7 +445,10 @@ class LLMNotesGenerator(BaseGenerator):
                         "generation_method": "llm",
                         "use_rlhf": self.use_rlhf,
                         "attempt": self.attempts_made,
-                        "rlhf_metadata": result["result"].get("rlhf_metadata", {})
+                        "rlhf_metadata": result["result"].get("rlhf_metadata", {}),
+                        "feedback_applied": bool(udfs_to_apply),
+                        "udfs_applied_count": len(udfs_to_apply),
+                        "session_id": session_id
                     }
                 )
             else:
@@ -309,6 +480,37 @@ class LLMNotesGenerator(BaseGenerator):
                 }
             )
 
+    def _apply_udfs_to_result(self, result: Dict[str, Any], udfs: List[str], feedback_context: Dict[str, Any]) -> Dict[str, Any]:
+        """Apply UDFs to the generation result"""
+        try:
+            # Execute each UDF and apply modifications
+            for udf_code in udfs:
+                try:
+                    # Create a local namespace for UDF execution
+                    local_vars = {}
+                    exec(udf_code, {"datetime": datetime}, local_vars)
+
+                    # Find the UDF function (it will be the last defined function)
+                    udf_func = None
+                    for var_name, var_value in local_vars.items():
+                        if callable(var_value) and var_name.startswith('apply_user_feedback'):
+                            udf_func = var_value
+                            break
+
+                    if udf_func:
+                        # Apply the UDF to the result data
+                        result["result"] = udf_func(result["result"], feedback_context.get('feedback_type', 'general'))
+
+                except Exception as e:
+                    logger.warning(f"Failed to apply UDF: {e}")
+                    continue
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Error applying UDFs: {e}")
+            return result
+
     def refine(self, previous_result: GenerationResult, feedback: List[str]) -> GenerationResult:
         """Refine LLM notes generation based on feedback"""
         logger.info(f"Refining LLM notes generation with feedback: {feedback}")

agents/reward_model.py

@@ -1,307 +1,131 @@
 """
-RLHF Reward Model for FinRyver
-Predicts quality scores for generated financial statements based on human feedback
+Enhanced Text-Based RLHF Reward Model for FinRyver
+Focuses on collecting and analyzing specific feedback content instead of predicting quality scores
 """
 import json
 import os
 import logging
-from typing import Dict, Any, List, Optional, Tuple
-import numpy as np
-from sklearn.ensemble import RandomForestRegressor
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import mean_squared_error, r2_score
-import joblib
+from typing import Dict, Any, List, Optional
 import time
 
 logger = logging.getLogger(__name__)
 
-class FinancialRewardModel:
+class TextBasedRewardModel:
     """
-    Reward model that predicts quality scores for financial statements
-    Uses traditional ML initially, can be upgraded to transformer-based models
+    Simple reward model that collects and analyzes text-based feedback
     """
-    
+
     def __init__(self, model_dir: str = "data/models"):
         self.model_dir = model_dir
-        self.model_path = os.path.join(model_dir, "reward_model.pkl")
-        self.feature_names_path = os.path.join(model_dir, "feature_names.json")
-        self.model_stats_path = os.path.join(model_dir, "model_stats.json")
-        
+        self.feedback_data_path = os.path.join(model_dir, "feedback_data.json")
+
         os.makedirs(model_dir, exist_ok=True)
-        
-        # Initialize model
-        self.model = RandomForestRegressor(
-            n_estimators=100,
-            max_depth=10,
-            random_state=42,
-            n_jobs=-1
-        )
-        
-        self.feature_names = []
+
+        self.feedback_data = []
         self.is_trained = False
-        self.model_version = "1.0"
-        
-        # Load existing model if available
-        self._load_model()
-        
-    def extract_features(self, statement_data: Dict[str, Any], statement_content: str = "") -> np.ndarray:
-        """Extract features from statement data for reward prediction"""
-        features = []
-        
-        # Basic metadata features
-        features.append(len(statement_content))  # Content length
-        features.append(statement_data.get("generation_time", 0))  # Generation time
-        features.append(1 if statement_data.get("statement_type") == "notes" else 0)
-        features.append(1 if statement_data.get("statement_type") == "balance_sheet" else 0)
-        features.append(1 if statement_data.get("statement_type") == "pnl" else 0)
-        features.append(1 if statement_data.get("statement_type") == "cash_flow" else 0)
-        
-        # Content-based features (simple heuristics)
-        if statement_content:
-            features.append(statement_content.count("$"))  # Number of monetary values
-            features.append(statement_content.count("\n"))  # Number of lines
-            features.append(len(statement_content.split()))  # Word count
-            features.append(statement_content.count("."))  # Number of sentences
-            features.append(statement_content.count(","))  # Number of commas (complexity indicator)
-            
-            # Financial keywords
-            financial_keywords = ["asset", "liability", "equity", "revenue", "expense", "cash", "account"]
-            keyword_count = sum(statement_content.lower().count(keyword) for keyword in financial_keywords)
-            features.append(keyword_count)
-            
-            # Professional language indicators
-            professional_words = ["accordance", "pursuant", "whereas", "therefore", "respective"]
-            professional_count = sum(statement_content.lower().count(word) for word in professional_words)
-            features.append(professional_count)
-        else:
-            # Default values if no content available
-            features.extend([0] * 7)
-        
-        # File-based features (if available)
-        metadata = statement_data.get("metadata", {})
-        features.append(metadata.get("file_size", 0))
-        features.append(metadata.get("num_accounts", 0))
-        features.append(metadata.get("complexity_score", 0))
-        
-        # Ensure we have consistent feature names
-        if not self.feature_names:
-            self.feature_names = [
-                "content_length", "generation_time", "is_notes", "is_balance_sheet", 
-                "is_pnl", "is_cash_flow", "monetary_values", "line_count", 
-                "word_count", "sentence_count", "comma_count", "financial_keywords",
-                "professional_words", "file_size", "num_accounts", "complexity_score"
-            ]
-        
-        return np.array(features).reshape(1, -1)
-        
-    def train_reward_model(self, training_data: List[Dict[str, Any]]) -> Dict[str, float]:
-        """Train reward model from human feedback data"""
-        if len(training_data) < 2:  # Lowered from 10 to 2 for testing
-            logger.warning(f"Insufficient training data: {len(training_data)} samples")
-            return {"error": "insufficient_data", "sample_count": len(training_data)}
-        
-        # Prepare training data
-        X = []
-        y = []
-        
-        for sample in training_data:
-            # Create dummy statement data for feature extraction
-            statement_data = {
-                "statement_type": sample.get("statement_type", "unknown"),
-                "generation_time": sample.get("generation_time", 0),
-                "metadata": sample.get("metadata", {})
-            }
-            
-            # Extract features
-            features = self.extract_features(statement_data, "")
-            X.append(features.flatten())
-            y.append(sample["reward_score"])
-        
-        X = np.array(X)
-        y = np.array(y)
-        
-        # Split data
-        if len(X) > 20:
-            X_train, X_test, y_train, y_test = train_test_split(
-                X, y, test_size=0.2, random_state=42
-            )
-        else:
-            X_train, X_test, y_train, y_test = X, X, y, y
-        
-        # Train model
-        logger.info(f"Training reward model with {len(X_train)} samples")
-        self.model.fit(X_train, y_train)
-        
-        # Evaluate model
-        train_pred = self.model.predict(X_train)
-        test_pred = self.model.predict(X_test)
-        
-        metrics = {
-            "train_mse": mean_squared_error(y_train, train_pred),
-            "test_mse": mean_squared_error(y_test, test_pred),
-            "train_r2": r2_score(y_train, train_pred),
-            "test_r2": r2_score(y_test, test_pred),
-            "sample_count": len(training_data),
-            "feature_importance": dict(zip(self.feature_names, self.model.feature_importances_))
+        self.model_version = "2.0-text-based"
+
+        # Load existing feedback data if available
+        self._load_feedback_data()
+
+    def collect_feedback(self, feedback_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Collect and store text-based feedback"""
+
+        # Validate that we have text feedback
+        text_feedback = []
+        if feedback_data.get('specific_errors', '').strip():
+            text_feedback.append(feedback_data['specific_errors'])
+        if feedback_data.get('missing_items', '').strip():
+            text_feedback.append(feedback_data['missing_items'])
+        if feedback_data.get('improvement_suggestions', '').strip():
+            text_feedback.append(feedback_data['improvement_suggestions'])
+
+        if not text_feedback:
+            return {"error": "No text feedback provided"}
+
+        # Store feedback
+        feedback_entry = {
+            "timestamp": time.time(),
+            "statement_id": feedback_data.get("statement_id"),
+            "reviewer_id": feedback_data.get("reviewer_id", "anonymous"),
+            "statement_type": feedback_data.get("statement_type"),
+            "specific_errors": feedback_data.get("specific_errors", ""),
+            "missing_items": feedback_data.get("missing_items", ""),
+            "improvement_suggestions": feedback_data.get("improvement_suggestions", ""),
+            "would_accept_for_audit": feedback_data.get("would_accept_for_audit", False),
+            "complexity_level": feedback_data.get("complexity_level", "medium")
         }
-        
-        self.is_trained = True
-        
-        # Save model
-        self._save_model(metrics)
-        
-        logger.info(f"Reward model trained. R2 score: {metrics['test_r2']:.3f}")
-        return metrics
-        
-    def predict_reward(self, statement_data: Dict[str, Any], statement_content: str = "") -> float:
-        """Predict reward score for a generated financial statement"""
-        if not self.is_trained:
-            logger.warning("Reward model not trained, returning default score")
-            return 3.0  # Default neutral score
-        
-        try:
-            features = self.extract_features(statement_data, statement_content)
-            reward = self.model.predict(features)[0]
-            
-            # Clamp to valid range [1, 5]
-            reward = max(1.0, min(5.0, reward))
-            
-            return float(reward)
-            
-        except Exception as e:
-            logger.error(f"Error predicting reward: {e}")
-            return 3.0  # Default score on error
-            
-    def predict_with_confidence(self, statement_data: Dict[str, Any], statement_content: str = "") -> Tuple[float, float]:
-        """Predict reward with confidence interval"""
-        if not self.is_trained:
-            return 3.0, 0.0
-        
-        try:
-            features = self.extract_features(statement_data, statement_content)
-            
-            # For Random Forest, we can get prediction from all trees
-            tree_predictions = [tree.predict(features)[0] for tree in self.model.estimators_]
-            
-            reward = np.mean(tree_predictions)
-            confidence = 1.0 / (1.0 + np.std(tree_predictions))  # Higher std = lower confidence
-            
-            reward = max(1.0, min(5.0, reward))
-            
-            return float(reward), float(confidence)
-            
-        except Exception as e:
-            logger.error(f"Error predicting reward with confidence: {e}")
-            return 3.0, 0.0
-            
-    def get_feature_importance(self) -> Dict[str, float]:
-        """Get feature importance from trained model"""
-        if not self.is_trained:
-            return {}
-        
-        return dict(zip(self.feature_names, self.model.feature_importances_))
-        
-    def get_model_stats(self) -> Dict[str, Any]:
-        """Get model training statistics"""
-        if os.path.exists(self.model_stats_path):
-            try:
-                with open(self.model_stats_path, "r") as f:
-                    return json.load(f)
-            except:
-                pass
-        return {"status": "not_trained"}
-        
-    def _save_model(self, training_stats: Dict[str, Any]):
-        """Save trained model and metadata"""
-        try:
-            # Save model
-            joblib.dump(self.model, self.model_path)
-            
-            # Save feature names
-            with open(self.feature_names_path, "w") as f:
-                json.dump(self.feature_names, f)
-            
-            # Save training stats
-            stats = {
-                "model_version": self.model_version,
-                "training_timestamp": time.time(),
-                "is_trained": True,
-                **training_stats
-            }
-            
-            with open(self.model_stats_path, "w") as f:
-                json.dump(stats, f, indent=2)
-                
-            logger.info("Reward model saved successfully")
-            
-        except Exception as e:
-            logger.error(f"Error saving model: {e}")
-            
-    def _load_model(self):
-        """Load existing trained model"""
-        try:
-            if os.path.exists(self.model_path) and os.path.exists(self.feature_names_path):
-                self.model = joblib.load(self.model_path)
-                
-                with open(self.feature_names_path, "r") as f:
-                    self.feature_names = json.load(f)
-                
-                self.is_trained = True
-                logger.info("Existing reward model loaded successfully")
-                
-        except Exception as e:
-            logger.warning(f"Could not load existing model: {e}")
-            self.is_trained = False
 
+        self.feedback_data.append(feedback_entry)
+        self._save_feedback_data()
 
-class RLHFTrainer:
-    """Coordinates RLHF training pipeline"""
-    
-    def __init__(self, feedback_manager, reward_model):
-        self.feedback_manager = feedback_manager
-        self.reward_model = reward_model
-        self.min_feedback_threshold = 2  # Lowered for testing (was 20)
-        
-    def should_retrain(self) -> bool:
-        """Determine if model should be retrained"""
-        stats = self.feedback_manager.get_feedback_stats()
-        
-        # Check if we have enough new feedback
-        total_feedback = stats.get("total_feedback", 0)
-        
-        # Get last training count
-        model_stats = self.reward_model.get_model_stats()
-        last_training_count = model_stats.get("sample_count", 0)
-        
-        new_feedback_count = total_feedback - last_training_count
-        
-        return (total_feedback >= self.min_feedback_threshold and 
-                new_feedback_count >= 2)  # At least 2 new samples (was 10)
-        
-    def retrain_model(self) -> Dict[str, Any]:
-        """Retrain reward model with latest feedback"""
-        training_data = self.feedback_manager.get_training_data()
-        
-        if len(training_data) < self.min_feedback_threshold:
-            return {
-                "status": "insufficient_data",
-                "current_count": len(training_data),
-                "required_count": self.min_feedback_threshold
-            }
-        
-        # Train model
-        metrics = self.reward_model.train_reward_model(training_data)
-        
         return {
             "status": "success",
-            "training_metrics": metrics,
-            "timestamp": time.time()
+            "feedback_stored": True,
+            "total_feedback": len(self.feedback_data)
+        }
+
+    def get_feedback_patterns(self) -> Dict[str, Any]:
+        """Get patterns and insights from collected feedback"""
+
+        if not self.feedback_data:
+            return {"error": "No feedback data available"}
+
+        # Analyze feedback patterns
+        patterns = {
+            "total_feedback": len(self.feedback_data),
+            "statement_types": {},
+            "common_issues": [],
+            "improvement_suggestions": [],
+            "acceptance_rate": 0.0
         }
-        
-    def periodic_training_check(self) -> Dict[str, Any]:
-        """Check if retraining is needed and perform if necessary"""
-        if self.should_retrain():
-            logger.info("Initiating automatic model retraining")
-            return self.retrain_model()
-        else:
-            return {"status": "no_retraining_needed"}
+
+        # Count statement types
+        statement_counts = {}
+        acceptance_count = 0
+
+        for feedback in self.feedback_data:
+            stmt_type = feedback.get("statement_type", "unknown")
+            statement_counts[stmt_type] = statement_counts.get(stmt_type, 0) + 1
+
+            if feedback.get("would_accept_for_audit"):
+                acceptance_count += 1
+
+            # Collect common issues
+            if feedback.get("specific_errors"):
+                patterns["common_issues"].append(feedback["specific_errors"])
+            if feedback.get("missing_items"):
+                patterns["common_issues"].append(feedback["missing_items"])
+            if feedback.get("improvement_suggestions"):
+                patterns["improvement_suggestions"].append(feedback["improvement_suggestions"])
+
+        patterns["statement_types"] = statement_counts
+        patterns["acceptance_rate"] = acceptance_count / len(self.feedback_data) if self.feedback_data else 0
+
+        return patterns
+
+    def get_recent_feedback(self, limit: int = 10) -> List[Dict[str, Any]]:
+        """Get recent feedback entries"""
+        return self.feedback_data[-limit:] if self.feedback_data else []
+
+    def _save_feedback_data(self):
+        """Save feedback data to disk"""
+        try:
+            with open(self.feedback_data_path, 'w') as f:
+                json.dump(self.feedback_data, f, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving feedback data: {e}")
+
+    def _load_feedback_data(self):
+        """Load feedback data from disk"""
+        try:
+            if os.path.exists(self.feedback_data_path):
+                with open(self.feedback_data_path, 'r') as f:
+                    self.feedback_data = json.load(f)
+                logger.info(f"Loaded {len(self.feedback_data)} feedback entries")
+        except Exception as e:
+            logger.warning(f"Error loading feedback data: {e}")
+            self.feedback_data = []
+
+# Backward compatibility alias
+FinancialRewardModel = TextBasedRewardModel

agents/rlhf_routes.py

@@ -7,7 +7,7 @@ from fastapi.responses import JSONResponse, HTMLResponse
 from typing import Optional, Dict, Any
 import logging
 from agents.feedback_manager import FeedbackManager
-from agents.reward_model import FinancialRewardModel, RLHFTrainer
+from agents.reward_model import TextBasedRewardModel
 from agents.rlhf_workflows import get_rlhf_manager
 
 logger = logging.getLogger(__name__)
@@ -17,37 +17,23 @@ rlhf_router = APIRouter(prefix="/rlhf", tags=["RLHF Feedback"])
 
 # Initialize components
 feedback_manager = FeedbackManager()
-reward_model = FinancialRewardModel()
-trainer = RLHFTrainer(feedback_manager, reward_model)
+reward_model = TextBasedRewardModel()
 
 @rlhf_router.post("/feedback")
 async def collect_feedback(
     statement_id: str = Form(...),
     reviewer_id: str = Form("anonymous"),
-    
-    # Technical accuracy metrics (1-5 scale)
-    calculation_accuracy: float = Form(..., ge=1, le=5),
-    account_classification: float = Form(..., ge=1, le=5),
-    statement_balance: float = Form(..., ge=1, le=5),
-    
-    # Compliance metrics (1-5 scale)
-    accounting_standards: float = Form(..., ge=1, le=5),
-    regulatory_compliance: float = Form(..., ge=1, le=5),
-    
-    # Quality metrics (1-5 scale)
-    completeness: float = Form(..., ge=1, le=5),
-    professional_presentation: float = Form(..., ge=1, le=5),
-    
-    # Qualitative feedback
-    specific_errors: str = Form(""),
+
+    # Primary text-based feedback (required)
+    specific_errors: str = Form(..., min_length=1),
     missing_items: str = Form(""),
     improvement_suggestions: str = Form(""),
-    
+
     # Binary approval
     would_accept_for_audit: bool = Form(False),
-    
+
     # Additional context
-    complexity_level: str = Form("medium")  # low, medium, high
+    complexity_level: str = Form("medium")
 ):
     """
     Collect detailed human feedback on generated financial statements
@@ -59,17 +45,10 @@ async def collect_feedback(
         if not statement_info:
             raise HTTPException(status_code=404, detail="Statement not found")
         
-        # Prepare feedback data
+        # Prepare feedback data (text-focused)
         feedback_data = {
             "statement_id": statement_id,
             "reviewer_id": reviewer_id,
-            "calculation_accuracy": calculation_accuracy,
-            "account_classification": account_classification,
-            "statement_balance": statement_balance,
-            "accounting_standards": accounting_standards,
-            "regulatory_compliance": regulatory_compliance,
-            "completeness": completeness,
-            "professional_presentation": professional_presentation,
             "specific_errors": specific_errors,
             "missing_items": missing_items,
             "improvement_suggestions": improvement_suggestions,
@@ -77,19 +56,16 @@ async def collect_feedback(
             "statement_type": statement_info.get("statement_type"),
             "complexity_level": complexity_level
         }
-        
-        # Store feedback
+
+        # Store feedback in both feedback manager and reward model
         feedback_id = feedback_manager.store_feedback(feedback_data)
-        
-        # Check if model should be retrained
-        retrain_result = trainer.periodic_training_check()
+        reward_model.collect_feedback(feedback_data)
         
         return {
             "status": "success",
             "feedback_id": feedback_id,
-            "message": "Feedback collected successfully",
-            "model_retrain_status": retrain_result.get("status"),
-            "overall_score": feedback_manager._compute_overall_score(feedback_data)
+            "message": "Text feedback collected successfully",
+            "feedback_stored": True
         }
         
     except Exception as e:
@@ -155,31 +131,33 @@ async def get_feedback_stats():
 @rlhf_router.post("/retrain")
 async def manual_retrain():
     """
-    Manually trigger model retraining
+    Get current feedback patterns (no retraining needed for text-based model)
     """
     try:
-        result = trainer.retrain_model()
+        feedback_patterns = reward_model.get_feedback_patterns()
         return {
             "status": "success",
-            "retrain_result": result
+            "message": "Text-based model doesn't require retraining",
+            "feedback_patterns": feedback_patterns
         }
     except Exception as e:
-        logger.error(f"Error during manual retrain: {e}")
+        logger.error(f"Error getting feedback patterns: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 @rlhf_router.get("/model-info")
 async def get_model_info():
     """
-    Get information about the current reward model
+    Get information about the text-based reward model
     """
     try:
+        feedback_patterns = reward_model.get_feedback_patterns()
+
         return {
             "status": "success",
-            "model_trained": reward_model.is_trained,
             "model_version": reward_model.model_version,
-            "feature_count": len(reward_model.feature_names),
-            "feature_names": reward_model.feature_names,
-            "model_stats": reward_model.get_model_stats()
+            "model_type": "text-based",
+            "feedback_collected": feedback_patterns.get("total_feedback", 0),
+            "feedback_patterns": feedback_patterns
         }
     except Exception as e:
         logger.error(f"Error getting model info: {e}")
@@ -222,101 +200,11 @@ def generate_review_html(statement_id: str, statement_info: Dict) -> str:
                 <input type="text" name="reviewer_id" placeholder="Enter your identifier">
             </div>
             
-            <h3>Technical Accuracy (1-5 scale)</h3>
-            
-            <div class="form-group">
-                <label>Calculation Accuracy:</label>
-                <select name="calculation_accuracy" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Major calculation errors</option>
-                    <option value="2">2 - Some calculation errors</option>
-                    <option value="3">3 - Minor calculation issues</option>
-                    <option value="4">4 - Mostly accurate calculations</option>
-                    <option value="5">5 - All calculations correct</option>
-                </select>
-            </div>
-            
-            <div class="form-group">
-                <label>Account Classification:</label>
-                <select name="account_classification" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Major classification errors</option>
-                    <option value="2">2 - Some classification errors</option>
-                    <option value="3">3 - Minor classification issues</option>
-                    <option value="4">4 - Mostly correct classification</option>
-                    <option value="5">5 - Perfect classification</option>
-                </select>
-            </div>
-            
-            <div class="form-group">
-                <label>Statement Balance/Reconciliation:</label>
-                <select name="statement_balance" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Does not balance</option>
-                    <option value="2">2 - Major balance issues</option>
-                    <option value="3">3 - Minor balance issues</option>
-                    <option value="4">4 - Mostly balanced</option>
-                    <option value="5">5 - Perfect balance</option>
-                </select>
-            </div>
-            
-            <h3>Compliance & Standards (1-5 scale)</h3>
-            
-            <div class="form-group">
-                <label>Accounting Standards Compliance:</label>
-                <select name="accounting_standards" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Major compliance issues</option>
-                    <option value="2">2 - Some compliance issues</option>
-                    <option value="3">3 - Minor compliance issues</option>
-                    <option value="4">4 - Mostly compliant</option>
-                    <option value="5">5 - Fully compliant</option>
-                </select>
-            </div>
-            
-            <div class="form-group">
-                <label>Regulatory Compliance:</label>
-                <select name="regulatory_compliance" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Major regulatory issues</option>
-                    <option value="2">2 - Some regulatory issues</option>
-                    <option value="3">3 - Minor regulatory issues</option>
-                    <option value="4">4 - Mostly compliant</option>
-                    <option value="5">5 - Fully compliant</option>
-                </select>
-            </div>
-            
-            <h3>Quality & Presentation (1-5 scale)</h3>
-            
-            <div class="form-group">
-                <label>Completeness:</label>
-                <select name="completeness" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Major items missing</option>
-                    <option value="2">2 - Some items missing</option>
-                    <option value="3">3 - Minor items missing</option>
-                    <option value="4">4 - Mostly complete</option>
-                    <option value="5">5 - Complete</option>
-                </select>
-            </div>
-            
-            <div class="form-group">
-                <label>Professional Presentation:</label>
-                <select name="professional_presentation" required>
-                    <option value="">Select rating</option>
-                    <option value="1">1 - Unprofessional</option>
-                    <option value="2">2 - Below standard</option>
-                    <option value="3">3 - Adequate</option>
-                    <option value="4">4 - Good presentation</option>
-                    <option value="5">5 - Excellent presentation</option>
-                </select>
-            </div>
-            
             <h3>Detailed Feedback</h3>
             
             <div class="form-group">
-                <label>Specific Errors (if any):</label>
-                <textarea name="specific_errors" rows="3" placeholder="Describe any specific errors found..."></textarea>
+                <label>Specific Errors (required):</label>
+                <textarea name="specific_errors" rows="4" placeholder="Describe any specific errors found..." required></textarea>
             </div>
             
             <div class="form-group">

agents/rlhf_workflows.py

@@ -19,7 +19,7 @@ from agents.simple_tools import (
     generate_llm_notes,
 )
 from agents.feedback_manager import FeedbackManager
-from agents.reward_model import FinancialRewardModel, RLHFTrainer
+from agents.reward_model import TextBasedRewardModel
 
 logger = logging.getLogger(__name__)
 
@@ -42,228 +42,31 @@ class RLHFFinancialAgentState(TypedDict):
     feedback_collected: Optional[bool]
 
 class RLHFWorkflowManager:
-    """Manages RLHF-enhanced workflows"""
-    
+    """Manages RLHF-enhanced workflows with text-based feedback"""
+
     def __init__(self):
         self.feedback_manager = FeedbackManager()
-        self.reward_model = FinancialRewardModel()
-        self.trainer = RLHFTrainer(self.feedback_manager, self.reward_model)
-        
-        # Check for model retraining on initialization
-        self._check_and_retrain()
-        
-    def _check_and_retrain(self):
-        """Check if model needs retraining"""
-        try:
-            result = self.trainer.periodic_training_check()
-            if result.get("status") == "success":
-                logger.info("Reward model retrained successfully")
-        except Exception as e:
-            logger.error(f"Error during model retraining check: {e}")
-    
-    def make_rlhf_workflow(self, tool_func, statement_type: str):
-        """Create RLHF-enhanced workflow"""
-        
-        def rlhf_node(state: RLHFFinancialAgentState) -> RLHFFinancialAgentState:
-            state["start_time"] = time.time()
-            state["statement_id"] = str(uuid.uuid4())
-            
-            try:
-                # Generate multiple candidates if reward model is trained
-                if self.reward_model.is_trained:
-                    candidates = self._generate_candidates(tool_func, state, num_candidates=3)
-                    state["candidates_generated"] = candidates
-                    
-                    # Select best candidate using reward model
-                    best_candidate, best_index = self._select_best_candidate(
-                        candidates, statement_type, state["file_path"]
-                    )
-                    
-                    state["result"] = best_candidate
-                    state["best_candidate_index"] = best_index
-                    
-                else:
-                    # Single generation if no trained model
-                    result = tool_func.invoke({"file_path": state["file_path"]})
-                    state["result"] = result
-                    state["candidates_generated"] = [result]
-                    state["best_candidate_index"] = 0
-                
-                # Predict quality score
-                if state["result"].get("status") == "success":
-                    predicted_quality, confidence = self._predict_quality(
-                        state["result"], statement_type, state["file_path"]
-                    )
-                    state["predicted_quality"] = predicted_quality
-                    state["confidence_score"] = confidence
-                    state["status"] = "success"
-                    
-                    # Store statement for potential feedback
-                    self._store_for_feedback(state, statement_type)
-                    
-                else:
-                    state["status"] = "error"
-                    state["error"] = state["result"].get("error", "Unknown error")
-                
-            except Exception as e:
-                state["status"] = "error"
-                state["error"] = str(e)
-                logger.error(f"Error in RLHF workflow: {e}")
-            
-            state["end_time"] = time.time()
-            return state
-        
-        # Create workflow graph
-        wf = StateGraph(RLHFFinancialAgentState)
-        wf.add_node("rlhf_run", rlhf_node)
-        wf.set_entry_point("rlhf_run")
-        wf.add_edge("rlhf_run", END)
-        return wf.compile()
-    
-    def _generate_candidates(self, tool_func, state: RLHFFinancialAgentState, num_candidates: int = 3) -> List[Dict[str, Any]]:
-        """Generate multiple candidates for comparison"""
-        candidates = []
-        
-        for i in range(num_candidates):
-            try:
-                result = tool_func.invoke({"file_path": state["file_path"]})
-                candidates.append({
-                    "index": i,
-                    "result": result,
-                    "timestamp": time.time()
-                })
-            except Exception as e:
-                logger.warning(f"Failed to generate candidate {i}: {e}")
-                candidates.append({
-                    "index": i,
-                    "result": {"status": "error", "error": str(e)},
-                    "timestamp": time.time()
-                })
-        
-        return candidates
-    
-    def _select_best_candidate(self, candidates: List[Dict[str, Any]], statement_type: str, file_path: str) -> tuple:
-        """Select best candidate using reward model"""
-        best_candidate = None
-        best_score = -1
-        best_index = 0
-        
-        for candidate in candidates:
-            if candidate["result"].get("status") == "success":
-                # Create statement data for reward prediction
-                statement_data = {
-                    "statement_type": statement_type,
-                    "file_path": file_path,
-                    "generation_time": 0,  # Could be calculated from timestamps
-                    "metadata": {}
-                }
-                
-                # Predict reward
-                predicted_reward, confidence = self.reward_model.predict_with_confidence(
-                    statement_data, ""
-                )
-                
-                # Weight by confidence
-                weighted_score = predicted_reward * confidence
-                
-                if weighted_score > best_score:
-                    best_score = weighted_score
-                    best_candidate = candidate["result"]
-                    best_index = candidate["index"]
-        
-        # Fallback to first successful candidate
-        if best_candidate is None:
-            for candidate in candidates:
-                if candidate["result"].get("status") == "success":
-                    best_candidate = candidate["result"]
-                    best_index = candidate["index"]
-                    break
-        
-        # Final fallback
-        if best_candidate is None and candidates:
-            best_candidate = candidates[0]["result"]
-            best_index = 0
-        
-        return best_candidate, best_index
-    
-    def _predict_quality(self, result: Dict[str, Any], statement_type: str, file_path: str) -> tuple:
-        """Predict quality score for generated statement"""
-        statement_data = {
-            "statement_type": statement_type,
-            "file_path": file_path,
-            "generation_time": 0,
-            "metadata": {}
-        }
-        
-        return self.reward_model.predict_with_confidence(statement_data, "")
-    
-    def _store_for_feedback(self, state: RLHFFinancialAgentState, statement_type: str):
-        """Store generated statement for feedback collection"""
-        try:
-            statement_data = {
-                "type": statement_type,
-                "file_path": state["file_path"],
-                "output_path": state["result"].get("output_path"),
-                "generation_time": state["end_time"] - state["start_time"],
-                "predicted_quality": state.get("predicted_quality"),
-                "confidence_score": state.get("confidence_score"),
-                "metadata": {
-                    "candidates_count": len(state.get("candidates_generated", [])),
-                    "best_candidate_index": state.get("best_candidate_index"),
-                    "workflow_version": "rlhf_v1"
-                }
-            }
-            
-            stored_id = self.feedback_manager.store_generated_statement(statement_data)
-            state["statement_id"] = stored_id
-            
-        except Exception as e:
-            logger.error(f"Error storing statement for feedback: {e}")
+        self.reward_model = TextBasedRewardModel()
 
-# Global RLHF manager instance
-rlhf_manager = RLHFWorkflowManager()
+    def collect_feedback(self, feedback_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Collect text-based feedback"""
+        return self.reward_model.collect_feedback(feedback_data)
 
-# RLHF-enhanced workflows
-rlhf_workflows = {
-    "notes": rlhf_manager.make_rlhf_workflow(generate_notes_full_pipeline_from_path, "notes"),
-    "pnl": rlhf_manager.make_rlhf_workflow(generate_pnl_statement, "pnl"),
-    "bs": rlhf_manager.make_rlhf_workflow(generate_balance_sheet, "balance_sheet"),
-    "cf": rlhf_manager.make_rlhf_workflow(generate_cash_flow_statement, "cash_flow"),
-    "notes-llm": rlhf_manager.make_rlhf_workflow(generate_llm_notes, "notes"),
-}
-
-def run_rlhf_workflow(file_path: str, kind: str) -> Dict[str, Any]:
-    """Run RLHF-enhanced workflow"""
-    state = RLHFFinancialAgentState(
-        messages=[HumanMessage(content=f"Run RLHF {kind} for {file_path}")],
-        file_path=file_path,
-        result={},
-        status="",
-        start_time=0,
-        end_time=0,
-        error="",
-        statement_id=None,
-        predicted_quality=None,
-        confidence_score=None,
-        candidates_generated=None,
-        best_candidate_index=None,
-        feedback_collected=False
-    )
-    
-    final_state = rlhf_workflows[kind].invoke(state)
-    
-    # Add RLHF metadata to result
-    if final_state["status"] == "success":
-        final_state["result"]["rlhf_metadata"] = {
-            "statement_id": final_state.get("statement_id"),
-            "predicted_quality": final_state.get("predicted_quality"),
-            "confidence_score": final_state.get("confidence_score"),
-            "candidates_generated": len(final_state.get("candidates_generated", [])),
-            "model_used": "rlhf_enhanced"
-        }
-    
-    return final_state
+    def get_feedback_patterns(self) -> Dict[str, Any]:
+        """Get feedback patterns and insights"""
+        return self.reward_model.get_feedback_patterns()
 
 def get_rlhf_manager() -> RLHFWorkflowManager:
-    """Get global RLHF manager instance"""
-    return rlhf_manager
+    """Get the RLHF workflow manager instance"""
+    return RLHFWorkflowManager()
+
+def run_rlhf_workflow(file_path: str, kind: str) -> Dict[str, Any]:
+    """Run RLHF-enhanced workflow (placeholder - simplified)"""
+    # For now, just return a basic structure
+    # This can be enhanced later with actual RLHF logic
+    return {
+        "status": "error",
+        "error": "RLHF workflow not implemented for this endpoint",
+        "file_path": file_path,
+        "kind": kind
+    }

agents/simple_tools.py

@@ -432,6 +432,31 @@ def generate_llm_notes(file_path: str, note_numbers: str = "") -> Dict[str, Any]
         input_json = "data/generated_notes/notes.json"
         output_excel = "data/generated_notes_excel/notes.xlsx"
 
+        # Check if the JSON file was created and has content
+        if not os.path.exists(input_json):
+            execution_time = round(time.time() - start_time, 2)
+            return {
+                "status": "error",
+                "error": "No notes JSON file was generated - LLM may have failed to produce any notes",
+                "execution_id": execution_id,
+                "execution_time": execution_time
+            }
+
+        # Check if JSON file has content
+        try:
+            with open(input_json, 'r', encoding='utf-8') as f:
+                json_content = json.load(f)
+            if isinstance(json_content, dict) and 'notes' in json_content and not json_content['notes']:
+                logger.warning("JSON file exists but contains no notes")
+        except Exception as e:
+            execution_time = round(time.time() - start_time, 2)
+            return {
+                "status": "error",
+                "error": f"Invalid JSON file generated: {str(e)}",
+                "execution_id": execution_id,
+                "execution_time": execution_time
+            }
+
         result3 = subprocess.run(
             ["python", "notes/llm_notes_excel_converter.py", input_json, output_excel],
             env=env,

app.py

@@ -5,18 +5,21 @@ import os
 import shutil
 import logging
 import json
+from dotenv import load_dotenv
+from agents.simple_tools import generate_notes_full_pipeline_from_path
 from agents.generator_validator import create_notes_pipeline, InteractiveFeedbackManager
 from agents.langgraph import run_workflow
 from agents.rlhf_workflows import run_rlhf_workflow
 from agents.rlhf_routes import rlhf_router
 
+# Load environment variables from .env file
+load_dotenv()
 
 # Configure logging for the application
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("financial_notes_api")
 
 
-
 app = FastAPI(
     title="Financial Notes Generator API",
     description="API for generating financial notes, balance sheets, cash flow statements, and P&L reports with RLHF capabilities and Interactive Feedback.",
@@ -212,8 +215,22 @@ async def generate_with_feedback(
         # Create pipeline with feedback integration
         pipeline = create_notes_pipeline(use_rlhf=False)
 
-        # Process through pipeline
-        generation_result, validation_result = pipeline.process(file_path)
+        # Prepare feedback context for the generator
+        feedback_context = {
+            'session_id': session_id,
+            'udfs': session.archived_udfs,  # Pass all archived UDFs
+            'feedback_history': [
+                {
+                    'text': f.feedback_text,
+                    'type': f.feedback_type,
+                    'iteration': f.iteration_number
+                } for f in session.feedback_history
+            ],
+            'current_iteration': session.current_iteration
+        }
+
+        # Process through pipeline with feedback context
+        generation_result, validation_result = pipeline.process(file_path, feedback_context=feedback_context)
 
         if generation_result.success and validation_result.is_valid:
             response = FileResponse(
@@ -250,31 +267,31 @@ async def generate_with_feedback(
         raise HTTPException(status_code=500, detail=f"Generation failed: {str(e)}")
 
 
+
 @router.post("/notes")
-async def notes_route(file: UploadFile = File(...), use_rlhf: bool = Query(False)):
-    file_path = f"data/input/{file.filename}"
-    os.makedirs("data/input", exist_ok=True)
-    with open(file_path, "wb") as buffer:
-        shutil.copyfileobj(file.file, buffer)
-    
-    # Choose workflow based on RLHF preference
-    if use_rlhf:
-        result = run_rlhf_workflow(file_path, "notes")
-    else:
-        result = run_workflow(file_path, "notes")
-    
-    if result["status"] == "success":
-        response = FileResponse(result["result"]["output_xlsx_path"], filename=os.path.basename(result["result"]["output_xlsx_path"]))
-        
-        # Add RLHF metadata to headers if available
-        if "rlhf_metadata" in result.get("result", {}):
-            rlhf_data = result["result"]["rlhf_metadata"]
-            response.headers["X-RLHF-Statement-ID"] = str(rlhf_data.get("statement_id", ""))
-            response.headers["X-RLHF-Quality-Score"] = str(rlhf_data.get("predicted_quality", ""))
-            response.headers["X-RLHF-Confidence"] = str(rlhf_data.get("confidence_score", ""))
-        
-        return response
-    raise HTTPException(status_code=500, detail=result["error"])
+async def notes_route(file: UploadFile = File(...)):
+    """Generate financial notes directly from uploaded file"""
+    try:
+        # Save uploaded file
+        file_path = f"data/input/{file.filename}"
+        os.makedirs("data/input", exist_ok=True)
+        with open(file_path, "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+
+        # Generate notes directly
+        result = generate_notes_full_pipeline_from_path(file_path)
+
+        if result["status"] == "success":
+            # Return the generated Excel file
+            output_path = result["output_xlsx_path"]
+            return FileResponse(output_path, filename=os.path.basename(output_path))
+
+        # If generation failed, raise HTTP exception
+        raise HTTPException(status_code=500, detail=result.get("error", "Notes generation failed"))
+
+    except Exception as e:
+        logger.error(f"Error in notes generation: {e}")
+        raise HTTPException(status_code=500, detail=f"Error generating notes: {str(e)}")
 
 @router.post("/pnl")
 async def pnl_route(file: UploadFile = File(...), use_rlhf: bool = Query(False)):

notes/data_extraction.py

@@ -164,6 +164,20 @@ def extract_trial_balance_data(
 	Returns a list of validated TrialBalanceRecord objects.
 	"""
 	try:
+		# First, try to find a sheet containing 'trial' or 'balance' in the name
+		excel_file = pd.ExcelFile(file_path)
+		trial_sheet_names = [name for name in excel_file.sheet_names if 'trial' in name.lower() or 'balance' in name.lower()]
+		
+		if trial_sheet_names:
+			# Use the first matching sheet
+			sheet_name = trial_sheet_names[0]
+			logger.info(f"Found trial balance sheet: {sheet_name}")
+			# For trial balance sheets, the data usually starts after 5-6 header rows
+			header_row = 5
+		else:
+			logger.warning(f"No trial balance sheet found, using default sheet index {sheet_name}")
+			header_row = header_row  # Use the passed parameter
+		
 		df_raw = pd.read_excel(file_path, sheet_name=sheet_name, header=header_row)
 	except Exception as e:
 		logger.error(f"Error reading Excel file: {e}")

notes/llm_notes_generator.py

@@ -26,7 +26,7 @@ from pydantic_settings import BaseSettings
 from utils.utils import convert_note_json_to_lakhs
 
 # Load environment variables
-load_dotenv()
+load_dotenv(dotenv_path=Path(__file__).parent.parent / '.env')
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)