Update notes/llm_notes_generator.py

notes/llm_notes_generator.py

@@ -10,6 +10,7 @@ class FlexibleFinancialNoteGenerator:
 	def generate_all_notes(self, trial_balance_path=None):
 		# Placeholder logic
 		return {"dummy": True}
+
 import json
 import os
 import logging
@@ -23,10 +24,11 @@ from typing import Dict, List, Any, Optional, Tuple
 import pandas as pd
 from pydantic import BaseModel, ValidationError
 from pydantic_settings import BaseSettings
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from utils.utils import convert_note_json_to_lakhs
 
 # Load environment variables
-load_dotenv()
+load_dotenv(dotenv_path=Path(__file__).parent.parent / '.env')
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
@@ -73,61 +75,32 @@ class FlexibleFinancialNoteGenerator:
             "X-Title": "Financial Note Generator"
         }
         self.note_templates = self.load_note_templates()
-        self.account_patterns = self._init_account_patterns()
+        # Updated model list with DeepSeek as first choice
         self.recommended_models = [
-            "mistralai/mixtral-8x7b-instruct",
-            "mistralai/mistral-7b-instruct-v0.2"
+            
+           "deepseek/deepseek-r1",
+            #"deepseek/deepseek-coder", 
+            "mistralai/mixtral-8x7b-instruct"
         ]
 
-    def _init_account_patterns(self) -> Dict[str, Dict[str, Any]]:
-        """Initialize account classification patterns."""
-        return {
-            "10": {
-                "keywords": ["security deposit", "long term advance", "deposit", "advance recoverable"],
-                "groups": ["Long Term Loans and Advances", "Non-Current Assets"],
-                "exclude_keywords": ["short term", "current", "trade"]
-            },
-            "11": {
-                "keywords": ["inventory", "stock", "raw material", "finished goods", "work in progress", "consumables"],
-                "groups": ["Inventories", "Current Assets"],
-                "exclude_keywords": ["advance", "deposit"]
-            },
-            "12": {
-                "keywords": ["trade receivable", "debtors", "accounts receivable", "sundry debtors"],
-                "groups": ["Trade Receivables", "Current Assets"],
-                "exclude_keywords": ["advance", "deposit"]
-            },
-            "13": {
-                "keywords": ["cash", "bank", "petty cash", "cash on hand", "current account", "savings account", "fixed deposit"],
-                "groups": ["Cash and Bank Balances", "Current Assets"],
-                "exclude_keywords": ["advance", "loan"]
-            },
-            "14": {
-                "keywords": ["prepaid", "advance", "short term", "employee advance", "supplier advance", "advance tax", "tds", "gst", "statutory"],
-                "groups": ["Short Term Loans and Advances", "Current Assets"],
-                "exclude_keywords": ["long term", "security deposit"]
-            },
-            "15": {
-                "keywords": ["interest accrued", "accrued income", "other current", "miscellaneous current"],
-                "groups": ["Other Current Assets", "Current Assets"],
-                "exclude_keywords": ["trade", "advance"]
-            }
-        }
-
     def load_note_templates(self) -> Dict[str, Any]:
-        """Load note templates from app.notes_template.py file."""
+        """Load note templates from notes_template.py file."""
         try:
-            from .notes_template import note_templates
+            # Add parent directory to path for imports when run as script
+            if __name__ == "__main__":
+                sys.path.append(str(Path(__file__).parent.parent))
+
+            from notes_template import note_templates
             return note_templates
         except ImportError as e:
-            logger.error(f"Error importing note_templates from app.notes_template: {e}")
+            logger.error(f"Error importing note_templates from notes_template: {e}")
             return {}
         except Exception as e:
             logger.error(f"Unexpected error loading note_templates: {e}")
             return {}
 
     def load_trial_balance(self, file_path: str = settings.trial_balance_json) -> Optional[Dict[str, Any]]:
-        """Load the classified trial balance from Excel or JSON."""
+        """Load the complete trial balance from Excel or JSON."""
         try:
             if file_path.endswith('.json'):
                 with open(file_path, 'r', encoding='utf-8') as f:
@@ -156,186 +129,155 @@ class FlexibleFinancialNoteGenerator:
             logger.error(f"Error loading trial balance: {e}")
             return None
 
-    def classify_accounts_by_note(self, trial_balance_data: Dict[str, Any], note_number: str) -> List[Dict[str, Any]]:
-        """Classify accounts based on note number and patterns"""
-        if not trial_balance_data or "accounts" not in trial_balance_data:
-            return []
-        
-        classified_accounts = []
-        patterns = self.account_patterns.get(note_number, {})
-        keywords = patterns.get("keywords", [])
-        groups = patterns.get("groups", [])
-        exclude_keywords = patterns.get("exclude_keywords", [])
-        
-        for account in trial_balance_data["accounts"]:
-            account_name = account.get("account_name", "").lower()
-            account_group = account.get("group", "")
-            
-            if any(exclude_word.lower() in account_name for exclude_word in exclude_keywords):
-                continue
-            
-            keyword_match = any(keyword.lower() in account_name for keyword in keywords)
-            group_match = account_group in groups
-            
-            if keyword_match or group_match:
-                classified_accounts.append(account)
-        
-        logger.info(f"Classified {len(classified_accounts)} accounts for Note {note_number}")
-        return classified_accounts
-    
-    def safe_amount_conversion(self, amount: Any, conversion_factor: float = 100000) -> float:
-        """Safely convert amount to lakhs"""
-        try:
-            if isinstance(amount, str):
-                cleaned = re.sub(r'[^\d.-]', '', amount)
-                amount_float = float(cleaned) if cleaned else 0.0
-            else:
-                amount_float = float(amount) if amount is not None else 0.0
-            return round(amount_float / conversion_factor, 2)
-        except (ValueError, TypeError):
-            return 0.0
-    
-    def calculate_totals(self, accounts: List[Dict[str, Any]], conversion_factor: float = 100000) -> Tuple[float, float]:
-        """Calculate totals with safe amount conversion"""
-        total_amount = 0.0
-        for account in accounts:
-            amount = self.safe_amount_conversion(account.get("amount", 0), 1)
-            total_amount += amount
-        total_lakhs = round(total_amount / conversion_factor, 2)
-        return total_amount, total_lakhs
-    
-    def categorize_accounts(self, accounts: List[Dict[str, Any]], note_number: str) -> Dict[str, List[Dict[str, Any]]]:
-        """Categorize accounts based on note-specific rules"""
-        categories = {
-            "prepaid_expenses": [],
-            "other_advances": [],
-            "advance_tax": [],
-            "statutory_balances": [],
-            "uncategorized": []
-        } if note_number == "14" else {}
-        
-        for account in accounts:
-            account_name = account.get("account_name", "").lower()
-            categorized = False
-            
-            if note_number == "14":
-                if "prepaid" in account_name:
-                    categories["prepaid_expenses"].append(account)
-                    categorized = True
-                elif any(word in account_name for word in ["advance tax", "tax advance", "income tax"]):
-                    categories["advance_tax"].append(account)
-                    categorized = True
-                elif any(word in account_name for word in ["tds", "gst", "statutory", "government", "vat", "pf", "esi"]):
-                    categories["statutory_balances"].append(account)
-                    categorized = True
-                elif any(word in account_name for word in ["advance", "deposit", "recoverable", "employee advance", "supplier advance"]):
-                    categories["other_advances"].append(account)
-                    categorized = True
-                
-                if not categorized:
-                    categories["uncategorized"].append(account)
-        
-        return categories
-    
-    def calculate_category_totals(self, categories: Dict[str, List[Dict[str, Any]]], conversion_factor: float = 100000) -> Tuple[Dict[str, Dict[str, Any]], float]:
-        """Calculate totals for each category"""
-        category_totals = {}
-        grand_total = 0.0
-        
-        for category_name, accounts in categories.items():
-            if not isinstance(accounts, list):
-                continue
-            total_amount = 0.0
-            for account in accounts:
-                amount = self.safe_amount_conversion(account.get("amount", 0), 1)
-                total_amount += amount
-            total_lakhs = round(total_amount / conversion_factor, 2)
-            category_totals[category_name] = {
-                "amount": total_amount,
-                "lakhs": total_lakhs,
-                "count": len(accounts),
-                "accounts": [acc.get("account_name", "") for acc in accounts]
-            }
-            grand_total += total_amount
-        
-        return category_totals, round(grand_total / conversion_factor, 2)
-    
-    def build_llm_prompt(self, note_number: str, trial_balance_data: Dict[str, Any], classified_accounts: List[Dict[str, Any]]) -> Optional[str]:
-        """Build dynamic LLM prompt based on note template and classified accounts"""
+    def build_llm_prompt(self, note_number: str, trial_balance_data: Dict[str, Any]) -> Optional[str]:
+        """Build comprehensive LLM prompt with strict JSON output requirements"""
         if note_number not in self.note_templates:
             return None
         
         template = self.note_templates[note_number]
-        total_amount, total_lakhs = self.calculate_totals(classified_accounts)
-        categories = self.categorize_accounts(classified_accounts, note_number)
-        category_totals, grand_total_lakhs = self.calculate_category_totals(categories)
+        all_accounts = trial_balance_data.get("accounts", [])
         
+        # Build context with full trial balance
         context = {
             "note_info": {
                 "number": note_number,
                 "title": template.get("title", ""),
                 "full_title": template.get("full_title", "")
             },
-            "financial_data": {
-                "total_accounts": len(classified_accounts),
-                "total_amount": total_amount,
-                "total_lakhs": total_lakhs,
-                "grand_total_lakhs": grand_total_lakhs
+            "trial_balance": {
+                "total_accounts": len(all_accounts),
+                "accounts": all_accounts
             },
-            "categories": category_totals,
-            "trial_balance": trial_balance_data,
             "current_date": datetime.now().strftime("%Y-%m-%d"),
             "financial_year": "2023-24"
         }
         
-        prompt = (
-            f"\nYou are a financial reporting AI system with two roles:\n"
-            f"1. ACCOUNTANT — You extract, compute, and classify data from the financial context and trial balance.\n"
-            f"2. AUDITOR — You review the Accountant’s output for accuracy, assumptions, and consistency with reporting standards.\n"
-            f"\nYour task is to generate a financial note titled: \"{template['full_title']}\" strictly following the JSON structure below, based on the provided financial context and trial balance data.\n"
-            f"\n---\n**CRITICAL RULES**\n"
-            f"- Respond ONLY with a valid JSON object (no markdown, no explanations).\n"
-            f"- If a value is unavailable or not calculable, use `0.0`.\n"
-            f"- Strictly Convert all ₹ amounts to lakhs by dividing by 100000 and round to 2 decimal places.\n"
-            f"- Ensure that category subtotals **match** the grand total.\n"
-            f"- Return a key `markdown_content` containing a markdown-formatted table for this note.\n"
-            f"- Validate that your JSON structure matches the `TEMPLATE STRUCTURE` exactly.\n"
-            f"- Perform intelligent classification: if an entry from the trial balance clearly fits a category, assign it logically.\n"
-            f"- If data is ambiguous, make a conservative estimate, and record it in an `assumptions` field inside the JSON.\n"
-            f"\n---\n**REFLECTION**\n"
-            f"- After generating the financial note, reflect on the process: Did you miss any data? Are there any uncertainties or assumptions that should be highlighted?\n"
-            f"- Explicitly mention any limitations, ambiguities, or areas where further information would improve accuracy in the `assumptions` field.\n"
-            f"\n**REFLEXION**\n"
-            f"- Before finalizing the output, review your own reasoning and calculations. Double-check that all ₹ amounts are converted to lakhs and that category subtotals match the grand total.\n"
-            f"- If you spot any inconsistencies or possible errors, correct them and note your corrections in the `assumptions` field.\n"
-            f"\n**TALES**\n"
-            f"- For each major category or unusual entry, briefly narrate (in the `assumptions` field) the story or logic behind its classification, especially if it required inference or was ambiguous.\n"
-            f"- Use the `assumptions` field to share any tales of how you mapped trial balance entries to categories, including any conservative estimates or judgment calls.\n"
-            f"\n---\n**TEMPLATE STRUCTURE**\n{json.dumps(template, indent=2)}\n"
-            f"\n---\n**TRIAL BALANCE & CONTEXT**\n{json.dumps(context, indent=2)}\n"
-            f"\n---\n**CATEGORY RULES FOR NOTE 14 (Short Term Loans and Advances):**\n"
-            f"- Categorize entries under:\n"
-            f"  - Unsecured, considered good:\n"
-            f"    - Prepaid Expenses\n"
-            f"    - Other Advances\n"
-            f"  - Other loans and advances:\n"
-            f"    - Advance Tax\n"
-            f"    - Balances with statutory/government authorities\n"
-            f"- Use logical inference to map trial balance entries into these subcategories\n"
-            f"- If values for March 31, 2023 are missing, default to 0\n"
-            f"- Ensure the sum of all subcategories = `Total`\n"
-            f"\n---\n**REQUIRED OUTPUT JSON FORMAT**\n"
-            f"- The JSON must include:\n"
-            f"  - All categories and subcategories with March 2024 and March 2023 values\n"
-            f"  - A computed `grand_total_lakhs`\n"
-            f"  - A `markdown_content` with the financial note table\n"
-            f"  - A `generated_on` timestamp\n"
-            f"  - An `assumptions` field (optional, if any data was inferred or missing)\n"
-            f"\n---\nGenerate the final JSON now:\n"
-        )
+        # Get note-specific classification guidance
+        classification_guide = self._get_classification_guide(note_number)
+        
+        prompt = f"""You are a senior financial analyst and chartered accountant with expertise in Indian accounting standards and Schedule III of the Companies Act 2013.
+
+🔴 CRITICAL INSTRUCTIONS - MUST FOLLOW EXACTLY:
+1. OUTPUT ONLY VALID JSON - NO MARKDOWN, NO EXPLANATIONS, NO TEXT OUTSIDE JSON
+2. START YOUR RESPONSE WITH {{ and END WITH }}
+3. DO NOT USE ```json``` CODE BLOCKS
+4. DO NOT ADD ANY COMMENTARY OR EXPLANATIONS
+
+🔴 REQUIRED JSON STRUCTURE - ALL FIELDS MANDATORY:
+{{
+  "title": "{template.get('title', '')}",
+  "full_title": "{template.get('full_title', '')}",
+  "structure": [
+    {{
+      "category": "In Lakhs",
+      "subcategories": [
+        {{ "label": "March 31, 2024", "value": 0.00 }},
+        {{ "label": "March 31, 2023", "value": 0.00 }}
+      ]
+    }},
+    {{
+      "category": "Category Name",
+      "subcategories": [
+        {{ "label": "Subcategory Item", "value": 0.00, "previous_value": 0.00 }}
+      ],
+      "total": 0.00,
+      "previous_total": 0.00
+    }}
+  ],
+  "metadata": {{
+    "note_number": {note_number},
+    "generated_on": "{datetime.now().isoformat()}"
+  }},
+  "assumptions": "List any assumptions made during classification"
+}}
+
+🔴 STRUCTURE ARRAY EXPLAINED:
+- First element: Header row with column labels (March 31, 2024, March 31, 2023)
+- Subsequent elements: Data categories with subcategories
+- Each data category must have:
+  * "category": Main category name
+  * "subcategories": Array of line items with "label", "value", "previous_value"
+  * "total": Sum of current year values in subcategories
+  * "previous_total": Sum of previous year values in subcategories
+
+🔴 YOUR TASK:
+1. Analyze ALL trial balance accounts provided below
+2. Identify accounts that belong to "{template['full_title']}"
+3. Classify into appropriate subcategories per Schedule III
+4. Convert all amounts to lakhs (₹ ÷ 100,000) with 2 decimal places
+5. Calculate accurate totals ensuring mathematical consistency
+6. Structure output in hierarchical "structure" array format
+
+🔴 MATHEMATICAL REQUIREMENTS:
+- All amounts MUST be in lakhs (divide original by 100,000)
+- All subtotals MUST equal the grand total exactly
+- Use 0.00 for March 2023 if data missing
+- Round to 2 decimal places consistently
+- Ensure "total" = sum of "value" in subcategories
+- Ensure "previous_total" = sum of "previous_value" in subcategories
+
+🔴 CLASSIFICATION GUIDANCE FOR NOTE {note_number}:
+{classification_guide}
+
+🔴 COMPLETE TRIAL BALANCE DATA:
+{json.dumps(context, indent=2)}
+
+🔴 TEMPLATE STRUCTURE TO FOLLOW:
+{json.dumps(template, indent=2)}
+
+🔴 VALIDATION RULES:
+- If no accounts match this note category, use empty categories with 0.00 totals
+- Ensure "metadata.note_number" exactly matches {note_number}
+- Document classification logic in "assumptions" field
+- Structure must have at least 2 elements (header + data)
+
+GENERATE VALID JSON NOW (NO ADDITIONAL TEXT):"""
         
         return prompt
-    
+
+    def _get_classification_guide(self, note_number: str) -> str:
+        """Get note-specific classification guidance"""
+        guides = {
+            "10": """
+**Note 10 - Long Term Loans and Advances:**
+- Include: Security deposits, long-term advances to suppliers/employees, deposits with utilities
+- Categories: Unsecured considered good, Unsecured considered doubtful, Doubtful (provision)
+- Exclude: Short-term advances, trade receivables, prepaid expenses under 1 year
+            """,
+            "11": """
+**Note 11 - Inventories:**
+- Include: Raw materials, work-in-progress, finished goods, stores and spares, consumables
+- Value at lower of cost or net realizable value
+- Exclude: Advances for inventory purchases (classify as advances)
+            """,
+            "12": """
+**Note 12 - Trade Receivables:**
+- Include: Amounts due from customers for goods/services, bills receivable
+- Categories: Unsecured considered good, Unsecured considered doubtful, Provision for doubtful debts
+- Exclude: Advances, deposits, other receivables
+            """,
+            "13": """
+**Note 13 - Cash and Cash Equivalents:**
+- Include: Cash on hand, balances with banks (current/savings), short-term deposits (≤3 months)
+- Separate: Cash and cash equivalents vs Other bank balances (FDs >3 months)
+- Show: Balances in current accounts, savings accounts, fixed deposits separately
+            """,
+            "14": """
+**Note 14 - Short Term Loans and Advances:**
+- Include: Prepaid expenses, advances to suppliers, employee advances, advance tax, TDS receivable
+- Categories: 
+  * Unsecured, considered good: Prepaid expenses, Other advances
+  * Other loans and advances: Advance tax, Balances with statutory/govt authorities
+- Exclude: Long-term advances, trade receivables
+            """,
+            "15": """
+**Note 15 - Other Current Assets:**
+- Include: Interest accrued, export incentives receivable, insurance claims, other miscellaneous current assets
+- Exclude: Items that fit into specific categories like trade receivables, advances, cash
+            """
+        }
+        return guides.get(note_number, f"**Note {note_number}:** Classify accounts logically based on their nature and the note title.")
+
     def call_openrouter_api(self, prompt: str) -> Optional[str]:
         """Make API call to OpenRouter with model fallback"""
         for model in self.recommended_models:
@@ -343,10 +285,13 @@ class FlexibleFinancialNoteGenerator:
             payload = {
                 "model": model,
                 "messages": [
-                    {"role": "system", "content": "You are a financial reporting expert. Always respond with valid JSON only."},
+                    {
+                        "role": "system", 
+                        "content": "You are an expert chartered accountant specializing in Indian accounting standards. You MUST respond with ONLY valid JSON, never with markdown code blocks or explanations. Start with { and end with }."
+                    },
                     {"role": "user", "content": prompt}
                 ],
-                "max_tokens": 8000,
+                "max_tokens": 12000,
                 "temperature": 0.1,
                 "top_p": 0.9
             }
@@ -355,13 +300,20 @@ class FlexibleFinancialNoteGenerator:
                     self.api_url,
                     headers=self.headers,
                     json=payload,
-                    timeout=30  # <-- Add timeout here!
+                    timeout=60
                 )
                 response.raise_for_status()
                 result = response.json()
                 content = result['choices'][0]['message']['content']
                 logger.info(f"Successful response from {model}")
                 return content
+            except requests.exceptions.HTTPError as e:
+                if e.response.status_code == 404:
+                    logger.warning(f"Model {model} not found (404), trying next model")
+                elif e.response.status_code == 402:
+                    logger.warning(f"Model {model} requires payment (402), trying next model")
+                else:
+                    logger.error(f"HTTP error with {model}: {e}")
             except Exception as e:
                 logger.error(f"Failed with {model}: {e}")
                 continue
@@ -369,64 +321,385 @@ class FlexibleFinancialNoteGenerator:
         return None
     
     def extract_json_from_markdown(self, response_text: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
-        """Extract JSON from response, handling markdown code blocks"""
+        """Extract JSON from response, handling markdown code blocks and cleaning"""
         response_text = response_text.strip()
+        
+        # CRITICAL FIX: Handle concatenated/duplicate JSON (e.g., "}{\n{")
+        # Find the first complete JSON object
+        json_objects = []
+        brace_count = 0
+        start_idx = -1
+        
+        for i, char in enumerate(response_text):
+            if char == '{':
+                if brace_count == 0:
+                    start_idx = i
+                brace_count += 1
+            elif char == '}':
+                brace_count -= 1
+                if brace_count == 0 and start_idx != -1:
+                    # Found complete JSON object
+                    potential_json = response_text[start_idx:i+1]
+                    try:
+                        parsed = json.loads(potential_json)
+                        json_objects.append((parsed, potential_json))
+                        # Use the first valid JSON object
+                        break
+                    except json.JSONDecodeError:
+                        continue
+        
+        if json_objects:
+            logger.info("Successfully extracted first valid JSON object from response")
+            return json_objects[0]
+        
+        # Fallback: Try original extraction methods
+        # Remove any leading/trailing text outside JSON
         json_patterns = [
             r'```json\s*(.*?)\s*```',
             r'```\s*(.*?)\s*```',
-            r'(\{.*\})'
+            r'(\{.*?\})'
         ]
         
         for pattern in json_patterns:
             match = re.search(pattern, response_text, re.DOTALL)
             if match:
                 try:
-                    json_data = json.loads(match.group(1))
-                    return json_data, match.group(1)
+                    json_content = match.group(1).strip()
+                    json_data = json.loads(json_content)
+                    return json_data, json_content
                 except json.JSONDecodeError:
                     continue
         
+        # Try parsing the entire response as JSON
         try:
             json_data = json.loads(response_text)
             return json_data, response_text
         except json.JSONDecodeError:
+            # Last attempt: find JSON-like structure
+            try:
+                start = response_text.find('{')
+                end = response_text.rfind('}') + 1
+                if start != -1 and end > start:
+                    json_part = response_text[start:end]
+                    json_data = json.loads(json_part)
+                    return json_data, json_part
+            except json.JSONDecodeError:
+                pass
+            
             return None, None
     
+    def validate_and_fix_json(self, json_data: Dict[str, Any], note_number: str) -> Dict[str, Any]:
+        """Validate JSON structure and auto-fix missing required fields"""
+        fixed_data = json_data.copy()
+        
+        # Get template for this note
+        template = self.note_templates.get(note_number, {})
+        
+        # Auto-fix title fields
+        if "title" not in fixed_data or not fixed_data["title"]:
+            fixed_data["title"] = template.get("title", f"Note {note_number}")
+            logger.info(f"Auto-fixed missing title field")
+        
+        if "full_title" not in fixed_data or not fixed_data["full_title"]:
+            fixed_data["full_title"] = template.get("full_title", f"{note_number}. {fixed_data.get('title', 'Financial Note')}")
+            logger.info(f"Auto-fixed missing full_title field")
+        
+        # Auto-fix or create metadata
+        if "metadata" not in fixed_data or not isinstance(fixed_data["metadata"], dict):
+            fixed_data["metadata"] = {}
+            logger.info("Auto-created metadata object")
+        
+        # CRITICAL FIX: Ensure note_number is correct integer, not 0.0
+        metadata_note_num = fixed_data["metadata"].get("note_number")
+        try:
+            # Convert note_number string to int
+            expected_note_num = int(note_number)
+            
+            # Check if metadata note_number is wrong (0, 0.0, or mismatch)
+            if (metadata_note_num is None or 
+                metadata_note_num == 0 or 
+                metadata_note_num == 0.0 or 
+                int(metadata_note_num) != expected_note_num):
+                
+                fixed_data["metadata"]["note_number"] = expected_note_num
+                logger.info(f"Auto-corrected metadata.note_number from {metadata_note_num} to {expected_note_num}")
+        except ValueError:
+            fixed_data["metadata"]["note_number"] = note_number
+            logger.info(f"Auto-set metadata.note_number to string: {note_number}")
+        
+        if "generated_on" not in fixed_data["metadata"]:
+            fixed_data["metadata"]["generated_on"] = datetime.now().isoformat()
+            logger.info("Auto-fixed missing metadata.generated_on field")
+        
+        # Auto-fix or create structure array
+        if "structure" not in fixed_data or not isinstance(fixed_data["structure"], list):
+            logger.warning("Structure array missing, creating default structure")
+            fixed_data["structure"] = [
+                {
+                    "category": "In Lakhs",
+                    "subcategories": [
+                        {"label": "March 31, 2024", "value": 0.00},
+                        {"label": "March 31, 2023", "value": 0.00}
+                    ]
+                },
+                {
+                    "category": "No data available",
+                    "subcategories": [
+                        {"label": "Items", "value": 0.00, "previous_value": 0.00}
+                    ],
+                    "total": 0.00,
+                    "previous_total": 0.00
+                }
+            ]
+        else:
+            # Validate and fix structure elements
+            if len(fixed_data["structure"]) == 0:
+                logger.warning("Empty structure array, adding default elements")
+                fixed_data["structure"] = [
+                    {
+                        "category": "In Lakhs",
+                        "subcategories": [
+                            {"label": "March 31, 2024", "value": 0.00},
+                            {"label": "March 31, 2023", "value": 0.00}
+                        ]
+                    }
+                ]
+            
+            # Ensure each structure element has required fields
+            for i, struct_elem in enumerate(fixed_data["structure"]):
+                if not isinstance(struct_elem, dict):
+                    continue
+                
+                if "category" not in struct_elem:
+                    struct_elem["category"] = f"Category {i}"
+                
+                if "subcategories" not in struct_elem or not isinstance(struct_elem["subcategories"], list):
+                    struct_elem["subcategories"] = []
+                
+                # For data rows (not header), ensure totals exist
+                if i > 0 and struct_elem.get("subcategories"):
+                    if "total" not in struct_elem:
+                        struct_elem["total"] = sum(
+                            sub.get("value", 0.0) 
+                            for sub in struct_elem["subcategories"] 
+                            if isinstance(sub, dict)
+                        )
+                    
+                    if "previous_total" not in struct_elem:
+                        struct_elem["previous_total"] = sum(
+                            sub.get("previous_value", 0.0) 
+                            for sub in struct_elem["subcategories"] 
+                            if isinstance(sub, dict)
+                        )
+        
+        # Auto-fix assumptions
+        if "assumptions" not in fixed_data:
+            fixed_data["assumptions"] = "Classification based on account names and standard accounting practices"
+            logger.info("Auto-added default assumptions")
+        
+        return fixed_data
+
+    def validate_json_structure(self, json_data: Dict[str, Any], note_number: str) -> Tuple[bool, str]:
+        """Validate that the JSON matches expected structure"""
+        required_fields = ["title", "full_title", "structure", "metadata", "assumptions"]
+        
+        # Check required fields
+        missing_fields = []
+        for field in required_fields:
+            if field not in json_data:
+                missing_fields.append(field)
+        
+        if missing_fields:
+            return False, f"Missing required fields: {', '.join(missing_fields)}"
+        
+        # Check metadata structure
+        if not isinstance(json_data.get("metadata"), dict):
+            return False, "metadata must be an object"
+        
+        metadata = json_data["metadata"]
+        if "note_number" not in metadata:
+            return False, "metadata.note_number is required"
+        
+        if str(metadata.get("note_number", "")) != str(note_number):
+            return False, f"Note number mismatch: expected {note_number}, got {metadata.get('note_number')}"
+        
+        # Check structure array
+        if not isinstance(json_data.get("structure"), list):
+            return False, "structure must be an array"
+        
+        if len(json_data["structure"]) == 0:
+            return False, "structure array cannot be empty"
+        
+        return True, "Validation passed"
+
+    def _generate_markdown_from_structure(self, json_data: Dict[str, Any]) -> str:
+        """Generate markdown table from structure array"""
+        try:
+            title = json_data.get("full_title", json_data.get("title", "Financial Note"))
+            structure = json_data.get("structure", [])
+            
+            if not structure:
+                return f"# {title}\n\n*No data available*"
+            
+            # Start markdown
+            md_lines = [f"# {title}\n"]
+            
+            # Get header row (first element)
+            header_elem = structure[0] if len(structure) > 0 else None
+            if header_elem and header_elem.get("subcategories"):
+                headers = [sub.get("label", "") for sub in header_elem["subcategories"]]
+                md_lines.append("| Particulars | " + " | ".join(headers) + " |")
+                md_lines.append("|" + "---|" * (len(headers) + 1))
+            
+            # Process data rows
+            for i in range(1, len(structure)):
+                elem = structure[i]
+                category = elem.get("category", "")
+                subcategories = elem.get("subcategories", [])
+                
+                # Add category header if exists
+                if category:
+                    md_lines.append(f"\n**{category}**\n")
+                
+                # Add subcategory rows
+                for sub in subcategories:
+                    label = sub.get("label", "")
+                    value = sub.get("value", 0.00)
+                    previous_value = sub.get("previous_value", 0.00)
+                    md_lines.append(f"| {label} | {value:.2f} | {previous_value:.2f} |")
+                
+                # Add total row if exists
+                if "total" in elem:
+                    total = elem.get("total", 0.00)
+                    previous_total = elem.get("previous_total", 0.00)
+                    md_lines.append(f"| **Total {category}** | **{total:.2f}** | **{previous_total:.2f}** |")
+            
+            # Add metadata
+            metadata = json_data.get("metadata", {})
+            md_lines.append(f"\n\n*Generated on: {metadata.get('generated_on', 'Unknown')}*")
+            
+            # Add assumptions if present
+            assumptions = json_data.get("assumptions", "")
+            if assumptions:
+                md_lines.append(f"\n\n**Assumptions:** {assumptions}")
+            
+            return "\n".join(md_lines)
+            
+        except Exception as e:
+            logger.error(f"Error generating markdown from structure: {e}")
+            return f"# {json_data.get('full_title', 'Financial Note')}\n\n*Error generating markdown table*"
+
     def save_generated_note(self, note_data: str, note_number: str, output_dir: str = settings.output_dir) -> bool:
-        """Save the generated note to file in both JSON and markdown formats"""
+        """Save the generated note to file with robust validation and auto-fixing"""
         Path(output_dir).mkdir(parents=True, exist_ok=True)
         json_output_path = f"{output_dir}/notes.json"
         raw_output_path = f"{output_dir}/notes_raw.txt"
         formatted_md_path = f"{output_dir}/notes_formatted.md"
         
         try:
+            # Always save raw response for debugging
             with open(raw_output_path, 'w', encoding='utf-8') as f:
                 f.write(note_data)
+            
+            # Extract and validate JSON
             json_data, json_string = self.extract_json_from_markdown(note_data)
+            
             if json_data:
+                # Auto-fix missing or incorrect fields
+                json_data = self.validate_and_fix_json(json_data, note_number)
+                
+                # Final validation
+                is_valid, validation_msg = self.validate_json_structure(json_data, note_number)
+                if not is_valid:
+                    logger.warning(f"JSON validation warning after auto-fix: {validation_msg}")
+                
+                # Convert to lakhs if needed
                 json_data = convert_note_json_to_lakhs(json_data)
+                
+                # Save JSON
                 with open(json_output_path, 'w', encoding='utf-8') as f:
                     json.dump(json_data, f, indent=2, ensure_ascii=False)
                 logger.info(f"JSON saved to {json_output_path}")
-                md_content = json_data.get('markdown_content')
+                
+                # Generate and save markdown
+                md_content = json_data.get('markdown_content', '')
                 if not md_content:
-                    md_content = f"# Note {note_number}\n\n```json\n{json.dumps(json_data, indent=2)}\n```"
+                    # Generate markdown from structure
+                    md_content = self._generate_markdown_from_structure(json_data)
+                    logger.info("Auto-generated markdown from structure array")
+                
                 with open(formatted_md_path, 'w', encoding='utf-8') as f:
                     f.write(md_content)
+                    
                 return True
             else:
+                # Create fallback JSON with all required fields
+                template = self.note_templates.get(note_number, {})
                 fallback_json = {
-                    "note_number": note_number,
+                    "title": template.get("title", f"Note {note_number}"),
+                    "full_title": template.get("full_title", f"{note_number}. Financial Note"),
+                    "structure": [
+                        {
+                            "category": "In Lakhs",
+                            "subcategories": [
+                                {"label": "March 31, 2024", "value": 0.00},
+                                {"label": "March 31, 2023", "value": 0.00}
+                            ]
+                        },
+                        {
+                            "category": "Error - No data",
+                            "subcategories": [
+                                {"label": "Could not parse response", "value": 0.00, "previous_value": 0.00}
+                            ],
+                            "total": 0.00,
+                            "previous_total": 0.00
+                        }
+                    ],
+                    "metadata": {
+                        "note_number": int(note_number) if note_number.isdigit() else note_number,
+                        "generated_on": datetime.now().isoformat()
+                    },
+                    "assumptions": "Failed to parse LLM response",
                     "raw_response": note_data,
-                    "error": "Could not parse JSON from response",
-                    "generated_on": datetime.now().isoformat()
+                    "error": "Could not parse JSON from response"
                 }
+                
                 with open(json_output_path, 'w', encoding='utf-8') as f:
                     json.dump(fallback_json, f, indent=2, ensure_ascii=False)
-                logger.warning(f"Fallback JSON saved to {json_output_path}")
+                logger.warning(f"Fallback JSON with required fields saved to {json_output_path}")
                 return False
+                
         except Exception as e:
             logger.error(f"Error saving files: {e}")
+            
+            # Emergency fallback
+            try:
+                template = self.note_templates.get(note_number, {})
+                emergency_json = {
+                    "title": template.get("title", f"Note {note_number}"),
+                    "full_title": template.get("full_title", f"{note_number}. Financial Note"),
+                    "structure": [
+                        {
+                            "category": "In Lakhs",
+                            "subcategories": [
+                                {"label": "March 31, 2024", "value": 0.00},
+                                {"label": "March 31, 2023", "value": 0.00}
+                            ]
+                        }
+                    ],
+                    "metadata": {
+                        "note_number": int(note_number) if note_number.isdigit() else note_number,
+                        "generated_on": datetime.now().isoformat()
+                    },
+                    "assumptions": "Emergency fallback due to processing error",
+                    "error": str(e)
+                }
+                with open(json_output_path, 'w', encoding='utf-8') as f:
+                    json.dump(emergency_json, f, indent=2, ensure_ascii=False)
+                logger.info(f"Emergency fallback JSON saved to {json_output_path}")
+            except Exception as emergency_error:
+                logger.error(f"Emergency fallback also failed: {emergency_error}")
+            
             return False
     
     def generate_note(self, note_number: str, trial_balance_path: str = settings.trial_balance_json) -> bool:
@@ -436,21 +709,25 @@ class FlexibleFinancialNoteGenerator:
             return False
         
         logger.info(f"Starting Note {note_number} generation...")
+        
+        # Load complete trial balance
         trial_balance = self.load_trial_balance(trial_balance_path)
         if not trial_balance:
             return False
         
-        classified_accounts = self.classify_accounts_by_note(trial_balance, note_number)
-        prompt = self.build_llm_prompt(note_number, trial_balance, classified_accounts)
+        # Build prompt with full trial balance
+        prompt = self.build_llm_prompt(note_number, trial_balance)
         if not prompt:
             logger.error("Failed to build prompt")
             return False
         
+        # Get LLM response
         response = self.call_openrouter_api(prompt)
         if not response:
             logger.error("Failed to get API response")
             return False
         
+        # Save the generated note
         success = self.save_generated_note(response, note_number)
         logger.info(f"Note {note_number} {'generated successfully' if success else 'generated with issues'}")
         return success
@@ -460,69 +737,230 @@ class FlexibleFinancialNoteGenerator:
         logger.info(f"Starting generation of all {len(self.note_templates)} notes...")
         results = {}
         all_notes = []
+        
+        # Load trial balance once
+        trial_balance = self.load_trial_balance(trial_balance_path)
+        if not trial_balance:
+            logger.error("Failed to load trial balance")
+            return {note: False for note in self.note_templates.keys()}
+        
         for note_number in self.note_templates.keys():
             logger.info(f"Processing Note {note_number}")
-            trial_balance = self.load_trial_balance(trial_balance_path)
-            if not trial_balance:
-                results[note_number] = False
-                continue
-            classified_accounts = self.classify_accounts_by_note(trial_balance, note_number)
-            prompt = self.build_llm_prompt(note_number, trial_balance, classified_accounts)
+            
+            # Build prompt for this note
+            prompt = self.build_llm_prompt(note_number, trial_balance)
             if not prompt:
                 results[note_number] = False
                 continue
+            
+            # Get LLM response
             response = self.call_openrouter_api(prompt)
             if not response:
                 results[note_number] = False
                 continue
+            
+            # Parse JSON response
             json_data, _ = self.extract_json_from_markdown(response)
             if json_data:
-                all_notes.append(json_data)
-                results[note_number] = True
+                # Auto-fix and validate
+                json_data = self.validate_and_fix_json(json_data, note_number)
+                is_valid, validation_msg = self.validate_json_structure(json_data, note_number)
+                
+                if is_valid:
+                    json_data = convert_note_json_to_lakhs(json_data)
+                    all_notes.append(json_data)
+                    results[note_number] = True
+                    logger.info(f"Note {note_number} processed successfully")
+                else:
+                    logger.warning(f"Note {note_number} validation failed even after auto-fix: {validation_msg}")
+                    # Still include it but mark as failed
+                    json_data = convert_note_json_to_lakhs(json_data)
+                    all_notes.append(json_data)
+                    results[note_number] = False
             else:
+                logger.error(f"Note {note_number}: Could not parse JSON from response")
+                # Create fallback note with new structure
+                template = self.note_templates.get(note_number, {})
+                fallback_note = {
+                    "title": template.get("title", f"Note {note_number}"),
+                    "full_title": template.get("full_title", f"{note_number}. Financial Note"),
+                    "structure": [
+                        {
+                            "category": "In Lakhs",
+                            "subcategories": [
+                                {"label": "March 31, 2024", "value": 0.00},
+                                {"label": "March 31, 2023", "value": 0.00}
+                            ]
+                        },
+                        {
+                            "category": "Error",
+                            "subcategories": [
+                                {"label": "Failed to generate from LLM response", "value": 0.00, "previous_value": 0.00}
+                            ],
+                            "total": 0.00,
+                            "previous_total": 0.00
+                        }
+                    ],
+                    "metadata": {
+                        "note_number": int(note_number) if note_number.isdigit() else note_number,
+                        "generated_on": datetime.now().isoformat()
+                    },
+                    "assumptions": "LLM response parsing failed",
+                    "error": "JSON parsing failed"
+                }
+                all_notes.append(fallback_note)
                 results[note_number] = False
+            
+            # Brief pause between API calls
             import time
-            time.sleep(1)
-        # Save all notes in one file
+            time.sleep(2)
+        
+        # Save all notes in consolidated file
         output_dir = settings.output_dir
         Path(output_dir).mkdir(parents=True, exist_ok=True)
+        
+        consolidated_output = {
+            "notes": all_notes,
+            "generation_summary": {
+                "total_notes": len(self.note_templates),
+                "successful_notes": sum(1 for success in results.values() if success),
+                "failed_notes": sum(1 for success in results.values() if not success),
+                "generated_on": datetime.now().isoformat(),
+                "results": results
+            }
+        }
+        
         with open(f"{output_dir}/notes.json", "w", encoding="utf-8") as f:
-            json.dump({"notes": all_notes}, f, indent=2, ensure_ascii=False)
+            json.dump(consolidated_output, f, indent=2, ensure_ascii=False)
+        
         successful = sum(1 for success in results.values() if success)
         total = len(results)
         logger.info(f"GENERATION SUMMARY: {successful}/{total} notes generated successfully")
         logger.info(f"All notes saved to {output_dir}/notes.json")
+        
         return results
 
 def main() -> None:
     """Main function to run the flexible note generator"""
     try:
+        # Initialize generator
         generator = FlexibleFinancialNoteGenerator()
         if not generator.note_templates:
-            logger.error("No note templates loaded. Check app/new.py")
+            logger.error("No note templates loaded. Check notes_template.py")
             return
-        
+
         logger.info(f"Loaded {len(generator.note_templates)} note templates")
-        choice = input("\nGenerate (1) specific note or (2) all notes? Enter 1 or 2: ").strip()
-        
-        if choice == "1":
-            available_notes = list(generator.note_templates.keys())
-            print(f"Available notes: {', '.join(available_notes)}")
-            note_number = input("Enter note number: ").strip()
-            if note_number in available_notes:
-                success = generator.generate_note(note_number)
-                logger.info(f"Note {note_number} {'generated successfully' if success else 'generated with issues'}")
+
+        # Check for command line arguments
+        if len(sys.argv) > 1:
+            # Command line mode
+            if len(sys.argv) < 3:
+                logger.error("Usage: python llm_notes_generator.py <mode> <note_numbers>")
+                logger.error("  mode: 'specific' or 'all'")
+                logger.error("  note_numbers: comma-separated note numbers (for specific mode)")
+                sys.exit(1)
+
+            mode = sys.argv[1].lower()
+            note_numbers = sys.argv[2] if len(sys.argv) > 2 else ""
+
+            if mode == "specific":
+                if not note_numbers:
+                    logger.error("Note numbers required for specific mode")
+                    sys.exit(1)
+
+                note_list = [n.strip() for n in note_numbers.split(",")]
+                all_notes = []
+                successful_notes = []
+
+                for note_number in note_list:
+                    if note_number in generator.note_templates:
+                        success = generator.generate_note(note_number)
+                        if success:
+                            # Load the generated note
+                            try:
+                                with open("data/generated_notes/notes.json", "r", encoding="utf-8") as f:
+                                    note_data = json.load(f)
+                                all_notes.append(note_data)
+                                successful_notes.append(note_number)
+                                logger.info(f"Note {note_number} generated successfully")
+                            except Exception as e:
+                                logger.error(f"Failed to load generated note {note_number}: {e}")
+                        else:
+                            logger.error(f"Failed to generate note {note_number}")
+                    else:
+                        logger.error(f"Note {note_number} not found in templates")
+
+                # Save consolidated notes
+                if all_notes:
+                    output_dir = settings.output_dir
+                    Path(output_dir).mkdir(parents=True, exist_ok=True)
+                    consolidated = {
+                        "notes": all_notes,
+                        "generation_summary": {
+                            "requested_notes": note_list,
+                            "successful_notes": successful_notes,
+                            "total_successful": len(successful_notes),
+                            "generated_on": datetime.now().isoformat()
+                        }
+                    }
+                    with open(f"{output_dir}/notes.json", "w", encoding="utf-8") as f:
+                        json.dump(consolidated, f, indent=2, ensure_ascii=False)
+                    logger.info(f"Consolidated notes saved to {output_dir}/notes.json")
+
+            elif mode == "all":
+                results = generator.generate_all_notes()
+                successful = sum(1 for success in results.values() if success)
+                total = len(results)
+                logger.info(f"{successful}/{total} notes generated successfully")
+                
+                # Print detailed results
+                for note, success in results.items():
+                    status = "✅ SUCCESS" if success else "❌ FAILED"
+                    logger.info(f"  Note {note}: {status}")
+                    
             else:
-                logger.error(f"Note {note_number} not found")
-        elif choice == "2":
-            results = generator.generate_all_notes()
-            successful = sum(1 for success in results.values() if success)
-            total = len(results)
-            logger.info(f"{successful}/{total} notes generated successfully")
+                logger.error("Invalid mode. Use 'specific' or 'all'")
+                sys.exit(1)
+                
         else:
-            logger.error("Invalid choice. Enter 1 or 2.")
+            # Interactive mode
+            choice = input("\nGenerate (1) specific note or (2) all notes? Enter 1 or 2: ").strip()
+
+            if choice == "1":
+                available_notes = list(generator.note_templates.keys())
+                print(f"Available notes: {', '.join(available_notes)}")
+                note_number = input("Enter note number: ").strip()
+                
+                if note_number in available_notes:
+                    success = generator.generate_note(note_number)
+                    logger.info(f"Note {note_number} {'generated successfully' if success else 'generated with issues'}")
+                else:
+                    logger.error(f"Note {note_number} not found")
+                    
+            elif choice == "2":
+                results = generator.generate_all_notes()
+                successful = sum(1 for success in results.values() if success)
+                total = len(results)
+                logger.info(f"{successful}/{total} notes generated successfully")
+                
+                # Print summary
+                print("\n" + "="*50)
+                print("GENERATION SUMMARY")
+                print("="*50)
+                for note, success in results.items():
+                    status = "✅ SUCCESS" if success else "❌ FAILED"
+                    print(f"Note {note}: {status}")
+                print("="*50)
+                
+            else:
+                logger.error("Invalid choice. Enter 1 or 2.")
+                
+    except KeyboardInterrupt:
+        logger.info("Generation interrupted by user")
+        sys.exit(0)
     except Exception as e:
         logger.error(f"Error: {e}", exc_info=True)
+        sys.exit(1)
 
 if __name__ == "__main__":
     main()