Update app.py

app.py

@@ -178,7 +178,252 @@ Do not name the company if name is not there and return just the report and noth
         st.error(f"An unexpected error occurred during Gemini report generation: {e}") # Catch other potential errors
         return None
 
+def chunk_transactions(transactions, batch_size=400):
+    """Split transactions into smaller batches for processing."""
+    batches = []
+    for i in range(0, len(transactions), batch_size):
+        batch = transactions[i:i + batch_size]
+        batches.append(batch)
+    st.info(f"Split {len(transactions)} transactions into {len(batches)} batches of up to {batch_size} transactions each.")
+    return batches
+
+def generate_batch_summary(model, json_data, start_date, end_date, statement_type, batch_num, total_batches):
+    """Generate a summary analysis for a batch of transactions."""
+    st.info(f"Processing batch {batch_num}/{total_batches} with {len(json_data['transactions'])} transactions...")
+    
+    prompt = f"""Analyze this batch of transactions (batch {batch_num} of {total_batches}) for the period from {start_date.strftime('%d/%m/%Y')} to {end_date.strftime('%d/%m/%Y')}.
+
+Transaction data:
+{json.dumps(json_data)}
+
+Create a structured summary focusing on aggregation and categorization. Return ONLY the following JSON structure:
+
+{{
+    "batch_info": {{
+        "batch_number": {batch_num},
+        "total_batches": {total_batches},
+        "transaction_count": {len(json_data['transactions'])},
+        "date_range": "{start_date.strftime('%d/%m/%Y')} to {end_date.strftime('%d/%m/%Y')}"
+    }},
+    "financial_summary": {{
+        "total_income": 0,
+        "total_expenses": 0,
+        "net_position": 0
+    }},
+    "income_breakdown": {{
+        "by_customer": {{}},
+        "by_month": {{}}
+    }},
+    "expense_breakdown": {{
+        "by_category": {{}},
+        "by_month": {{}}
+    }},
+    "key_transactions": [
+        // Top 5 largest transactions (income and expense)
+    ],
+    "monthly_totals": {{
+        // Format: "YYYY-MM": {{"income": 0, "expenses": 0, "net": 0}}
+    }}
+}}
+
+Focus on numerical aggregation and categorization. Be precise with calculations."""
+
+    try:
+        response = model.generate_content([prompt])
+        time.sleep(4)
+        return response.text
+    except exceptions.ServiceUnavailable as e:
+        if e.response.status_code == 504:
+            st.error(f"Batch {batch_num} timed out. Skipping this batch.")
+            return None
+        else:
+            st.error(f"API error processing batch {batch_num}: {e}")
+            raise
+    except Exception as e:
+        st.error(f"Error processing batch {batch_num}: {e}")
+        return None
+
+def consolidate_batch_summaries(batch_summaries, start_date, end_date, statement_type):
+    """Combine multiple batch summaries into aggregated data structure."""
+    st.info(f"Consolidating {len(batch_summaries)} batch summaries...")
+    
+    consolidated = {
+        "total_batches": len(batch_summaries),
+        "total_transactions": 0,
+        "date_range": f"{start_date.strftime('%d/%m/%Y')} to {end_date.strftime('%d/%m/%Y')}",
+        "financial_summary": {
+            "total_income": 0,
+            "total_expenses": 0,
+            "net_position": 0
+        },
+        "income_breakdown": {
+            "by_customer": {},
+            "by_month": {}
+        },
+        "expense_breakdown": {
+            "by_category": {},
+            "by_month": {}
+        },
+        "key_transactions": [],
+        "monthly_totals": {}
+    }
+    
+    # Process each batch summary
+    for batch_data in batch_summaries:
+        if not batch_data:
+            continue
+            
+        try:
+            # Extract JSON from response if needed
+            if isinstance(batch_data, str):
+                start_idx = batch_data.find('{')
+                end_idx = batch_data.rfind('}') + 1
+                if start_idx != -1 and end_idx > start_idx:
+                    json_str = batch_data[start_idx:end_idx]
+                    batch_data = json.loads(json_str)
+                else:
+                    st.warning("Could not extract JSON from batch summary")
+                    continue
+            
+            # Aggregate financial summary
+            if 'financial_summary' in batch_data:
+                fs = batch_data['financial_summary']
+                consolidated['financial_summary']['total_income'] += fs.get('total_income', 0)
+                consolidated['financial_summary']['total_expenses'] += fs.get('total_expenses', 0)
+            
+            # Aggregate transaction count
+            if 'batch_info' in batch_data:
+                consolidated['total_transactions'] += batch_data['batch_info'].get('transaction_count', 0)
+            
+            # Merge income breakdown by customer
+            if 'income_breakdown' in batch_data:
+                for customer, amount in batch_data['income_breakdown'].get('by_customer', {}).items():
+                    consolidated['income_breakdown']['by_customer'][customer] = \
+                        consolidated['income_breakdown']['by_customer'].get(customer, 0) + amount
+                
+                # Merge income by month
+                for month, amount in batch_data['income_breakdown'].get('by_month', {}).items():
+                    consolidated['income_breakdown']['by_month'][month] = \
+                        consolidated['income_breakdown']['by_month'].get(month, 0) + amount
+            
+            # Merge expense breakdown by category
+            if 'expense_breakdown' in batch_data:
+                for category, amount in batch_data['expense_breakdown'].get('by_category', {}).items():
+                    consolidated['expense_breakdown']['by_category'][category] = \
+                        consolidated['expense_breakdown']['by_category'].get(category, 0) + amount
+                
+                # Merge expenses by month
+                for month, amount in batch_data['expense_breakdown'].get('by_month', {}).items():
+                    consolidated['expense_breakdown']['by_month'][month] = \
+                        consolidated['expense_breakdown']['by_month'].get(month, 0) + amount
+            
+            # Collect key transactions
+            if 'key_transactions' in batch_data:
+                consolidated['key_transactions'].extend(batch_data.get('key_transactions', []))
+            
+            # Merge monthly totals
+            if 'monthly_totals' in batch_data:
+                for month, totals in batch_data['monthly_totals'].items():
+                    if month not in consolidated['monthly_totals']:
+                        consolidated['monthly_totals'][month] = {"income": 0, "expenses": 0, "net": 0}
+                    
+                    consolidated['monthly_totals'][month]['income'] += totals.get('income', 0)
+                    consolidated['monthly_totals'][month]['expenses'] += totals.get('expenses', 0)
+                    consolidated['monthly_totals'][month]['net'] += totals.get('net', 0)
+            
+        except json.JSONDecodeError as e:
+            st.warning(f"Could not parse batch summary JSON: {e}")
+            continue
+        except Exception as e:
+            st.warning(f"Error processing batch summary: {e}")
+            continue
+    
+    # Calculate final net position
+    consolidated['financial_summary']['net_position'] = \
+        consolidated['financial_summary']['total_income'] - consolidated['financial_summary']['total_expenses']
+    
+    st.success(f"Successfully consolidated data from {len(batch_summaries)} batches covering {consolidated['total_transactions']} transactions.")
+    return consolidated
+
+def generate_final_report(model, consolidated_data, statement_type):
+    """Generate the final comprehensive report using consolidated batch data."""
+    st.info("Generating final comprehensive report from consolidated data...")
+    
+    prompt = f"""Using this consolidated financial data, generate a comprehensive {statement_type} report:
+
+Consolidated Data:
+{json.dumps(consolidated_data, indent=2)}
 
+Generate a detailed {statement_type} report with the following requirements:
+
+1. **Professional Format**: Use standard South African accounting format and terminology
+2. **Clear Structure**: Organize with proper headings, subheadings, and sections
+3. **Comprehensive Analysis**: Include:
+   - Executive Summary
+   - Detailed breakdown by categories/customers
+   - Monthly trend analysis
+   - Key performance indicators
+   - Notable transactions and patterns
+4. **Visual Elements**: Use tables, proper formatting for better readability
+5. **Insights**: Provide meaningful business insights based on the data
+6. **Currency**: Use "R" for South African Rand where appropriate
+
+Return the report in well-formatted Markdown. Do not include company name if not available.
+Focus on creating a professional, comprehensive financial statement that provides clear insights into the business performance."""
+
+    try:
+        response = model.generate_content([prompt])
+        time.sleep(6)
+        st.success("Final comprehensive report generated successfully!")
+        return response.text
+    except exceptions.ServiceUnavailable as e:
+        if e.response.status_code == 504:
+            st.error("Final report generation timed out. The consolidated data might be too large.")
+            return None
+        else:
+            st.error(f"API error generating final report: {e}")
+            raise
+    except Exception as e:
+        st.error(f"Error generating final report: {e}")
+        return None
+
+def generate_batched_financial_report(model, filtered_transactions, start_date, end_date, statement_type, batch_size=400):
+    """Main function to generate financial report using batch processing."""
+    st.info(f"Starting batched financial report generation for {len(filtered_transactions)} transactions...")
+    
+    # Step 1: Split transactions into batches
+    transaction_batches = chunk_transactions(filtered_transactions, batch_size)
+    
+    # Step 2: Process each batch
+    batch_summaries = []
+    progress_bar = st.progress(0)
+    status_text = st.empty()
+    
+    for i, batch in enumerate(transaction_batches):
+        progress = (i + 1) / len(transaction_batches)
+        progress_bar.progress(progress)
+        status_text.text(f"Processing batch {i + 1} of {len(transaction_batches)}...")
+        
+        batch_json = {"transactions": batch}
+        summary = generate_batch_summary(model, batch_json, start_date, end_date, statement_type, i + 1, len(transaction_batches))
+        
+        if summary:
+            batch_summaries.append(summary)
+    
+    progress_bar.progress(1.0)
+    status_text.text("All batches processed!")
+    
+    if not batch_summaries:
+        st.error("No batch summaries were successfully generated.")
+        return None
+    
+    # Step 3: Consolidate batch summaries
+    consolidated_data = consolidate_batch_summaries(batch_summaries, start_date, end_date, statement_type)
+    
+    # Step 4: Generate final comprehensive report
+    final_report = generate_final_report(model, consolidated_data, statement_type)
+    
+    return final_report
 # Install required libraries:
 # pip install fpdf2 beautifulsoup4 markdown
 
@@ -571,12 +816,12 @@ def main():
     statement_type = st.selectbox("Select Financial Statement", ["Income Statement", "Cashflow Statement", "Balance Sheet"])
 
     if st.button("Generate Financial Report"):
-        st.info(f"User clicked 'Generate Financial Report' for {statement_type} from {start_date} to {end_date}.") # Log button click
+        st.info(f"User clicked 'Generate Financial Report' for {statement_type} from {start_date} to {end_date}.")
         if not all_transactions:
-            st.error("No transactions available to generate report. Please upload files first.") # Log
+            st.error("No transactions available to generate report. Please upload files first.")
         else:
             # Filter transactions by date
-            st.info(f"Filtering {len(all_transactions)} transactions for the period {start_date} to {end_date}...") # Log filtering
+            st.info(f"Filtering {len(all_transactions)} transactions for the period {start_date} to {end_date}...")
             filtered_transactions = []
             for transaction in all_transactions:
                 try:
@@ -584,53 +829,56 @@ def main():
                     if start_date <= transaction_date <= end_date:
                         filtered_transactions.append(transaction)
                 except (ValueError, TypeError):
-                    st.warning(f"Could not parse date for transaction, skipping: {transaction}") # Log problematic transactions
+                    st.warning(f"Could not parse date for transaction, skipping: {transaction}")
                     continue
 
             if not filtered_transactions:
-                st.warning("No transactions found within the selected date range. Please adjust dates or upload relevant files.") # Log
+                st.warning("No transactions found within the selected date range. Please adjust dates or upload relevant files.")
             else:
-                st.info(f"Found {len(filtered_transactions)} transactions within the selected date range.") # Log filtered count
+                st.info(f"Found {len(filtered_transactions)} transactions within the selected date range.")
                 try:
                     model1 = configure_gemini1(api_key)
-                    combined_json = {"transactions": filtered_transactions}
-                    with st.spinner("Generating financial report..."):
-                        report_text = generate_financial_report(model1, combined_json, start_date, end_date, statement_type)
-                        if report_text:
-                            st.success("Financial report generated successfully by Gemini!") # Log report text ready
-                            
-                            # Display the report as markdown
-                            st.markdown("### Financial Report Preview")
-                            st.markdown(report_text)
-                
-                            # Create PDF from markdown
-                            try:
-                                st.info("Attempting to generate PDF from the report markdown.") # Log PDF start
-                                pdf_buffer = create_pdf_report(report_text)
-                                st.download_button(
-                                    label="Download Financial Report as PDF",
-                                    data=pdf_buffer.getvalue(),
-                                    file_name=f"{statement_type.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d')}.pdf",
-                                    mime="application/pdf"
-                                )
-                                st.success("PDF download button enabled.") # Log
-                            except Exception as e:
-                                st.error(f"Error generating PDF for download: {str(e)}") # Log PDF error
-                                st.info("For better PDF generation, please ensure NotoSans fonts are installed in the same directory.")
-                                st.exception(e) # Show traceback
-                except exceptions.ServiceUnavailable as e:
-                    if e.response.status_code == 504:
-                        st.error("Error generating report: Gemini API timed out (504). Please try reducing the time period for the report.")
+                    
+                    # Decide whether to use batched or regular processing
+                    if len(filtered_transactions) > 600:
+                        st.info(f"Large dataset detected ({len(filtered_transactions)} transactions). Using batched processing...")
+                        with st.spinner("Generating batched financial report..."):
+                            report_text = generate_batched_financial_report(
+                                model1, filtered_transactions, start_date, end_date, statement_type
+                            )
                     else:
-                        st.error(f"Error generating financial report due to Gemini API issue: {str(e)}") # Log API error
-                        st.exception(e) # Show traceback
+                        st.info("Using standard processing for smaller dataset...")
+                        combined_json = {"transactions": filtered_transactions}
+                        with st.spinner("Generating financial report..."):
+                            report_text = generate_financial_report(model1, combined_json, start_date, end_date, statement_type)
+                    
+                    if report_text:
+                        st.success("Financial report generated successfully!")
+                        
+                        # Display the report as markdown
+                        st.markdown("### Financial Report Preview")
+                        st.markdown(report_text)
+            
+                        # Create PDF from markdown
+                        try:
+                            st.info("Generating PDF from the report...")
+                            pdf_buffer = create_pdf_report(report_text)
+                            st.download_button(
+                                label="Download Financial Report as PDF",
+                                data=pdf_buffer.getvalue(),
+                                file_name=f"{statement_type.replace(' ', '_')}_{datetime.now().strftime('%Y%m%d')}.pdf",
+                                mime="application/pdf"
+                            )
+                            st.success("PDF download ready.")
+                        except Exception as e:
+                            st.error(f"Error generating PDF: {str(e)}")
+                            st.exception(e)
+                            
                 except Exception as e:
-                    st.error(f"An unexpected error occurred while generating the financial report: {str(e)}") # Log general error
+                    st.error(f"Error generating financial report: {str(e)}")
                     if "504" in str(e):
-                        st.info("The Gemini API might be overloaded. Consider generating reports for smaller time periods.")
-                    elif len(filtered_transactions) > 500:
-                        st.info("For large datasets, consider generating reports for smaller time periods.")
-                    st.exception(e) # Show traceback
+                        st.info("Consider using a smaller date range or fewer transactions.")
+                    st.exception(e)
 
 if __name__ == "__main__":
     main()