Update app.py

app.py

@@ -1,574 +1,960 @@
-import gradio as gr
+https://huggingface.co/spaces/Basementup/Seamstressesedout/edit/main/app.py
+
+
 import os
-import json
-import tempfile
-from datetime import datetime
-from typing import List, Dict
-import PyPDF2
-from difflib import SequenceMatcher
 import re
-import requests
-
-# ============================================================================
-# OPTIONAL: Hugging Face zero-shot classifier (CPU mode, loads lazily)
-# ============================================================================
-zero_shot_classifier = None
-
-def get_classifier():
-    global zero_shot_classifier
-    if zero_shot_classifier is None:
-        try:
-            from transformers import pipeline
-            zero_shot_classifier = pipeline(
-                "zero-shot-classification",
-                model="facebook/bart-large-mnli",
-                device=-1
-            )
-        except Exception as e:
-            print(f"Warning: Could not load zero-shot classifier: {e}")
-    return zero_shot_classifier
-
-
-# ============================================================================
-# MODULE 1: Document Parser
-# ============================================================================
-
-class DocumentParser:
-
-    @staticmethod
-    def extract_text_from_pdf(file_path: str) -> str:
-        try:
-            with open(file_path, 'rb') as f:
-                reader = PyPDF2.PdfReader(f)
-                text = ""
-                for page in reader.pages:
-                    extracted = page.extract_text()
-                    if extracted:
-                        text += extracted + "\n"
-            return text
-        except Exception as e:
-            return f"Error reading PDF: {str(e)}"
-
-    @staticmethod
-    def extract_text_from_file(file_path: str) -> str:
-        if not file_path:
-            return "Error: No file path provided"
-
-        try:
-            lower_path = file_path.lower()
-
-            if lower_path.endswith('.pdf'):
-                return DocumentParser.extract_text_from_pdf(file_path)
-
-            if lower_path.endswith('.docx'):
-                from docx import Document as DocxDocument
-                doc = DocxDocument(file_path)
-                paragraphs = [p.text.strip() for p in doc.paragraphs if p.text and p.text.strip()]
-                return "\n\n".join(paragraphs)
-
-            if lower_path.endswith('.txt'):
-                with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-                    return f.read()
-
-            return "Error reading file: Unsupported file type"
-
-        except Exception as e:
-            return f"Error reading file: {str(e)}"
-
-    @staticmethod
-    def parse_document(text: str) -> Dict:
-        text = text.replace('\r\n', '\n').replace('\r', '\n')
-        text = re.sub(r'\n[ \t]*\n+', '\n\n', text).strip()
-
+import asyncio
+import pandas as pd
+import numpy as np
+import gradio as gr
+import pdfplumber
+import dateparser
+import edge_tts
+from datetime import datetime
+from sentence_transformers import SentenceTransformer, util
+from docx import Document
+from docx.shared import Pt, Inches, RGBColor
+from docx.enum.text import WD_ALIGN_PARAGRAPH
+from docx.oxml.ns import qn
+from docx.oxml import OxmlElement
+from itertools import groupby
+from openai import OpenAI # Import OpenAI library
+
+# Force model to cache properly on Hugging Face
+os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+os.environ["SENTENCE_TRANSFORMERS_HOME"] = "/tmp/model_cache"
+
+# Initialize the Sentence Transformer model for deduplication
+model_deduplication = SentenceTransformer('all-MiniLM-L6-v2')
+
+# Initialize OpenAI client (will use environment variable OPENAI_API_KEY if set)
+# For Hugging Face Spaces, you might need to set this as a Space Secret
+client = OpenAI()
+
+# --- Utility Functions ---
+
+def extract_text_from_file(file_path):
+    """Extracts text from PDF, DOCX, or TXT files."""
+    ext = os.path.splitext(file_path)[1].lower()
+    text = ""
+    try:
+        if ext == ".pdf":
+            with pdfplumber.open(file_path) as pdf:
+                for page in pdf.pages:
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += page_text + "\n"
+        elif ext == ".docx":
+            doc = Document(file_path)
+            text = "\n".join([para.text for para in doc.paragraphs])
+        elif ext == ".txt":
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                text = f.read()
+    except Exception as e:
+        print(f"Error extracting from {file_path}: {e}")
+    return text
+
+def extract_date(text):
+    """Attempts to extract a date from a text snippet."""
+    date_patterns = [
+        r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b',
+        r'\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b',
+        r'\b\d{4}-\d{2}-\d{2}\b'
+    ]
+    for pattern in date_patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            parsed_date = dateparser.parse(match.group())
+            if parsed_date:
+                return parsed_date
+    return None
+
+def detect_breach(text):
+    """Detects if a text snippet describes a breach or violation."""
+    breach_keywords = [
+        "breach", "violation", "failure", "non-compliance", "infringement", 
+        "unauthorized", "leak", "compromise", "illegal", "incident", "misconduct"
+    ]
+    for keyword in breach_keywords:
+        if keyword.lower() in text.lower():
+            return True
+    return False
+
+def create_docx(sections, output_path="Master_Evidence_Document.docx"):
+    """Creates a professionally formatted DOCX file with cover, TOC, and monthly grouping."""
+    doc = Document()
+    
+    # Style settings
+    style = doc.styles['Normal']
+    font = style.font
+    font.name = 'Calibri'
+    font.size = Pt(11)
+    style.paragraph_format.space_after = Pt(6)
+    
+    # Cover page
+    title = doc.add_heading('Master Evidence Document', 0)
+    title.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    subtitle = doc.add_heading('Consolidated Chronology and Identified Breaches', level=1)
+    subtitle.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    date_para = doc.add_paragraph()
+    date_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    date_para.add_run(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    doc.add_page_break()
+    
+    # Table of Contents
+    doc.add_heading('Table of Contents', level=1)
+    para = doc.add_paragraph()
+    run = para.add_run()
+    fldChar1 = OxmlElement('w:fldChar')
+    fldChar1.set(qn('w:fldCharType'), 'begin')
+    run._r.append(fldChar1)
+    instrText = OxmlElement('w:instrText')
+    instrText.set(qn('xml:space'), 'preserve')
+    instrText.text = ' TOC \\o "1-3" \\h \\z \\u '
+    run._r.append(instrText)
+    fldChar2 = OxmlElement('w:fldChar')
+    fldChar2.set(qn('w:fldCharType'), 'separate')
+    run._r.append(fldChar2)
+    run2 = para.add_run('Right-click and choose "Update Field" to generate the Table of Contents')
+    fldChar3 = OxmlElement('w:fldChar')
+    fldChar3.set(qn('w:fldCharType'), 'end')
+    run2._r.append(fldChar3)
+    doc.add_page_break()
+    
+    # Group dated sections by month
+    dated_sections = [s for s in sections if s['date']]
+    undated_sections = [s for s in sections if not s['date']]
+    
+    def get_month_key(section):
+        return section['date'].strftime('%B %Y')
+    
+    grouped = {}
+    for month, group in groupby(dated_sections, key=get_month_key):
+        grouped[month] = list(group)
+    
+    for month, items in grouped.items():
+        month_heading = doc.add_heading(month, level=2)
+        for item in items:
+            p = doc.add_paragraph()
+            date_run = p.add_run(f"🔹 {item['date'].strftime('%Y-%m-%d')}  ")
+            date_run.bold = True
+            date_run.font.size = Pt(10)
+            if item['is_breach']:
+                breach_run = p.add_run("⚠️ BREACH IDENTIFIED  ")
+                breach_run.bold = True
+                breach_run.font.color.rgb = RGBColor(0xCC, 0x00, 0x00)
+            p.add_run(item['content'])
+        doc.add_paragraph("_" * 40)
+        doc.add_paragraph()
+    
+    # Undated sections
+    if undated_sections:
+        doc.add_heading('Undated Entries', level=2)
+        for item in undated_sections:
+            p = doc.add_paragraph()
+            if item['is_breach']:
+                breach_run = p.add_run("⚠️ BREACH IDENTIFIED  ")
+                breach_run.bold = True
+                breach_run.font.color.rgb = RGBColor(0xCC, 0x00, 0x00)
+            p.add_run(item['content'])
+    
+    doc.save(output_path)
+    return output_path
+
+# --- LLM Integration for PhD-level English and Book Generation ---
+
+def generate_phd_level_text(prompt, max_tokens=2000):
+    """Generates PhD-level English text using an LLM."""
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o-mini", # Using gpt-4o-mini as a powerful and cost-effective option
+            messages=[
+                {"role": "system", "content": "You are an expert academic writer, specializing in producing PhD-level English. Your writing is characterized by precise vocabulary, complex sentence structures, rigorous argumentation, and adherence to scholarly conventions. You are writing a book based on provided evidence. Maintain a formal, objective, and authoritative tone. Ensure logical flow and coherence across paragraphs and sections. Avoid colloquialisms, contractions, and overly simplistic language. Focus on clarity, conciseness, and intellectual depth. Structure your output as a well-organized academic text, suitable for a scholarly publication."},
+                {"role": "user", "content": prompt}
+            ],
+            max_tokens=max_tokens,
+            temperature=0.7, # Adjust for creativity vs. factual accuracy
+            top_p=1,
+            frequency_penalty=0,
+            presence_penalty=0
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        print(f"LLM generation error: {e}")
+        return f"Error generating text: {e}"
+
+def generate_book_chapter(title, content_sections):
+    """Generates a single book chapter from processed content sections."""
+    chapter_prompt = f"""Generate a comprehensive and rigorously argued chapter titled '{title}'.
+    
+    Integrate the following key evidence and arguments, presented as individual sections. Synthesize these points into a cohesive narrative, ensuring smooth transitions and a logical progression of ideas. Your writing should reflect PhD-level academic standards, with precise language, complex sentence structures, and a formal tone. Expand on the evidence where necessary to provide context, analysis, and critical discussion. Ensure that the chapter contributes meaningfully to a larger scholarly work.
+    
+    Evidence sections:
+    """
+    for i, section in enumerate(content_sections):
+        chapter_prompt += f"\nSection {i+1}:\n{section['content']}\n"
+        if section['is_breach']:
+            chapter_prompt += "(Note: This section highlights a potential breach or violation.)\n"
+    
+    chapter_prompt += """
+    Structure the chapter with an introduction, several analytical paragraphs, and a concluding summary that ties back to the chapter's main argument. Ensure proper academic phrasing and avoid repetition. The goal is to transform raw evidence into a polished, scholarly narrative.
+    """
+    return generate_phd_level_text(chapter_prompt)
+
+# --- Core Logic with Enhanced Deduplication ---
+
+def process_files(files, deduplication_threshold=0.85):
+    if not files:
+        return "No files uploaded.", []
+
+    all_sections = []
+    for file in files:
+        file_path = file.name
+        file_name = os.path.basename(file_path)
+        text = extract_text_from_file(file_path)
+        # Split by paragraphs and also by sentences for finer granularity in deduplication
         paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()]
-
-        sentences = []
-        for para in paragraphs:
-            parts = re.split(r'(?<=[.!?])\s+', para)
-            sentences.extend([s.strip() for s in parts if s.strip()])
-
-        return {
-            'full_text': text,
-            'paragraphs': paragraphs,
-            'sentences': sentences,
-            'word_count': len(text.split()),
-            'char_count': len(text)
-        }
-
-
-# ============================================================================
-# MODULE 2: Forensic Analyzer
-# ============================================================================
-
-class ForensicAnalyzer:
-
-    RISK_PATTERNS = {
-        'denial_trap': r"\b(deny|denies|denied|denial|no such|never|not applicable)\b",
-        'admission_by_silence': r"\b(fail|failed|failure|omit|omitted|omission|no response|no reply)\b",
-        'procedural_hedging': r"\b(without prejudice|subject to|conditional|provided that|notwithstanding)\b",
-        'burden_flip': r"\b(claimant must|burden|onus|responsibility|incumbent upon|required to prove)\b"
-    }
-
-    @staticmethod
-    def find_duplicates(docs: List[Dict]) -> List[Dict]:
-        duplicates = []
-        for i, doc1 in enumerate(docs):
-            for j, doc2 in enumerate(docs[i + 1:], start=i + 1):
-                for para1 in doc1['paragraphs']:
-                    for para2 in doc2['paragraphs']:
-                        if len(para1) < 20 or len(para2) < 20:
-                            continue
-                        ratio = SequenceMatcher(None, para1, para2).ratio()
-                        if ratio > 0.85:
-                            duplicates.append({
-                                'doc1': i, 'doc2': j,
-                                'similarity': ratio,
-                                'text': para1[:50000] + "...",
-                                'type': 'duplicate' if ratio > 0.95 else 'near_duplicate'
-                            })
-        return duplicates
-
-    @staticmethod
-    def find_unique_content(docs: List[Dict]) -> List[Dict]:
-        unique_items = []
-        for i, doc in enumerate(docs):
-            for para in doc['paragraphs']:
-                if len(para) < 20:
-                    continue
-                is_unique = True
-                for j, other_doc in enumerate(docs):
-                    if i == j:
-                        continue
-                    for other_para in other_doc['paragraphs']:
-                        if SequenceMatcher(None, para, other_para).ratio() > 0.8:
-                            is_unique = False
-                            break
-                    if not is_unique:
-                        break
-                if is_unique:
-                    unique_items.append({
-                        'doc_index': i,
-                        'content': para,
-                        'length': len(para),
-                        'type': 'unique_paragraph'
-                    })
-        return unique_items
-
-    @staticmethod
-    def detect_risk_language(text: str) -> List[Dict]:
-        findings = []
-        for risk_type, pattern in ForensicAnalyzer.RISK_PATTERNS.items():
-            for match in re.finditer(pattern, text, re.IGNORECASE):
-                start = max(0, match.start() - 50)
-                end = min(len(text), match.end() + 50)
-                findings.append({
-                    'type': risk_type,
-                    'matched_text': match.group(),
-                    'context': text[start:end],
-                    'position': match.start()
-                })
-        return findings
-
-    @staticmethod
-    def analyze_documents(docs: List[Dict]) -> Dict:
-        return {
-            'duplicates': ForensicAnalyzer.find_duplicates(docs),
-            'unique_content': ForensicAnalyzer.find_unique_content(docs),
-            'risk_language': [
-                item for doc in docs
-                for item in ForensicAnalyzer.detect_risk_language(doc['full_text'])
-            ]
-        }
-
-
-# ============================================================================
-# MODULE 3 & 4: Document Merger
-# ============================================================================
-
-class DocumentMerger:
-
-    @staticmethod
-    def select_best_content(duplicates, docs):
-        best_versions = []
-        processed = set()
-        for dup in duplicates:
-            key = tuple(sorted([dup['doc1'], dup['doc2']]))
-            if key in processed:
-                continue
-            processed.add(key)
-            doc1_text = docs[dup['doc1']]['full_text']
-            doc2_text = docs[dup['doc2']]['full_text']
-            best_versions.append(doc1_text if len(doc1_text) > len(doc2_text) else doc2_text)
-        return best_versions
-
-    @staticmethod
-    def merge_documents(docs, analysis):
-        output = []
-        output.append("MASTER EVIDENCE DOCUMENT")
-        output.append("=" * 80)
-        output.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-        output.append(f"Source Documents: {len(docs)}")
-        output.append("=" * 80 + "\n")
-
-        output.append("EXECUTIVE SUMMARY")
-        output.append("-" * 80)
-        output.append(f"Total Duplicates Found: {len(analysis['duplicates'])}")
-        output.append(f"Unique Content Items: {len(analysis['unique_content'])}")
-        output.append(f"Risk Language Instances: {len(analysis['risk_language'])}\n")
-
-        output.append("CONSOLIDATED CONTENT")
-        output.append("-" * 80)
-        best_versions = DocumentMerger.select_best_content(analysis['duplicates'], docs)
-        if best_versions:
-            for i, content in enumerate(best_versions, 1):
-                output.append(f"\n[Section {i}]")
-                output.append(content[:50000] + "..." if len(content) > 500 else content)
-        else:
-            output.append("No duplicate content found across documents.")
-
-        output.append("\n\nUNIQUE FINDINGS")
-        output.append("-" * 80)
-        for item in analysis['unique_content'][:100]:
-            output.append(f"\n[From Document {item['doc_index'] + 1}]")
-            c = item['content']
-            output.append(c[:50000] + "..." if len(c) > 300 else c)
-
-        output.append("\n\nRISK LANGUAGE ANALYSIS")
-        output.append("-" * 80)
-        risk_summary = {}
-        for risk in analysis['risk_language']:
-            risk_summary[risk['type']] = risk_summary.get(risk['type'], 0) + 1
-        if risk_summary:
-            for risk_type, count in risk_summary.items():
-                output.append(f"{risk_type.replace('_', ' ').title()}: {count} instances")
+        for p in paragraphs:
+            all_sections.append({
+                "content": p,
+                "source": file_name,
+                "date": extract_date(p),
+                "is_breach": detect_breach(p)
+            })
+
+    if not all_sections:
+        return "No content extracted.", []
+
+    # Aggressive Deduplication using SentenceTransformer with strict thresholding
+    contents = [s["content"] for s in all_sections]
+    embeddings = model_deduplication.encode(contents, convert_to_tensor=True)
+    unique_sections = []
+    processed_indices = set()
+
+    for i in range(len(all_sections)):
+        if i in processed_indices:
+            continue
+
+        # Find all similar sections using a very strict threshold for "crazy" deduplication
+        cosine_scores = util.cos_sim(embeddings[i], embeddings)[0]
+        # We use a slightly lower threshold for grouping but will use LLM to pick the absolute best
+        similar_indices = [j for j, score in enumerate(cosine_scores) if score > 0.75]
+        
+        if len(similar_indices) > 1:
+            # If multiple similar sections, use LLM to pick the single best phrasing
+            candidates = [all_sections[idx]["content"] for idx in similar_indices]
+            selection_prompt = "From the following similar excerpts, select the single best, most comprehensive, and academically rigorous version. Return ONLY the selected text, nothing else.\n\n" + "\n---\n".join(candidates)
+            best_content = generate_phd_level_text(selection_prompt, max_tokens=1000)
+            
+            # Use the metadata from the first occurrence but updated content
+            best_section = all_sections[i].copy()
+            best_section["content"] = best_content
+            unique_sections.append(best_section)
         else:
-            output.append("No risk language patterns detected.")
-
-        output.append("\n" + "=" * 80)
-        output.append("END OF MASTER EVIDENCE DOCUMENT")
-        return "\n".join(output)
-
-
-# ============================================================================
-# AI SUMMARISATION
-# ============================================================================
-
-def ai_summarise(merged_doc: str, analysis: Dict) -> str:
+            unique_sections.append(all_sections[i])
+            
+        for idx in similar_indices:
+            processed_indices.add(idx)
+
+    # Sort by date
+    dated = [s for s in unique_sections if s["date"]]
+    undated = [s for s in unique_sections if not s["date"]]
+    dated.sort(key=lambda x: x["date"])
+    final_sections = dated + undated
+
+    # Text output for preview
+    output_text = "# Master Evidence Document\n\n"
+    for s in final_sections:
+        prefix = ""
+        if s["date"]:
+            prefix += f"**[{s['date'].strftime('%Y-%m-%d')}]** "
+        if s["is_breach"]:
+            prefix += "⚠️ **[BREACH IDENTIFIED]** "
+        output_text += f"{prefix}{s['content']}\n\n---\n\n"
+
+    return output_text, final_sections
+
+# --- TTS ---
+
+async def generate_tts_audio(text):
     try:
-        hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
-
-        risk_summary = {}
-        for risk in analysis.get('risk_language', []):
-            risk_summary[risk['type']] = risk_summary.get(risk['type'], 0) + 1
-
-        # Condensed input for BART summarisation model
-        input_text = f"""
-Forensic document analysis results:
-- Duplicates found: {len(analysis.get('duplicates', []))}
-- Unique content items: {len(analysis.get('unique_content', []))}
-- Risk language: {json.dumps(risk_summary)}
-
-Document excerpt:
-{merged_doc[:50000]}
-""".strip()
-
-        headers = {"Content-Type": "application/json"}
-        if hf_token:
-            headers["Authorization"] = f"Bearer {hf_token}"
-
-        response = requests.post(
-            "https://router.huggingface.co/hf-inference/models/facebook/bart-large-cnn",
-            headers=headers,
-            json={
-                "inputs": input_text,
-                "parameters": {
-                    "max_length": 400,
-                    "min_length": 100,
-                    "do_sample": False
-                }
-            }
-        )
+        short_text = text[:5000] # Limit text for TTS
+        communicate = edge_tts.Communicate(short_text, "en-US-GuyNeural")
+        output_file = "output_audio.mp3"
+        await communicate.save(output_file)
+        return output_file
+    except Exception as e:
+        print(f"TTS error: {e}")
+        return None
 
-        if response.status_code == 200:
-            data = response.json()
-            if isinstance(data, list) and data:
-                summary = data[0].get('summary_text', '')
-                risk_lines = "\n".join([
-                    f"- {k.replace('_', ' ').title()}: {v} instances"
-                    for k, v in risk_summary.items()
-                ])
-                return f"AI SUMMARY\n{'=' * 60}\n\n{summary}\n\nRisk Language Breakdown:\n{risk_lines}"
-            return "Summary could not be extracted from model response."
-        elif response.status_code == 503:
-            return "Model is loading — please wait 20 seconds and try again."
+# --- Main Runner ---
+
+def create_book_docx(title, content, output_path="PhD_Level_Book.docx"):
+    """Creates a professionally formatted DOCX for the generated book."""
+    doc = Document()
+    
+    # Cover page
+    title_para = doc.add_heading(title, 0)
+    title_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    doc.add_page_break()
+    
+    # Split content by markdown headers for chapters
+    chapters = re.split(r'\n# ', content)
+    for chapter in chapters:
+        if not chapter.strip(): continue
+        lines = chapter.split('\n', 1)
+        chap_title = lines[0].strip()
+        chap_body = lines[1].strip() if len(lines) > 1 else ""
+        
+        doc.add_heading(chap_title, level=1)
+        doc.add_paragraph(chap_body)
+        doc.add_page_break()
+        
+    doc.save(output_path)
+    return output_path
+
+def run_app(files, book_title="A Scholarly Examination", chapter_titles_str="Introduction, Analysis, Conclusion"):
+    output_text, final_sections = process_files(files)
+    if not final_sections:
+        return output_text, None, None, "No book generated.", None
+    
+    # Generate Evidence DOCX
+    try:
+        evidence_docx_path = create_docx(final_sections, "Master_Evidence_Document.docx")
+    except Exception as e:
+        print(f"Evidence DOCX error: {e}")
+        evidence_docx_path = None
+    
+    # Generate Book Content
+    book_content = ""
+    chapter_titles = [t.strip() for t in chapter_titles_str.split(',') if t.strip()]
+    sections_per_chapter = len(final_sections) // len(chapter_titles) if len(chapter_titles) > 0 else 1
+    
+    start_idx = 0
+    for i, title in enumerate(chapter_titles):
+        end_idx = start_idx + sections_per_chapter
+        if i == len(chapter_titles) - 1:
+            chapter_sections = final_sections[start_idx:]
         else:
-            return f"Summarisation unavailable (status {response.status_code}): {response.text}"
-
+            chapter_sections = final_sections[start_idx:end_idx]
+        
+        if chapter_sections:
+            book_content += f"\n# {title}\n"
+            generated_chapter = generate_book_chapter(title, chapter_sections)
+            book_content += generated_chapter + "\n"
+        start_idx = end_idx
+
+    # Generate Book DOCX
+    try:
+        book_docx_path = create_book_docx(book_title, book_content)
     except Exception as e:
-        return f"Summarisation error: {str(e)}"
-
+        print(f"Book DOCX error: {e}")
+        book_docx_path = None
 
-# ============================================================================
-# EXPORT FUNCTIONS
-# ============================================================================
-
-def export_txt(merged_doc: str, analysis_report: str) -> str:
+    # Generate Audio
     try:
-        tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.txt', mode='w', encoding='utf-8')
-        tmp.write("SEAMSTRESSES FRAMEWORK — EXPORT\n")
-        tmp.write(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
-        tmp.write("=" * 80 + "\n\n")
-        tmp.write(merged_doc)
-        tmp.write("\n\n" + "=" * 80 + "\n\n")
-        tmp.write(analysis_report)
-        tmp.close()
-        return tmp.name
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        audio_path = loop.run_until_complete(generate_tts_audio(book_content))
+        loop.close()
     except Exception as e:
-        print(f"TXT export error: {e}")
-        return None
+        print(f"Audio error: {e}")
+        audio_path = None
+    
+    return output_text, audio_path, evidence_docx_path, book_content, book_docx_path
 
+# --- Gradio UI ---
 
-def export_pdf(merged_doc: str, analysis_report: str) -> str:
-    try:
-        from reportlab.lib.pagesizes import A4
-        from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
-        from reportlab.lib.units import mm
-        from reportlab.lib import colors
-        from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, HRFlowable
-        from reportlab.lib.enums import TA_CENTER
-
-        tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.pdf')
-        tmp.close()
-
-        doc = SimpleDocTemplate(
-            tmp.name, pagesize=A4,
-            rightMargin=20*mm, leftMargin=20*mm,
-            topMargin=20*mm, bottomMargin=20*mm
-        )
+with gr.Blocks(title="File Seamstresses Framework - PhD Edition") as demo:
+    gr.Markdown("# 🧵 File Seamstresses Framework - PhD Edition")
+    gr.Markdown("Intelligent merging, aggressive deduplication, chronology tracking, breach placement, and **PhD-level book generation** with TTS.")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            file_input = gr.File(label="Upload Files", file_count="multiple")
+            book_title_input = gr.Textbox(label="Book Title", value="A Scholarly Examination of Evidence")
+            chapter_titles_input = gr.Textbox(label="Chapter Titles (comma-separated)", value="Introduction, Background, Analysis of Breaches, Case Studies, Conclusion")
+            process_btn = gr.Button("🧵 Process, Sew, and Write Book", variant="primary")
+            
+        with gr.Column(scale=2):
+            gr.Markdown("## Processed Evidence Preview")
+            text_output = gr.Markdown(label="Consolidated Evidence")
+            with gr.Row():
+                audio_output = gr.Audio(label="Audio Version (TTS)", type="filepath")
+                file_output = gr.File(label="Download Master Evidence Document (.docx)")
+            
+            gr.Markdown("## Generated Book Content")
+            book_output = gr.Markdown(label="PhD-Level Book Draft")
+            book_file_output = gr.File(label="Download PhD-Level Book (.docx)")
+
+    process_btn.click(
+        fn=run_app,
+        inputs=[file_input, book_title_input, chapter_titles_input],
+        outputs=[text_output, audio_output, file_output, book_output, book_file_output]
+    )
 
-        styles = getSampleStyleSheet()
-        title_style = ParagraphStyle('Title', parent=styles['Heading1'], fontSize=16, alignment=TA_CENTER, spaceAfter=6)
-        heading_style = ParagraphStyle('Heading', parent=styles['Heading2'], fontSize=11, textColor=colors.HexColor('#1a1a2e'), spaceAfter=4)
-        body_style = ParagraphStyle('Body', parent=styles['Normal'], fontSize=9, leading=14, spaceAfter=3)
-
-        story = []
-        story.append(Paragraph("Seamstresses Framework", title_style))
-        story.append(Paragraph("Forensic Document Analysis Report", heading_style))
-        story.append(Paragraph(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", body_style))
-        story.append(HRFlowable(width="100%", thickness=1, color=colors.grey))
-        story.append(Spacer(1, 10))
-
-        story.append(Paragraph("Master Document", heading_style))
-        for block in re.split(r'\n\s*\n', merged_doc):
-            block = block.strip()
-            if not block:
-                continue
-            safe = block.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br/>')
-            if set(block) <= {'=', '-'}:
-                story.append(HRFlowable(width="100%", thickness=0.5, color=colors.lightgrey))
-            elif block.isupper() and len(block) > 3 and '\n' not in block and not block.startswith('['):
-                story.append(Paragraph(safe, heading_style))
-            else:
-                story.append(Paragraph(safe, body_style))
-
-        story.append(Spacer(1, 12))
-        story.append(Paragraph("Analysis Report", heading_style))
-        story.append(HRFlowable(width="100%", thickness=0.5, color=colors.lightgrey))
-        for block in re.split(r'\n\s*\n', analysis_report):
-            block = block.strip()
-            if not block:
-                continue
-            safe = block.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br/>')
-            if block.isupper() and len(block) > 3 and '\n' not in block:
-                story.append(Paragraph(safe, heading_style))
-            else:
-                story.append(Paragraph(safe, body_style))
-
-        doc.build(story)
-        return tmp.name
+# Launch
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0")
 
+import os
+import re
+import asyncio
+import pandas as pd
+import numpy as np
+import gradio as gr
+import pdfplumber
+import dateparser
+import edge_tts
+from datetime import datetime
+from sentence_transformers import SentenceTransformer, util
+from docx import Document
+from docx.shared import Pt, Inches, RGBColor
+from docx.enum.text import WD_ALIGN_PARAGRAPH
+from docx.oxml.ns import qn
+from docx.oxml import OxmlElement
+from itertools import groupby
+
+# Force model to cache properly on Hugging Face
+os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+os.environ["SENTENCE_TRANSFORMERS_HOME"] = "/tmp/model_cache"
+
+# Initialize the Sentence Transformer model
+model = SentenceTransformer('all-MiniLM-L6-v2')
+
+# --- Utility Functions ---
+
+def extract_text_from_file(file_path):
+    """Extracts text from PDF, DOCX, or TXT files."""
+    ext = os.path.splitext(file_path)[1].lower()
+    text = ""
+    try:
+        if ext == ".pdf":
+            with pdfplumber.open(file_path) as pdf:
+                for page in pdf.pages:
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += page_text + "\n"
+        elif ext == ".docx":
+            doc = Document(file_path)
+            text = "\n".join([para.text for para in doc.paragraphs])
+        elif ext == ".txt":
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                text = f.read()
+    except Exception as e:
+        print(f"Error extracting from {file_path}: {e}")
+    return text
+
+def extract_date(text):
+    """Attempts to extract a date from a text snippet."""
+    date_patterns = [
+        r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b',
+        r'\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b',
+        r'\b\d{4}-\d{2}-\d{2}\b'
+    ]
+    for pattern in date_patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            parsed_date = dateparser.parse(match.group())
+            if parsed_date:
+                return parsed_date
+    return None
+
+def detect_breach(text):
+    """Detects if a text snippet describes a breach or violation."""
+    breach_keywords = [
+        "breach", "violation", "failure", "non-compliance", "infringement", 
+        "unauthorized", "leak", "compromise", "illegal", "incident", "misconduct"
+    ]
+    for keyword in breach_keywords:
+        if keyword.lower() in text.lower():
+            return True
+    return False
+
+def create_docx(sections, output_path="Master_Evidence_Document.docx"):
+    """Creates a professionally formatted DOCX file with cover, TOC, and monthly grouping."""
+    doc = Document()
+    
+    # Style settings
+    style = doc.styles['Normal']
+    font = style.font
+    font.name = 'Calibri'
+    font.size = Pt(11)
+    style.paragraph_format.space_after = Pt(6)
+    
+    # Cover page
+    title = doc.add_heading('Master Evidence Document', 0)
+    title.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    subtitle = doc.add_heading('Consolidated Chronology and Identified Breaches', level=1)
+    subtitle.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    date_para = doc.add_paragraph()
+    date_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    date_para.add_run(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    doc.add_page_break()
+    
+    # Table of Contents
+    doc.add_heading('Table of Contents', level=1)
+    para = doc.add_paragraph()
+    run = para.add_run()
+    fldChar1 = OxmlElement('w:fldChar')
+    fldChar1.set(qn('w:fldCharType'), 'begin')
+    run._r.append(fldChar1)
+    instrText = OxmlElement('w:instrText')
+    instrText.set(qn('xml:space'), 'preserve')
+    instrText.text = ' TOC \\o "1-3" \\h \\z \\u '
+    run._r.append(instrText)
+    fldChar2 = OxmlElement('w:fldChar')
+    fldChar2.set(qn('w:fldCharType'), 'separate')
+    run._r.append(fldChar2)
+    run2 = para.add_run('Right-click and choose "Update Field" to generate the Table of Contents')
+    fldChar3 = OxmlElement('w:fldChar')
+    fldChar3.set(qn('w:fldCharType'), 'end')
+    run2._r.append(fldChar3)
+    doc.add_page_break()
+    
+    # Group dated sections by month
+    dated_sections = [s for s in sections if s['date']]
+    undated_sections = [s for s in sections if not s['date']]
+    
+    def get_month_key(section):
+        return section['date'].strftime('%B %Y')
+    
+    grouped = {}
+    for month, group in groupby(dated_sections, key=get_month_key):
+        grouped[month] = list(group)
+    
+    for month, items in grouped.items():
+        month_heading = doc.add_heading(month, level=2)
+        for item in items:
+            p = doc.add_paragraph()
+            date_run = p.add_run(f"🔹 {item['date'].strftime('%Y-%m-%d')}  ")
+            date_run.bold = True
+            date_run.font.size = Pt(10)
+            if item['is_breach']:
+                breach_run = p.add_run("⚠️ BREACH IDENTIFIED  ")
+                breach_run.bold = True
+                breach_run.font.color.rgb = RGBColor(0xCC, 0x00, 0x00)
+            p.add_run(item['content'])
+        doc.add_paragraph("_" * 40)
+        doc.add_paragraph()
+    
+    # Undated sections
+    if undated_sections:
+        doc.add_heading('Undated Entries', level=2)
+        for item in undated_sections:
+            p = doc.add_paragraph()
+            if item['is_breach']:
+                breach_run = p.add_run("⚠️ BREACH IDENTIFIED  ")
+                breach_run.bold = True
+                breach_run.font.color.rgb = RGBColor(0xCC, 0x00, 0x00)
+            p.add_run(item['content'])
+    
+    doc.save(output_path)
+    return output_path
+
+# --- Core Logic ---
+
+def process_files(files):
+    if not files:
+        return "No files uploaded.", []
+
+    all_sections = []
+    for file in files:
+        file_path = file.name
+        file_name = os.path.basename(file_path)
+        text = extract_text_from_file(file_path)
+        paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()]
+        for p in paragraphs:
+            all_sections.append({
+                "content": p,
+                "source": file_name,
+                "date": extract_date(p),
+                "is_breach": detect_breach(p)
+            })
+
+    if not all_sections:
+        return "No content extracted.", []
+
+    # Deduplication
+    contents = [s["content"] for s in all_sections]
+    embeddings = model.encode(contents, convert_to_tensor=True)
+    unique_sections = []
+    processed_indices = set()
+    for i in range(len(all_sections)):
+        if i in processed_indices: 
+            continue
+        cosine_scores = util.cos_sim(embeddings[i], embeddings)[0]
+        similar_indices = [j for j, score in enumerate(cosine_scores) if score > 0.85]
+        best_idx = i
+        max_len = len(all_sections[i]["content"])
+        for idx in similar_indices:
+            if len(all_sections[idx]["content"]) > max_len:
+                max_len = len(all_sections[idx]["content"])
+                best_idx = idx
+            processed_indices.add(idx)
+        unique_sections.append(all_sections[best_idx])
+
+    # Sort by date
+    dated = [s for s in unique_sections if s["date"]]
+    undated = [s for s in unique_sections if not s["date"]]
+    dated.sort(key=lambda x: x["date"])
+    final_sections = dated + undated
+
+    # Text output for preview
+    output_text = "# Master Evidence Document\n\n"
+    for s in final_sections:
+        prefix = ""
+        if s["date"]: 
+            prefix += f"**[{s['date'].strftime('%Y-%m-%d')}]** "
+        if s["is_breach"]: 
+            prefix += "⚠️ **[BREACH IDENTIFIED]** "
+        output_text += f"{prefix}{s['content']}\n\n---\n\n"
+
+    return output_text, final_sections
+
+# --- TTS ---
+
+async def generate_tts_audio(text):
+    try:
+        short_text = text[:5000]
+        communicate = edge_tts.Communicate(short_text, "en-US-GuyNeural")
+        output_file = "output_audio.mp3"
+        await communicate.save(output_file)
+        return output_file
     except Exception as e:
-        print(f"PDF export error: {e}")
+        print(f"TTS error: {e}")
         return None
 
+# --- Main Runner ---
 
-def export_docx(merged_doc: str, analysis_report: str, summary: str) -> str:
+def run_app(files):
+    output_text, final_sections = process_files(files)
+    if not final_sections:
+        return output_text, None, None
+    
+    # Generate DOCX
     try:
-        from docx import Document as DocxDocument
-        from docx.enum.text import WD_ALIGN_PARAGRAPH
-
-        tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.docx')
-        tmp.close()
-
-        doc = DocxDocument()
-
-        title = doc.add_heading('Seamstresses Framework', 0)
-        title.alignment = WD_ALIGN_PARAGRAPH.CENTER
-        sub = doc.add_heading('Forensic Document Analysis Report', level=2)
-        sub.alignment = WD_ALIGN_PARAGRAPH.CENTER
-        doc.add_paragraph(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-        doc.add_paragraph()
+        docx_path = create_docx(final_sections)
+    except Exception as e:
+        print(f"DOCX error: {e}")
+        docx_path = None
+    
+    # Generate Audio
+    try:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        audio_path = loop.run_until_complete(generate_tts_audio(output_text))
+        loop.close()
+    except Exception as e:
+        print(f"Audio error: {e}")
+        audio_path = None
+    
+    return output_text, audio_path, docx_path
 
-        # AI Summary
-        if summary and not summary.startswith("AI summarisation"):
-            doc.add_heading('AI Summary', level=1)
-            doc.add_paragraph(summary)
-            doc.add_paragraph()
-
-        # Master document
-        doc.add_heading('Master Document', level=1)
-        for line in merged_doc.split('\n'):
-            if line.strip():
-                if line.startswith('===') or line.startswith('---'):
-                    doc.add_paragraph('─' * 60)
-                elif line.isupper() and len(line) > 3 and not line.startswith('['):
-                    doc.add_heading(line, level=2)
-                else:
-                    doc.add_paragraph(line)
+# --- Gradio UI ---
 
-        doc.add_page_break()
+with gr.Blocks(title="File Seamstresses Framework") as demo:
+    gr.Markdown("# 🧵 File Seamstresses Framework")
+    gr.Markdown("Intelligent merging, deduplication, chronology tracking, breach placement, and TTS.")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            file_input = gr.File(label="Upload Files", file_count="multiple")
+            process_btn = gr.Button("🧵 Process and Sew Files", variant="primary")
+            
+        with gr.Column(scale=2):
+            text_output = gr.Markdown(label="Preview")
+            with gr.Row():
+                audio_output = gr.Audio(label="Audio Version (TTS)", type="filepath")
+                file_output = gr.File(label="Download Master Document (.docx)")
 
-        # Analysis report
-        doc.add_heading('Analysis Report', level=1)
-        for line in analysis_report.split('\n'):
-            if line.strip():
-                if line.isupper() and len(line) > 3:
-                    doc.add_heading(line, level=2)
-                else:
-                    doc.add_paragraph(line)
+    process_btn.click(
+        fn=run_app,
+        inputs=[file_input],
+        outputs=[text_output, audio_output, file_output]
+    )
 
-        doc.save(tmp.name)
-        return tmp.name
+# Launch
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0")
 
+import os
+import re
+import asyncio
+import pandas as pd
+import numpy as np
+import gradio as gr
+import pdfplumber
+import dateparser
+import edge_tts
+from datetime import datetime
+from sentence_transformers import SentenceTransformer, util
+from docx import Document
+from docx.shared import Pt, Inches, RGBColor
+from docx.enum.text import WD_ALIGN_PARAGRAPH
+from docx.oxml.ns import qn
+from docx.oxml import OxmlElement
+from itertools import groupby
+
+# Force model to cache properly on Hugging Face
+os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+os.environ["SENTENCE_TRANSFORMERS_HOME"] = "/tmp/model_cache"
+
+# Initialize the Sentence Transformer model
+model = SentenceTransformer('all-MiniLM-L6-v2')
+
+# --- Utility Functions ---
+
+def extract_text_from_file(file_path):
+    """Extracts text from PDF, DOCX, or TXT files."""
+    ext = os.path.splitext(file_path)[1].lower()
+    text = ""
+    try:
+        if ext == ".pdf":
+            with pdfplumber.open(file_path) as pdf:
+                for page in pdf.pages:
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += page_text + "\n"
+        elif ext == ".docx":
+            doc = Document(file_path)
+            text = "\n".join([para.text for para in doc.paragraphs])
+        elif ext == ".txt":
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                text = f.read()
+    except Exception as e:
+        print(f"Error extracting from {file_path}: {e}")
+    return text
+
+def extract_date(text):
+    """Attempts to extract a date from a text snippet."""
+    date_patterns = [
+        r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b',
+        r'\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b',
+        r'\b\d{4}-\d{2}-\d{2}\b'
+    ]
+    for pattern in date_patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            parsed_date = dateparser.parse(match.group())
+            if parsed_date:
+                return parsed_date
+    return None
+
+def detect_breach(text):
+    """Detects if a text snippet describes a breach or violation."""
+    breach_keywords = [
+        "breach", "violation", "failure", "non-compliance", "infringement", 
+        "unauthorized", "leak", "compromise", "illegal", "incident", "misconduct"
+    ]
+    for keyword in breach_keywords:
+        if keyword.lower() in text.lower():
+            return True
+    return False
+
+def create_docx(sections, output_path="Master_Evidence_Document.docx"):
+    """Creates a professionally formatted DOCX file with cover, TOC, and monthly grouping."""
+    doc = Document()
+    
+    # Style settings
+    style = doc.styles['Normal']
+    font = style.font
+    font.name = 'Calibri'
+    font.size = Pt(11)
+    style.paragraph_format.space_after = Pt(6)
+    
+    # Cover page
+    title = doc.add_heading('Master Evidence Document', 0)
+    title.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    subtitle = doc.add_heading('Consolidated Chronology and Identified Breaches', level=1)
+    subtitle.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    date_para = doc.add_paragraph()
+    date_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
+    date_para.add_run(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    doc.add_page_break()
+    
+    # Table of Contents
+    doc.add_heading('Table of Contents', level=1)
+    para = doc.add_paragraph()
+    run = para.add_run()
+    fldChar1 = OxmlElement('w:fldChar')
+    fldChar1.set(qn('w:fldCharType'), 'begin')
+    run._r.append(fldChar1)
+    instrText = OxmlElement('w:instrText')
+    instrText.set(qn('xml:space'), 'preserve')
+    instrText.text = ' TOC \\o "1-3" \\h \\z \\u '
+    run._r.append(instrText)
+    fldChar2 = OxmlElement('w:fldChar')
+    fldChar2.set(qn('w:fldCharType'), 'separate')
+    run._r.append(fldChar2)
+    run2 = para.add_run('Right-click and choose "Update Field" to generate the Table of Contents')
+    fldChar3 = OxmlElement('w:fldChar')
+    fldChar3.set(qn('w:fldCharType'), 'end')
+    run2._r.append(fldChar3)
+    doc.add_page_break()
+    
+    # Group dated sections by month
+    dated_sections = [s for s in sections if s['date']]
+    undated_sections = [s for s in sections if not s['date']]
+    
+    def get_month_key(section):
+        return section['date'].strftime('%B %Y')
+    
+    grouped = {}
+    for month, group in groupby(dated_sections, key=get_month_key):
+        grouped[month] = list(group)
+    
+    for month, items in grouped.items():
+        month_heading = doc.add_heading(month, level=2)
+        for item in items:
+            p = doc.add_paragraph()
+            date_run = p.add_run(f"🔹 {item['date'].strftime('%Y-%m-%d')}  ")
+            date_run.bold = True
+            date_run.font.size = Pt(10)
+            if item['is_breach']:
+                breach_run = p.add_run("⚠️ BREACH IDENTIFIED  ")
+                breach_run.bold = True
+                breach_run.font.color.rgb = RGBColor(0xCC, 0x00, 0x00)
+            p.add_run(item['content'])
+        doc.add_paragraph("_" * 40)
+        doc.add_paragraph()
+    
+    # Undated sections
+    if undated_sections:
+        doc.add_heading('Undated Entries', level=2)
+        for item in undated_sections:
+            p = doc.add_paragraph()
+            if item['is_breach']:
+                breach_run = p.add_run("⚠️ BREACH IDENTIFIED  ")
+                breach_run.bold = True
+                breach_run.font.color.rgb = RGBColor(0xCC, 0x00, 0x00)
+            p.add_run(item['content'])
+    
+    doc.save(output_path)
+    return output_path
+
+# --- Core Logic ---
+
+def process_files(files):
+    if not files:
+        return "No files uploaded.", []
+
+    all_sections = []
+    for file in files:
+        file_path = file.name
+        file_name = os.path.basename(file_path)
+        text = extract_text_from_file(file_path)
+        paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()]
+        for p in paragraphs:
+            all_sections.append({
+                "content": p,
+                "source": file_name,
+                "date": extract_date(p),
+                "is_breach": detect_breach(p)
+            })
+
+    if not all_sections:
+        return "No content extracted.", []
+
+    # Deduplication
+    contents = [s["content"] for s in all_sections]
+    embeddings = model.encode(contents, convert_to_tensor=True)
+    unique_sections = []
+    processed_indices = set()
+    for i in range(len(all_sections)):
+        if i in processed_indices: 
+            continue
+        cosine_scores = util.cos_sim(embeddings[i], embeddings)[0]
+        similar_indices = [j for j, score in enumerate(cosine_scores) if score > 0.85]
+        best_idx = i
+        max_len = len(all_sections[i]["content"])
+        for idx in similar_indices:
+            if len(all_sections[idx]["content"]) > max_len:
+                max_len = len(all_sections[idx]["content"])
+                best_idx = idx
+            processed_indices.add(idx)
+        unique_sections.append(all_sections[best_idx])
+
+    # Sort by date
+    dated = [s for s in unique_sections if s["date"]]
+    undated = [s for s in unique_sections if not s["date"]]
+    dated.sort(key=lambda x: x["date"])
+    final_sections = dated + undated
+
+    # Text output for preview
+    output_text = "# Master Evidence Document\n\n"
+    for s in final_sections:
+        prefix = ""
+        if s["date"]: 
+            prefix += f"**[{s['date'].strftime('%Y-%m-%d')}]** "
+        if s["is_breach"]: 
+            prefix += "⚠️ **[BREACH IDENTIFIED]** "
+        output_text += f"{prefix}{s['content']}\n\n---\n\n"
+
+    return output_text, final_sections
+
+# --- TTS ---
+
+async def generate_tts_audio(text):
+    try:
+        short_text = text[:5000]
+        communicate = edge_tts.Communicate(short_text, "en-US-GuyNeural")
+        output_file = "output_audio.mp3"
+        await communicate.save(output_file)
+        return output_file
     except Exception as e:
-        print(f"DOCX export error: {e}")
+        print(f"TTS error: {e}")
         return None
 
+# --- Main Runner ---
 
-# ============================================================================
-# MAIN PROCESSING FUNCTION
-# ============================================================================
-
-def process_documents(file1, file2, file3):
+def run_app(files):
+    output_text, final_sections = process_files(files)
+    if not final_sections:
+        return output_text, None, None
+    
+    # Generate DOCX
     try:
-        raw_inputs = [file1, file2, file3]
-        files = []
-        for f in raw_inputs:
-            if f is None:
-                continue
-            if isinstance(f, dict):
-                path = f.get('path') or f.get('name')
-            elif hasattr(f, 'name'):
-                path = f.name
-            elif isinstance(f, str):
-                path = f
-            else:
-                path = str(f)
-            if path:
-                files.append(path)
-
-        if not files:
-            return "Please upload at least one document.", "", "", None, None, None
-
-        docs = []
-        file_names = []
-        for file_path in files:
-            text = DocumentParser.extract_text_from_file(file_path)
-            if not text or text.startswith("Error"):
-                return f"Error: {text}", "", "", None, None, None
-            docs.append(DocumentParser.parse_document(text))
-            file_names.append(os.path.basename(file_path))
-
-        analysis = ForensicAnalyzer.analyze_documents(docs)
-        merged_doc = DocumentMerger.merge_documents(docs, analysis)
-
-        # Analysis report
-        analysis_report = f"FORENSIC ANALYSIS REPORT\n{'=' * 80}\n\n"
-        analysis_report += f"FILES ANALYZED: {', '.join(file_names)}\n\n"
-        analysis_report += f"DUPLICATE ANALYSIS\n{'-' * 80}\n"
-        analysis_report += f"Total Duplicates Found: {len(analysis['duplicates'])}\n\n"
-        for dup in analysis['duplicates'][:50]:
-            analysis_report += f"- Documents {dup['doc1']+1} & {dup['doc2']+1}: {dup['similarity']:.1%} similarity\n"
-        analysis_report += f"\nUNIQUE CONTENT\n{'-' * 80}\n"
-        analysis_report += f"Total Unique Items: {len(analysis['unique_content'])}\n\n"
-        for item in analysis['unique_content'][:50]:
-            analysis_report += f"- From Document {item['doc_index']+1}: {item['content'][:50000]}...\n"
-        analysis_report += f"\nRISK LANGUAGE DETECTED\n{'-' * 80}\n"
-        risk_types = {}
-        for risk in analysis['risk_language']:
-            risk_types[risk['type']] = risk_types.get(risk['type'], 0) + 1
-        for risk_type, count in risk_types.items():
-            analysis_report += f"- {risk_type.replace('_', ' ').title()}: {count} instances\n"
-
-        # AI Summary
-        summary = ai_summarise(merged_doc, analysis)
-
-        # Exports
-        txt_path = export_txt(merged_doc, analysis_report)
-        pdf_path = export_pdf(merged_doc, analysis_report)
-        docx_path = export_docx(merged_doc, analysis_report, summary)
-
-        return merged_doc, analysis_report, summary, txt_path, pdf_path, docx_path
-
+        docx_path = create_docx(final_sections)
     except Exception as e:
-        import traceback
-        return f"Error: {str(e)}\n\n{traceback.format_exc()}", "", "", None, None, None
-
-
-# ============================================================================
-# GRADIO INTERFACE
-# ============================================================================
+        print(f"DOCX error: {e}")
+        docx_path = None
+    
+    # Generate Audio
+    try:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        audio_path = loop.run_until_complete(generate_tts_audio(output_text))
+        loop.close()
+    except Exception as e:
+        print(f"Audio error: {e}")
+        audio_path = None
+    
+    return output_text, audio_path, docx_path
 
-with gr.Blocks(title="Seamstresses Framework", theme=gr.themes.Soft(primary_hue="blue", secondary_hue="cyan")) as demo:
-    gr.Markdown("""
-    # 🧵 Seamstresses Framework
-    ## Forensic Document Analysis, Intelligent Merging & AI Summarisation
-    Upload up to 3 legal documents to deduplicate, analyse risk language, and generate an AI-powered summary.
-    """)
+# --- Gradio UI ---
 
+with gr.Blocks(title="File Seamstresses Framework") as demo:
+    gr.Markdown("# 🧵 File Seamstresses Framework")
+    gr.Markdown("Intelligent merging, deduplication, chronology tracking, breach placement, and TTS.")
+    
     with gr.Row():
-        with gr.Column():
-            gr.Markdown("### Upload Documents")
-            file1 = gr.File(label="Document 1", file_types=[".pdf", ".txt", ".docx"], type="filepath")
-            file2 = gr.File(label="Document 2", file_types=[".pdf", ".txt", ".docx"], type="filepath")
-            file3 = gr.File(label="Document 3 (Optional)", file_types=[".pdf", ".txt", ".docx"], type="filepath")
-            analyze_btn = gr.Button("🔍 Analyze, Merge & Summarise", variant="primary", size="lg")
-
-    with gr.Tabs():
-        with gr.TabItem("📄 Master Document"):
-            merged_output = gr.Textbox(label="Merged Document", lines=30, max_lines=200)
-
-        with gr.TabItem("🔬 Analysis Report"):
-            analysis_output = gr.Textbox(label="Forensic Analysis", lines=30, max_lines=200)
-
-        with gr.TabItem("🤖 AI Summary"):
-            summary_output = gr.Textbox(label="AI-Powered Summary", lines=30, max_lines=200)
-
-        with gr.TabItem("📥 Export"):
-            gr.Markdown("### Download Your Reports")
+        with gr.Column(scale=1):
+            file_input = gr.File(label="Upload Files", file_count="multiple")
+            process_btn = gr.Button("🧵 Process and Sew Files", variant="primary")
+            
+        with gr.Column(scale=2):
+            text_output = gr.Markdown(label="Preview")
             with gr.Row():
-                txt_file = gr.File(label="📄 Plain Text (.txt)", interactive=False)
-                pdf_file = gr.File(label="📕 PDF (.pdf)", interactive=False)
-                docx_file = gr.File(label="📘 Word Document (.docx)", interactive=False)
-            gr.Markdown("""
-            > **Word document** includes AI summary, merged document, and full analysis.
-            > **PDF & TXT** contain the merged document and analysis report.
-            """)
-
-    analyze_btn.click(
-        fn=process_documents,
-        inputs=[file1, file2, file3],
-        outputs=[merged_output, analysis_output, summary_output, txt_file, pdf_file, docx_file]
-    )
+                audio_output = gr.Audio(label="Audio Version (TTS)", type="filepath")
+                file_output = gr.File(label="Download Master Document (.docx)")
 
-    gr.Markdown("""
-    ---
-    ### Risk Language Detected
-    - **Denial Trap** — deny, denied, never, not applicable
-    - **Admission by Silence** — failure, omission, no response
-    - **Procedural Hedging** — without prejudice, notwithstanding, subject to
-    - **Burden Flip** — burden, onus, required to prove, claimant must
-    """)
+    process_btn.click(
+        fn=run_app,
+        inputs=[file_input],
+        outputs=[text_output, audio_output, file_output]
+    )
 
+# Launch
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(server_name="0.0.0.0")