Create app.py

app.py

@@ -0,0 +1,1994 @@
+import gradio as gr
+import torch
+from transformers import T5ForConditionalGeneration, T5Tokenizer, AutoTokenizer, AutoModelForSeq2SeqLM
+from bs4 import BeautifulSoup, NavigableString, Tag
+import re
+import time
+import random
+import nltk
+from nltk.tokenize import sent_tokenize
+
+# Download required NLTK data
+try:
+    nltk.download('punkt', quiet=True)
+except:
+    pass
+
+# Try to import spaCy but make it optional
+try:
+    import spacy
+    SPACY_AVAILABLE = True
+except:
+    print("spaCy not available, using NLTK for sentence processing")
+    SPACY_AVAILABLE = False
+
+class HumanLikeVariations:
+    """Add human-like variations and intentional imperfections"""
+    
+    def __init__(self):
+        # Common human writing patterns - EXPANDED for Originality AI
+        self.casual_transitions = [
+             "So, ", "Well, ", "Now, ", "Actually, ", "Basically, ", 
+             "You know, ", "I mean, ", "Thing is, ", "Honestly, ",
+             "Look, ", "Listen, ", "See, ", "Okay, ", "Right, ",
+             "Anyway, ", "Besides, ", "Plus, ", "Also, ", "Oh, ",
+             "Hey, ", "Alright, ", "Sure, ", "Fine, ", "Obviously, ",
+             "Clearly, ", "Seriously, ", "Literally, ", "Frankly, ",
+             "To be honest, ", "Truth is, ", "In fact, ", "Believe it or not, ",
+             "Here's the thing, ", "Let me tell you, ", "Get this, ",
+             "Funny thing is, ", "Interestingly, ", "Surprisingly, ",
+             "Let's be real here, ", "Can we talk about ", "Quick question: ",
+             "Real talk: ", "Hot take: ", "Unpopular opinion: ", "Fun fact: ",
+             "Pro tip: ", "Side note: ", "Random thought: ", "Food for thought: ",
+             "Just saying, ", "Not gonna lie, ", "For what it's worth, ",
+             "If you ask me, ", "Between you and me, ", "Here's my take: ",
+             "Let's face it, ", "No kidding, ", "Seriously though, ",
+             "But wait, ", "Hold on, ", "Check this out: ", "Guess what? "
+        ]
+        
+        self.filler_phrases = [
+            "kind of", "sort of", "pretty much", "basically", "actually",
+            "really", "just", "quite", "rather", "fairly", "totally",
+            "definitely", "probably", "maybe", "perhaps", "somehow",
+            "somewhat", "literally", "seriously", "honestly", "frankly",
+            "simply", "merely", "purely", "truly", "genuinely",
+            "absolutely", "completely", "entirely", "utterly", "practically",
+            "virtually", "essentially", "fundamentally", "generally", "typically",
+            "usually", "normally", "often", "sometimes", "occasionally",
+            "apparently", "evidently", "obviously", "clearly", "seemingly",
+            "arguably", "potentially", "possibly", "likely", "unlikely",
+            "more or less", "give or take", "so to speak", "if you will",
+            "per se", "as such", "in a way", "to some extent", "to a degree",
+            "I kid you not", "no joke", "for real", "not gonna lie",
+            "I'm telling you", "trust me", "believe me", "I swear",
+            "hands down", "without a doubt", "100%", "straight up",
+            "I think", "I feel like", "I guess", "I suppose", "seems like",
+            "appears to be", "might be", "could be", "tends to", "tends to be",
+            "in my experience", "from what I've seen", "as far as I know",
+            "to the best of my knowledge", "if I'm not mistaken", "correct me if I'm wrong",
+            "you know what", "here's the deal", "bottom line", "at any rate",
+            "all in all", "when you think about it", "come to think of it",
+            "now that I think about it", "if we're being honest", "to be fair"
+        ]
+        
+        self.human_connectors = [
+            ", which means", ", so", ", because", ", since", ", although",
+            ". That's why", ". This means", ". So basically,", ". The thing is,",
+            ", and honestly", ", but here's the thing", ", though", ", however",
+            ". Plus,", ". Also,", ". Besides,", ". Moreover,", ". Furthermore,",
+            ", which is why", ", and that's because", ", given that", ", considering",
+            ". In other words,", ". Put simply,", ". To clarify,", ". That said,",
+            ", you see", ", you know", ", right?", ", okay?", ", yeah?",
+            ". Here's why:", ". Let me explain:", ". Think about it:",
+            ", if you ask me", ", in my opinion", ", from my perspective",
+            ". On the flip side,", ". On the other hand,", ". Conversely,",
+            ", not to mention", ", let alone", ", much less", ", aside from",
+            ". What's more,", ". Even better,", ". Even worse,", ". The catch is,",
+            ", believe it or not", ", surprisingly enough", ", interestingly enough",
+            ". Long story short,", ". Bottom line is,", ". Point being,",
+            ", as you might expect", ", as it turns out", ", as luck would have it",
+            ". And get this:", ". But wait, there's more:", ". Here's the kicker:",
+            ", and here's why", ", and here's the thing", ", but here's what happened",
+            ". Spoiler alert:", ". Plot twist:", ". Reality check:",
+            ", at the end of the day", ", when all is said and done", ", all things considered",
+            ". Make no mistake,", ". Don't get me wrong,", ". Let's not forget,",
+            ", between you and me", ", off the record", ", just between us",
+            ". And honestly?", ". But seriously,", ". And you know what?",
+            ", which brings me to", ". This reminds me of", ", speaking of which",
+            ". Funny enough,", ". Weird thing is,", ". Strange but true:",
+            ", and I mean", ". I'm not kidding when I say", ", and trust me on this"
+        ]
+        
+        # NEW: Common human typos and variations
+        self.common_typos = {
+            "the": ["teh", "th", "hte"],
+            "and": ["adn", "nad", "an"],
+            "that": ["taht", "htat", "tha"],
+            "with": ["wiht", "wtih", "iwth"],
+            "have": ["ahve", "hvae", "hav"],
+            "from": ["form", "fro", "frmo"],
+            "they": ["tehy", "thye", "htey"],
+            "which": ["whihc", "wich", "whcih"],
+            "their": ["thier", "theri", "tehir"],
+            "would": ["woudl", "wuold", "woul"],
+            "there": ["tehre", "theer", "ther"],
+            "could": ["coudl", "cuold", "coud"],
+            "people": ["poeple", "peopel", "pepole"],
+            "through": ["thorugh", "throught", "trhough"],
+            "because": ["becuase", "becasue", "beacuse"],
+            "before": ["beofre", "befroe", "befor"],
+            "different": ["differnt", "differnet", "diferent"],
+            "between": ["bewteen", "betwen", "betewen"],
+            "important": ["improtant", "importnat", "importan"],
+            "information": ["infromation", "informaiton", "informaton"]
+        }
+        
+        # NEW: Human-like sentence starters for variety
+        self.varied_starters = [
+            "When it comes to", "As for", "Regarding", "In terms of",
+            "With respect to", "Concerning", "Speaking of", "About",
+            "If we look at", "Looking at", "Considering", "Given",
+            "Taking into account", "Bear in mind that", "Keep in mind",
+            "It's worth noting that", "It should be noted that",
+            "One thing to consider is", "An important point is",
+            "What's interesting is", "What stands out is",
+            "The key here is", "The main thing is", "The point is",
+            "Here's what matters:", "Here's the deal:", "Here's something:",
+            "Let's not forget", "We should remember", "Don't forget",
+            "Think about it this way:", "Look at it like this:",
+            "Consider this:", "Picture this:", "Imagine this:",
+            "You might wonder", "You might ask", "You may think",
+            "Some people say", "Many believe", "It's often said",
+            "Research shows", "Studies indicate", "Evidence suggests",
+            "Experience tells us", "History shows", "Time has shown"
+        ]
+    
+    def add_human_touch(self, text):
+        """Add subtle human-like imperfections - MORE AGGRESSIVE"""
+        sentences = text.split('. ')
+        modified_sentences = []
+        
+        for i, sent in enumerate(sentences):
+            if not sent.strip():
+                continue
+            
+            # Occasionally start with casual transition (25% chance - increased)
+            if i > 0 and random.random() < 0.25 and len(sent.split()) > 5:
+                transition = random.choice(self.casual_transitions)
+                sent = transition + sent[0].lower() + sent[1:] if len(sent) > 1 else sent
+            
+            # Add filler words occasionally (20% chance - increased)
+            if random.random() < 0.2 and len(sent.split()) > 8:
+                words = sent.split()
+                # Add multiple fillers sometimes
+                num_fillers = random.randint(1, 2)
+                for _ in range(num_fillers):
+                    if len(words) > 4:
+                        insert_pos = random.randint(2, len(words)-2)
+                        filler = random.choice(self.filler_phrases)
+                        words.insert(insert_pos, filler)
+                sent = ' '.join(words)
+            
+            # Add varied sentence starters (15% chance)
+            if i > 0 and random.random() < 0.15 and len(sent.split()) > 10:
+                starter = random.choice(self.varied_starters)
+                sent = starter + " " + sent[0].lower() + sent[1:] if len(sent) > 1 else sent
+            
+            # Occasionally use contractions (35% chance - increased)
+            if random.random() < 0.35:
+                sent = self.apply_contractions(sent)
+            
+            # Add occasional comma splices (10% chance) - common human error
+            if random.random() < 0.1 and ',' in sent and len(sent.split()) > 10:
+                # Replace a period with comma sometimes
+                parts = sent.split(', ')
+                if len(parts) > 2:
+                    join_idx = random.randint(1, len(parts)-1)
+                    parts[join_idx-1] = parts[join_idx-1] + ','
+                    sent = ' '.join(parts)
+            
+            # NEW: Add parenthetical thoughts (8% chance)
+            if random.random() < 0.08 and len(sent.split()) > 15:
+                parentheticals = [
+                    "(and that's saying something)",
+                    "(which is pretty interesting)",
+                    "(trust me on this one)",
+                    "(I've seen this firsthand)",
+                    "(no joke)",
+                    "(seriously)",
+                    "(and for good reason)",
+                    "(believe it or not)",
+                    "(surprisingly enough)",
+                    "(which makes sense)",
+                    "(go figure)",
+                    "(who knew?)",
+                    "(makes you think)",
+                    "(worth considering)"
+                ]
+                words = sent.split()
+                insert_pos = random.randint(len(words)//3, 2*len(words)//3)
+                parenthetical = random.choice(parentheticals)
+                words.insert(insert_pos, parenthetical)
+                sent = ' '.join(words)
+            
+            # NEW: Occasionally add rhetorical questions (5% chance)
+            if random.random() < 0.05 and i < len(sentences) - 1:
+                rhetorical_questions = [
+                    "Makes sense, right?",
+                    "Pretty cool, huh?",
+                    "Interesting, isn't it?",
+                    "Who would've thought?",
+                    "Sound familiar?",
+                    "See what I mean?",
+                    "Get the picture?",
+                    "Following along?",
+                    "Crazy, right?",
+                    "Wild, isn't it?"
+                ]
+                sent = sent + " " + random.choice(rhetorical_questions)
+            
+            modified_sentences.append(sent)
+        
+        return '. '.join(modified_sentences)
+    
+    def apply_contractions(self, text):
+        """Apply common contractions - EXPANDED"""
+        contractions = {
+            "it is": "it's", "that is": "that's", "there is": "there's",
+            "he is": "he's", "she is": "she's", "what is": "what's",
+            "where is": "where's", "who is": "who's", "how is": "how's",
+            "cannot": "can't", "will not": "won't", "do not": "don't",
+            "does not": "doesn't", "did not": "didn't", "could not": "couldn't",
+            "should not": "shouldn't", "would not": "wouldn't", "is not": "isn't",
+            "are not": "aren't", "was not": "wasn't", "were not": "weren't",
+            "have not": "haven't", "has not": "hasn't", "had not": "hadn't",
+            "I am": "I'm", "you are": "you're", "we are": "we're",
+            "they are": "they're", "I have": "I've", "you have": "you've",
+            "we have": "we've", "they have": "they've", "I will": "I'll",
+            "you will": "you'll", "he will": "he'll", "she will": "she'll",
+            "we will": "we'll", "they will": "they'll", "I would": "I'd",
+            "you would": "you'd", "he would": "he'd", "she would": "she'd",
+            "we would": "we'd", "they would": "they'd", "could have": "could've",
+            "should have": "should've", "would have": "would've", "might have": "might've",
+            "must have": "must've", "there has": "there's", "here is": "here's",
+            "let us": "let's", "that will": "that'll", "who will": "who'll"
+        }
+        
+        for full, contr in contractions.items():
+            if random.random() < 0.8:  # 80% chance to apply each contraction
+                text = re.sub(r'\b' + full + r'\b', contr, text, flags=re.IGNORECASE)
+        
+        return text
+    
+    def add_minor_errors(self, text):
+        """Add very minor, human-like errors - MORE REALISTIC"""
+        # Occasionally miss Oxford comma (15% chance)
+        if random.random() < 0.15:
+            text = re.sub(r'(\w+), (\w+), and', r'\1, \2 and', text)
+        
+        # Sometimes use 'which' instead of 'that' (8% chance)
+        if random.random() < 0.08:
+            text = text.replace(' that ', ' which ', 1)
+        
+        # NEW: Add very occasional typos (3% chance per sentence)
+        sentences = text.split('. ')
+        for i, sent in enumerate(sentences):
+            if random.random() < 0.03 and len(sent.split()) > 10:
+                words = sent.split()
+                # Pick a random word to potentially typo
+                word_idx = random.randint(0, len(words)-1)
+                word = words[word_idx].lower()
+                
+                # Only typo common words
+                if word in self.common_typos and random.random() < 0.5:
+                    typo = random.choice(self.common_typos[word])
+                    # Preserve original capitalization
+                    if words[word_idx][0].isupper():
+                        typo = typo[0].upper() + typo[1:]
+                    words[word_idx] = typo
+                    sentences[i] = ' '.join(words)
+        
+        text = '. '.join(sentences)
+        
+        # NEW: Occasionally double a word (2% chance)
+        if random.random() < 0.02:
+            words = text.split()
+            if len(words) > 20:
+                # Pick a small common word to double
+                small_words = ['the', 'a', 'an', 'is', 'was', 'are', 'were', 'to', 'of', 'in', 'on']
+                for idx, word in enumerate(words):
+                    if word.lower() in small_words and random.random() < 0.1:
+                        words[idx] = word + ' ' + word
+                        break
+                text = ' '.join(words)
+        
+        # NEW: Mix up common homophones occasionally (3% chance)
+        if random.random() < 0.03:
+            homophones = [
+                ('their', 'there'), ('your', 'you\'re'), ('its', 'it\'s'),
+                ('then', 'than'), ('to', 'too'), ('effect', 'affect')
+            ]
+            for pair in homophones:
+                if pair[0] in text and random.random() < 0.3:
+                    text = text.replace(pair[0], pair[1], 1)
+                    break
+        
+        return text
+    
+    def add_originality_specific_patterns(self, text):
+        """Add patterns that Originality AI associates with human writing"""
+        # 1. Add personal touches and opinions
+        if random.random() < 0.1:
+            personal_phrases = [
+                "In my view, ", "From my perspective, ", "I believe ",
+                "It seems to me that ", "I've found that ", "In my experience, ",
+                "I tend to think ", "My take is that ", "I'd argue that ",
+                "Personally, I think ", "If you ask me, ", "The way I see it, "
+            ]
+            sentences = text.split('. ')
+            if len(sentences) > 3:
+                idx = random.randint(1, len(sentences)-2)
+                sentences[idx] = random.choice(personal_phrases) + sentences[idx][0].lower() + sentences[idx][1:]
+                text = '. '.join(sentences)
+        
+        # 2. Add conversational asides
+        if random.random() < 0.08:
+            asides = [
+                " - and this is important - ",
+                " - bear with me here - ",
+                " - stay with me - ",
+                " - and I mean this - ",
+                " - no exaggeration - ",
+                " - true story - ",
+                " - I'm serious - ",
+                " - think about it - ",
+                " - and here's why - "
+            ]
+            words = text.split()
+            if len(words) > 20:
+                pos = random.randint(10, len(words)-10)
+                words.insert(pos, random.choice(asides))
+                text = ' '.join(words)
+        
+        # 3. Add emphatic repetition (human pattern)
+        if random.random() < 0.05:
+            emphatic_words = ['very', 'really', 'truly', 'absolutely', 'totally']
+            sentences = text.split('. ')
+            if sentences:
+                sent_idx = random.randint(0, len(sentences)-1)
+                words = sentences[sent_idx].split()
+                if len(words) > 5:
+                    # Find an adjective or adverb to emphasize
+                    for i, word in enumerate(words):
+                        if i > 0 and i < len(words)-1:
+                            # Add emphasis
+                            if random.random() < 0.3:
+                                emphasis = random.choice(emphatic_words)
+                                words.insert(i, emphasis)
+                                # Sometimes repeat for extra emphasis
+                                if random.random() < 0.3:
+                                    words.insert(i, emphasis + ',')
+                                break
+                    sentences[sent_idx] = ' '.join(words)
+                    text = '. '.join(sentences)
+        
+        return text
+
+class SelectiveGrammarFixer:
+    """Minimal grammar fixes to maintain human-like quality while fixing critical errors"""
+    
+    def __init__(self):
+        self.nlp = None
+        self.human_variations = HumanLikeVariations()
+    
+    def fix_incomplete_sentences_only(self, text):
+        """Fix only incomplete sentences without over-correcting"""
+        if not text:
+            return text
+        
+        sentences = text.split('. ')
+        fixed_sentences = []
+        
+        for i, sent in enumerate(sentences):
+            sent = sent.strip()
+            if not sent:
+                continue
+            
+            # Only fix if sentence is incomplete
+            if sent and sent[-1] not in '.!?':
+                # Check if it's the last sentence
+                if i == len(sentences) - 1:
+                    # Add period if it's clearly a statement
+                    if not sent.endswith(':') and not sent.endswith(','):
+                        sent += '.'
+                else:
+                    # Middle sentences should have periods
+                    sent += '.'
+            
+            # Fix cut-off words (very short last word without punctuation)
+            words = sent.split()
+            if len(words) > 3:
+                last_word = words[-1].rstrip('.!?')
+                if len(last_word) <= 2 and last_word.isalpha():
+                    # Check if it has vowels (real word vs cut-off)
+                    if not any(c in 'aeiouAEIOU' for c in last_word):
+                        # Likely a cut-off word, remove it
+                        words = words[:-1]
+                        sent = ' '.join(words)
+                        if sent and sent[-1] not in '.!?':
+                            sent += '.'
+            
+            # Ensure first letter capitalization ONLY after sentence endings
+            if i > 0 and sent and sent[0].islower():
+                # Check if previous sentence ended with punctuation
+                if fixed_sentences and fixed_sentences[-1].rstrip().endswith(('.', '!', '?')):
+                    sent = sent[0].upper() + sent[1:]
+            elif i == 0 and sent and sent[0].islower():
+                # First sentence should be capitalized
+                sent = sent[0].upper() + sent[1:]
+            
+            fixed_sentences.append(sent)
+        
+        result = ' '.join(fixed_sentences)
+        
+        # Add human-like variations
+        result = self.human_variations.add_human_touch(result)
+        result = self.human_variations.add_minor_errors(result)
+        result = self.human_variations.add_originality_specific_patterns(result)
+        
+        return result
+    
+    def fix_basic_punctuation_errors(self, text):
+        """Fix only the most egregious punctuation errors"""
+        if not text:
+            return text
+        
+        # Fix double spaces (human-like error)
+        text = re.sub(r'\s{2,}', ' ', text)
+        
+        # Fix space before punctuation (common error)
+        text = re.sub(r'\s+([.,!?;:])', r'\1', text)
+        
+        # Fix missing space after punctuation (human-like)
+        text = re.sub(r'([.,!?])([A-Z])', r'\1 \2', text)
+        
+        # Fix accidental double punctuation
+        text = re.sub(r'([.!?])\1+', r'\1', text)
+        
+        # Fix "i" capitalization (common human error to fix)
+        text = re.sub(r'\bi\b', 'I', text)
+        
+        return text
+    
+    def preserve_natural_variations(self, text):
+        """Keep some natural human-like variations"""
+        # Don't fix everything - leave some variety
+        # Only fix if really broken
+        if text.count('.') == 0 and len(text.split()) > 20:
+            # Long text with no periods - needs fixing
+            words = text.split()
+            # Add periods every 15-25 words naturally (more variation)
+            new_text = []
+            for i, word in enumerate(words):
+                new_text.append(word)
+                if i > 0 and i % random.randint(12, 25) == 0:
+                    if word[-1] not in '.!?,;:':
+                        new_text[-1] = word + '.'
+                        # Capitalize next word if it's not an acronym
+                        if i + 1 < len(words) and words[i + 1][0].islower():
+                            # Check if it's not likely an acronym
+                            if not words[i + 1].isupper():
+                                words[i + 1] = words[i + 1][0].upper() + words[i + 1][1:]
+            text = ' '.join(new_text)
+        
+        return text
+    
+    def smart_fix(self, text):
+        """Apply minimal fixes to maintain human-like quality"""
+        # Apply fixes in order of importance
+        text = self.fix_basic_punctuation_errors(text)
+        text = self.fix_incomplete_sentences_only(text)
+        text = self.preserve_natural_variations(text)
+        
+        return text
+
+class EnhancedDipperHumanizer:
+    def __init__(self):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        print(f"Using device: {self.device}")
+        
+        # Clear GPU cache
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        
+        # Initialize grammar fixer
+        self.grammar_fixer = SelectiveGrammarFixer()
+        
+        # Try to load spaCy if available
+        self.nlp = None
+        self.use_spacy = False
+        if SPACY_AVAILABLE:
+            try:
+                self.nlp = spacy.load("en_core_web_sm")
+                self.use_spacy = True
+                print("spaCy loaded successfully")
+            except:
+                print("spaCy model not found, using NLTK for sentence splitting")
+        
+        try:
+            # Load Dipper paraphraser WITHOUT 8-bit quantization for better performance
+            print("Loading Dipper paraphraser model...")
+            self.tokenizer = T5Tokenizer.from_pretrained('google/t5-v1_1-xxl')
+            self.model = T5ForConditionalGeneration.from_pretrained(
+                "kalpeshk2011/dipper-paraphraser-xxl",
+                device_map="auto",  # This will distribute across 4xL40S automatically
+                torch_dtype=torch.float16,
+                low_cpu_mem_usage=True
+            )
+            print("Dipper model loaded successfully!")
+            self.is_dipper = True
+            
+        except Exception as e:
+            print(f"Error loading Dipper model: {str(e)}")
+            print("Falling back to Flan-T5-XL...")
+            self.is_dipper = False
+            
+            # Fallback to Flan-T5-XL
+            try:
+                self.model = T5ForConditionalGeneration.from_pretrained(
+                    "google/flan-t5-xl",
+                    torch_dtype=torch.float16,
+                    low_cpu_mem_usage=True,
+                    device_map="auto"
+                )
+                self.tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-xl")
+                print("Loaded Flan-T5-XL as fallback")
+            except:
+                raise Exception("Could not load any model. Please check your system resources.")
+        
+        # Load BART as secondary model
+        try:
+            print("Loading BART model for additional variation...")
+            self.bart_model = AutoModelForSeq2SeqLM.from_pretrained(
+                "eugenesiow/bart-paraphrase",
+                torch_dtype=torch.float16,
+                device_map="auto"  # Distribute across GPUs
+            )
+            self.bart_tokenizer = AutoTokenizer.from_pretrained("eugenesiow/bart-paraphrase")
+            self.use_bart = True
+            print("BART model loaded successfully")
+        except:
+            print("BART model not available")
+            self.use_bart = False
+    
+    def preserve_keywords(self, text, keywords):
+        """Mark keywords to preserve them during paraphrasing"""
+        if not keywords:
+            return text, {}
+        
+        # Create a mapping of placeholders to keywords
+        keyword_map = {}
+        modified_text = text
+        
+        # Sort keywords by length (longest first) to avoid partial replacements
+        sorted_keywords = sorted(keywords, key=len, reverse=True)
+        
+        for i, keyword in enumerate(sorted_keywords):
+            # Use unique markers that won't be confused
+            placeholder = f"__KW{i:03d}__"  # e.g., __KW001__
+            
+            # Find all occurrences of the keyword (case-insensitive)
+            pattern = r'\b' + re.escape(keyword) + r'\b'
+            matches = list(re.finditer(pattern, modified_text, flags=re.IGNORECASE))
+            
+            if matches:
+                # Replace all occurrences with the placeholder
+                for match in reversed(matches):  # Reverse to maintain positions
+                    original_keyword = match.group(0)
+                    start, end = match.span()
+                    modified_text = modified_text[:start] + placeholder + modified_text[end:]
+                    # Store the original case version
+                    keyword_map[placeholder] = original_keyword
+        
+        return modified_text, keyword_map
+    
+    def restore_keywords_robust(self, text, keyword_map):
+        """Restore keywords with more flexible pattern matching"""
+        if not keyword_map:
+            return text
+        
+        restored_text = text
+        
+        # Debug: print what we're working with
+        print(f"Restoring keywords in text: {restored_text[:100]}...")
+        print(f"Keyword map: {keyword_map}")
+        
+        # First pass: Direct placeholder replacement
+        for placeholder, keyword in keyword_map.items():
+            if placeholder in restored_text:
+                print(f"Found exact placeholder {placeholder}, replacing with {keyword}")
+                restored_text = restored_text.replace(placeholder, keyword)
+        
+        # Second pass: Handle any mangled placeholders
+        # The model might alter placeholders in various ways
+        for placeholder, keyword in keyword_map.items():
+            # Extract the number from placeholder
+            match = re.search(r'__KW(\d+)__', placeholder)
+            if match:
+                num = match.group(1)
+                
+                # Various patterns the model might create
+                patterns = [
+                    f'__KW{num}__',
+                    f'__ KW{num}__',
+                    f'__KW {num}__',
+                    f'__ KW {num} __',
+                    f'_KW{num}_',
+                    f'_kw{num}_',  # lowercase with single underscore
+                    f'KW{num}',
+                    f'KW {num}',
+                    f'__kw{num}__',  # lowercase variant
+                    f'__Kw{num}__',  # mixed case
+                    f'__ kw{num}__',
+                    f'__KW{num}_',   # missing underscore
+                    f'_KW{num}__',   # missing underscore
+                    f'kw{num}',      # just lowercase
+                    f'___',          # Sometimes model reduces to just underscores
+                    f'____',         # Various underscore patterns
+                    f'_____',
+                    f'__ __',
+                    f'___ ___',
+                ]
+                
+                for pattern in patterns:
+                    if pattern in restored_text:
+                        print(f"Found pattern '{pattern}', replacing with {keyword}")
+                        restored_text = restored_text.replace(pattern, keyword)
+        
+        # Third pass: Use regex to catch any remaining variations
+        # This catches cases where the model might have added characters
+        for placeholder, keyword in keyword_map.items():
+            match = re.search(r'__KW(\d+)__', placeholder)
+            if match:
+                num = match.group(1)
+                # Regex to match various mangled versions including single underscore
+                regex_patterns = [
+                    rf'_+\s*[Kk][Ww]\s*{num}\s*_*',  # Any underscores, case insensitive
+                    rf'[Kk][Ww]\s*{num}(?!\d)',       # KW followed by the number
+                    rf'__?\s*[Kk][Ww]\s*{num}\s*__?', # Optional underscores
+                    rf'_[Kk][Ww]{num}_',              # Single underscore version
+                    rf'_+\s*{num}\s*_*',              # Just the number with underscores
+                    rf'__+',                          # Multiple underscores (fallback)
+                ]
+                
+                for pattern in regex_patterns:
+                    matches = list(re.finditer(pattern, restored_text, flags=re.IGNORECASE))
+                    if matches:
+                        print(f"Found regex pattern '{pattern}' {len(matches)} times")
+                        # Replace from end to beginning to maintain positions
+                        for match in reversed(matches):
+                            restored_text = restored_text[:match.start()] + keyword + restored_text[match.end():]
+        
+        # Fourth pass: Look for common patterns where model mangles placeholders
+        # Sometimes the model turns __KW002__ into things like "___ University" or "___ College__"
+        underscore_patterns = [
+            (r'___+\s*[Uu]niversity', keyword + ' University') if 'universit' in keyword.lower() else None,
+            (r'___+\s*[Cc]ollege__?', keyword + ' College') if 'college' in keyword.lower() else None,
+            (r'___+\s*[Ss]chool', keyword + ' School') if 'school' in keyword.lower() else None,
+            (r'___+', keyword),  # Generic underscore replacement
+        ]
+        
+        for pattern_tuple in underscore_patterns:
+            if pattern_tuple:
+                pattern, replacement = pattern_tuple
+                if re.search(pattern, restored_text):
+                    print(f"Found underscore pattern '{pattern}', replacing with {replacement}")
+                    restored_text = re.sub(pattern, replacement, restored_text)
+        
+        # Final safety check: Look for any remaining placeholder-like patterns
+        remaining_underscores = re.findall(r'_{2,}', restored_text)
+        if remaining_underscores:
+            print(f"Warning: Found remaining underscore patterns: {remaining_underscores}")
+            # If we still have multiple underscores and we have keywords, do a simple replacement
+            # This is aggressive but necessary when model completely mangles placeholders
+            if '___' in restored_text and keyword_map:
+                # Replace the first occurrence of multiple underscores with each keyword
+                for placeholder, keyword in keyword_map.items():
+                    if '___' in restored_text:
+                        restored_text = restored_text.replace('___', keyword, 1)
+        
+        # Log final result
+        print(f"Final restored text: {restored_text[:100]}...")
+        
+        return restored_text
+    
+    def should_skip_element(self, element, text):
+        """Determine if an element should be skipped from paraphrasing"""
+        if not text or len(text.strip()) < 3:
+            return True
+            
+        # Skip JavaScript code inside script tags
+        parent = element.parent
+        if parent and parent.name in ['script', 'style', 'noscript']:
+            return True
+            
+        # Skip headings (h1-h6)
+        if parent and parent.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title']:
+            return True
+            
+        # Skip content inside <strong> and <b> tags
+        if parent and parent.name in ['strong', 'b']:
+            return True
+            
+        # Skip table content
+        if parent and (parent.name in ['td', 'th'] or any(p.name == 'table' for p in parent.parents)):
+            return True
+            
+        # Special handling for content inside tables
+        # Skip if it's inside strong/b/h1-h6 tags AND also inside a table
+        if parent:
+            # Check if we're inside a table
+            is_in_table = any(p.name == 'table' for p in parent.parents)
+            if is_in_table:
+                # If we're in a table, skip any text that's inside formatting tags
+                if parent.name in ['strong', 'b', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'em', 'i']:
+                    return True
+                # Also check if parent's parent is a formatting tag
+                if parent.parent and parent.parent.name in ['strong', 'b', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                    return True
+            
+        # Skip table of contents
+        if parent:
+            parent_text = str(parent).lower()
+            if any(toc in parent_text for toc in ['table of contents', 'toc-', 'contents']):
+                return True
+                
+        # Skip CTAs and buttons
+        if parent and parent.name in ['button', 'a']:
+            return True
+            
+        # Skip if parent has onclick or other event handlers
+        if parent and parent.attrs:
+            event_handlers = ['onclick', 'onchange', 'onsubmit', 'onload', 'onmouseover', 'onmouseout']
+            if any(handler in parent.attrs for handler in event_handlers):
+                return True
+            
+        # Special check for testimonial cards - check up to 3 levels of ancestors
+        if parent:
+            ancestors_to_check = []
+            current = parent
+            for _ in range(3):  # Check up to 3 levels up
+                if current:
+                    ancestors_to_check.append(current)
+                    current = current.parent
+            
+            # Check if any ancestor has testimonial-card class
+            for ancestor in ancestors_to_check:
+                if ancestor and ancestor.get('class'):
+                    classes = ancestor.get('class', [])
+                    if isinstance(classes, list):
+                        if any('testimonial-card' in str(cls) for cls in classes):
+                            return True
+                    elif isinstance(classes, str) and 'testimonial-card' in classes:
+                        return True
+        
+        # Skip if IMMEDIATE parent or element itself has skip-worthy classes/IDs
+        skip_indicators = [
+            'cta-', 'button', 'btn', 'heading', 'title', 'caption', 
+            'toc-', 'contents', 'quiz', 'tip', 'note', 'alert', 
+            'warning', 'info', 'success', 'error', 'code', 'pre',
+            'stats-grid', 'testimonial-card', 'highlight-box',
+            'cta-box', 'quiz-container', 'news-box', 'contact-form',
+            'faq-question', 'sidebar', 'widget', 'banner', 'news-section',
+            'author-intro', 'testimonial', 'review', 'feedback',
+            'floating-', 'stat-', 'progress-', 'option', 'results',
+            'question-container', 'quiz-', 'faq-',
+            'comparision-tables', 'process-flowcharts', 'infographics', 'cost-breakdown'
+        ]
+        
+        # Check only immediate parent and grandparent (not all ancestors)
+        elements_to_check = [parent]
+        if parent and parent.parent:
+            elements_to_check.append(parent.parent)
+            
+        for elem in elements_to_check:
+            if not elem:
+                continue
+                
+            # Check element's class
+            elem_class = elem.get('class', [])
+            if isinstance(elem_class, list):
+                class_str = ' '.join(str(cls).lower() for cls in elem_class)
+                if any(indicator in class_str for indicator in skip_indicators):
+                    return True
+                    
+            # Check element's ID
+            elem_id = elem.get('id', '')
+            if any(indicator in str(elem_id).lower() for indicator in skip_indicators):
+                return True
+                
+        # Skip short phrases that might be UI elements
+        word_count = len(text.split())
+        if word_count <= 5:
+            ui_patterns = [
+                'click', 'download', 'learn more', 'read more', 'sign up', 
+                'get started', 'try now', 'buy now', 'next', 'previous', 
+                'back', 'continue', 'submit', 'cancel', 'get now', 'book your',
+                'check out:', 'see also:', 'related:', 'question', 'of'
+            ]
+            if any(pattern in text.lower() for pattern in ui_patterns):
+                return True
+                
+        # Skip very short content in styled containers
+        if parent and parent.name in ['div', 'section', 'aside', 'blockquote']:
+            style = parent.get('style', '')
+            if 'border' in style or 'background' in style:
+                if word_count <= 20:
+                    # But don't skip if it's inside a paragraph
+                    if not any(p.name == 'p' for p in parent.parents):
+                        return True
+                    
+        return False
+    
+    def is_likely_acronym_or_proper_noun(self, word):
+        """Check if a word is likely an acronym or part of a proper noun"""
+        # Common acronyms and abbreviations
+        acronyms = {'MBA', 'CEO', 'USA', 'UK', 'GMAT', 'GRE', 'SAT', 'ACT', 'PhD', 'MD', 'IT', 'AI', 'ML'}
+        
+        # Check if it's in our acronym list
+        if word.upper() in acronyms:
+            return True
+        
+        # Check if it's all caps (likely acronym)
+        if word.isupper() and len(word) > 1:
+            return True
+        
+        # Check if it follows patterns like "Edition", "Focus", etc. that often come after proper nouns
+        proper_noun_continuations = {
+            'Edition', 'Version', 'Series', 'Focus', 'System', 'Method', 'School', 
+            'University', 'College', 'Institute', 'Academy', 'Center', 'Centre'
+        }
+        
+        if word in proper_noun_continuations:
+            return True
+        
+        return False
+    
+    def clean_model_output_enhanced(self, text):
+        """Enhanced cleaning that preserves more natural structure"""
+        if not text:
+            return ""
+        
+        # Store original for fallback
+        original = text
+        
+        # Remove ONLY clear model artifacts
+        text = re.sub(r'^lexical\s*=\s*\d+\s*,\s*order\s*=\s*\d+\s*', '', text, flags=re.IGNORECASE)
+        text = re.sub(r'<sent>\s*', '', text, flags=re.IGNORECASE)
+        text = re.sub(r'\s*</sent>', '', text, flags=re.IGNORECASE)
+        
+        # Only remove clear prefixes
+        if text.lower().startswith('paraphrase:'):
+            text = text[11:].strip()
+        elif text.lower().startswith('rewrite:'):
+            text = text[8:].strip()
+        
+        # Remove leading non-letter characters carefully
+        # IMPORTANT: Preserve keyword placeholders
+        if not re.match(r'^__KW\d+__', text):
+            # Only remove if it doesn't start with a placeholder
+            text = re.sub(r'^[^a-zA-Z_]+', '', text)
+        
+        # If we accidentally removed too much, use original
+        if len(text) < len(original) * 0.5:
+            text = original
+        
+        return text.strip()
+    
+    def paraphrase_with_dipper(self, text, lex_diversity=60, order_diversity=20, keywords=None):
+        """Paraphrase text using Dipper model with sentence-level processing"""
+        if not text or len(text.strip()) < 3:
+            return text
+        
+        # Preserve keywords
+        text_with_placeholders, keyword_map = self.preserve_keywords(text, keywords)
+        
+        # Add debug logging
+        if keyword_map:
+            print(f"Debug: Created keyword map: {keyword_map}")
+            print(f"Debug: Text with placeholders: {text_with_placeholders[:100]}...")
+        
+        # Split into sentences for better control
+        sentences = self.split_into_sentences_advanced(text_with_placeholders)
+        paraphrased_sentences = []
+        
+        for sentence in sentences:
+            if len(sentence.strip()) < 3:
+                paraphrased_sentences.append(sentence)
+                continue
+                
+            try:
+                # Adjust diversity based on presence of keywords
+                has_keywords = any(placeholder in sentence for placeholder in keyword_map.keys())
+                if has_keywords:
+                    # Use MODERATE diversity when keywords are present to avoid mangling
+                    lex_diversity = 40  # Reduced from 70
+                    order_diversity = 10  # Reduced from 20
+                elif len(sentence.split()) < 10:
+                    lex_diversity = 70  # Reduced from 80
+                    order_diversity = 25  # Reduced from 30
+                else:
+                    lex_diversity = 85  # Slightly reduced from 90
+                    order_diversity = 35  # Slightly reduced from 40
+                
+                lex_code = int(100 - lex_diversity)
+                order_code = int(100 - order_diversity)
+                
+                # Format input for Dipper
+                if self.is_dipper:
+                    input_text = f"lexical = {lex_code}, order = {order_code} <sent> {sentence} </sent>"
+                else:
+                    input_text = f"paraphrase: {sentence}"
+                
+                # Tokenize
+                inputs = self.tokenizer(
+                    input_text,
+                    return_tensors="pt",
+                    max_length=512,
+                    truncation=True,
+                    padding=True
+                )
+                
+                # Move to device
+                if hasattr(self.model, 'device_map') and self.model.device_map:
+                    device = next(iter(self.model.device_map.values()))
+                    inputs = {k: v.to(device) for k, v in inputs.items()}
+                else:
+                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                
+                # Generate with appropriate variation based on keywords
+                original_length = len(sentence.split())
+                max_new_length = int(original_length * 1.3)  # Reduced from 1.4
+                
+                # Adjust temperature based on keywords
+                temp = 0.9 if has_keywords else 1.1  # Lower temp for keywords
+                top_p_val = 0.95 if has_keywords else 0.9
+                
+                with torch.no_grad():
+                    outputs = self.model.generate(
+                        **inputs,
+                        max_length=max_new_length + 20,
+                        min_length=max(5, int(original_length * 0.7)),
+                        do_sample=True,
+                        top_p=top_p_val,
+                        temperature=temp,
+                        no_repeat_ngram_size=3,
+                        num_beams=3 if has_keywords else 2,  # More beams for stability with keywords
+                        early_stopping=True
+                    )
+                
+                # Decode
+                paraphrased = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+                
+                # Clean model artifacts
+                paraphrased = self.clean_model_output_enhanced(paraphrased)
+                
+                # Fix incomplete sentences
+                paraphrased = self.fix_incomplete_sentence_smart(paraphrased, sentence)
+                
+                # Ensure reasonable length
+                if len(paraphrased.split()) > max_new_length:
+                    paraphrased = ' '.join(paraphrased.split()[:max_new_length])
+                
+                paraphrased_sentences.append(paraphrased)
+                
+            except Exception as e:
+                print(f"Error paraphrasing sentence: {str(e)}")
+                paraphrased_sentences.append(sentence)
+        
+        # Join sentences back
+        result = ' '.join(paraphrased_sentences)
+        
+        # Debug before restoration
+        if keyword_map:
+            print(f"Debug: Result before restoration: {result[:100]}...")
+            print(f"Debug: Checking for placeholders...")
+            for placeholder in keyword_map.keys():
+                if placeholder in result:
+                    print(f"Debug: Found placeholder {placeholder} in result")
+                else:
+                    # Check for mangled versions
+                    if '___' in result:
+                        print(f"Debug: Found underscores ___ instead of {placeholder}")
+        
+        # Restore keywords AFTER joining all sentences
+        result = self.restore_keywords_robust(result, keyword_map)
+        
+        # Debug after restoration
+        if keyword_map:
+            print(f"Debug: Result after restoration: {result[:100]}...")
+        
+        # Apply minimal grammar fixes with human variations
+        result = self.grammar_fixer.smart_fix(result)
+        
+        return result
+    
+    def fix_incomplete_sentence_smart(self, generated, original):
+        """Smarter sentence completion that maintains natural flow"""
+        if not generated or not generated.strip():
+            return original
+        
+        generated = generated.strip()
+        
+        # Check if the sentence seems complete semantically
+        words = generated.split()
+        if len(words) >= 3:
+            # Check if last word is a good ending word
+            last_word = words[-1].lower().rstrip('.,!?;:')
+            
+            # Common ending words that might not need punctuation fix
+            ending_words = {
+                'too', 'also', 'well', 'though', 'however',
+                'furthermore', 'moreover', 'indeed', 'anyway',
+                'regardless', 'nonetheless', 'therefore', 'thus'
+            }
+            
+            # If it ends with a good word, just add appropriate punctuation
+            if last_word in ending_words:
+                if generated[-1] not in '.!?':
+                    generated += '.'
+                return generated
+        
+        # Check for cut-off patterns
+        if len(words) > 0:
+            last_word = words[-1]
+            
+            # Remove if it's clearly cut off (1-2 chars, no vowels)
+            # But don't remove valid short words like "is", "of", "to", etc.
+            short_valid_words = {'is', 'of', 'to', 'in', 'on', 'at', 'by', 'or', 'if', 'so', 'up', 'no', 'we', 'he', 'me', 'be', 'do', 'go'}
+            if (len(last_word) <= 2 and 
+                last_word.lower() not in short_valid_words and
+                not any(c in 'aeiouAEIOU' for c in last_word)):
+                words = words[:-1]
+                generated = ' '.join(words)
+        
+        # Add ending punctuation based on context
+        if generated and generated[-1] not in '.!?:,;':
+            # Check original ending
+            orig_stripped = original.strip()
+            if orig_stripped.endswith('?'):
+                # Check if generated seems like a question
+                question_words = ['what', 'why', 'how', 'when', 'where', 'who', 'which', 'is', 'are', 'do', 'does', 'can', 'could', 'would', 'should']
+                first_word = generated.split()[0].lower() if generated.split() else ''
+                if first_word in question_words:
+                    generated += '?'
+                else:
+                    generated += '.'
+            elif orig_stripped.endswith('!'):
+                # Check if generated seems exclamatory
+                exclaim_words = ['amazing', 'incredible', 'fantastic', 'terrible', 'awful', 'wonderful', 'excellent']
+                if any(word in generated.lower() for word in exclaim_words):
+                    generated += '!'
+                else:
+                    generated += '.'
+            elif orig_stripped.endswith(':'):
+                generated += ':'
+            else:
+                generated += '.'
+        
+        # Ensure first letter is capitalized ONLY if it's sentence start
+        # Don't capitalize words like "iPhone" or "eBay" or placeholders
+        if generated and generated[0].islower() and not self.is_likely_acronym_or_proper_noun(generated.split()[0]) and not generated.startswith('__KW'):
+            generated = generated[0].upper() + generated[1:]
+        
+        return generated
+    
+    def split_into_sentences_advanced(self, text):
+        """Advanced sentence splitting using spaCy or NLTK"""
+        if self.use_spacy and self.nlp:
+            doc = self.nlp(text)
+            sentences = [sent.text.strip() for sent in doc.sents]
+        else:
+            # Fallback to NLTK
+            try:
+                sentences = sent_tokenize(text)
+            except:
+                # Final fallback to regex
+                sentences = re.split(r'(?<=[.!?])\s+', text)
+        
+        # Clean up sentences
+        return [s for s in sentences if s and len(s.strip()) > 0]
+    
+    def paraphrase_with_bart(self, text, keywords=None):
+        """Additional paraphrasing with BART for more variation"""
+        if not self.use_bart or not text or len(text.strip()) < 3:
+            return text
+            
+        try:
+            # Preserve keywords
+            text_with_placeholders, keyword_map = self.preserve_keywords(text, keywords)
+            
+            # Process in smaller chunks for BART
+            sentences = self.split_into_sentences_advanced(text_with_placeholders)
+            paraphrased_sentences = []
+            
+            for sentence in sentences:
+                if len(sentence.split()) < 5:
+                    paraphrased_sentences.append(sentence)
+                    continue
+                    
+                inputs = self.bart_tokenizer(
+                    sentence,
+                    return_tensors='pt',
+                    max_length=128,
+                    truncation=True
+                )
+                
+                # Move to appropriate device
+                if hasattr(self.bart_model, 'device_map') and self.bart_model.device_map:
+                    device = next(iter(self.bart_model.device_map.values()))
+                    inputs = {k: v.to(device) for k, v in inputs.items()}
+                else:
+                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                
+                original_length = len(sentence.split())
+                
+                with torch.no_grad():
+                    outputs = self.bart_model.generate(
+                        **inputs,
+                        max_length=int(original_length * 1.4) + 10,
+                        min_length=max(5, int(original_length * 0.6)),
+                        num_beams=2,
+                        temperature=1.1,  # Higher temperature
+                        do_sample=True,
+                        top_p=0.9,
+                        early_stopping=True
+                    )
+                
+                paraphrased = self.bart_tokenizer.decode(outputs[0], skip_special_tokens=True)
+                
+                # Fix incomplete sentences
+                paraphrased = self.fix_incomplete_sentence_smart(paraphrased, sentence)
+                
+                paraphrased_sentences.append(paraphrased)
+            
+            result = ' '.join(paraphrased_sentences)
+            
+            # Restore keywords AFTER joining all sentences
+            result = self.restore_keywords_robust(result, keyword_map)
+            
+            # Apply minimal grammar fixes
+            result = self.grammar_fixer.smart_fix(result)
+            
+            return result
+            
+        except Exception as e:
+            print(f"Error in BART paraphrasing: {str(e)}")
+            return text
+    
+    def apply_sentence_variation(self, text):
+        """Apply natural sentence structure variations - MORE AGGRESSIVE"""
+        sentences = self.split_into_sentences_advanced(text)
+        varied_sentences = []
+        
+        for i, sentence in enumerate(sentences):
+            # Skip empty sentences
+            if not sentence.strip():
+                continue
+            
+            # MORE aggressive variations
+            # Combine short sentences more often (50% chance)
+            if (i < len(sentences) - 1 and 
+                len(sentence.split()) < 15 and 
+                len(sentences[i+1].split()) < 15 and 
+                random.random() < 0.5):
+                
+                connectors = [', and', ', but', '; however,', '. Also,', '. Plus,', ', so', ', which means',
+                             ' - and', ' - but', '; meanwhile,', '. That said,', ', yet', ' - though']
+                connector = random.choice(connectors)
+                
+                # Handle the next sentence properly
+                next_sent = sentences[i+1].strip()
+                if next_sent:
+                    combined = f"{sentence.rstrip('.')}{connector} {next_sent[0].lower()}{next_sent[1:]}"
+                    varied_sentences.append(combined)
+                    sentences[i+1] = ""  # Mark as processed
+            
+            elif sentence:  # Only process non-empty sentences
+                # Split very long sentences more aggressively
+                if len(sentence.split()) > 18 and ',' in sentence:
+                    parts = sentence.split(', ', 1)
+                    if len(parts) == 2 and len(parts[1].split()) > 6:
+                        # 70% chance to split
+                        if random.random() < 0.7:
+                            varied_sentences.append(parts[0] + '.')
+                            # Ensure second part starts with capital
+                            if parts[1]:
+                                varied_sentences.append(parts[1][0].upper() + parts[1][1:])
+                        else:
+                            varied_sentences.append(sentence)
+                    else:
+                        varied_sentences.append(sentence)
+                else:
+                    # Add natural variations more often (35% chance)
+                    if i > 0 and random.random() < 0.35:
+                        # Sometimes add a transition
+                        transitions = ['Furthermore, ', 'Additionally, ', 'Moreover, ', 'Also, ', 
+                                     'Besides, ', 'What\'s more, ', 'In addition, ', 'Not only that, ',
+                                     'To add to that, ', 'On top of that, ', 'Beyond that, ']
+                        transition = random.choice(transitions)
+                        if sentence[0].isupper():
+                            sentence = transition + sentence[0].lower() + sentence[1:]
+                    
+                    # Add mid-sentence interruptions (10% chance)
+                    if random.random() < 0.1 and len(sentence.split()) > 12:
+                        interruptions = [
+                            " - and this is crucial - ",
+                            " - believe me - ",
+                            " - no kidding - ",
+                            " (and yes, I mean it) ",
+                            " - stay with me here - ",
+                            " - and I'm not exaggerating - "
+                        ]
+                        words = sentence.split()
+                        pos = random.randint(len(words)//3, 2*len(words)//3)
+                        words.insert(pos, random.choice(interruptions))
+                        sentence = ' '.join(words)
+                    
+                    varied_sentences.append(sentence)
+        
+        # Post-process for additional human patterns
+        result = ' '.join([s for s in varied_sentences if s])
+        
+        # Add occasional fragments for human touch (5% chance)
+        if random.random() < 0.05:
+            fragments = [
+                "Crazy, I know.",
+                "Wild stuff.",
+                "Makes you think.",
+                "Pretty interesting.",
+                "Go figure.",
+                "Who knew?",
+                "There you have it.",
+                "Food for thought.",
+                "Just saying.",
+                "Worth considering."
+            ]
+            sentences = result.split('. ')
+            if len(sentences) > 3:
+                insert_pos = random.randint(1, len(sentences)-1)
+                sentences.insert(insert_pos, random.choice(fragments))
+                result = '. '.join(sentences)
+        
+        return result
+    
+    def fix_punctuation(self, text):
+        """Comprehensive punctuation and formatting fixes"""
+        if not text:
+            return ""
+        
+        # First, clean any remaining model artifacts
+        text = self.clean_model_output_enhanced(text)
+        
+        # Fix weird symbols and characters using safe replacements
+        text = text.replace('<>', '')  # Remove empty angle brackets
+        
+        # Normalize quotes - use replace instead of regex for problematic characters
+        text = text.replace('«', '"').replace('»', '"')
+        text = text.replace('„', '"').replace('"', '"').replace('"', '"')
+        text = text.replace(''', "'").replace(''', "'")
+        text = text.replace('–', '-').replace('—', '-')
+        
+        # Fix colon issues
+        text = re.sub(r'\.:', ':', text)  # Remove period before colon
+        text = re.sub(r':\s*\.', ':', text)  # Remove period after colon
+        
+        # Fix basic spacing
+        text = re.sub(r'\s+', ' ', text)  # Multiple spaces to single
+        text = re.sub(r'\s+([.,!?;:])', r'\1', text)  # Remove space before punctuation
+        text = re.sub(r'([.,!?;:])\s*([.,!?;:])', r'\1', text)  # Remove double punctuation
+        text = re.sub(r'([.!?])\s*\1+', r'\1', text)  # Remove repeated punctuation
+        
+        # Fix colons
+        text = re.sub(r':\s*([.,!?])', ':', text)  # Remove punctuation after colon
+        text = re.sub(r'([.,!?])\s*:', ':', text)  # Remove punctuation before colon
+        text = re.sub(r':+', ':', text)  # Multiple colons to one
+        
+        # Fix quotes and parentheses
+        text = re.sub(r'"\s*([^"]*?)\s*"', r'"\1"', text)
+        text = re.sub(r"'\s*([^']*?)\s*'", r"'\1'", text)
+        text = re.sub(r'\(\s*([^)]*?)\s*\)', r'(\1)', text)
+        
+        # Fix sentence capitalization more carefully
+        # Split on ACTUAL sentence endings only
+        sentences = re.split(r'(?<=[.!?])\s+', text)
+        fixed_sentences = []
+        
+        for i, sentence in enumerate(sentences):
+            if not sentence:
+                continue
+            
+            # Only capitalize the first letter if it's actually lowercase
+            # and not part of a special case (like iPhone, eBay, etc.)
+            words = sentence.split()
+            if words:
+                first_word = words[0]
+                # Check if it's not an acronym or proper noun that should stay lowercase
+                if (first_word[0].islower() and 
+                    not self.is_likely_acronym_or_proper_noun(first_word) and 
+                    not first_word.startswith('__KW') and
+                    not first_word.startswith('_kw')):
+                    # Only capitalize if it's a regular word
+                    sentence = first_word[0].upper() + first_word[1:] + ' ' + ' '.join(words[1:])
+            
+            fixed_sentences.append(sentence)
+        
+        text = ' '.join(fixed_sentences)
+        
+        # Fix common issues
+        text = re.sub(r'\bi\b', 'I', text)  # Capitalize 'I'
+        text = re.sub(r'\.{2,}', '.', text)  # Multiple periods to one
+        text = re.sub(r',{2,}', ',', text)  # Multiple commas to one
+        text = re.sub(r'\s*,\s*,\s*', ', ', text)  # Double commas with spaces
+        
+        # Remove weird artifacts
+        text = re.sub(r'\b(CHAPTER\s+[IVX]+|SECTION\s+\d+)\b[^\w]*', '', text, flags=re.IGNORECASE)
+        
+        # Fix abbreviations
+        text = re.sub(r'\betc\s*\.\s*\.', 'etc.', text)
+        text = re.sub(r'\be\.g\s*\.\s*[,\s]', 'e.g., ', text)
+        text = re.sub(r'\bi\.e\s*\.\s*[,\s]', 'i.e., ', text)
+        
+        # Fix numbers with periods (like "1. " at start of lists)
+        text = re.sub(r'(\d+)\.\s+', r'\1. ', text)
+        
+        # Fix bold/strong tags punctuation
+        text = self.fix_bold_punctuation(text)
+        
+        # Clean up any remaining issues
+        text = re.sub(r'\s+([.,!?;:])', r'\1', text)  # Final space cleanup
+        text = re.sub(r'([.,!?;:])\s{2,}', r'\1 ', text)  # Fix multiple spaces after punctuation
+        
+        # Ensure ending punctuation
+        text = text.strip()
+        if text and text[-1] not in '.!?':
+            # Don't add period if it ends with colon (likely a list header)
+            if not text.endswith(':'):
+                text += '.'
+        
+        return text
+    
+    def fix_bold_punctuation(self, text):
+        """Fix punctuation issues around bold/strong tags"""
+        # Check if this is likely a list item with colon pattern
+        def is_list_item_with_colon(text):
+            # Pattern: starts with or contains <strong>Text:</strong> or <b>Text:</b>
+            list_pattern = r'^\s*(?:[-•*▪▫◦‣⁃]\s*)?<(?:strong|b)>[^<]+:</(?:strong|b)>'
+            return bool(re.search(list_pattern, text))
+        
+        # If it's a list item with colon, preserve the format
+        if is_list_item_with_colon(text):
+            # Just clean up spacing but preserve the colon inside bold
+            text = re.sub(r'<(strong|b)>\s*([^:]+)\s*:\s*</\1>', r'<\1>\2:</\1>', text)
+            return text
+        
+        # Pattern to find bold/strong content
+        bold_pattern = r'<(strong|b)>(.*?)</\1>'
+        
+        def fix_bold_match(match):
+            tag = match.group(1)
+            content = match.group(2).strip()
+            
+            if not content:
+                return f'<{tag}></{tag}>'
+            
+            # Check if this is a list header (contains colon at the end)
+            if content.endswith(':'):
+                # Preserve list headers with colons
+                return f'<{tag}>{content}</{tag}>'
+            
+            # Remove any periods at the start or end of bold content
+            content = content.strip('.')
+            
+            # Check if this bold text is at the start of a sentence
+            # (preceded by nothing, or by '. ', '! ', '? ')
+            start_pos = match.start()
+            is_sentence_start = (start_pos == 0 or 
+                               (start_pos > 2 and text[start_pos-2:start_pos] in ['. ', '! ', '? ', '\n\n']))
+            
+            # Capitalize first letter if it's at sentence start
+            if is_sentence_start and content and content[0].isalpha():
+                content = content[0].upper() + content[1:]
+            
+            return f'<{tag}>{content}</{tag}>'
+        
+        # Fix bold/strong tags
+        text = re.sub(bold_pattern, fix_bold_match, text)
+        
+        # Fix spacing around bold/strong tags (but not for list items)
+        if not is_list_item_with_colon(text):
+            text = re.sub(r'\.\s*<(strong|b)>', r'. <\1>', text)  # Period before bold
+            text = re.sub(r'</(strong|b)>\s*\.', r'</\1>.', text)  # Period after bold
+            text = re.sub(r'([.!?])\s*<(strong|b)>', r'\1 <\2>', text)  # Space after sentence end
+            text = re.sub(r'</(strong|b)>\s+([a-z])', lambda m: f'</{m.group(1)}> {m.group(2)}', text)  # Keep lowercase after bold if mid-sentence
+            
+            # Remove duplicate periods around bold tags
+            text = re.sub(r'\.\s*</(strong|b)>\s*\.', r'</\1>.', text)
+            text = re.sub(r'\.\s*<(strong|b)>\s*\.', r'. <\1>', text)
+            
+            # Fix cases where bold content ends a sentence
+            # If bold is followed by a new sentence (capital letter), add period
+            text = re.sub(r'</(strong|b)>\s+([A-Z])', r'</\1>. \2', text)
+        
+        # Don't remove these for list items
+        if not is_list_item_with_colon(text):
+            text = re.sub(r'<(strong|b)>\s*:\s*</\1>', ':', text)  # Remove empty bold colons
+            text = re.sub(r'<(strong|b)>\s*\.\s*</\1>', '.', text)  # Remove empty bold periods
+        
+        return text
+    
+    def extract_text_from_html(self, html_content):
+        """Extract text elements from HTML with skip logic"""
+        soup = BeautifulSoup(html_content, 'html.parser')
+        text_elements = []
+        
+        # Get all text nodes using string instead of text (fixing deprecation)
+        for element in soup.find_all(string=True):
+            # Skip script, style, and noscript content completely
+            if element.parent.name in ['script', 'style', 'noscript']:
+                continue
+            
+            text = element.strip()
+            if text and not self.should_skip_element(element, text):
+                text_elements.append({
+                    'text': text,
+                    'element': element
+                })
+        
+        return soup, text_elements
+    
+    def validate_and_fix_html(self, html_text):
+        """Fix common HTML syntax errors after processing"""
+        
+        # Fix DOCTYPE
+        html_text = re.sub(r'<!\s*DOCTYPE', '<!DOCTYPE', html_text, flags=re.IGNORECASE)
+        
+        # Fix spacing issues
+        html_text = re.sub(r'>\s+<', '><', html_text)  # Remove extra spaces between tags
+        html_text = re.sub(r'\s+>', '>', html_text)  # Remove spaces before closing >
+        html_text = re.sub(r'<\s+', '<', html_text)  # Remove spaces after opening <
+        
+        # Fix common word errors that might occur during processing
+        html_text = html_text.replace('down loaded', 'downloaded')
+        html_text = html_text.replace('But your document', 'Your document')
+        
+        return html_text
+    
+    def wrap_keywords_in_paragraphs(self, soup, keywords):
+        """Wrap keywords with <strong> tags inside <p> tags only"""
+        if not keywords:
+            return
+        
+        # Find all paragraph tags
+        for p_tag in soup.find_all('p'):
+            # Skip paragraphs that are inside special elements
+            # Check if paragraph is inside any of these elements
+            skip_parents = ['div.author-intro', 'div.cta-box', 'div.testimonial-card', 
+                          'div.news-box', 'button', 'a', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+                          'div.quiz-container', 'div.question-container', 'div.results']
+            
+            # Check if this paragraph should be skipped
+            should_skip = False
+            for parent in p_tag.parents:
+                # Check by class
+                if parent.name == 'div' and parent.get('class'):
+                    classes = parent.get('class', [])
+                    if isinstance(classes, list):
+                        class_str = ' '.join(str(cls) for cls in classes)
+                    else:
+                        class_str = str(classes)
+                    
+                    if any(skip_class in class_str for skip_class in 
+                          ['author-intro', 'cta-box', 'testimonial-card', 'news-box', 
+                           'quiz-container', 'question-container', 'results', 'stats-grid',
+                           'toc-', 'comparison-tables']):
+                        should_skip = True
+                        break
+                
+                # Check by tag name
+                if parent.name in ['button', 'a', 'blockquote', 'details', 'summary']:
+                    should_skip = True
+                    break
+            
+            if should_skip:
+                continue
+            
+            # Additional check: Skip if paragraph has specific classes
+            p_classes = p_tag.get('class', [])
+            if isinstance(p_classes, list):
+                p_class_str = ' '.join(str(cls) for cls in p_classes)
+            else:
+                p_class_str = str(p_classes)
+            
+            if any(skip_class in p_class_str for skip_class in ['testimonial-card', 'quiz-', 'stat-']):
+                continue
+            
+            # Process only if this is a regular content paragraph
+            # Get all text nodes in this paragraph
+            for text_node in p_tag.find_all(string=True):
+                # Skip if already inside a strong or b tag
+                if text_node.parent.name in ['strong', 'b', 'em', 'i', 'span', 'a']:
+                    continue
+                
+                # Skip if the text node's immediate parent isn't the p tag
+                # (to avoid nested elements)
+                if text_node.parent != p_tag:
+                    continue
+                
+                original_text = str(text_node)
+                
+                # Skip very short text nodes
+                if len(original_text.strip()) < 20:
+                    continue
+                
+                modified_text = original_text
+                
+                # Check each keyword
+                for keyword in keywords:
+                    # Use word boundaries for accurate matching
+                    pattern = r'\b' + re.escape(keyword) + r'\b'
+                    
+                    # Find all matches (case-insensitive)
+                    matches = list(re.finditer(pattern, modified_text, flags=re.IGNORECASE))
+                    
+                    # Replace from end to beginning to maintain positions
+                    for match in reversed(matches):
+                        start, end = match.span()
+                        matched_text = match.group(0)
+                        # Wrap with strong tag
+                        modified_text = (modified_text[:start] + 
+                                       f'<strong>{matched_text}</strong>' + 
+                                       modified_text[end:])
+                
+                # If text was modified, replace the text node
+                if modified_text != original_text:
+                    # Parse the modified text to create new nodes
+                    new_soup = BeautifulSoup(modified_text, 'html.parser')
+                    # Replace the text node with the new nodes
+                    for new_node in reversed(new_soup.contents):
+                        text_node.insert_after(new_node)
+                    text_node.extract()
+    
+    def add_natural_flow_variations(self, text):
+        """Add more natural flow and rhythm variations for Originality AI"""
+        sentences = self.split_into_sentences_advanced(text)
+        enhanced_sentences = []
+        
+        for i, sentence in enumerate(sentences):
+            if not sentence.strip():
+                continue
+            
+            # Add stream-of-consciousness elements (10% chance)
+            if random.random() < 0.1 and len(sentence.split()) > 10:
+                stream_elements = [
+                    " - wait, let me back up - ",
+                    " - actually, scratch that - ",
+                    " - or maybe I should say - ",
+                    " - hmm, how do I put this - ",
+                    " - okay, here's the thing - ",
+                    " - you know what I mean? - "
+                ]
+                words = sentence.split()
+                pos = random.randint(len(words)//4, 3*len(words)//4)
+                words.insert(pos, random.choice(stream_elements))
+                sentence = ' '.join(words)
+            
+            # Add human-like self-corrections (5% chance)
+            if random.random() < 0.05:
+                corrections = [
+                    " - or rather, ",
+                    " - well, actually, ",
+                    " - I mean, ",
+                    " - or should I say, ",
+                    " - correction: "
+                ]
+                words = sentence.split()
+                if len(words) > 8:
+                    pos = random.randint(len(words)//2, len(words)-3)
+                    correction = random.choice(corrections)
+                    # Repeat a concept with variation
+                    repeated_word_idx = random.randint(max(0, pos-5), pos-1)
+                    if repeated_word_idx < len(words):
+                        words.insert(pos, correction)
+                sentence = ' '.join(words)
+            
+            # Add thinking-out-loud patterns (8% chance)
+            if random.random() < 0.08 and i > 0:
+                thinking_patterns = [
+                    "Come to think of it, ",
+                    "Actually, you know what? ",
+                    "Wait, here's a thought: ",
+                    "Oh, and another thing - ",
+                    "Speaking of which, ",
+                    "This reminds me, ",
+                    "Now that I mention it, ",
+                    "Funny you should ask, because "
+                ]
+                pattern = random.choice(thinking_patterns)
+                sentence = pattern + sentence[0].lower() + sentence[1:] if len(sentence) > 1 else sentence
+            
+            enhanced_sentences.append(sentence)
+        
+        return ' '.join(enhanced_sentences)
+    
+    def process_html(self, html_content, primary_keywords="", secondary_keywords="", progress_callback=None):
+        """Main processing function with progress callback"""
+        if not html_content.strip():
+            return "Please provide HTML content."
+        
+        # Store all script and style content to preserve it
+        script_placeholder = "###SCRIPT_PLACEHOLDER_{}###"
+        style_placeholder = "###STYLE_PLACEHOLDER_{}###"
+        preserved_scripts = []
+        preserved_styles = []
+        
+        # Temporarily replace script and style tags with placeholders
+        soup_temp = BeautifulSoup(html_content, 'html.parser')
+        
+        # Preserve all script tags
+        for idx, script in enumerate(soup_temp.find_all('script')):
+            placeholder = script_placeholder.format(idx)
+            preserved_scripts.append(str(script))
+            script.replace_with(placeholder)
+        
+        # Preserve all style tags
+        for idx, style in enumerate(soup_temp.find_all('style')):
+            placeholder = style_placeholder.format(idx)
+            preserved_styles.append(str(style))
+            style.replace_with(placeholder)
+        
+        # Get the modified HTML
+        html_content = str(soup_temp)
+        
+        # Combine keywords and clean them
+        all_keywords = []
+        if primary_keywords:
+            # Clean and validate each keyword
+            for k in primary_keywords.split(','):
+                cleaned = k.strip()
+                if cleaned and len(cleaned) > 1:  # Skip empty or single-char keywords
+                    all_keywords.append(cleaned)
+        if secondary_keywords:
+            for k in secondary_keywords.split(','):
+                cleaned = k.strip()
+                if cleaned and len(cleaned) > 1:
+                    all_keywords.append(cleaned)
+        
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_keywords = []
+        for k in all_keywords:
+            if k.lower() not in seen:
+                seen.add(k.lower())
+                unique_keywords.append(k)
+        all_keywords = unique_keywords
+        
+        try:
+            # Extract text elements
+            soup, text_elements = self.extract_text_from_html(html_content)
+            
+            total_elements = len(text_elements)
+            print(f"Found {total_elements} text elements to process (after filtering)")
+            if all_keywords:
+                print(f"Preserving keywords: {all_keywords}")
+            
+            # Process each text element
+            processed_count = 0
+            
+            for i, element_info in enumerate(text_elements):
+                original_text = element_info['text']
+                
+                # Skip placeholders
+                if "###SCRIPT_PLACEHOLDER_" in original_text or "###STYLE_PLACEHOLDER_" in original_text:
+                    continue
+                
+                # Skip very short texts
+                if len(original_text.split()) < 3:
+                    continue
+                
+                # Debug: Check if keywords are in this text
+                text_has_keywords = any(keyword.lower() in original_text.lower() for keyword in all_keywords)
+                if text_has_keywords:
+                    print(f"Debug: Processing text with keywords: {original_text[:50]}...")
+                
+                # First pass with Dipper (with adjusted diversity)
+                paraphrased_text = self.paraphrase_with_dipper(
+                    original_text,
+                    keywords=all_keywords
+                )
+                
+                # Verify no placeholders remain
+                if '__KW' in paraphrased_text or '___' in paraphrased_text:
+                    print(f"Warning: Placeholder or underscores found in paraphrased text: {paraphrased_text[:100]}...")
+                    # Try to restore again with the enhanced function
+                    temp_map = {}
+                    for j, keyword in enumerate(all_keywords):
+                        temp_map[f'__KW{j:03d}__'] = keyword
+                    paraphrased_text = self.restore_keywords_robust(paraphrased_text, temp_map)
+                
+                # Second pass with BART for longer texts (increased probability)
+                if self.use_bart and len(paraphrased_text.split()) > 8:
+                    # 50% chance to use BART for more variation (reduced from 60%)
+                    if random.random() < 0.5:
+                        paraphrased_text = self.paraphrase_with_bart(
+                            paraphrased_text,
+                            keywords=all_keywords
+                        )
+                
+                # Apply sentence variation
+                paraphrased_text = self.apply_sentence_variation(paraphrased_text)
+                
+                # Add natural flow variations
+                paraphrased_text = self.add_natural_flow_variations(paraphrased_text)
+                
+                # Fix punctuation and formatting
+                paraphrased_text = self.fix_punctuation(paraphrased_text)
+                
+                # Final check for any remaining placeholders or underscores
+                if '___' in paraphrased_text or '__KW' in paraphrased_text:
+                    print(f"Error: Unresolved placeholders in final text")
+                    # Use original text if we can't resolve placeholders
+                    paraphrased_text = original_text
+                
+                # Final quality check
+                if paraphrased_text and len(paraphrased_text.split()) >= 3:
+                    element_info['element'].replace_with(NavigableString(paraphrased_text))
+                    processed_count += 1
+                
+                # Progress update
+                if progress_callback:
+                    progress_callback(i + 1, total_elements)
+                
+                if i % 10 == 0 or i == total_elements - 1:
+                    progress = (i + 1) / total_elements * 100
+                    print(f"Progress: {progress:.1f}%")
+            
+            # Wrap keywords with <strong> tags in paragraphs
+            self.wrap_keywords_in_paragraphs(soup, all_keywords)
+            
+            # Post-process the entire HTML to fix bold/strong formatting
+            result = str(soup)
+            result = self.post_process_html(result)
+            
+            # Final safety check for any remaining placeholders or underscores
+            if '__KW' in result or re.search(r'_{3,}', result):
+                print("Warning: Found placeholders or multiple underscores in final HTML output")
+                # Attempt to clean them with keywords
+                for i, keyword in enumerate(all_keywords):
+                    result = result.replace(f'__KW{i:03d}__', keyword)
+                    result = re.sub(r'_{3,}', keyword, result, count=1)
+            
+            # Restore all script tags
+            for idx, script_content in enumerate(preserved_scripts):
+                placeholder = script_placeholder.format(idx)
+                result = result.replace(placeholder, script_content)
+            
+            # Restore all style tags
+            for idx, style_content in enumerate(preserved_styles):
+                placeholder = style_placeholder.format(idx)
+                result = result.replace(placeholder, style_content)
+            
+            # Validate and fix HTML syntax
+            result = self.validate_and_fix_html(result)
+            
+            # Count skipped elements properly
+            all_text_elements = soup.find_all(string=True)
+            skipped = len([e for e in all_text_elements if e.strip() and e.parent.name not in ['script', 'style', 'noscript']]) - total_elements
+            
+            print(f"Successfully processed {processed_count} text elements")
+            print(f"Skipped {skipped} elements (headings, CTAs, tables, testimonials, strong/bold tags, etc.)")
+            print(f"Preserved {len(preserved_scripts)} script tags and {len(preserved_styles)} style tags")
+            
+            return result
+            
+        except Exception as e:
+            import traceback
+            error_msg = f"Error processing HTML: {str(e)}\n{traceback.format_exc()}"
+            print(error_msg)
+            # Return original HTML with error message prepended as HTML comment
+            return f"<!-- {error_msg} -->\n{html_content}"
+    
+    def post_process_html(self, html_text):
+        """Post-process the entire HTML to fix formatting issues"""
+        # Fix empty angle brackets that might appear
+        html_text = re.sub(r'<>\s*([^<>]+?)\s*(?=\.|\s|<)', r'\1', html_text)  # Remove <> around text
+        html_text = re.sub(r'<>', '', html_text)  # Remove any remaining empty <>
+        
+        # Fix double angle brackets around bold tags
+        html_text = re.sub(r'<<b>>', '<b>', html_text)
+        html_text = re.sub(r'<</b>>', '</b>', html_text)
+        html_text = re.sub(r'<<strong>>', '<strong>', html_text)
+        html_text = re.sub(r'<</strong>>', '</strong>', html_text)
+        
+        # Fix periods around bold/strong tags
+        html_text = re.sub(r'\.\s*<(b|strong)>', '. <\1>', html_text)  # Period before bold
+        html_text = re.sub(r'</(b|strong)>\s*\.', '</\1>.', html_text)  # Period after bold
+        html_text = re.sub(r'\.<<(b|strong)>>', '. <\1>', html_text)  # Fix double bracket cases
+        html_text = re.sub(r'</(b|strong)>>\.', '</\1>.', html_text)
+        
+        # Fix periods after colons
+        html_text = re.sub(r':\s*\.', ':', html_text)
+        html_text = re.sub(r'\.:', ':', html_text)
+        
+        # Check if a line is a list item
+        def process_line(line):
+            # Check if this line contains a list pattern with bold
+            list_pattern = r'(?:^|\s)(?:[-•*▪▫◦‣⁃]\s*)?<(?:strong|b)>[^<]+:</(?:strong|b)>'
+            if re.search(list_pattern, line):
+                # This is a list item, preserve the colon format
+                return line
+            
+            # Not a list item, apply regular fixes
+            # Remove periods immediately inside bold tags
+            line = re.sub(r'<(strong|b)>\s*\.\s*([^<]+)\s*\.\s*</\1>', r'<\1>\2</\1>', line)
+            
+            # Fix sentence endings with bold
+            line = re.sub(r'</(strong|b)>\s*([.!?])', r'</\1>\2', line)
+            
+            return line
+        
+        # Process line by line to preserve list formatting
+        lines = html_text.split('\n')
+        processed_lines = [process_line(line) for line in lines]
+        html_text = '\n'.join(processed_lines)
+        
+        # Fix sentence starts with bold
+        def fix_bold_sentence_start(match):
+            pre_context = match.group(1)
+            tag = match.group(2)
+            content = match.group(3)
+            
+            # Skip if this is part of a list item with colon
+            full_match = match.group(0)
+            if ':' in full_match and '</' + tag + '>' in full_match:
+                return full_match
+            
+            # Check if this should start with capital
+            if pre_context == '' or pre_context.endswith(('.', '!', '?', '>')):
+                if content and content[0].islower():
+                    content = content[0].upper() + content[1:]
+            
+            return f'{pre_context}<{tag}>{content}'
+        
+        # Look for bold/strong tags and check their context
+        html_text = re.sub(r'(^|.*?)(<(?:strong|b)>)([a-zA-Z])', fix_bold_sentence_start, html_text)
+        
+        # Clean up spacing around bold tags (but preserve list formatting)
+        # Split into segments to handle list items separately
+        segments = re.split(r'(<(?:strong|b)>[^<]*:</(?:strong|b)>)', html_text)
+        cleaned_segments = []
+        
+        for i, segment in enumerate(segments):
+            if i % 2 == 1:  # This is a list item pattern
+                cleaned_segments.append(segment)
+            else:
+                # Apply spacing fixes to non-list segments
+                segment = re.sub(r'\s+<(strong|b)>', r' <\1>', segment)
+                segment = re.sub(r'</(strong|b)>\s+', r'</\1> ', segment)
+                # Fix punctuation issues
+                segment = re.sub(r'([.,!?;:])\s*([.,!?;:])', r'\1', segment)
+                # Fix periods inside/around bold
+                segment = re.sub(r'\.<(strong|b)>\.', '. <\1>', segment)
+                segment = re.sub(r'\.</(strong|b)>\.', '</\1>.', segment)
+                cleaned_segments.append(segment)
+        
+        html_text = ''.join(cleaned_segments)
+        
+        # Final cleanup
+        html_text = re.sub(r'\.{2,}', '.', html_text)  # Multiple periods
+        html_text = re.sub(r',{2,}', ',', html_text)  # Multiple commas
+        html_text = re.sub(r':{2,}', ':', html_text)  # Multiple colons
+        html_text = re.sub(r'\s+([.,!?;:])', r'\1', html_text)  # Space before punctuation
+        
+        # Fix empty bold tags (but not those with just colons)
+        html_text = re.sub(r'<(strong|b)>\s*</\1>', '', html_text)
+        
+        # Fix specific patterns in lists/stats
+        # Pattern like "5,000+" should not have period after
+        html_text = re.sub(r'(\d+[,\d]*\+?)\s*\.\s*\n', r'\1\n', html_text)
+        
+        # Clean up any remaining double brackets
+        html_text = re.sub(r'<<', '<', html_text)
+        html_text = re.sub(r'>>', '>', html_text)
+        
+        # Apply final minimal grammar fixes
+        html_text = self.grammar_fixer.smart_fix(html_text)
+        
+        return html_text
+
+# Initialize the humanizer
+humanizer = EnhancedDipperHumanizer()
+
+def humanize_html(html_input, primary_keywords="", secondary_keywords="", progress=gr.Progress()):
+    """Gradio interface function with progress updates"""
+    if not html_input:
+        return "Please provide HTML content to humanize."
+    
+    progress(0, desc="Starting processing...")
+    start_time = time.time()
+    
+    # Create a wrapper to update progress
+    def progress_callback(current, total):
+        if total > 0:
+            progress(current / total, desc=f"Processing: {current}/{total} elements")
+    
+    # Pass progress callback to process_html
+    result = humanizer.process_html(
+        html_input, 
+        primary_keywords, 
+        secondary_keywords,
+        progress_callback=progress_callback
+    )
+    
+    processing_time = time.time() - start_time
+    print(f"Processing completed in {processing_time:.2f} seconds")
+    progress(1.0, desc="Complete!")
+    
+    return result
+
+# Create Gradio interface with queue
+iface = gr.Interface(
+    fn=humanize_html,
+    inputs=[
+        gr.Textbox(
+            lines=10,
+            placeholder="Paste your HTML content here...",
+            label="HTML Input"
+        ),
+        gr.Textbox(
+            placeholder="Enter primary keywords separated by commas (e.g., GMAT Focus Edition, MBA, Data Insights)",
+            label="Primary Keywords (preserved exactly)"
+        ),
+        gr.Textbox(
+            placeholder="Enter secondary keywords separated by commas (e.g., test preparation, business school)",
+            label="Secondary Keywords (preserved exactly)"
+        )
+    ],
+    outputs=gr.Textbox(
+        lines=10,
+        label="Humanized HTML Output"
+    ),
+    title="Enhanced Dipper AI Humanizer - Optimized for Originality AI",
+    description="""
+    Ultra-aggressive humanizer optimized to achieve 100% human scores on both Undetectable AI and Originality AI.
+    
+    Key Features:
+    - Maximum diversity settings (90% lexical, 40% order) for natural variation
+    - Enhanced human patterns: personal opinions, self-corrections, thinking-out-loud
+    - Natural typos, contractions, and conversational flow
+    - Stream-of-consciousness elements and rhetorical questions
+    - Originality AI-specific optimizations: varied sentence starters, emphatic repetitions
+    - Fixed placeholder system that preserves keywords
+    - Keywords inside <p> tags are automatically wrapped with <strong> tags
+    - Skips content in <strong>, <b>, and heading tags (including inside tables)
+    - Designed to pass the strictest AI detection systems
+    
+    The tool creates genuinely human-like writing patterns that fool even the most sophisticated detectors!
+    
+    ⚠️ Note: Processing may take 5-10 minutes for large HTML documents.
+    """,
+    examples=[
+        ["""<article>
+<h1>The Benefits of Regular Exercise</h1>
+<div class="author-intro">By John Doe, Fitness Expert | 10 years experience</div>
+<p>Regular exercise is essential for maintaining good health. It helps improve cardiovascular fitness, strengthens muscles, and enhances mental well-being. Studies have shown that people who exercise regularly have lower risks of chronic diseases.</p>
+<p>Additionally, exercise can boost mood and energy levels. It releases endorphins, which are natural mood elevators. Even moderate activities like walking can make a significant difference in overall health.</p>
+</article>""", "cardiovascular fitness, mental well-being, chronic diseases", "exercise, health, endorphins"]
+    ],
+    theme="default"
+)
+
+if __name__ == "__main__":
+    # Enable queue for better handling of long-running processes
+    iface.queue(max_size=10)
+    iface.launch(share=True)