Update app.py

app.py

@@ -1,114 +1,4 @@
-def run_tool(source_url, target_url, anchor_text, smart_rewrite, plain_text, suggest_alternative_anchor):
-    if not source_url or not target_url or not anchor_text:
-        return "❌ Please provide Source URL, Target URL, and Anchor Text."
-
-    # Auto-correct swapped inputs
-    warn = ""
-    if looks_like_url(anchor_text) and not looks_like_url(target_url):
-        anchor_text, target_url = target_url, anchor_text
-        warn = "ℹ️ Detected swapped inputs. I used the URL as Target URL and the text as Anchor.\n\n"
-
-    source_url = normalize_url(source_url)
-    target_url = normalize_url(target_url)
-
-    try:
-        # Get source blocks for later use
-        source_blocks = get_text_blocks(source_url)
-        
-        # Only get 1 result, not multiple!
-        results = suggest_insertions(source_url, target_url, anchor_text, top_k=1, suggest_alternative=suggest_alternative_anchor)
-        res = results[0]
-    except Exception as e:
-        return f"❌ Error processing the page: {str(e)}"
-
-    if "error" in res:
-        return f"❌ {res['error']}"
-    
-    # Detect language from the original sentence
-    original_sentence = res['best_sentence_original']
-    detected_lang = detect_language(original_sentence)
-    language_name = get_language_name(detected_lang)
-    print(f"Detected language: {language_name} ({detected_lang})")
-
-    # Process original anchor suggestion
-    draft_html = res["best_sentence_with_anchor"]
-
-    # Check if anchor was already present in the article
-    anchor_was_present = res.get("anchor_was_present", False)
-    keyword_in_article = res.get("keyword_in_article", False)
-    
-    # If anchor is present in the article (even if not in the best sentence)
-    if keyword_in_article:
-        # Anchor exists somewhere in article
-        if anchor_was_present:
-            # Anchor is in the suggested sentence - just show where to add the link
-            final_output = to_plain_text(draft_html) if plain_text else draft_html
-            result = warn + f"✅ **Anchor text '{anchor_text}' found in article!**\n\n"
-            result += f"🔗 Add link here:\n\n"
-            result += f"{final_output}"
-        else:
-            # Anchor is in article but not in this sentence
-            if smart_rewrite:
-                g = gpt_rewrite(draft_html, anchor_text, target_url, style="neutral", language=language_name)
-                final_html = g["sentence_html"]
-            else:
-                final_html = draft_html
-            
-            polished = gpt_validate_and_polish(final_html, anchor_text, target_url, language=language_name)
-            final_html = polished.get("sentence_html", final_html)
-            final_output = to_plain_text(final_html) if plain_text else final_html
-            
-            result = warn + f"✅ **Anchor text '{anchor_text}' found in article!**\n\n"
-            result += f"🔗 Add link here:\n\n"
-            result += f"{final_output}"
-    else:
-        # Anchor doesn't exist in article at all - need to add it
-        if smart_rewrite:
-            g = gpt_rewrite(draft_html, anchor_text, target_url, style="neutral", language=language_name)
-            final_html = g["sentence_html"]
-        else:
-            final_html = draft_html
-
-        polished = gpt_validate_and_polish(final_html, anchor_text, target_url, language=language_name)
-        final_html = polished.get("sentence_html", final_html)
-        final_output = to_plain_text(final_html) if plain_text else final_html
-
-        result = warn + f"⚠️ **Anchor text '{anchor_text}' not found in article**\n\n"
-        result += f"🔗 Result 1 - Suggested placement:\n\n"
-        result += f"Change this sentence: {original_sentence}\n\n"
-        result += f"With this one: {final_output}"
-        
-        # Show alternative if requested and available
-        if suggest_alternative_anchor and res.get("alternative_anchor"):
-            alt_anchor = res["alternative_anchor"]
-            alt_content = res.get("alternative_sentence", "")  # This now contains position info + content
-            
-            if alt_content:
-                # The content already has the link included from GPT
-                # Parse the position text and actual content
-                if "[Insert after paragraph" in alt_content:
-                    parts = alt_content.split("\n\n", 1)
-                    position_and_sentence = parts[0] if len(parts) > 0 else ""
-                    actual_content = parts[1] if len(parts) > 1 else alt_content
-                else:
-                    position_and_sentence = ""
-                    actual_content = alt_content
-                
-                alt_output = to_plain_text(actual_content) if plain_text else actual_content
-                
-                # Add alternative as Result 2
-                result += f"\n\n{'='*50}\n\n"
-                result += f"🔗 Result 2 - Suggested new anchor with placement:\n"
-                result += f"💡 Using keyword: '{alt_anchor}'\n"
-                
-                if position_and_sentence:
-                    result += f"\n{position_and_sentence}\n\n"
-                
-                result += f"Add this: {alt_output}"
-    
-    return result:
-                    # No paragraph info available
-                    result += f"\n{alt_output}"import os, re, json, requests, urllib.parse
+import os, re, json, requests, urllib.parse
 import torch, torch.nn.functional as F
 from bs4 import BeautifulSoup
 from transformers import AutoTokenizer, AutoModel
@@ -417,11 +307,11 @@ def find_alternative_anchor(blocks, target_url, original_anchor):
         chosen_keyword = result.get("chosen_keyword", keywords[0] if keywords else original_anchor)
         new_content = result.get("new_content", "")
         insert_after = result.get("insert_after_paragraph", 0)
-        after_sentence = result.get("after_sentence", "")
+        reasoning = result.get("reasoning", "")
         
         # Format the response for compatibility
         # Return: (anchor_text, formatted_content_with_position)
-        position_text = f"[Insert after paragraph {insert_after + 1}]\nAfter this sentence: {after_sentence}"
+        position_text = f"[Insert after paragraph {insert_after + 1}]: {reasoning}"
         
         return chosen_keyword, f"{position_text}\n\n{new_content}"
         
@@ -645,7 +535,7 @@ def gpt_rewrite(sentence_html, anchor_text, target_url, style="neutral", languag
         "(2) Do NOT use an em dash or any dash. "
         '(3) Avoid phrases like "for details", "click here", "learn more", "visit", "read more". '
         "Prefer integrating the anchor as part of the sentence. "
-        f"(4) Write in {language} and preserve ALL special characters (ć, đ, š, ž, č, etc.). "
+        f"(4) Write in {language} and preserve ALL special characters (ć, č, š, ž, đ, etc.). "
         "Return a compact JSON object with key sentence_html only."
     )
     
@@ -770,7 +660,21 @@ def gpt_get_search_keywords(target_content, target_url):
     
     return obj.get("keywords", ["related content"])
 
-ates the most suitable keyword. "
+def gpt_generate_content_with_keyword(source_blocks, keywords, target_url, language="English"):
+    """
+    Generate new content with the best keyword and specify where to insert it.
+    """
+    if not OPENAI_API_KEY or not keywords:
+        return None
+    
+    # Create cache key
+    source_preview = " ".join(source_blocks[:3])[:500]
+    cache_key = hashlib.md5(f"generate_{source_preview}_{str(keywords)}_{target_url}_{language}".encode()).hexdigest()
+    
+    system = (
+        f"You are a skilled content writer writing in {language}. "
+        "Given an article and a list of keywords related to a target page, "
+        "create a NATURAL addition to the article that incorporates the most suitable keyword. "
         "The addition should flow seamlessly with the existing content. "
         "\n\nYOUR TASK:\n"
         "1. Choose the ONE keyword that fits most naturally with the article's context\n"
@@ -891,8 +795,8 @@ def run_tool(source_url, target_url, anchor_text, smart_rewrite, plain_text, sug
 
         result = warn + f"⚠️ **Anchor text '{anchor_text}' not found in article**\n\n"
         result += f"🔗 Result 1 - Suggested placement:\n\n"
-        result += f"Change this sentence: {original_sentence}\n\n"
-        result += f"With this one: {final_output}"
+        result += f"Original: {original_sentence}\n\n"
+        result += f"Suggested: {final_output}"
         
         # Show alternative if requested and available
         if suggest_alternative_anchor and res.get("alternative_anchor"):
@@ -901,47 +805,24 @@ def run_tool(source_url, target_url, anchor_text, smart_rewrite, plain_text, sug
             
             if alt_content:
                 # Parse if there's position information
-                paragraph_num = None
-                actual_content = alt_content
-                
                 if "[Insert after paragraph" in alt_content:
                     parts = alt_content.split("\n\n", 1)
                     position_info = parts[0] if len(parts) > 0 else ""
                     actual_content = parts[1] if len(parts) > 1 else alt_content
-                    
-                    # Extract paragraph number from position_info
-                    import re
-                    match = re.search(r'\[Insert after paragraph (\d+)\]', position_info)
-                    if match:
-                        paragraph_num = int(match.group(1)) - 1  # Convert to 0-based index
+                else:
+                    position_info = ""
+                    actual_content = alt_content
                 
                 # The content already has the link included from GPT
                 alt_output = to_plain_text(actual_content) if plain_text else actual_content
                 
                 # Add alternative as Result 2
                 result += f"\n\n{'='*50}\n\n"
-                result += f"🔗 Result 2 - Suggested new anchor with placement:\n"
+                result += f"🔗 Result 2 - Suggested new content to add:\n"
                 result += f"💡 Using keyword: '{alt_anchor}'\n"
-                
-                # Try to get the actual paragraph sentence to show where to insert
-                if paragraph_num is not None and paragraph_num < len(source_blocks):
-                    try:
-                        # Get the last sentence of the target paragraph
-                        target_paragraph = source_blocks[paragraph_num]
-                        # Split into sentences and get the last one
-                        sentences = re.split(r'(?<=[.!?])\s+', target_paragraph.strip())
-                        last_sentence = sentences[-1] if sentences else target_paragraph[:100] + "..."
-                        
-                        result += f"\n[Insert after paragraph {paragraph_num + 1}]\n"
-                        result += f"After this sentence: {last_sentence}\n\n"
-                        result += f"Add this: {alt_output}"
-                    except:
-                        # Fallback if we can't get the paragraph
-                        result += f"\n[Insert after paragraph {paragraph_num + 1}]\n\n"
-                        result += f"{alt_output}"
-                else:
-                    # No paragraph info available
-                    result += f"\n{alt_output}"
+                if position_info:
+                    result += f"📍 {position_info}\n"
+                result += f"\n{alt_output}"
     
     return result
 
@@ -998,7 +879,7 @@ with gr.Blocks(title=f"Link Insertion Helper • GPT: {gpt_status}") as demo:
     
     gr.Markdown("""
     ### Features:
-    - 🌍 **Auto Language Detection**: Preserves special characters (ć, đ, š, ž, č, etc.)
+    - 🌍 **Auto Language Detection**: Preserves special characters (ć, č, š, ž, đ, etc.)
     - 💾 **Smart Caching**: Caches embeddings and API responses for faster repeated queries
     - 🎯 **Main Content Focus**: Prioritizes first 5-7 paragraphs, ignores author bios
     - 🔄 **Alternative Anchor Suggestion**: When your anchor isn't in the article, suggests better anchors from existing text