deploy_tmp/title_generator.py

import re
from html.parser import HTMLParser
import os
import requests
import json
import os
from html.parser import HTMLParser

class HTMLTitleExtractor(HTMLParser):
    """Extract <title> and body text from HTML"""
    def __init__(self):
        super().__init__()
        self.title = ""
        self.text_content = []
        self.in_title = False
        self.in_body = False
        self.skip_tags = {'script', 'style', 'meta', 'link', 'noscript'}
        self.skip = False
    
    def handle_starttag(self, tag, attrs):
        if tag == 'title':
            self.in_title = True
        if tag == 'body':
            self.in_body = True
        if tag in self.skip_tags:
            self.skip = True
    
    def handle_endtag(self, tag):
        if tag == 'title':
            self.in_title = False
        if tag == 'body':
            self.in_body = False
        if tag in self.skip_tags:
            self.skip = False
    
    def handle_data(self, data):
        if self.in_title:
            self.title = data.strip()
        elif self.in_body and not self.skip:
            text = data.strip()
            if text:
                self.text_content.append(text)

def extract_title_and_text(html_content):
    parser = HTMLTitleExtractor()
    parser.feed(html_content)
    return parser.title, " ".join(parser.text_content)

def generate_title_with_t5(text, title_tag=None):
    """Generate high-quality, concise repo title using Gemma summarization"""
    return generate_t5_summary(text, context=title_tag, max_len=8, min_len=2)

_FREE_MODEL_CHAIN_TITLE = [
    "google/gemma-4-31b-it:free",
    "qwen/qwen3-next-80b-a3b-instruct:free",
    "qwen/qwen3-coder:free",
    "meta-llama/llama-3.3-8b-instruct:free"
]

def generate_t5_summary(text, context=None, max_len=30, min_len=5):
    """
    Generic function to generate summaries (both titles and descriptions) using OpenRouter.
    Tries a chain of free models with 2-second delays between fallbacks.
    (Keeps old function name 'generate_t5_summary' for compatibility with other files)
    """
    import time
    openrouter_key = os.environ.get("OPENROUTER_API_KEY", "")
    if not openrouter_key:
        print("Warning: OPENROUTER_API_KEY not set. Using fallback logic.")
        return None

    # Clean text
    text = ' '.join(text.split())
    if not text:
        return None
        
    system_prompt = "You are an AI tasked with summarization. Output ONLY the summary text requested with no markdown, quotes, preamble, or conversational filler."
    if context:
        user_prompt = f"Context: {context}\nSummarize the purpose of this content concisely (around {max_len} words): {text[:1000]}"
    else:
        user_prompt = f"Summarize this text concisely: {text[:1000]}"

    headers = {
        "Authorization": f"Bearer {openrouter_key}",
        "Content-Type": "application/json",
        "HTTP-Referer": "https://webcraft.com",
        "X-Title": "WebCraft AI Backend"
    }
    messages = [
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": user_prompt}
    ]

    for attempt, model in enumerate(_FREE_MODEL_CHAIN_TITLE):
        if attempt > 0:
            print(f"  Waiting 2s before title fallback attempt {attempt+1}/{len(_FREE_MODEL_CHAIN_TITLE)}: {model}")
            time.sleep(2)

        try:
            response = requests.post(
                url="https://openrouter.ai/api/v1/chat/completions",
                headers=headers,
                data=json.dumps({"model": model, "messages": messages}),
                timeout=60
            )
        except Exception as e:
            print(f"DEBUG: OpenRouter Request Error ({model}): {e}")
            continue

        if response.status_code == 200:
            try:
                summary = response.json()['choices'][0]['message']['content'].strip()
                if summary.startswith('"') and summary.endswith('"'):
                    summary = summary[1:-1]
                return summary
            except (KeyError, Exception) as e:
                print(f"DEBUG: Parse error ({model}): {e}")
                continue
        else:
            tag = "Primary" if attempt == 0 else "Fallback"
            print(f"  OpenRouter {tag} Error ({model}): Code {response.status_code}")
            try:
                print(f"  Reason: {response.json().get('error', {}).get('message', response.text)}")
            except:
                print(f"  Reason: {response.text}")

    print("All OpenRouter free models exhausted for summarization.")
    
    # ==========================
    # ULTIMATE FAILOVER: Hugging Face
    # ==========================
    hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN", "")
    if hf_token:
        hf_model = "microsoft/Phi-3.5-mini-instruct"
        print(f"  [Ultimate Failover] Trying Hugging Face Inference: {hf_model}")
        try:
            from huggingface_hub import InferenceClient
            client = InferenceClient(api_key=hf_token, timeout=30)
            reply = client.chat_completion(model=hf_model, messages=messages, max_tokens=100)
            summary = reply.choices[0].message.content.strip()
            if summary.startswith('"') and summary.endswith('"'):
                summary = summary[1:-1]
            return summary
        except Exception as e:
            print(f"  Hugging Face Request Error: {e}")

    print("  Using local fallback.")
    return None




def get_repo_title(file_path, openai_api_key=None):
    
    # Read HTML file with multiple encodings
    html_content = None
    for enc in ['utf-8', 'latin-1', 'cp1252', 'iso-8859-1']:
        try:
            with open(file_path, 'r', encoding=enc) as f:
                html_content = f.read()
            break
        except:
            continue
    
    if not html_content:
        with open(file_path, 'rb') as f:
            html_content = f.read().decode('utf-8', errors='ignore')
    
    title_tag, text_content = extract_title_and_text(html_content)
    
    results = {
        "title_tag": sanitize_for_repo_name(title_tag) if title_tag else None,
        "ai_title": None,
        "fallback": None
    }

    # Generate AI title using T5 (The Free Way)
    ai_title = generate_title_with_t5(text_content, title_tag)
    if ai_title:
        results["ai_title"] = sanitize_for_repo_name(ai_title)

    # Determine fallback
    fallback = os.path.splitext(os.path.basename(file_path))[0]
    if fallback.lower() in ['index', '', 'home']:
        folder_name = os.path.basename(os.path.dirname(file_path))
        is_temp_dir = folder_name.startswith('tmp') or re.match(r'^[a-z0-9]{8,}$', folder_name)
        if folder_name.strip() and not is_temp_dir:
            fallback = folder_name
        else:
            fallback = "my-webcraft-site"
    
    results["fallback"] = sanitize_for_repo_name(fallback)
    
    return results


def sanitize_for_repo_name(title):
    """Convert title to a clean repository name slug"""
    # 1. Lowercase and replace non-alphanumeric with spaces
    name = title.lower()
    name = re.sub(r'[^a-z0-9]', ' ', name)
    
    # 2. Tokenize and remove stop words for a cleaner slug
    stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'is', 'at', 'on', 'of', 'to', 'for', 'with', 'in', 'has', 'was', 'were'}
    tokens = [t for t in name.split() if t not in stop_words]
    
    # 3. Join with hyphens
    slug = '-'.join(tokens)
    
    # 4. Handle length: Don't cut words in the middle
    if len(slug) > 40:
        # Cut at the last hyphen before the limit
        shortened = slug[:40]
        if '-' in shortened:
            slug = shortened.rsplit('-', 1)[0]
        else:
            slug = shortened
            
    # Final cleanup
    slug = re.sub(r'-+', '-', slug).strip('-')
    
    if not slug:
        slug = "untitled-project"
        
    return slug