import re from html.parser import HTMLParser import os import requests import json import os from html.parser import HTMLParser class HTMLTitleExtractor(HTMLParser): """Extract and body text from HTML""" def __init__(self): super().__init__() self.title = "" self.text_content = [] self.in_title = False self.in_body = False self.skip_tags = {'script', 'style', 'meta', 'link', 'noscript'} self.skip = False def handle_starttag(self, tag, attrs): if tag == 'title': self.in_title = True if tag == 'body': self.in_body = True if tag in self.skip_tags: self.skip = True def handle_endtag(self, tag): if tag == 'title': self.in_title = False if tag == 'body': self.in_body = False if tag in self.skip_tags: self.skip = False def handle_data(self, data): if self.in_title: self.title = data.strip() elif self.in_body and not self.skip: text = data.strip() if text: self.text_content.append(text) def extract_title_and_text(html_content): parser = HTMLTitleExtractor() parser.feed(html_content) return parser.title, " ".join(parser.text_content) def generate_title_with_t5(text, title_tag=None): """Generate high-quality, concise repo title using Gemma summarization""" return generate_t5_summary(text, context=title_tag, max_len=8, min_len=2) _FREE_MODEL_CHAIN_TITLE = [ "google/gemma-4-31b-it:free", "qwen/qwen3-next-80b-a3b-instruct:free", "qwen/qwen3-coder:free", "meta-llama/llama-3.3-8b-instruct:free" ] def generate_t5_summary(text, context=None, max_len=30, min_len=5): """ Generic function to generate summaries (both titles and descriptions) using OpenRouter. Tries a chain of free models with 2-second delays between fallbacks. (Keeps old function name 'generate_t5_summary' for compatibility with other files) """ import time openrouter_key = os.environ.get("OPENROUTER_API_KEY", "") if not openrouter_key: print("Warning: OPENROUTER_API_KEY not set. Using fallback logic.") return None # Clean text text = ' '.join(text.split()) if not text: return None system_prompt = "You are an AI tasked with summarization. Output ONLY the summary text requested with no markdown, quotes, preamble, or conversational filler." if context: user_prompt = f"Context: {context}\nSummarize the purpose of this content concisely (around {max_len} words): {text[:1000]}" else: user_prompt = f"Summarize this text concisely: {text[:1000]}" headers = { "Authorization": f"Bearer {openrouter_key}", "Content-Type": "application/json", "HTTP-Referer": "https://webcraft.com", "X-Title": "WebCraft AI Backend" } messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt} ] for attempt, model in enumerate(_FREE_MODEL_CHAIN_TITLE): if attempt > 0: print(f" Waiting 2s before title fallback attempt {attempt+1}/{len(_FREE_MODEL_CHAIN_TITLE)}: {model}") time.sleep(2) try: response = requests.post( url="https://openrouter.ai/api/v1/chat/completions", headers=headers, data=json.dumps({"model": model, "messages": messages}), timeout=60 ) except Exception as e: print(f"DEBUG: OpenRouter Request Error ({model}): {e}") continue if response.status_code == 200: try: summary = response.json()['choices'][0]['message']['content'].strip() if summary.startswith('"') and summary.endswith('"'): summary = summary[1:-1] return summary except (KeyError, Exception) as e: print(f"DEBUG: Parse error ({model}): {e}") continue else: tag = "Primary" if attempt == 0 else "Fallback" print(f" OpenRouter {tag} Error ({model}): Code {response.status_code}") try: print(f" Reason: {response.json().get('error', {}).get('message', response.text)}") except: print(f" Reason: {response.text}") print("All OpenRouter free models exhausted for summarization.") # ========================== # ULTIMATE FAILOVER: Hugging Face # ========================== hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN", "") if hf_token: hf_model = "microsoft/Phi-3.5-mini-instruct" print(f" [Ultimate Failover] Trying Hugging Face Inference: {hf_model}") try: from huggingface_hub import InferenceClient client = InferenceClient(api_key=hf_token, timeout=30) reply = client.chat_completion(model=hf_model, messages=messages, max_tokens=100) summary = reply.choices[0].message.content.strip() if summary.startswith('"') and summary.endswith('"'): summary = summary[1:-1] return summary except Exception as e: print(f" Hugging Face Request Error: {e}") print(" Using local fallback.") return None def get_repo_title(file_path, openai_api_key=None): # Read HTML file with multiple encodings html_content = None for enc in ['utf-8', 'latin-1', 'cp1252', 'iso-8859-1']: try: with open(file_path, 'r', encoding=enc) as f: html_content = f.read() break except: continue if not html_content: with open(file_path, 'rb') as f: html_content = f.read().decode('utf-8', errors='ignore') title_tag, text_content = extract_title_and_text(html_content) results = { "title_tag": sanitize_for_repo_name(title_tag) if title_tag else None, "ai_title": None, "fallback": None } # Generate AI title using T5 (The Free Way) ai_title = generate_title_with_t5(text_content, title_tag) if ai_title: results["ai_title"] = sanitize_for_repo_name(ai_title) # Determine fallback fallback = os.path.splitext(os.path.basename(file_path))[0] if fallback.lower() in ['index', '', 'home']: folder_name = os.path.basename(os.path.dirname(file_path)) is_temp_dir = folder_name.startswith('tmp') or re.match(r'^[a-z0-9]{8,}$', folder_name) if folder_name.strip() and not is_temp_dir: fallback = folder_name else: fallback = "my-webcraft-site" results["fallback"] = sanitize_for_repo_name(fallback) return results def sanitize_for_repo_name(title): """Convert title to a clean repository name slug""" # 1. Lowercase and replace non-alphanumeric with spaces name = title.lower() name = re.sub(r'[^a-z0-9]', ' ', name) # 2. Tokenize and remove stop words for a cleaner slug stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'is', 'at', 'on', 'of', 'to', 'for', 'with', 'in', 'has', 'was', 'were'} tokens = [t for t in name.split() if t not in stop_words] # 3. Join with hyphens slug = '-'.join(tokens) # 4. Handle length: Don't cut words in the middle if len(slug) > 40: # Cut at the last hyphen before the limit shortened = slug[:40] if '-' in shortened: slug = shortened.rsplit('-', 1)[0] else: slug = shortened # Final cleanup slug = re.sub(r'-+', '-', slug).strip('-') if not slug: slug = "untitled-project" return slug