import re
from html.parser import HTMLParser
import os
import requests
import json
import os
from html.parser import HTMLParser
class HTMLTitleExtractor(HTMLParser):
"""Extract
and body text from HTML"""
def __init__(self):
super().__init__()
self.title = ""
self.text_content = []
self.in_title = False
self.in_body = False
self.skip_tags = {'script', 'style', 'meta', 'link', 'noscript'}
self.skip = False
def handle_starttag(self, tag, attrs):
if tag == 'title':
self.in_title = True
if tag == 'body':
self.in_body = True
if tag in self.skip_tags:
self.skip = True
def handle_endtag(self, tag):
if tag == 'title':
self.in_title = False
if tag == 'body':
self.in_body = False
if tag in self.skip_tags:
self.skip = False
def handle_data(self, data):
if self.in_title:
self.title = data.strip()
elif self.in_body and not self.skip:
text = data.strip()
if text:
self.text_content.append(text)
def extract_title_and_text(html_content):
parser = HTMLTitleExtractor()
parser.feed(html_content)
return parser.title, " ".join(parser.text_content)
def generate_title_with_t5(text, title_tag=None):
"""Generate high-quality, concise repo title using Gemma summarization"""
return generate_t5_summary(text, context=title_tag, max_len=8, min_len=2)
_FREE_MODEL_CHAIN_TITLE = [
"google/gemma-4-31b-it:free",
"qwen/qwen3-next-80b-a3b-instruct:free",
"qwen/qwen3-coder:free",
"meta-llama/llama-3.3-8b-instruct:free"
]
def generate_t5_summary(text, context=None, max_len=30, min_len=5):
"""
Generic function to generate summaries (both titles and descriptions) using OpenRouter.
Tries a chain of free models with 2-second delays between fallbacks.
(Keeps old function name 'generate_t5_summary' for compatibility with other files)
"""
import time
openrouter_key = os.environ.get("OPENROUTER_API_KEY", "")
if not openrouter_key:
print("Warning: OPENROUTER_API_KEY not set. Using fallback logic.")
return None
# Clean text
text = ' '.join(text.split())
if not text:
return None
system_prompt = "You are an AI tasked with summarization. Output ONLY the summary text requested with no markdown, quotes, preamble, or conversational filler."
if context:
user_prompt = f"Context: {context}\nSummarize the purpose of this content concisely (around {max_len} words): {text[:1000]}"
else:
user_prompt = f"Summarize this text concisely: {text[:1000]}"
headers = {
"Authorization": f"Bearer {openrouter_key}",
"Content-Type": "application/json",
"HTTP-Referer": "https://webcraft.com",
"X-Title": "WebCraft AI Backend"
}
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]
for attempt, model in enumerate(_FREE_MODEL_CHAIN_TITLE):
if attempt > 0:
print(f" Waiting 2s before title fallback attempt {attempt+1}/{len(_FREE_MODEL_CHAIN_TITLE)}: {model}")
time.sleep(2)
try:
response = requests.post(
url="https://openrouter.ai/api/v1/chat/completions",
headers=headers,
data=json.dumps({"model": model, "messages": messages}),
timeout=60
)
except Exception as e:
print(f"DEBUG: OpenRouter Request Error ({model}): {e}")
continue
if response.status_code == 200:
try:
summary = response.json()['choices'][0]['message']['content'].strip()
if summary.startswith('"') and summary.endswith('"'):
summary = summary[1:-1]
return summary
except (KeyError, Exception) as e:
print(f"DEBUG: Parse error ({model}): {e}")
continue
else:
tag = "Primary" if attempt == 0 else "Fallback"
print(f" OpenRouter {tag} Error ({model}): Code {response.status_code}")
try:
print(f" Reason: {response.json().get('error', {}).get('message', response.text)}")
except:
print(f" Reason: {response.text}")
print("All OpenRouter free models exhausted for summarization.")
# ==========================
# ULTIMATE FAILOVER: Hugging Face
# ==========================
hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN", "")
if hf_token:
hf_model = "microsoft/Phi-3.5-mini-instruct"
print(f" [Ultimate Failover] Trying Hugging Face Inference: {hf_model}")
try:
from huggingface_hub import InferenceClient
client = InferenceClient(api_key=hf_token, timeout=30)
reply = client.chat_completion(model=hf_model, messages=messages, max_tokens=100)
summary = reply.choices[0].message.content.strip()
if summary.startswith('"') and summary.endswith('"'):
summary = summary[1:-1]
return summary
except Exception as e:
print(f" Hugging Face Request Error: {e}")
print(" Using local fallback.")
return None
def get_repo_title(file_path, openai_api_key=None):
# Read HTML file with multiple encodings
html_content = None
for enc in ['utf-8', 'latin-1', 'cp1252', 'iso-8859-1']:
try:
with open(file_path, 'r', encoding=enc) as f:
html_content = f.read()
break
except:
continue
if not html_content:
with open(file_path, 'rb') as f:
html_content = f.read().decode('utf-8', errors='ignore')
title_tag, text_content = extract_title_and_text(html_content)
results = {
"title_tag": sanitize_for_repo_name(title_tag) if title_tag else None,
"ai_title": None,
"fallback": None
}
# Generate AI title using T5 (The Free Way)
ai_title = generate_title_with_t5(text_content, title_tag)
if ai_title:
results["ai_title"] = sanitize_for_repo_name(ai_title)
# Determine fallback
fallback = os.path.splitext(os.path.basename(file_path))[0]
if fallback.lower() in ['index', '', 'home']:
folder_name = os.path.basename(os.path.dirname(file_path))
is_temp_dir = folder_name.startswith('tmp') or re.match(r'^[a-z0-9]{8,}$', folder_name)
if folder_name.strip() and not is_temp_dir:
fallback = folder_name
else:
fallback = "my-webcraft-site"
results["fallback"] = sanitize_for_repo_name(fallback)
return results
def sanitize_for_repo_name(title):
"""Convert title to a clean repository name slug"""
# 1. Lowercase and replace non-alphanumeric with spaces
name = title.lower()
name = re.sub(r'[^a-z0-9]', ' ', name)
# 2. Tokenize and remove stop words for a cleaner slug
stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'is', 'at', 'on', 'of', 'to', 'for', 'with', 'in', 'has', 'was', 'were'}
tokens = [t for t in name.split() if t not in stop_words]
# 3. Join with hyphens
slug = '-'.join(tokens)
# 4. Handle length: Don't cut words in the middle
if len(slug) > 40:
# Cut at the last hyphen before the limit
shortened = slug[:40]
if '-' in shortened:
slug = shortened.rsplit('-', 1)[0]
else:
slug = shortened
# Final cleanup
slug = re.sub(r'-+', '-', slug).strip('-')
if not slug:
slug = "untitled-project"
return slug